{description} '
+ 'The same environment powers training and evaluation.
'
+ '1,000 training tasks250 test tasks'
+ f'{capability}Isolated workspaces'
+ '
')
+ with gr.Tab("Explore the environment"):
+ with gr.Row(equal_height=False):
+ with gr.Column(scale=4, min_width=310, elem_id="task-panel"):
+ gr.Markdown("### 1. Choose a task")
+ with gr.Row():
+ split = gr.Dropdown([("Test · 250 tasks", "test"), ("Train · 1,000 tasks", "train")],
+ value="test", label="Dataset")
+ index = gr.Number(value=0, minimum=0, maximum=999, precision=0, label="Task index", info="0–249")
+ with gr.Row():
+ inspect = gr.Button("Load task", variant="primary")
+ random = gr.Button("Random task")
+ badge = gr.Markdown("Load a task to see its difficulty.", elem_id="task-badge")
+ instruction = gr.Textbox(label="Your task", lines=18, max_lines=30,
+ interactive=False, elem_id="task-instructions")
+ with gr.Accordion("Task details", open=False):
+ metadata = gr.JSON(label="Task metadata")
+ with gr.Column(scale=6, min_width=400, elem_id="workspace-panel"):
+ if arm == "whitebox":
+ ui = WhiteboxUI()
+ gr.Markdown("### 2. Work with the data\nStart a workspace, inspect the files, and use the tools below.")
+ with gr.Row():
+ begin = gr.Button("Start workspace", variant="primary")
+ close = gr.Button("Close workspace")
+ state = gr.Textbox(label="Workspace status", value="Not started", interactive=False)
+ tool = gr.Dropdown(["bash", "read", "write", "edit", "grep", "glob", "ls"],
+ value="bash", label="Tool")
+ command = gr.Textbox(value="ls -la /workdir", label="Shell command", lines=3)
+ path = gr.Textbox(value="/workdir", label="File or directory path", visible=False)
+ content = gr.Textbox(label="File content / replacement text", lines=5, visible=False)
+ old = gr.Textbox(label="Text to replace", visible=False)
+ execute = gr.Button("Run tool", variant="primary")
+ output = gr.Code(label="Output", language=None, interactive=False, lines=16, elem_id="tool-console")
+ gr.Markdown("### 3. Submit your answer")
+ answer = gr.Textbox(label="Final answer", placeholder="Enter your answer, or leave blank to grade answer.txt.")
+ submit = gr.Button("Submit and grade", variant="primary")
+ gr.Markdown("Your workspace is isolated from other users. Close it when finished; idle sessions expire after 20 minutes.",
+ elem_classes="agent-footnote")
+ tool.change(tool_inputs, tool, [command, path, content, old], api_visibility="private")
+ begin.click(ui.start, [split, index], [instruction, output, state], api_name="start_task")
+ execute.click(ui.run, [tool, command, path, content, old], output, api_name="run_tool")
+ submit.click(ui.grade, answer, [output, state], api_name="grade_task")
+ close.click(ui.close, outputs=[output, state], api_name="close_task")
+ def unload(request: gr.Request):
+ ui.close(request)
+ demo.unload(unload)
+ if use_hf:
+ with gr.Accordion("Let a model solve this task", open=False):
+ hf_provider, hf_model = provider_demo.controls()
+ hf_run = gr.Button("Run model on this task", variant="primary")
+ hf_summary = gr.JSON(label="Agent result")
+ hf_transcript = gr.Code(label="Agent conversation", language="json", lines=14)
+ def run_hf_whitebox(s, i, p, m, oauth_token: gr.OAuthToken, request: gr.Request):
+ yield from provider_demo.whitebox(ui, s, i, p, m, oauth_token, request)
+ hf_run.click(run_hf_whitebox, [split, index, hf_provider, hf_model],
+ [hf_transcript, hf_summary], concurrency_limit=4,
+ concurrency_id="interactive-agents", api_name="run_hf_agent")
+ else:
+ gr.Markdown("### 2. Connect your agent\nUse an OpenAI-compatible model endpoint to run a task.")
+ with gr.Row():
+ harness = gr.Dropdown([("OpenCode", "opencode"), ("Claude Code", "claude-code"),
+ ("Codex", "codex"), ("Mini-SWE-Agent", "mini-swe-agent")],
+ value="opencode", label="Agent harness")
+ model = gr.Textbox(value="Qwen/Qwen3.5-2B", label="Model name")
+ url = gr.Textbox(label="Inference endpoint", placeholder="https://your-endpoint/v1")
+ key = gr.Textbox(label="Inference API key", type="password", placeholder="Only if your endpoint requires authentication")
+ run = gr.Button("Run agent on this task", variant="primary")
+ gr.Markdown("### 3. Inspect the result")
+ summary = gr.JSON(label="Score and run statistics")
+ with gr.Accordion("Agent conversation", open=True):
+ transcript = gr.Code(label="Conversation", language="json", interactive=False, lines=20)
+ run.click(blackbox_run, [split, index, harness, url, model, key], [transcript, summary],
+ concurrency_limit=4, api_name="run_agent")
+ if use_hf:
+ with gr.Accordion("Use your Hugging Face account", open=True):
+ hf_provider, hf_model = provider_demo.controls()
+ hf_run = gr.Button("Run with HF Inference Providers", variant="primary")
+ hf_run.click(provider_demo.blackbox,
+ [split, index, harness, hf_provider, hf_model], [transcript, summary],
+ concurrency_limit=4, concurrency_id="interactive-agents", api_name="run_hf_agent")
+ def load(s, i):
+ return preview(arm, s, i)
+ inspect.click(load, [split, index], [instruction, metadata], api_name="preview_task").then(
+ task_badge, metadata, badge, api_visibility="private")
+ split.change(lambda s: gr.update(value=0, info="0–999" if s == "train" else "0–249"),
+ split, index, api_visibility="private").then(load, [split, index], [instruction, metadata],
+ api_visibility="private").then(task_badge, metadata, badge, api_visibility="private")
+ random.click(lambda s: secrets.randbelow(1000 if s == "train" else 250), split, index,
+ api_visibility="private").then(load, [split, index], [instruction, metadata], api_visibility="private").then(
+ task_badge, metadata, badge, api_visibility="private")
+ demo.load(lambda: preview(arm, "test", 0), outputs=[instruction, metadata], api_visibility="private").then(
+ task_badge, metadata, badge, api_visibility="private")
+ with gr.Tab("Training & evaluation API"):
+ name = "harbor_env" if arm == "blackbox" else "white_box_bash"
+ gr.Markdown(f"### One environment, both datasets\nUse this Space's endpoint for training and evaluation. "
+ "Select the task split per request; run model inference on your own endpoint.\n\n"
+ f"| Task operation | Route |\n| --- | --- |\n| Available datasets | `GET /{name}/splits` |\n"
+ f"| Number of tasks | `POST /{name}/num_tasks` |\n| Task instructions | `POST /{name}/task` |\n"
+ f"| Task range | `POST /{name}/task_range` |\n\n"
+ "The native OpenEnv MCP client executes environment actions. Training and evaluation "
+ "share the task catalog while using separate model endpoints and sandbox sessions.")
+ demo.queue(max_size=1024, default_concurrency_limit=8)
+ return gr.mount_gradio_app(app, demo, path="/", theme=OPENENV_GRADIO_THEME, css=OPENENV_GRADIO_CSS + UI_CSS)
diff --git a/04-data-agent/hf/runtime/eval_evidence.py b/04-data-agent/hf/runtime/eval_evidence.py
new file mode 100644
index 0000000..b7d59b6
--- /dev/null
+++ b/04-data-agent/hf/runtime/eval_evidence.py
@@ -0,0 +1,62 @@
+"""Preserve eval evidence across Job and Space restarts."""
+import hashlib
+import json
+import os
+from pathlib import Path
+
+from common import MODEL, REVISION, write_json
+
+
+def persist_trial(args, result):
+ """Save harness-version evidence alongside each graded capture, before its trace."""
+ import httpx
+ trial = getattr(result, "trial_name", "")
+ if not trial or Path(trial).name != trial:
+ return
+ output = Path(args.capture_dir).parent
+ try:
+ response = httpx.get(args.server.rstrip("/") + "/trial/" + trial + "/result", timeout=30)
+ response.raise_for_status()
+ write_json(output / "trials" / trial / "result.json", response.json())
+ except Exception as exc:
+ # Preserve the graded result even if metadata retrieval fails. The final
+ # audit retries retrieval and refuses to publish an unverified baseline.
+ write_json(output / "trial-evidence-errors" / (trial + ".json"),
+ {"trial": trial, "error_type": type(exc).__name__})
+
+
+def restore_whitebox(output, prefix):
+ from huggingface_hub import HfApi
+ origin = output / "resume-origin"
+ origin.mkdir()
+ HfApi().sync_bucket(prefix, str(origin), include=["attempts.jsonl", "eval_config.json", "captures/**"], quiet=True)
+ config = json.loads((origin / "eval_config.json").read_text())
+ expected = {"model": MODEL, "revision": REVISION, "pass_k": 1,
+ "temperature": 0.8, "top_p": 1.0, "max_output_tokens_per_call": 4096,
+ "max_episode_completion_tokens": 16384, "max_model_calls": 17,
+ "toolsets": ["bash", "seta"]}
+ if any(config.get(k) != v for k, v in expected.items()):
+ raise ValueError("Saved baseline protocol differs from the requested evaluation")
+ rows = []
+ graded = set()
+ for line in (origin / "attempts.jsonl").read_text().splitlines():
+ row = json.loads(line)
+ if row.get("capture_file"):
+ relative = Path("captures") / Path(row["capture_file"]).name
+ saved = origin / relative
+ if not saved.is_file():
+ raise ValueError("Restored attempt is missing its captured tokens")
+ target = output / relative
+ target.parent.mkdir(exist_ok=True)
+ target.write_bytes(saved.read_bytes())
+ row["capture_file"] = str(target)
+ if row.get("reward") in (0, 1) and row.get("tito_pass"):
+ graded.add((row["harness"], row["index"]))
+ rows.append(row)
+ (output / "attempts.jsonl").write_text("".join(json.dumps(row) + "\n" for row in rows))
+ write_json(output / "eval_config.json", config)
+ write_json(output / "eval_resume.json", {"origin": prefix, "restored_graded": len(graded),
+ "original_attempts_sha256": hashlib.sha256((origin / "attempts.jsonl").read_bytes()).hexdigest(),
+ "selection": "First graded result, including zeros; only capture paths rebound"})
+ from score_comparison import summarize
+ summarize("whitebox", output)
diff --git a/04-data-agent/hf/runtime/eval_opencode.py b/04-data-agent/hf/runtime/eval_opencode.py
new file mode 100644
index 0000000..01a5fec
--- /dev/null
+++ b/04-data-agent/hf/runtime/eval_opencode.py
@@ -0,0 +1,150 @@
+"""Pass@1 for the native standalone OpenCode client, with a separate ledger per backend."""
+from __future__ import annotations
+import argparse
+import concurrent.futures
+import hashlib
+import json
+import math
+import os
+from pathlib import Path
+import time
+from common import RUN, MODEL, configure, write_json
+configure()
+from data_agent_env import DataAgentEnv, opencode_agent_turns, to_trace_entries
+from openenv.core.harness.capture.validate import validate_training_turn
+
+
+def audit(result):
+ entries = opencode_agent_turns(to_trace_entries(result))
+ if result.rollout_type != "train" or not entries:
+ raise ValueError("No TiTO training turns")
+ for entry in entries:
+ validate_training_turn(entry['prompt_token_ids'], entry['completion_token_ids'],
+ entry['per_token_logps'], entry['loss_mask'])
+ logps = [p for e in entries for p in e['per_token_logps']]
+ if not any(abs(p) > 1e-8 for p in logps):
+ raise ValueError("All log probabilities are zero")
+ return {"tito_pass": True, "agent_turns": len(entries),
+ "supervised_tokens": sum(sum(e['loss_mask']) for e in entries),
+ "forwarded_tokens": sum(len(e['loss_mask']) for e in entries)}
+
+
+def summarize(ledger, expected, manifest, ungraded):
+ selected = {}
+ if ledger.exists():
+ for line in ledger.read_text().splitlines():
+ row=json.loads(line)
+ if row.get('correctness') is not None: selected.setdefault(row['index'],row)
+ difficulty={}
+ for i,row in selected.items():
+ tier=manifest[i]['difficulty']; d=difficulty.setdefault(tier,{'graded':0,'correct':0})
+ d['graded']+=1;d['correct']+=int(row['correctness'] >= 1.0)
+ correct=sum(int(r['correctness']>=1.0) for r in selected.values())
+ result={'metric':'pass@1','implementation':'standalone-opencode','graded_cells':len(selected),
+ 'expected_cells':expected,'complete':len(selected)==expected,'correct':correct,
+ 'pass_at_1':correct/len(selected) if selected else None,'difficulty':difficulty,
+ 'ungraded_attempts':ungraded,'tito_pass':bool(selected) and all(r.get('tito_pass') for r in selected.values())}
+ result['comparison_ready']=result['complete'] and result['tito_pass']
+ return selected,result
+
+
+def evaluate(args):
+ out=Path(args.out);out.mkdir(parents=True,exist_ok=True)
+ tasks=sorted(json.loads((RUN/'test_manifest.json').read_text())['tasks'],key=lambda t:t['name'])
+ limit=args.limit or 250
+ if limit > len(tasks): raise ValueError('Limit exceeds frozen test set')
+ order=list(range(limit))
+ # Spread the ramp over the frozen set instead of measuring only the leading easy tasks.
+ import random
+ random.Random(42).shuffle(order)
+ all_scores={}
+ for backend in args.backends.split(','):
+ if backend not in {'daytona','hf','e2b'}:raise ValueError('Unsupported sandbox backend')
+ dest=out/backend;dest.mkdir(exist_ok=True)
+ ledger=dest/'results.jsonl';failed=dest/'ungraded.jsonl'
+ ungraded=len(failed.read_text().splitlines()) if failed.exists() else 0
+ selected,scores=summarize(ledger,limit,tasks,ungraded)
+ backend_limit = getattr(args, backend + '_concurrency', None)
+ concurrency_limit = args.concurrency if backend_limit is None else backend_limit
+ if not 1 <= concurrency_limit <= 100:
+ raise ValueError('Backend concurrency must be between 1 and 100')
+ phases=[(min(8,concurrency_limit),min(8,limit)),
+ (min(32 if backend == 'daytona' else 16,concurrency_limit),min(32,limit)),
+ (concurrency_limit,limit)]
+ if getattr(args, 'no_ramp', False):
+ phases=[(concurrency_limit,limit)]
+ scalability=[]
+ write_json(dest/'configuration.json',{'backend':backend,'concurrency_limit':concurrency_limit,
+ 'phases':phases,'expected_cells':limit})
+ def one(index):
+ start=time.monotonic()
+ client=DataAgentEnv(args.server,message_timeout_s=1800)
+ try:
+ result=client.run_rollout(split='test',index=index,llm_url=args.vllm_url,
+ model=args.model,api_key=os.environ.get(args.api_key_env,''),sandbox=backend,
+ agent_step_limit=17,agent_timeout_s=600,require_tokens=True,timeout_s=1800)
+ path=dest/'captures'/f'{index}-{result.metadata["rollout_id"]}.json'
+ write_json(path,result.model_dump())
+ if result.metadata.get('error') or result.correctness is None:
+ return {'index':index,'graded':False,'error':result.metadata.get('error','ungraded'),
+ 'capture_file':str(path),'elapsed_s':time.monotonic()-start}
+ if result.metadata.get('task_id') != tasks[index]['name']:
+ raise ValueError('Returned task identity differs from frozen manifest')
+ if result.metadata.get('opencode_version') != '1.18.31':
+ raise ValueError('Unverified OpenCode version')
+ # Once graded, a failed audit stops the cohort; never retry a scored zero.
+ proof=audit(result)
+ return {'index':index,'task_id':tasks[index]['name'],'backend':backend,
+ 'correctness':result.correctness,'reward':result.reward,'capture_file':str(path),
+ 'elapsed_s':time.monotonic()-start,'opencode_version':'1.18.31',**proof}
+ finally:client.close()
+ for stage,(concurrency,count) in enumerate(phases):
+ remaining=[i for i in order if i not in selected][:count]
+ if not remaining:continue
+ start=time.monotonic();before=len(selected);attempts=0
+ write_json(dest/'progress.json',{'stage':stage,'concurrency':concurrency,
+ 'graded_before':before,'tasks_this_stage':len(remaining),'started_at':time.time()})
+ for attempt in range(3):
+ pending=[i for i in remaining if i not in selected]
+ if not pending:break
+ with concurrent.futures.ThreadPoolExecutor(max_workers=concurrency) as pool:
+ futures={pool.submit(one,i):i for i in pending}
+ for future in concurrent.futures.as_completed(futures):
+ i=futures[future];attempts+=1
+ # Transport failures carry no returned grade and can be retried.
+ try:row=future.result()
+ except (ValueError,AssertionError):raise
+ except Exception as e:row={'index':i,'graded':False,'error_type':type(e).__name__}
+ target=ledger if row.get('correctness') is not None else failed
+ with target.open('a') as f:f.write(json.dumps(row)+'\n')
+ if target==failed:ungraded+=1
+ selected,scores=summarize(ledger,limit,tasks,ungraded)
+ write_json(dest/'scores.json',scores)
+ print(json.dumps({'backend':backend,'stage':stage,'graded':len(selected),'latest_index':i,
+ 'latest_graded':target==ledger,'ungraded_attempts':ungraded}),flush=True)
+ if len(selected)-before < len(remaining) and attempt<2:time.sleep(5)
+ elapsed=time.monotonic()-start
+ scalability.append({'stage':stage,'concurrency':concurrency,'new_graded':len(selected)-before,
+ 'attempts':attempts,'elapsed_s':elapsed,'graded_per_minute':(len(selected)-before)*60/elapsed})
+ write_json(dest/'scalability.json',scalability)
+ if len(selected)-before < .9*len(remaining):
+ raise RuntimeError(f'{backend} failed the coverage gate at concurrency {concurrency}')
+ if limit == 250 and scores["comparison_ready"]:
+ from native_grading_audit import verify
+ verify(ledger, dest)
+ if backend == "daytona":
+ write_json(out / "verification.json", json.loads((dest / "verification.json").read_text()))
+ all_scores[backend]=scores
+ write_json(out/'canonical_scores.json',all_scores)
+ if not scores['comparison_ready']:raise RuntimeError(f'{backend} baseline incomplete or TiTO invalid')
+ return all_scores
+
+if __name__=='__main__':
+ p=argparse.ArgumentParser(description=__doc__)
+ p.add_argument('--server',required=True);p.add_argument('--vllm-url',required=True)
+ p.add_argument('--model',default=MODEL);p.add_argument('--api-key-env',default='HF_TOKEN')
+ p.add_argument('--backends',default='daytona,hf');p.add_argument('--concurrency',type=int,default=32)
+ p.add_argument('--daytona-concurrency',type=int);p.add_argument('--hf-concurrency',type=int)
+ p.add_argument('--no-ramp',action='store_true')
+ p.add_argument('--limit',type=int,default=250);p.add_argument('--out',required=True)
+ print(json.dumps(evaluate(p.parse_args())),flush=True)
diff --git a/04-data-agent/hf/runtime/harbor_service.py b/04-data-agent/hf/runtime/harbor_service.py
new file mode 100644
index 0000000..a32cd5c
--- /dev/null
+++ b/04-data-agent/hf/runtime/harbor_service.py
@@ -0,0 +1,36 @@
+"""Apply the example's train/eval admission policy without rewriting OpenEnv source."""
+from functools import wraps
+import os
+
+from service_policy import admission, output_limit
+
+
+def install():
+ from openenv.harbor import rollout, serving
+
+ if getattr(rollout.run_rollout, "_data_agent_policy", False):
+ return
+ original = rollout.run_rollout
+
+ @wraps(original)
+ async def run(**kwargs):
+ dataset = kwargs.get("dataset", "")
+ observer = kwargs.get("on_session_created")
+
+ def session_created(session_id):
+ session = kwargs["registry"].get(session_id)
+ if session is None:
+ raise RuntimeError("Rollout session disappeared before policy setup")
+ session.metadata["max_output_tokens"] = output_limit(dataset)
+ if observer is not None:
+ observer(session_id)
+
+ async with admission.slot(dataset):
+ return await original(**{**kwargs, "on_session_created": session_created})
+
+ run._data_agent_policy = True
+ rollout.run_rollout = run
+ # A protected Space's app endpoint cannot authenticate a sandbox's model
+ # credential as an HF token. Publish only capture through the existing tunnel.
+ public_url = serving.space_public_url
+ serving.space_public_url = lambda: "" if os.environ.get("OPENENV_CAPTURE_TRANSPORT") == "tunnel" else public_url()
diff --git a/04-data-agent/hf/runtime/inference_providers.py b/04-data-agent/hf/runtime/inference_providers.py
new file mode 100644
index 0000000..a677de3
--- /dev/null
+++ b/04-data-agent/hf/runtime/inference_providers.py
@@ -0,0 +1,152 @@
+"""HF OAuth model selection and temporary credentials for interactive agent demos.
+
+The experiment's vLLM path does not import this module. Provider availability comes
+from HF's live catalog; a visitor credential is never replaced by a Space secret.
+"""
+from contextlib import contextmanager
+from dataclasses import dataclass, field
+import secrets
+import threading
+import time
+
+import httpx
+from fastapi import HTTPException, Request
+from fastapi.responses import JSONResponse, StreamingResponse
+
+ROUTER = "https://router.huggingface.co/v1"
+
+
+class Catalog:
+ def __init__(self, ttl=300):
+ self.ttl = ttl
+ self._rows = {}
+ self._updated = 0
+ self._lock = threading.Lock()
+
+ def rows(self):
+ with self._lock:
+ if time.monotonic() - self._updated < self.ttl and self._rows:
+ return dict(self._rows)
+ response = httpx.get(ROUTER + "/models", timeout=20)
+ response.raise_for_status()
+ rows = {}
+ for model in response.json()["data"]:
+ for provider in model.get("providers", []):
+ if provider.get("status") == "live" and provider.get("supports_tools") is True:
+ rows[(provider["provider"], model["id"])] = provider
+ if not rows:
+ raise ValueError("No live providers with tool calling were returned")
+ self._rows, self._updated = rows, time.monotonic()
+ return dict(rows)
+
+ def select(self, provider, model):
+ if (provider, model) not in self.rows():
+ raise ValueError("Choose a live provider/model pair from the catalog")
+ return model + ":" + provider
+
+
+catalog = Catalog()
+
+
+def visitor_token(oauth):
+ if oauth is None or not oauth.token or oauth.expires_at <= time.time():
+ raise ValueError("Sign in with Hugging Face to use Inference Providers")
+ if "inference-api" not in oauth.scope.split():
+ raise ValueError("Sign in again and allow Inference Providers access")
+ return oauth.token
+
+
+@dataclass
+class Lease:
+ model: str
+ token: str = field(repr=False)
+ expires: float
+ remaining: int = 32 # Includes native capture capability probes.
+
+
+class VisitorCredentials:
+ """Keep OAuth credentials out of Harbor's long-lived upstream client cache."""
+ def __init__(self):
+ self._leases = {}
+ self._lock = threading.Lock()
+
+ @contextmanager
+ def issue(self, oauth, model):
+ token = visitor_token(oauth)
+ key = secrets.token_urlsafe(32)
+ with self._lock:
+ self._leases = {k: v for k, v in self._leases.items() if v.expires > time.time()}
+ self._leases[key] = Lease(model, token, min(oauth.expires_at, time.time() + 660))
+ try:
+ yield key
+ finally:
+ with self._lock:
+ self._leases.pop(key, None)
+
+ def get(self, key, consume=False):
+ with self._lock:
+ lease = self._leases.get(key)
+ if lease is None or lease.expires <= time.time():
+ self._leases.pop(key, None)
+ raise HTTPException(401, "Interactive inference session expired")
+ if consume:
+ if lease.remaining <= 0:
+ raise HTTPException(429, "Interactive model-call limit reached")
+ lease.remaining -= 1
+ return lease
+
+
+credentials = VisitorCredentials()
+
+
+def mount_provider_relay(app):
+ """Only an opaque, short-lived key reaches Harbor; the HF token stays here."""
+ def lease_for(request, consume=False):
+ value = request.headers.get("authorization", "")
+ key = value[7:] if value.lower().startswith("bearer ") else ""
+ return credentials.get(key, consume)
+
+ @app.get("/hf-inference/v1/models")
+ async def models(request: Request):
+ lease = lease_for(request)
+ return {"object": "list", "data": [{"id": lease.model, "object": "model"}]}
+
+ @app.post("/hf-inference/v1/chat/completions")
+ async def chat(request: Request):
+ lease = lease_for(request)
+ body = await request.json()
+ if body.get("model") != lease.model:
+ raise HTTPException(400, "This inference session is bound to the selected model")
+ lease_for(request, consume=True)
+ body["max_tokens"] = min(int(body.get("max_tokens") or 4096), 4096)
+ if "max_completion_tokens" in body:
+ body["max_completion_tokens"] = min(int(body["max_completion_tokens"]), 4096)
+ body.pop("max_tokens")
+ client = httpx.AsyncClient(timeout=120)
+ try:
+ upstream = await client.send(client.build_request("POST", ROUTER + "/chat/completions",
+ headers={"Authorization": "Bearer " + lease.token}, json=body), stream=True)
+ except httpx.HTTPError:
+ await client.aclose()
+ raise HTTPException(502, "The selected inference provider could not be reached") from None
+ if upstream.status_code >= 400:
+ status = upstream.status_code
+ await upstream.aclose()
+ await client.aclose()
+ message = {401: "Sign in again to renew your inference access",
+ 402: "Your HF account needs inference credits",
+ 403: "Your account cannot access this provider or model",
+ 429: "This provider is rate limited; try again later"}.get(status,
+ "The provider rejected the model request; try another model")
+ return JSONResponse({"error": {"message": message}}, status_code=status)
+
+ async def chunks():
+ try:
+ async for chunk in upstream.aiter_bytes():
+ yield chunk
+ finally:
+ await upstream.aclose()
+ await client.aclose()
+ return StreamingResponse(chunks(), media_type=upstream.headers.get("content-type", "application/json"))
+
+ return app
diff --git a/04-data-agent/hf/runtime/job.py b/04-data-agent/hf/runtime/job.py
new file mode 100644
index 0000000..a8f3f63
--- /dev/null
+++ b/04-data-agent/hf/runtime/job.py
@@ -0,0 +1,397 @@
+"""HF GPU task runner using the frozen native evaluators and trainers."""
+from __future__ import annotations
+
+import argparse
+import concurrent.futures
+import json
+import os
+from pathlib import Path
+import signal
+import subprocess
+import sys
+import time
+
+from common import ROOT, RUN, TOOLS, TRAIN_PY, ENV_PY, MODEL, REVISION, configure, ready, start, write_json
+
+
+def inference_url():
+ return "http://127.0.0.1:" + os.environ.get("LOCAL_INFERENCE_PORT", "8000")
+
+
+def job_endpoint():
+ if os.environ.get("LOCAL_RUNTIME") == "1":
+ return os.environ.get("SLURM_JOB_ID", os.environ["RUN_OWNER"]), inference_url()
+ from huggingface_hub import HfApi
+ api = HfApi()
+ for job in api.list_jobs(namespace=os.environ.get("HF_JOB_NAMESPACE", "HuggingEnvs"), labels={"experiment": "data-agent-daytona"}):
+ if job.environment.get("RUN_OWNER") == os.environ["RUN_OWNER"]:
+ info = api.inspect_job(job_id=job.id, namespace=os.environ.get("HF_JOB_NAMESPACE", "HuggingEnvs"))
+ urls = info.status.expose_urls
+ if not urls or len(urls) != 1:
+ raise RuntimeError("Expected exactly one exposed vLLM endpoint")
+ return job.id, urls[0].rstrip("/")
+ raise RuntimeError("Could not identify this Job through its unique owner")
+
+
+def serving(args, output, processes):
+ env = dict(os.environ)
+ env.update(MODEL=os.environ.get("CHECKPOINT_MODEL", MODEL), TRL_PROD=str(ROOT), VENV=str(ROOT / ".venv312"),
+ PORT=os.environ.get("LOCAL_INFERENCE_PORT", "8000"), TP_SIZE="1", DP_SIZE=str(args.dp), MAX_MODEL_LEN="131072",
+ GPU_MEMORY_UTILIZATION="0.85" if args.role == "train" else "0.90",
+ TOOL_CALL_PARSER="qwen3_xml", REASONING_PARSER="qwen3", ENABLE_THINKING="0",
+ ENFORCE_EAGER="0" if args.role == "train" or os.environ.get("LOCAL_RUNTIME") == "1" else "1", TUNNEL="none", SHORT_NAME="daytona-hf",
+ VLLM_LOG=str(output / "vllm.log"), VLLM_SERVER_DEV_MODE="1", VLLM_USE_DEEP_GEMM="0",
+ VLLM_DEEP_GEMM_WARMUP="skip", VLLM_USE_FLASHINFER_SAMPLER="0", READY_TIMEOUT_SEC="1200")
+ extra = ['--dtype bfloat16', '--generation-config vllm', '--logprobs-mode processed_logprobs',
+ '--return-tokens-as-token-ids', '--no-enable-prefix-caching',
+ '--limit-mm-per-prompt {"image":0,"video":0}', '--gdn-prefill-backend triton',
+ '--override-generation-config {"temperature":0.8,"top_p":1.0,"top_k":-1}',
+ '--served-model-name Qwen/Qwen3.5-2B']
+ if env["MODEL"] == MODEL:
+ extra += ["--revision " + REVISION]
+ if args.role == "train":
+ extra += ['--weight-transfer-config {"backend":"nccl"}']
+ env["CUDA_VISIBLE_DEVICES"] = os.environ.get("INFERENCE_GPU", "0")
+ if args.dp > 1:
+ extra += ["--data-parallel-rpc-port " + os.environ.get("VLLM_DP_RPC_PORT", "8950")]
+ env["EXTRA_VLLM_ARGS"] = " ".join(extra)
+ proc = start(["bash", RUN / "eval-source/serve_vllm_tunnel.sh"], output / "serving.log", env)
+ processes.append(proc)
+ ready(inference_url() + "/health", proc)
+ from openenv.core.harness.capture.validate_llm import validate_llm
+ report = validate_llm(inference_url() + "/v1", MODEL)
+ if not report.trainable:
+ raise RuntimeError("Inference preflight did not establish exact token capture")
+ job_id, public = job_endpoint()
+ ready(public + "/health", proc, headers={"Authorization": "Bearer " + os.environ["HF_TOKEN"]}, seconds=120)
+ return job_id, public
+
+
+def bridge(output, processes):
+ if os.environ.get("LOCAL_RUNTIME") == "1":
+ server = "http://127.0.0.1:" + os.environ["LOCAL_ENV_PORT"]
+ proc = start([ENV_PY, ROOT / "hf/runtime/local_environment.py"], output / "environment.log")
+ processes.append(proc)
+ ready(server + "/health", proc, seconds=300)
+ import httpx
+ info = httpx.get(server + "/deployment", timeout=30).raise_for_status().json()
+ if info["bundle_sha256"] != os.environ["BUNDLE_SHA256"] or info["train_tasks"] != 1000 or info["test_tasks"] != 250:
+ raise RuntimeError("Local environment task/source identity mismatch")
+ write_json(output / "space_identity.json", info)
+ os.environ["SPACE_URL"] = server
+ return server
+ proc = start([ENV_PY, ROOT / "hf/runtime/auth_bridge.py"], output / "bridge.log")
+ processes.append(proc)
+ ready("http://127.0.0.1:8100/health", proc, seconds=300)
+ import httpx
+ info = httpx.get("http://127.0.0.1:8100/deployment", timeout=30).raise_for_status().json()
+ expected = os.environ.get("SPACE_BUNDLE_SHA256", os.environ["BUNDLE_SHA256"])
+ if info["bundle_sha256"] != expected:
+ raise RuntimeError("Space and Job runtime bundle hashes differ")
+ if os.environ.get("COMPARISON_ARM") == "opencode" and os.environ.get("EVAL_SUITE") != "harbor":
+ if (info.get("implementation") != "standalone-opencode" or info.get("train_tasks") != 1000
+ or info.get("test_tasks") != 250 or info.get("opencode_version") != "1.18.31"
+ or info.get("output_tokens") != {"train": 16384, "test": 4096}):
+ raise RuntimeError("Standalone training service contract differs from the comparison")
+ write_json(output / "space_identity.json", info)
+ return "http://127.0.0.1:8100"
+
+
+def blackbox_audit(output, server):
+ from openenv.harbor.models import HarborRolloutResult
+ from smoke_multiharness_tito import audit
+ import httpx
+ selected = {}
+ for path in sorted((output / "traces").glob("*.jsonl")):
+ for line in path.read_text().splitlines():
+ row = json.loads(line)
+ if row.get("reward") in (0, 1) and row.get("n_turns", 0) > 0:
+ selected.setdefault((row["harness"], row["index"]), row)
+
+ def check(item):
+ (harness, index), row = item
+ result = HarborRolloutResult.model_validate_json(Path(row["capture_file"]).read_text())
+ report, _ = audit(result, token_budget=131072)
+ if not report["tito_pass"]:
+ raise RuntimeError(f"TiTO failure: {harness}/{index}")
+ trial = row["trial_name"]
+ path = output / "trials" / trial / "result.json"
+ if not path.exists():
+ native = httpx.get(server + "/trial/" + trial + "/result", timeout=60).raise_for_status().json()
+ write_json(path, native)
+ return {"harness": harness, "index": index, **report}
+
+ with concurrent.futures.ThreadPoolExecutor(max_workers=16) as pool:
+ reports = list(pool.map(check, selected.items()))
+ pins = json.loads((ROOT / "hf/configs/deployment.json").read_text())["harness_pins"]
+ counts = {h: sum(r["harness"] == h for r in reports) for h in pins}
+ write_json(output / "final_tito.json", {"counts": counts, "tito_pass": bool(reports), "reports": reports})
+ from score_comparison import summarize
+ scores = summarize("blackbox", output)
+ # Require actual harness versions for partial smokes too.
+ for harness, versions in scores["harness_versions"].items():
+ if versions and set(versions) != {pins[harness]}:
+ raise RuntimeError(f"Unverified harness version: {harness}")
+ return scores
+
+
+def evaluate(args, output, server, public):
+ suite = "blackbox" if os.environ.get("EVAL_SUITE") == "harbor" else args.arm
+ if suite == "opencode":
+ command = [ENV_PY, ROOT / "hf/runtime/eval_opencode.py", "--server", server,
+ "--vllm-url", public + "/v1", "--model", MODEL, "--out", output,
+ "--concurrency", os.environ.get("EVAL_CONCURRENCY", "35"),
+ "--limit", str(args.limit or (2 if args.phase == "smoke" else 250)),
+ "--backends", os.environ.get("EVAL_BACKENDS", "daytona,hf"),
+ "--daytona-concurrency", os.environ.get("EVAL_DAYTONA_CONCURRENCY", "35"),
+ "--hf-concurrency", os.environ.get("EVAL_HF_CONCURRENCY", "35")]
+ if os.environ.get("EVAL_NO_RAMP") == "1":
+ command += ["--no-ramp"]
+ if args.phase == "baseline" and os.environ.get("EVAL_NO_RAMP") != "1":
+ smoke = list(command)
+ smoke[smoke.index("--out")+1] = output / "smoke"
+ smoke[smoke.index("--limit")+1] = "2"
+ smoke[smoke.index("--concurrency")+1] = "2"
+ smoke[smoke.index("--daytona-concurrency")+1] = "2"
+ smoke[smoke.index("--hf-concurrency")+1] = "2"
+ process = start(smoke, output / "smoke-opencode.log")
+ if process.wait() != 0:
+ raise RuntimeError("Standalone backend rollout smoke failed; full baseline held")
+ process = start(command, output / "eval-opencode.log")
+ if process.wait() != 0:
+ raise RuntimeError("Standalone OpenCode evaluation failed")
+ return
+ ceiling = int(os.environ.get("EVAL_CONCURRENCY", "35"))
+ phases = [(8, 8), (min(32, ceiling), 32), (ceiling, 100)] if args.phase == "ramp" else [(8 if args.phase == "smoke" else ceiling, args.limit)]
+ if args.phase == "baseline":
+ # One immutable first-graded ledger spans the ramp and full evaluation.
+ # Later passes revisit only infrastructure failures, including scored zeros
+ # in the resume set so they can never become best-of-N samples.
+ phases = [(8, 8), (min(32, ceiling), 32), (ceiling, 100)] + [(ceiling, 0)] * 4
+ elif args.phase == "checkpoint":
+ phases = [(ceiling, 0)] * 4
+ if args.phase == "smoke":
+ phases = [(8, 8 if suite == "blackbox" else 2)]
+ records = []
+ scores = json.loads((output / "canonical_scores.json").read_text()) if (output / "canonical_scores.json").exists() else {}
+ for phase_index, (concurrency, limit) in enumerate(phases):
+ if (output / "canonical_scores.json").exists() and json.loads((output / "canonical_scores.json").read_text()).get("comparison_ready"):
+ break
+ write_json(output / "eval_progress.json", {"stage": phase_index, "concurrency": concurrency,
+ "max_new_rollouts": limit, "started_at": time.time(), "phase": args.phase})
+ before = json.loads((output / "canonical_scores.json").read_text()).get("graded_cells", 0) if (output / "canonical_scores.json").exists() else 0
+ if suite == "blackbox":
+ arms = [{"name": "model", "base_url": public + "/v1", "model": MODEL, "api_key_env": "HF_TOKEN"}]
+ write_json(output / "arms.json", arms)
+ indices = "@" + str(RUN / "test_indices.txt")
+ if args.phase == "smoke":
+ indices = ",".join((RUN / "test_indices.txt").read_text().replace(",", " ").split()[:2])
+ limit = 0
+ cmd = [TRAIN_PY, "-u", RUN / "eval-source/eval_concurrent.py", "--server", server,
+ "--arms", output / "arms.json", "--harnesses", "opencode,claude-code,codex,mini-swe-agent",
+ "--split", str(RUN / "datasets/test"), "--indices", indices, "--repeat", "1",
+ "--temperature", "0.8", "--reward-key", "correctness,reward", "--sandbox", "daytona",
+ "--agent-timeout", "600", "--agent-step-limit", "17", "--max-retries", "3",
+ "--trace-dir", output / "traces", "--capture-dir", output / "captures", "--progress-every", "1",
+ "--concurrency", str(concurrency), "--server-concurrency", str(concurrency),
+ "--sandbox-concurrency", str(concurrency), "--max-new-rollouts", str(limit), "--out", output / "results.json"]
+ if (output / "traces/eval_config.json").exists():
+ cmd += ["--resume"]
+ else:
+ cmd = [TRAIN_PY, "-u", TOOLS / "eval_whitebox_native.py", "--run", RUN, "--server", server,
+ "--vllm-url", inference_url() + "/v1", "--out", output, "--concurrency", str(concurrency),
+ "--max-new-rollouts", str(limit)]
+ began = time.monotonic()
+ proc = start(cmd, output / f"eval-stage{phase_index}-c{concurrency}.log")
+ code = proc.wait()
+ if code not in (0, 2):
+ raise RuntimeError(f"Native evaluator failed: {code}")
+ if suite == "blackbox":
+ scores = blackbox_audit(output, server)
+ scores["training_arm"] = args.arm
+ scores["arm"] = args.arm
+ scores["evaluation_suite"] = "harbor"
+ write_json(output / "canonical_scores.json", scores)
+ else:
+ from score_comparison import summarize
+ scores = summarize("whitebox", output)
+ elapsed = time.monotonic() - began
+ graded = scores["graded_cells"] - before
+ expected = (8 if suite == "blackbox" else 2) if args.phase == "smoke" else limit
+ if expected and graded < 0.9 * expected:
+ raise RuntimeError(f"Scale gate failed: only {graded}/{expected} graded")
+ hourly = {"a100-large": 2.5, "a100x4": 10, "h200x2": 10, "h200": 5}.get(os.environ["JOB_FLAVOR"])
+ records.append({"stage": phase_index, "concurrency": concurrency, "new_graded": graded, "elapsed_s": elapsed,
+ "graded_per_minute": graded * 60 / elapsed,
+ "compute_usd_per_1000": hourly * elapsed / 3600 * 1000 / graded if graded and hourly else None})
+ write_json(output / "scalability.json", records)
+ print(json.dumps(records[-1]), flush=True)
+ if args.phase in ("baseline", "checkpoint") and not scores["comparison_ready"]:
+ raise RuntimeError("Full eval coverage/TiTO/version gate did not pass")
+ if args.phase == "smoke" and scores["graded_cells"] != (8 if suite == "blackbox" else 2):
+ raise RuntimeError("Smoke did not grade every requested cell")
+ write_json(output / "eval_progress.json", {"finished_at": time.time(), "graded_cells": scores["graded_cells"],
+ "comparison_ready": scores["comparison_ready"], "phase": args.phase})
+
+
+def training_command(args, output, server):
+ save = 2 if args.phase == "smoke" else 50
+ config = json.loads((ROOT / "hf/configs/deployment.json").read_text())
+ harnesses = config["arms"][args.arm].get("training_harnesses", ["opencode"])
+ schedule = "reference_schedule.json" if len(harnesses) > 1 else "opencode_schedule.json"
+ if args.arm in {"blackbox", "opencode"}:
+ entrypoint = "train_standalone_comparison.py" if args.arm == "opencode" else "train_harbor_multi.py"
+ cmd = [TRAIN_PY, "-u", RUN / "source/HuggingEnvs/04-data-agent/train" / entrypoint,
+ "--server", server, "--vllm-url", inference_url(), "--model", MODEL,
+ "--model-revision", REVISION, "--split", RUN / "datasets/train", "--harnesses", ",".join(harnesses),
+ "--sandbox", "daytona", "--harness-schedule", RUN / schedule,
+ "--task-indices", "@" + str(RUN / "train_indices.txt"), "--learning-rate", "3e-6",
+ "--num-generations", "8", "--max-inflight", "32", "--max-staleness", "4", "--grad-accum", "4",
+ "--atomic-rollouts", "--max-outstanding-rollouts", "16", "--max-row-tokens", "131072",
+ "--per-device-batch-size", "4", "--reward-key", "reward", "--agent-step-limit", "17",
+ "--agent-timeout", "600", "--token-budget", "40960", "--max-completion-length", "16384",
+ "--dtype", "bfloat16", "--top-p", "1.0", "--temperature", "0.8", "--audit-dir", output / "audit",
+ "--project", f"daytona-{args.arm}-qwen35-2b", "--save-steps", str(save), "--output-dir", output / "run"]
+ else:
+ cmd = [TRAIN_PY, "-u", TOOLS / "train_whitebox_daytona.py", "--run", RUN, "--server", server,
+ "--vllm-url", inference_url(), "--output-dir", output / "run", "--save-steps", str(save)]
+ return cmd
+
+
+def train(args, output, server, public, publisher):
+ os.environ["ROLLOUT_LLM_URL"] = public
+ os.environ["ROLLOUT_LLM_API_KEY"] = os.environ["HF_TOKEN"]
+ for key in ["TRACKIO_SPACE_ID", "TRACKIO_SERVER_URL", "TRACKIO_BUCKET_ID", "TRACKIO_DATASET_ID"]:
+ os.environ.pop(key, None)
+ os.environ["TRACKIO_STORAGE_MODE"] = "jsonl"
+ os.environ["TRACKIO_DIR"] = str(output / "trackio")
+ cmd = training_command(args, output, server)
+ write_json(output / "training_recipe.json", {"arm": args.arm, "phase": args.phase,
+ "command": list(map(str, cmd)), "space_bundle_sha256": os.environ.get("SPACE_BUNDLE_SHA256", os.environ["BUNDLE_SHA256"]),
+ "job_bundle_sha256": os.environ["BUNDLE_SHA256"], "initialization": "pinned base; then native full-state restore for smoke"})
+ env = {**os.environ, "CUDA_VISIBLE_DEVICES": os.environ.get("TRAIN_GPU", "1")}
+ if args.phase == "smoke":
+ restored = output / "remote-resume/checkpoint-2"
+ for steps, name, extra in [(2, "first", []), (4, "resumed", ["--resume-from-checkpoint", restored])]:
+ proc = start(cmd + ["--max-steps", str(steps)] + extra, output / f"train-{name}.log", env)
+ if proc.wait() != 0:
+ raise RuntimeError(f"{name} training smoke failed")
+ publisher.sync()
+ if steps == 2:
+ from checkpoint_store import restore
+ restore(publisher.dest + "/run/checkpoint-2", restored, arm=args.arm,
+ bundle_sha256=os.environ["BUNDLE_SHA256"])
+ from training_smoke import validate
+ validate(output, args.arm)
+ else:
+ if not os.environ.get("VERIFIED_SMOKE_MANIFEST"):
+ raise RuntimeError("Long training requires a verified optimizer/save/resume manifest")
+ proof = json.loads(Path(os.environ["VERIFIED_SMOKE_MANIFEST"]).read_text())
+ if not (proof.get("passed") and proof.get("arm") == args.arm and
+ proof.get("bundle_sha256") == os.environ["BUNDLE_SHA256"] and
+ proof.get("remote_restore_verified") and proof.get("tito_pass") and proof.get("weights_updated")):
+ raise RuntimeError("Training smoke provenance or optimizer evidence does not match this run")
+ cmd += ["--max-steps", "1000", "--max-train-seconds", "82200", "--checkpoint-max-seconds", "3600"]
+ if os.environ.get("RESUME_CHECKPOINT"):
+ cmd += ["--resume-from-checkpoint", os.environ["RESUME_CHECKPOINT"]]
+ proc = start(cmd, output / "train.log", env)
+ if proc.wait() != 0:
+ raise RuntimeError("Trainer exited with an error")
+
+
+def main():
+ p = argparse.ArgumentParser(description=__doc__)
+ p.add_argument("--role", choices=["eval", "train", "coordinator"], required=True)
+ p.add_argument("--arm", choices=["blackbox", "whitebox", "opencode"], required=True)
+ p.add_argument("--phase", default="smoke")
+ p.add_argument("--dp", type=int, default=1)
+ p.add_argument("--limit", type=int, default=0)
+ args = p.parse_args()
+ configure()
+ output = ROOT / "outputs" / os.environ["RUN_OWNER"]
+ output.mkdir(parents=True, exist_ok=True)
+ if args.role == "coordinator":
+ if args.phase == "qualify":
+ from qualify_training import run
+ run(output)
+ elif args.phase == "setup":
+ from setup_pipeline import run
+ run(output)
+ else:
+ from coordinator import run
+ run(output, args.arm)
+ return
+ processes = []
+ from artifacts import Publisher
+ publisher = Publisher(output)
+ publisher.start()
+ from telemetry import Telemetry
+ telemetry = Telemetry(output, inference_url())
+ telemetry.start()
+ status = {"arm": args.arm, "phase": args.phase, "started_at": time.time(), "passed": False}
+ logger = None
+ logger_stop = output / "trackio-stop"
+ write_json(output / "status.json", status)
+ try:
+ if args.role == "eval" and args.phase == "checkpoint":
+ from checkpoint_store import restore_model
+ source = os.environ["CHECKPOINT_PREFIX"]
+ sha = os.environ["CHECKPOINT_MANIFEST_SHA"]
+ model = output / "inference-model"
+ manifest = restore_model(source, model, arm=args.arm,
+ bundle_sha256=os.environ["BUNDLE_SHA256"], manifest_sha256=sha)
+ if manifest["step"] != int(os.environ["CHECKPOINT_STEP"]):
+ raise ValueError("Checkpoint optimizer step differs from the queued evaluation")
+ os.environ["CHECKPOINT_MODEL"] = str(model)
+ write_json(output / "checkpoint_evaluation.json", {"source": source, "manifest_sha256": sha,
+ "step": manifest["step"], "bundle_sha256": os.environ["BUNDLE_SHA256"]})
+ if args.role == "train" and args.phase != "smoke":
+ from checkpoint_store import download_json
+ proof_path = output / "verified_smoke.json"
+ download_json(os.environ["SMOKE_PREFIX"], "training_smoke_verified.json", proof_path)
+ os.environ["VERIFIED_SMOKE_MANIFEST"] = str(proof_path)
+ server = bridge(output, processes)
+ if args.role == "train" and args.arm in {"blackbox", "opencode"}:
+ from service_contract import check
+ write_json(output / "service_contract.json", check(server, "", args.arm))
+ job_id, public = serving(args, output, processes)
+ write_json(output / "services.json", {"job_id": job_id, "public_vllm": public, "server": server,
+ "space": os.environ["SPACE_URL"], "tp": 1, "dp": args.dp, "flavor": os.environ["JOB_FLAVOR"]})
+ if args.role == "eval":
+ if os.environ.get("RESUME_EVAL_PREFIX"):
+ if args.arm != "whitebox" or args.phase != "baseline":
+ raise ValueError("This saved baseline restore is for whitebox baseline evaluations")
+ from eval_evidence import restore_whitebox
+ restore_whitebox(output, os.environ["RESUME_EVAL_PREFIX"])
+ evaluate(args, output, server, public)
+ else:
+ logger_env = {**os.environ, "TRACKIO_DIR": str(output / "trackio"),
+ "TRAINING_SMOKE": "1" if args.phase == "smoke" else "0"}
+ logger = start([TRAIN_PY, ROOT / "hf/runtime/logging_sync.py", "--out", output,
+ "--arm", args.arm, "--watch", "--stop-file", logger_stop],
+ output / "trackio-sync.log", logger_env)
+ try:
+ train(args, output, server, public, publisher)
+ finally:
+ logger_stop.touch()
+ if logger.wait(timeout=180) != 0:
+ raise RuntimeError("Training Trackio persistence failed")
+ status["passed"] = True
+ except Exception as exc:
+ status["error_type"] = type(exc).__name__
+ raise
+ finally:
+ status["finished_at"] = time.time()
+ write_json(output / "status.json", status)
+ telemetry.finish()
+ if logger is not None and logger.poll() is None:
+ os.killpg(logger.pid, signal.SIGTERM)
+ for process in reversed(processes):
+ if process.poll() is None:
+ os.killpg(process.pid, signal.SIGTERM)
+ publisher.finish()
+ print(json.dumps(status), flush=True)
+
+
+if __name__ == "__main__":
+ main()
diff --git a/04-data-agent/hf/runtime/local_entry.py b/04-data-agent/hf/runtime/local_entry.py
new file mode 100644
index 0000000..e6c14a3
--- /dev/null
+++ b/04-data-agent/hf/runtime/local_entry.py
@@ -0,0 +1,39 @@
+"""Use the same audited HF runner with local GPUs and a local native environment."""
+import json
+import os
+from pathlib import Path
+import sys
+
+from dotenv import dotenv_values
+
+values = dotenv_values(os.environ["LOCAL_ENV_FILE"])
+os.environ["HF_TOKEN"] = values.get("HF_API_KEY") or values["HF_TOKEN"]
+for key in ["DAYTONA_API_KEY", "DAYTONA_API_URL", "DAYTONA_TARGET", "E2B_API_KEY"]:
+ if values.get(key):
+ os.environ[key] = values[key]
+root = Path(os.environ["REPRO_ROOT"])
+config = json.loads((root / "hf/configs/deployment.json").read_text())
+gpu_ids = os.environ["CUDA_VISIBLE_DEVICES"].split(",")
+if len(gpu_ids) != 2:
+ raise RuntimeError("The local recipe requires two allocated GPUs")
+arm = sys.argv[sys.argv.index("--arm") + 1]
+role = sys.argv[sys.argv.index("--role") + 1]
+job = os.environ.get("SLURM_JOB_ID", str(os.getpid()))
+port_seed = int(job) % 1000
+os.environ.update(LOCAL_RUNTIME="1", RUN_OWNER=f"local-{role}-{arm}-{job}",
+ RUN_ID=config["run_id"], COMPARISON_ARM=arm,
+ BUNDLE_SHA256=json.loads((root / "local_manifest.json").read_text())["sha256"],
+ ARTIFACT_BUCKET=config["resources"]["artifacts_bucket"], JOB_FLAVOR="hopper-prod-2h100",
+ INFERENCE_GPU=gpu_ids[0], TRAIN_GPU=gpu_ids[1],
+ LOCAL_INFERENCE_PORT=str(12000 + port_seed), LOCAL_ENV_PORT=str(14000 + port_seed),
+ DATA_AGENT_CAPTURE_PORT=str(16000 + port_seed), VLLM_DP_RPC_PORT=str(26000 + port_seed),
+ EVAL_BACKENDS="daytona", EVAL_CONCURRENCY="50", EVAL_DAYTONA_CONCURRENCY="50", EVAL_NO_RAMP="1",
+ SANDBOX_CAPACITY="50" if role == "eval" else "16", TRAIN_RESERVED_SANDBOXES="8",
+ HF_HOME=os.environ.get("HF_HOME", str(root / "cache/huggingface")))
+from job import main
+try:
+ main()
+finally:
+ import subprocess
+ subprocess.run([str(root / 'OpenEnv/.venv/bin/python'), str(root / 'hf/runtime/cleanup_local.py')],
+ timeout=240, check=True)
diff --git a/04-data-agent/hf/runtime/local_environment.py b/04-data-agent/hf/runtime/local_environment.py
new file mode 100644
index 0000000..e21d22a
--- /dev/null
+++ b/04-data-agent/hf/runtime/local_environment.py
@@ -0,0 +1,56 @@
+"""Bind the frozen native OpenEnv service to loopback inside its Slurm allocation."""
+import os
+from common import RUN, ROOT, configure
+
+configure()
+arm = os.environ["COMPARISON_ARM"]
+os.environ.update(ENABLE_WEB_INTERFACE="false", MAX_CONCURRENT_ENVS="128")
+if arm == "opencode":
+ os.environ.update(DATA_AGENT_SPLITS="train,test", DATA_AGENT_SANDBOX="daytona",
+ DATA_AGENT_FROZEN_TASKS_DIR=str(RUN / "datasets"), DATA_AGENT_CAPTURE_EXPOSE="gradio",
+ DATA_AGENT_MAX_CONCURRENT=os.environ.get("SANDBOX_CAPACITY", "64"))
+ from data_agent_env.server.app import app
+ from data_agent_env.tasks import rows_for
+ counts = {split: len(rows_for(split)) for split in ("train", "test")}
+elif arm == "blackbox":
+ os.environ.update(OPENENV_DATASETS=",".join(str(RUN / "datasets" / split) for split in ("train", "test")),
+ OPENENV_CAPTURE_TRANSPORT="tunnel", OPENENV_EXPOSE="gradio", OPENENV_MAX_OUTPUT_TOKENS="16384",
+ OPENENV_CAPTURE_PORT=os.environ["DATA_AGENT_CAPTURE_PORT"],
+ OPENENV_HARBOR_TRIALS_DIR=str(ROOT / "outputs" / os.environ["RUN_OWNER"] / "trials"))
+ from harbor_service import install
+ install()
+ from harbor_env.server.app import app
+ from fastapi import HTTPException
+ from pathlib import Path
+ import json
+
+ @app.get("/trial/{name}/result")
+ def trial_result(name: str):
+ if Path(name).name != name or name in {".", ".."}:
+ raise HTTPException(400)
+ path = Path(os.environ["OPENENV_HARBOR_TRIALS_DIR"]) / name / "result.json"
+ if not path.is_file():
+ raise HTTPException(404)
+ return json.loads(path.read_text())
+ counts = {"train": 1000, "test": 250}
+else:
+ os.environ.update(WHITE_BOX_BASH_TASK_SOURCE="harbor-frozen",
+ DAYTONA_WHITEBOX_TRIALS=str(ROOT / "outputs" / os.environ["RUN_OWNER"] / "trials"),
+ WHITE_BOX_BASH_MAX_SESSIONS="16", WHITE_BOX_BASH_MAX_CONCURRENT_ENVS="128")
+ from whitebox_bash.server.app import app
+ # The same frozen manifest-backed provider used by the Space.
+ from whitebox_bash.tasks import num_tasks
+ counts = {split: num_tasks(split) for split in ("train", "test")}
+
+
+@app.get("/deployment")
+def deployment():
+ return {"arm": arm, "implementation": "standalone-opencode" if arm == "opencode" else "harbor" if arm == "blackbox" else "whitebox-seta",
+ "bundle_sha256": os.environ["BUNDLE_SHA256"], "owner": os.environ["RUN_OWNER"],
+ "train_tasks": counts["train"], "test_tasks": counts["test"], "local": True}
+
+
+if __name__ == "__main__":
+ import uvicorn
+ uvicorn.run(app, host="127.0.0.1", port=int(os.environ["LOCAL_ENV_PORT"]),
+ ws_ping_interval=20, ws_ping_timeout=None, timeout_keep_alive=120)
diff --git a/04-data-agent/hf/runtime/logging_sync.py b/04-data-agent/hf/runtime/logging_sync.py
new file mode 100644
index 0000000..f8a6758
--- /dev/null
+++ b/04-data-agent/hf/runtime/logging_sync.py
@@ -0,0 +1,136 @@
+"""Durable metrics replay through native Trackio, without optimizer network calls."""
+from __future__ import annotations
+
+import json
+import os
+from pathlib import Path
+import time
+
+from common import configure, write_json
+
+
+def checkpoint_scores(output):
+ """The logging process polls score artifacts; the optimizer never waits on HF."""
+ from huggingface_hub import HfApi
+ from huggingface_hub.errors import EntryNotFoundError
+ from checkpoint_store import bucket_location, download_json
+ dest = output / "checkpoint-scores"
+ dest.mkdir(exist_ok=True)
+ api = HfApi()
+ try:
+ baseline = os.environ.get("BASELINE_PREFIX")
+ if baseline and not (dest / "step-000000.json").exists():
+ scores = download_json(baseline, "canonical_scores.json", output / "baseline_scores.json", api)
+ write_json(dest / "step-000000.json", {"step": 0, "scores": scores, "source": {"job_id": os.environ.get("BASELINE_JOB")}})
+ source = os.environ.get("COORDINATION_PREFIX")
+ if source:
+ bucket, prefix = bucket_location(source + "/scores")
+ for item in api.list_bucket_tree(bucket, prefix=prefix, recursive=False):
+ name = Path(item.path).name
+ if name.startswith("step-") and name.endswith(".json") and not (dest / name).exists():
+ api.download_bucket_files(bucket, files=[(item.path, str(dest / name))], raise_on_missing_files=True)
+ except EntryNotFoundError:
+ pass # No completed checkpoint evaluation yet.
+ except Exception as exc:
+ write_json(output / "trackio_remote_error.json", {"time": time.time(), "type": type(exc).__name__})
+ return [json.loads(p.read_text()) for p in sorted(dest.glob("step-*.json"))]
+
+
+def sync_metrics(output: Path, arm: str, step=0, smoke=False):
+ configure()
+ import trackio_multi4 as native
+ training_smoke = os.environ.get("TRAINING_SMOKE") == "1"
+ project = f"daytona-{arm}-qwen35-2b" + ("-integration" if smoke else "-smoke" if training_smoke else "")
+ owner = os.environ.get("RUN_OWNER", "local")
+ metadata = {"arm": arm, "sandbox": "daytona", "hf_run": os.environ.get("RUN_ID"),
+ "bundle_sha256": os.environ.get("BUNDLE_SHA256"), "flavor": os.environ.get("JOB_FLAVOR"),
+ "integration_test": smoke or training_smoke}
+ events = []
+ if smoke:
+ events.append(native.event(project, owner, 0, {"integration/offline_online_roundtrip": 1.0}, metadata))
+ else:
+ for path in [output / "audit/metrics.jsonl", output / "run/metrics.jsonl"]:
+ for row in native.read_metrics(path):
+ events.append(native.event(project, owner, row["step"],
+ native.scalars({k: v for k, v in row.items() if k != "step"}, "train/"), metadata))
+ for record in checkpoint_scores(output):
+ score = record["scores"]
+ if not score.get("comparison_ready"):
+ continue
+ values = {"eval/pass_at_1": score["average_pass_at_1"], "eval/graded_cells": score["graded_cells"]}
+ for harness, item in score["harnesses"].items():
+ values[f"eval/{harness}/pass_at_1"] = item["pass_at_1"]
+ for level, detail in item["difficulty"].items():
+ values[f"eval/{harness}/{level}/pass_at_1"] = detail["pass_at_1"]
+ events.append(native.event(project, owner, record["step"], values, metadata, identity=record["source"]))
+ scores = native.read_json(output / "canonical_scores.json", {})
+ if scores.get("comparison_ready"):
+ values = {"eval/pass_at_1": scores["average_pass_at_1"], "eval/graded_cells": scores["graded_cells"]}
+ for h, score in scores["harnesses"].items():
+ values[f"eval/{h}/pass_at_1"] = score["pass_at_1"]
+ for level, item in score["difficulty"].items():
+ values[f"eval/{h}/{level}/pass_at_1"] = item["pass_at_1"]
+ events.append(native.event(project, "evaluation-curve", step, values, metadata))
+ if not events:
+ return
+ ledger = output / "trackio-events.jsonl"
+ existing = {json.loads(line)["log_id"] for line in ledger.read_text().splitlines() if line.strip()} if ledger.exists() else set()
+ events = [event for event in events if event["log_id"] not in existing]
+ if not events:
+ return
+ native.import_events(events)
+ native.backup_project(project, output / "trackio-backup")
+ with ledger.open("a") as stream:
+ for event in events:
+ stream.write(json.dumps(event) + "\n")
+ if not os.environ.get("TRACKIO_SPACE"):
+ from trackio.sqlite_storage import SQLiteStorage
+ write_json(output / "trackio_verified.json", {"passed": True, "project": project,
+ "run": owner, "local_database": str(SQLiteStorage.get_project_db_path(project)),
+ "mode": "offline", "remote_storage": "run artifact bucket", "native_remote_readback": False,
+ "updated_at": time.time(), "unique_events": len(existing) + len(events)})
+ return
+ config = {"logging": {"project": project, "space_id": os.environ["TRACKIO_SPACE"],
+ "bucket_id": "HuggingEnvs/data-agent-daytona-trackio"}}
+ from trackio.deploy import sync_incremental
+ from trackio.remote_client import RemoteClient
+ sync_incremental(project, os.environ["TRACKIO_SPACE"], private=False, pending_only=False)
+ client = RemoteClient(os.environ["TRACKIO_SPACE"], hf_token=os.environ["HF_TOKEN"],
+ httpx_kwargs={"timeout": 60})
+ configuration = native.configuration_records(project)
+ if configuration:
+ client.predict(api_name="/bulk_log", logs=configuration, hf_token=os.environ["HF_TOKEN"])
+ if smoke:
+ from trackio.remote_client import RemoteClient
+ from trackio.sqlite_storage import SQLiteStorage
+ client = RemoteClient(os.environ["TRACKIO_SPACE"], hf_token=os.environ["HF_TOKEN"],
+ httpx_kwargs={"timeout": 60})
+ # Native read-back, not merely an accepted upload request.
+ runs = client.predict(api_name="/get_runs_for_project", project=project)
+ assert owner in str(runs), f"Trackio read-back did not contain integration run {owner}"
+ logs = client.predict(api_name="/get_logs", project=project, run=owner, run_id=None, scalar_only=True)
+ assert logs and "integration/offline_online_roundtrip" in str(logs)
+ write_json(output / "trackio_verified.json", {"passed": True, "project": project,
+ "run": owner, "local_database": str(SQLiteStorage.get_project_db_path(project)),
+ "space": os.environ["TRACKIO_SPACE"], "native_remote_readback": True})
+
+
+if __name__ == "__main__":
+ import argparse
+ p = argparse.ArgumentParser()
+ p.add_argument("--out", type=Path, required=True)
+ p.add_argument("--arm", required=True)
+ p.add_argument("--step", type=int, default=0)
+ p.add_argument("--smoke", action="store_true")
+ p.add_argument("--watch", action="store_true")
+ p.add_argument("--stop-file", type=Path)
+ a = p.parse_args()
+ a.out.mkdir(parents=True, exist_ok=True)
+ if a.watch and a.stop_file is None:
+ p.error("--watch requires --stop-file")
+ while True:
+ sync_metrics(a.out, a.arm, a.step, a.smoke)
+ write_json(a.out / "trackio_collector.json", {"last_success": time.time(), "pid": os.getpid()})
+ if not a.watch or a.stop_file.exists():
+ break
+ time.sleep(30)
diff --git a/04-data-agent/hf/runtime/native_grading_audit.py b/04-data-agent/hf/runtime/native_grading_audit.py
new file mode 100644
index 0000000..1fd79c5
--- /dev/null
+++ b/04-data-agent/hf/runtime/native_grading_audit.py
@@ -0,0 +1,57 @@
+"""Verify frozen native grading parameters and rescore the unchanged first answers."""
+import hashlib
+import json
+from pathlib import Path
+
+from common import RUN, write_json
+
+
+def verify(ledger, output):
+ from data_agent_env.task import DataAgentTask
+ from data_agent_env.tasks import _frozen_rows
+ from data_agent_env.verifier import grade_rollout
+ import data_agent_env
+
+ selected = {}
+ encoded = Path(ledger).read_bytes()
+ for line in encoded.decode().splitlines():
+ row = json.loads(line)
+ if row.get('correctness') is not None:
+ selected.setdefault(row['index'], row)
+ if set(selected) != set(range(250)):
+ raise ValueError('Native grading qualification requires all 250 fixed first answers')
+ tasks = {}
+ checked = 0
+ for split in ('train', 'test'):
+ rows = _frozen_rows(str(RUN / 'datasets'), split)
+ for index, row in enumerate(rows):
+ task = DataAgentTask.from_row(row)
+ if (task.atol, task.rtol) != (row['atol'], row['rtol']):
+ raise ValueError('Native task parsing changed a frozen grading tolerance')
+ checked += 1
+ if split == 'test': tasks[index] = task
+ if checked != 1250:
+ raise ValueError('Unexpected fixed training/test task count')
+ reports = []
+ for index, row in sorted(selected.items()):
+ captured = Path(row['capture_file']).read_bytes()
+ result = json.loads(captured)
+ task = tasks[index]
+ if result['metadata']['task_id'] != task.task_id or row['task_id'] != task.task_id:
+ raise ValueError('Native capture/task identity mismatch')
+ answer, source = result.get('answer'), result.get('answer_source')
+ grade = grade_rollout(task, lambda _: answer if source == 'file' else None,
+ ('/answer',), final_message=answer if source == 'chat' else None)
+ if grade.correctness != result['correctness'] or grade.correctness != row['correctness']:
+ raise ValueError('Rescoring changed a first graded native answer')
+ reports.append({'index': index, 'task_id': task.task_id, 'correctness': grade.correctness,
+ 'capture_sha256': hashlib.sha256(captured).hexdigest()})
+ if Path(ledger).read_bytes() != encoded:
+ raise ValueError('First-graded ledger changed during verification')
+ package = Path(data_agent_env.__file__).parent
+ proof = {'passed': True, 'parameters_verified': checked, 'original_graded_records_preserved': True,
+ 'ledger_sha256': hashlib.sha256(encoded).hexdigest(), 'reports': reports,
+ 'runtime_files': {name: hashlib.sha256((package / name).read_bytes()).hexdigest()
+ for name in ('task.py', 'tasks.py', 'verifier.py', 'grader.py')}}
+ write_json(Path(output) / 'verification.json', proof)
+ return proof
diff --git a/04-data-agent/hf/runtime/native_tool_smoke.py b/04-data-agent/hf/runtime/native_tool_smoke.py
new file mode 100644
index 0000000..20ec067
--- /dev/null
+++ b/04-data-agent/hf/runtime/native_tool_smoke.py
@@ -0,0 +1,60 @@
+"""Real HTTP/MCP bash/SETA contract smoke; oracle answers are never sent to a model."""
+import argparse
+import concurrent.futures
+import json
+import time
+from pathlib import Path
+
+
+def one(server, index, correct, run):
+ from harbor.models.task.task import Task
+ from whitebox_bash import white_box_bash_env
+ manifest = json.loads((run / 'test_manifest.json').read_text())
+ native = Task(run / 'datasets/test/tasks' / manifest['tasks'][index]['name'])
+ env = white_box_bash_env(server, toolsets='bash,seta', step_limit=30)()
+ checks = {}
+ start = time.monotonic()
+ try:
+ prompt = env.reset(split='test', index=index)
+ checks['exact_task_instruction'] = prompt == (native.paths.task_dir / 'instruction.md').read_text()
+ checks['write'] = '[error]' not in env.write(path='contract.txt', content='alpha\nbeta\n')
+ checks['read'] = 'alpha\nbeta' in env.read(path='contract.txt')
+ checks['edit'] = '[error]' not in env.edit(path='contract.txt', old='beta', new='gamma')
+ checks['bash_same_filesystem'] = 'gamma' in env.bash(command='cat contract.txt')
+ checks['grep'] = 'gamma' in env.grep(pattern='gamma', path='contract.txt')
+ checks['glob'] = 'contract.txt' in env.glob(pattern='contract.*')
+ checks['ls'] = 'contract.txt' in env.ls(path='.')
+ checks['nonzero_command_preserved'] = '7' in env.bash(command='exit 7')
+ answer = native.config.verifier.env['EXPECTED_ANSWER'] if correct else '__known_wrong_contract_answer__'
+ env.submit_solution(answer=answer)
+ reward = env.get_reward()
+ checks['frozen_verifier'] = reward == float(correct)
+ return {'index':index, 'expected_correct':correct, 'checks':checks,
+ 'passed':all(checks.values()), 'elapsed_s':time.monotonic()-start}
+ except Exception as exc:
+ return {'index':index, 'passed':False, 'checks':checks, 'error_type':type(exc).__name__}
+ finally:
+ if env._session is not None:
+ try:
+ env.get_reward()
+ except Exception:
+ pass
+ env._mcp.close()
+
+
+def main():
+ p=argparse.ArgumentParser(description=__doc__)
+ p.add_argument('--run',type=Path,required=True)
+ p.add_argument('--server',required=True)
+ p.add_argument('--concurrency',type=int,default=4)
+ args=p.parse_args()
+ with concurrent.futures.ThreadPoolExecutor(max_workers=args.concurrency) as pool:
+ results=list(pool.map(lambda x: one(args.server,x//2,bool(x%2),args.run),range(8)))
+ report={'passed':all(r['passed'] for r in results),'expected':8,'results':results}
+ (args.run/'whitebox_tools_smoke.json').write_text(json.dumps(report,indent=2)+'\n')
+ print(json.dumps(report))
+ return 0 if report['passed'] else 2
+
+
+if __name__ == '__main__':
+ raise SystemExit(main())
diff --git a/04-data-agent/hf/runtime/opencode_space.py b/04-data-agent/hf/runtime/opencode_space.py
new file mode 100644
index 0000000..697f978
--- /dev/null
+++ b/04-data-agent/hf/runtime/opencode_space.py
@@ -0,0 +1,110 @@
+"""Space entry point for envs/blackbox-opencode, independent of Harbor execution."""
+import hmac
+import os
+from common import ROOT, RUN, configure
+
+configure()
+os.environ.update(DATA_AGENT_SPLITS="train,test", DATA_AGENT_SANDBOX="daytona",
+ DATA_AGENT_FROZEN_TASKS_DIR=str(RUN / "datasets"), DATA_AGENT_CAPTURE_EXPOSE="gradio",
+ DATA_AGENT_MAX_CONCURRENT=os.environ.get("SANDBOX_CAPACITY", "32"),
+ HF_SANDBOX_NAMESPACE="HuggingEnvs", ENABLE_WEB_INTERFACE="false",
+ RUN_OWNER=os.environ.get("SPACE_ID", "standalone-opencode").replace("/", "-"))
+from data_agent_env.server.app import app
+from data_agent_env.server.environment import DataAgentEnvironment
+from data_agent_env.tasks import task_at, rows_for, _public
+from data_agent_env.sandbox import BACKENDS, available
+from data_agent_env.server.rollout import OPENCODE_VERSION
+
+
+@app.get("/deployment")
+def deployment():
+ import hashlib
+ return {"arm": "opencode", "implementation": "standalone-opencode", "mode": "shared",
+ "source": "HuggingEnvs/04-data-agent/envs/blackbox-opencode",
+ "bundle_sha256": os.environ.get("BUNDLE_SHA256"),
+ "train_tasks": len(rows_for("train")), "test_tasks": len(rows_for("test")),
+ "test_manifest_sha256": hashlib.sha256((RUN / "test_manifest.json").read_bytes()).hexdigest(),
+ "sandboxes": {"supported": list(BACKENDS), "usable": available()},
+ "sandbox_capacity": int(os.environ["DATA_AGENT_MAX_CONCURRENT"]),
+ "opencode_version": OPENCODE_VERSION, "output_tokens": {"train": 16384, "test": 4096},
+ "interactive_ui": True, "trackio": False}
+
+
+class ServiceAuth:
+ """Native execution RPCs use the owner credential; the public UI supplies its own inference."""
+ def __init__(self, app): self.app = app
+ async def __call__(self, scope, receive, send):
+ path = scope.get("path", "")
+ protected = (scope["type"] == "websocket" or path in {"/step", "/reset", "/state", "/mcp"}
+ or path.startswith("/mcp/"))
+ if protected:
+ headers = dict(scope.get("headers", []))
+ token = os.environ.get("HF_TOKEN", "")
+ if not token or not hmac.compare_digest(headers.get(b"authorization", b""), ("Bearer " + token).encode()):
+ if scope["type"] == "websocket":
+ await send({"type": "websocket.close", "code": 1008})
+ else:
+ from starlette.responses import JSONResponse
+ await JSONResponse({"detail": "Authentication required"}, status_code=401)(scope, receive, send)
+ return
+ await self.app(scope, receive, send)
+
+app.add_middleware(ServiceAuth)
+
+@app.on_event("startup")
+async def startup():
+ import anyio.to_thread
+ anyio.to_thread.current_default_thread_limiter().total_tokens = 256
+ for split in ("train", "test"): rows_for(split)
+
+
+def preview(split, index):
+ task = task_at(split, int(index))
+ return task.instruction, f"**{task.difficulty_tier.capitalize()}** · {task.task_id}"
+
+
+def rollout(split, index, backend, url, model, key):
+ import json
+ if not url.strip() or not model.strip():
+ raise gr.Error("Enter an inference endpoint and model to run this task.")
+ result = json.loads(DataAgentEnvironment()._run_rollout(split, int(index), url.strip(),
+ model.strip(), backend, 17, 600, False, key))
+ if result["reward"] is None:
+ return "The rollout could not be graded. Check the endpoint and try again.", {}
+ dialogue = "\n\n".join(f"### Turn {i+1}\n{t['text']}\n" +
+ ("```json\n" + json.dumps(t['tool_calls'], indent=2) + "\n```" if t['tool_calls'] else "")
+ for i,t in enumerate(result['turns']))
+ return dialogue, {k:result[k] for k in ('correctness','reward','answer','answer_source','n_tool_calls','timed_out')}
+
+import gradio as gr
+from environment_ui import UI_CSS
+with gr.Blocks(title="Data Agent Blackbox OpenCode Env") as demo:
+ gr.HTML('''
HUGGINGENVS · DATA AGENT
+
Blackbox OpenCode
Give OpenCode a real data-analysis task. It explores the tables,
+ runs its own tools, and submits an answer in an isolated sandbox.
+
Daytona · HF · E2B
+ 1,000 training tasks250 test tasks
''')
+ with gr.Row():
+ with gr.Column(scale=5, elem_id="task-panel"):
+ split = gr.Radio(["train", "test"], value="test", label="Task set")
+ index = gr.Number(value=2, precision=0, minimum=0, maximum=249, label="Task index")
+ badge = gr.Markdown()
+ instruction = gr.Textbox(label="Task", lines=16, interactive=False, elem_id="task-instructions")
+ with gr.Column(scale=4, elem_id="workspace-panel"):
+ backend = gr.Dropdown(["daytona", "hf", "e2b"], value="daytona", label="Sandbox")
+ url = gr.Textbox(label="OpenAI-compatible inference URL", placeholder="https://…/v1")
+ model = gr.Textbox(label="Model", value="Qwen/Qwen3.5-2B")
+ key = gr.Textbox(label="Inference API key", type="password")
+ run = gr.Button("Run OpenCode", variant="primary")
+ score = gr.JSON(label="Result")
+ transcript = gr.Markdown(label="Agent activity")
+ split.change(lambda s: gr.update(maximum=999 if s == "train" else 249, value=2), split, index)
+ for event in (split.change, index.change): event(preview, [split,index], [instruction,badge], api_name=False)
+ demo.load(preview, [split,index], [instruction,badge], api_name=False)
+ run.click(rollout, [split,index,backend,url,model,key], [transcript,score], concurrency_limit=2, api_name=False)
+app = gr.mount_gradio_app(app, demo, path="/", css=UI_CSS)
+
+if __name__ == "__main__":
+ import uvicorn
+ uvicorn.run(app, host="0.0.0.0", port=7860, ws_ping_interval=20, ws_ping_timeout=None,
+ timeout_keep_alive=120)
diff --git a/04-data-agent/hf/runtime/preflight.py b/04-data-agent/hf/runtime/preflight.py
new file mode 100644
index 0000000..bcd145f
--- /dev/null
+++ b/04-data-agent/hf/runtime/preflight.py
@@ -0,0 +1,34 @@
+"""Exercise the actual frozen import and CLI boundaries in a fresh HF Job."""
+from common import ROOT, RUN, TOOLS, TRAIN_PY, ENV_PY, configure, verify_bundle, write_json
+import argparse
+import json
+import os
+import subprocess
+import sys
+import time
+
+
+def main():
+ argparse.ArgumentParser().parse_known_args()
+ configure()
+ count = verify_bundle()
+ subprocess.run([str(ENV_PY), str(ROOT / "hf/runtime/ui_smoke.py"), "--help"], check=True, stdout=subprocess.DEVNULL)
+ scripts = [RUN / "source/HuggingEnvs/04-data-agent/train/train_harbor_multi.py",
+ TOOLS / "train_whitebox_daytona.py", TOOLS / "eval_whitebox_native.py",
+ RUN / "eval-source/eval_concurrent.py"]
+ for script in scripts:
+ subprocess.run([str(TRAIN_PY), str(script), "--help"], check=True, stdout=subprocess.DEVNULL)
+ subprocess.run([str(ENV_PY), "-c", "from openenv.harbor.serving import HarborService; from whitebox_bash.server.environment import WhiteBoxBashEnvironment; from daytona_whitebox_backend import load_frozen_tasks; assert len(load_frozen_tasks('train'))==1000; assert len(load_frozen_tasks('test'))==250"], check=True)
+ subprocess.run([str(ENV_PY), str(ROOT / "hf/runtime/check_task_schedule.py")], check=True)
+ result = {"passed": True, "verified_files": count, "checked_at": time.time(),
+ "bundle_sha256": os.environ["BUNDLE_SHA256"], "role": "preflight"}
+ out = ROOT / "outputs/preflight"
+ write_json(out / "result.json", result)
+ from huggingface_hub import HfApi
+ dest = "hf://buckets/" + os.environ["ARTIFACT_BUCKET"] + "/" + os.environ["RUN_ID"] + "/preflight"
+ HfApi().sync_bucket(str(out), dest, quiet=True)
+ print(json.dumps(result), flush=True)
+
+
+if __name__ == "__main__":
+ main()
diff --git a/04-data-agent/hf/runtime/provider_demo.py b/04-data-agent/hf/runtime/provider_demo.py
new file mode 100644
index 0000000..c466828
--- /dev/null
+++ b/04-data-agent/hf/runtime/provider_demo.py
@@ -0,0 +1,158 @@
+"""Optional HF-signed-in demos; these conversations are separate from eval ledgers."""
+import inspect
+import json
+import os
+import time
+from typing import get_type_hints
+
+import gradio as gr
+
+from inference_providers import catalog, credentials, visitor_token
+
+
+def enabled():
+ # Never expose Gradio's local mocked-login flow in an unconfigured deployment.
+ return os.environ.get("SYSTEM") == "spaces" and all(os.environ.get(k) for k in
+ ("SPACE_ID", "OAUTH_CLIENT_ID", "OAUTH_CLIENT_SECRET", "OAUTH_SCOPES", "OPENID_PROVIDER_URL"))
+
+
+def provider_choices():
+ try:
+ providers = sorted({p for p, _ in catalog.rows()})
+ return gr.update(choices=providers, value=None), gr.update(choices=[], value=None)
+ except Exception:
+ raise gr.Error("The HF model catalog is unavailable. Try loading it again.") from None
+
+
+def model_choices(provider):
+ if not provider:
+ return gr.update(choices=[], value=None)
+ try:
+ models = sorted(m for p, m in catalog.rows() if p == provider)
+ return gr.update(choices=models, value=None)
+ except Exception:
+ raise gr.Error("The HF model catalog is unavailable. Try loading it again.") from None
+
+
+def model_details(provider, model):
+ if not provider or not model:
+ return "Choose a provider and model."
+ try:
+ row = catalog.rows()[(provider, model)]
+ except (KeyError, ValueError):
+ return "This model is no longer available from that provider. Refresh the catalog."
+ price = row.get("pricing", {})
+ text = "Tool calling supported. "
+ if "input" in price and "output" in price:
+ text += f"Per million tokens: ${price['input']:g} input · ${price['output']:g} output. "
+ if row.get("context_length"):
+ text += f"Context: {row['context_length']:,} tokens. "
+ return text + "Usage is charged to your HF account."
+
+
+def selected(oauth, provider, model):
+ try:
+ visitor_token(oauth)
+ return catalog.select(provider, model)
+ except ValueError as exc:
+ raise gr.Error(str(exc)) from None
+
+
+def controls():
+ gr.LoginButton()
+ gr.Markdown("Sign in and choose a model that can use tools. Inference uses your HF account's credits; "
+ "a demo runs for at most 10 minutes and 17 agent turns.")
+ refresh = gr.Button("Load available models")
+ with gr.Row():
+ provider = gr.Dropdown([], label="Inference provider", interactive=True)
+ model = gr.Dropdown([], label="Model", interactive=True)
+ detail = gr.Markdown("Choose a provider and model.")
+ refresh.click(provider_choices, outputs=[provider, model], api_visibility="private")
+ provider.change(model_choices, provider, model, api_visibility="private")
+ model.change(model_details, [provider, model], detail, api_visibility="private")
+ return provider, model
+
+
+def blackbox(split, index, harness, provider, model, oauth_token: gr.OAuthToken):
+ from environment_ui import blackbox_run, LOCAL
+ target = selected(oauth_token, provider, model)
+ with credentials.issue(oauth_token, target) as key:
+ yield from blackbox_run(split, index, harness, LOCAL + "/hf-inference/v1", target, key)
+
+
+def tool_schemas(env):
+ """Derive the demo schema from the environment's existing declared tool surface."""
+ from whitebox_bash.tools import specs_for
+ from pydantic import ConfigDict, create_model
+ tools, methods = [], {}
+ for spec in specs_for("bash,seta"):
+ method = getattr(env, spec.name)
+ params = inspect.signature(method).parameters
+ hints = get_type_hints(method)
+ arguments = create_model(spec.name, __config__=ConfigDict(extra="forbid"), **{
+ name: (hints[name], ... if p.default is inspect.Parameter.empty else p.default)
+ for name, p in params.items()})
+ tools.append({"type": "function", "function": {"name": spec.name, "description": spec.summary,
+ "parameters": arguments.model_json_schema()}})
+ methods[spec.name] = method
+ return tools, methods
+
+
+def whitebox(ui, split, index, provider, model, oauth, request):
+ from openai import OpenAI
+ from environment_ui import LOCAL
+ target = selected(oauth, provider, model)
+ # Own the same browser workspace lock as manual actions for the whole episode.
+ prompt, _, _ = ui.start(split, index, request)
+ session = ui.session(request)
+ with session.lock:
+ try:
+ env = session.env
+ if env is None:
+ raise gr.Error("The workspace was closed. Start the agent again.")
+ tools, methods = tool_schemas(env)
+ messages = [{"role": "system", "content": "You are a data-analysis agent in an isolated workspace. "
+ "Use the tools to inspect the data and compute the answer. Call submit_solution with the answer itself."},
+ {"role": "user", "content": prompt}]
+ deadline = time.monotonic() + 600
+ yield "Starting the agent…", {"state": "running", "model": target}
+ with credentials.issue(oauth, target) as key, OpenAI(base_url=LOCAL + "/hf-inference/v1", api_key=key,
+ timeout=120, max_retries=0) as client:
+ for turn in range(17):
+ if time.monotonic() >= deadline:
+ break
+ response = client.chat.completions.create(model=target, messages=messages, tools=tools,
+ tool_choice="auto", max_tokens=4096, temperature=0.8,
+ timeout=min(120, deadline-time.monotonic()))
+ msg = response.choices[0].message
+ messages.append(msg.model_dump(exclude_none=True))
+ for call in msg.tool_calls or []:
+ name = call.function.name
+ try:
+ if name not in methods:
+ raise ValueError("Unknown tool")
+ args = json.loads(call.function.arguments)
+ if not isinstance(args, dict):
+ raise ValueError("Tool arguments must be an object")
+ result = str(methods[name](**args))
+ except (ValueError, TypeError):
+ result = "[error] Invalid tool name or arguments; use the provided schema."
+ messages.append({"role": "tool", "tool_call_id": call.id, "content": result})
+ if env._reward is not None:
+ break
+ session.used_at = time.monotonic()
+ yield json.dumps(messages, indent=2, ensure_ascii=False)[-120000:], {
+ "state": "running", "model": target, "turns": turn + 1}
+ if not msg.tool_calls or env._reward is not None:
+ break
+ reward = env.get_reward()
+ yield json.dumps(messages, indent=2, ensure_ascii=False)[-120000:], {
+ "state": "finished", "reward": reward, "model": target,
+ "capture_level": "text", "training_eligible": False}
+ except gr.Error:
+ raise
+ except Exception as exc:
+ raise gr.Error(f"The demo stopped ({type(exc).__name__}). Check your HF inference credits or try another model.") from None
+ finally:
+ ui.dispose(session)
+ session.used_at = time.monotonic()
diff --git a/04-data-agent/hf/runtime/qualify_training.py b/04-data-agent/hf/runtime/qualify_training.py
new file mode 100644
index 0000000..280f867
--- /dev/null
+++ b/04-data-agent/hf/runtime/qualify_training.py
@@ -0,0 +1,96 @@
+"""Run both disposable optimizer smokes without restarting active eval Spaces."""
+import argparse
+import json
+import os
+from pathlib import Path
+import sys
+import time
+
+from common import ROOT, write_json
+from checkpoint_store import download_json
+from coordinator import TERMINAL
+
+
+def run(output):
+ from huggingface_hub import HfApi
+ from huggingface_hub.errors import EntryNotFoundError
+ sys.path.insert(0, str(ROOT / "hf"))
+ import deploy
+ api = HfApi()
+ config = json.loads((ROOT / "hf/configs/deployment.json").read_text())
+ policy = config["training_smoke"]
+ output.mkdir(parents=True, exist_ok=True)
+ bundle = json.loads(Path("/bundle/bundle.json").read_text())
+ bundle.update(repo=os.environ["BUNDLE_REPO"], revision=os.environ["BUNDLE_REVISION"])
+ write_json(output / "bundle_uploaded.json", bundle)
+ namespace = config["namespace"]
+ destination = "hf://buckets/" + os.environ["ARTIFACT_BUCKET"] + "/" + os.environ["RUN_ID"] + "/jobs/" + os.environ["RUN_OWNER"]
+ state = {"phase": "qualifying", "bundle_sha256": bundle["sha256"], "jobs": {}, "passed": False,
+ "long_training_launched": False, "standalone_gate": "wait for baseline Daytona cohort to release Space capacity"}
+
+ def persist():
+ state["updated_at"] = time.time()
+ write_json(output / "qualification.json", state)
+ api.sync_bucket(str(output), destination, quiet=True)
+
+ def launch(arm):
+ if arm in state["jobs"]:
+ return
+ matches = [j for j in api.list_jobs(namespace=namespace, labels={"role": "train", "arm": arm, "phase": "smoke"})
+ if j.environment.get("BUNDLE_SHA256") == bundle["sha256"]]
+ if len(matches) > 1:
+ raise RuntimeError("Multiple smoke jobs for the same arm and bundle")
+ if matches:
+ state["jobs"][arm] = {"id": matches[0].id, "stage": matches[0].status.stage}
+ return
+ state["jobs"][arm] = {"stage": "submitting"}
+ persist()
+ args = argparse.Namespace(role="train", arm=arm, phase="smoke", flavor=config["compute"]["training_flavor"], timeout="2h",
+ dp=1, limit=0, resume_eval_owner=None, training_job=None, baseline_job=None, smoke_job=None,
+ baseline_job_map=None, space_bundle_sha=policy["space_bundle_pins"][arm])
+ secrets = {k: os.environ[k] for k in ["HF_TOKEN", "DAYTONA_API_KEY", "DAYTONA_API_URL", "DAYTONA_TARGET"] if os.environ.get(k)}
+ value = deploy.submit(api, config, secrets, output, args)
+ state["jobs"][arm] = {"id": value["id"], "stage": value["stage"]}
+
+ try:
+ peers = [j for j in api.list_jobs(namespace=namespace, labels={"role": "coordinator", "phase": "qualify"})
+ if j.status.stage not in TERMINAL and j.environment.get("BUNDLE_SHA256") == bundle["sha256"]]
+ if peers and min(peers, key=lambda j: j.id).environment["RUN_OWNER"] != os.environ["RUN_OWNER"]:
+ raise RuntimeError("Another coordinator owns qualification for this bundle")
+ launch("whitebox")
+ while True:
+ if "opencode" not in state["jobs"]:
+ baseline = api.inspect_job(job_id=policy["opencode_wait_for_daytona_baseline_job"], namespace=namespace)
+ source = "hf://buckets/" + baseline.environment["ARTIFACT_BUCKET"] + "/" + baseline.environment["RUN_ID"] + "/jobs/" + baseline.environment["RUN_OWNER"] + "/daytona"
+ try:
+ scores = download_json(source, "scores.json", output / "daytona-baseline-scores.json", api)
+ except EntryNotFoundError:
+ scores = {}
+ state["daytona_baseline_graded"] = scores.get("graded_cells", 0)
+ if scores.get("comparison_ready") and scores.get("expected_cells") == 250:
+ launch("opencode")
+ elif baseline.status.stage in TERMINAL:
+ raise RuntimeError("Daytona baseline ended without complete TiTO-qualified coverage")
+ complete = len(state["jobs"]) == 2
+ for arm, item in state["jobs"].items():
+ job = api.inspect_job(job_id=item["id"], namespace=namespace)
+ item["stage"] = job.status.stage
+ if job.status.stage in TERMINAL and job.status.stage != "COMPLETED":
+ raise RuntimeError(f"{arm} optimizer smoke ended {job.status.stage}")
+ complete = complete and job.status.stage == "COMPLETED"
+ if job.status.stage == "COMPLETED" and "proof" not in item:
+ source = "hf://buckets/" + job.environment["ARTIFACT_BUCKET"] + "/" + job.environment["RUN_ID"] + "/jobs/" + job.environment["RUN_OWNER"]
+ proof = download_json(source, "training_smoke_verified.json", output / f"{arm}-verified.json", api)
+ if not proof.get("passed") or proof.get("bundle_sha256") != bundle["sha256"]:
+ raise RuntimeError("Smoke evidence does not match the qualification bundle")
+ item["proof"] = proof
+ persist()
+ if complete:
+ state.update(phase="complete", passed=True)
+ persist()
+ return
+ time.sleep(60)
+ except Exception as exc:
+ state.update(phase="needs_attention", error_type=type(exc).__name__, error=str(exc))
+ persist()
+ raise
diff --git a/04-data-agent/hf/runtime/service_contract.py b/04-data-agent/hf/runtime/service_contract.py
new file mode 100644
index 0000000..5c7448f
--- /dev/null
+++ b/04-data-agent/hf/runtime/service_contract.py
@@ -0,0 +1,32 @@
+"""Check the remote training API without launching a sandbox or inference request."""
+import asyncio
+import json
+
+
+def validate_tools(response, arm):
+ tools = response.get("data", {}).get("observation", {}).get("tools", [])
+ tool = next((item for item in tools if item.get("name") == "run_rollout"), None)
+ if tool is None:
+ raise ValueError("Environment did not advertise run_rollout")
+ properties = tool.get("input_schema", {}).get("properties", {})
+ required = {"sampling", "llm_url", "model"}
+ required |= {"agent_timeout_sec", "agent_step_limit"} if arm == "blackbox" else {"require_tokens", "agent_timeout_s"}
+ missing = required - properties.keys()
+ if missing:
+ raise ValueError("Deployed environment lacks training arguments: " + ", ".join(sorted(missing))
+ + ". Deploy the matching environment bundle before starting training.")
+ return {"passed": True, "arm": arm, "arguments": sorted(properties)}
+
+
+async def _probe(url, token, arm):
+ from websockets.asyncio.client import connect
+ url = url.rstrip("/").replace("https://", "wss://", 1).replace("http://", "ws://", 1)
+ headers = {"Authorization": "Bearer " + token} if token else {}
+ async with connect(url + "/ws", additional_headers=headers, open_timeout=30) as socket:
+ await socket.send(json.dumps({"type": "step", "data": {"type": "list_tools"}}))
+ response = json.loads(await asyncio.wait_for(socket.recv(), timeout=30))
+ return validate_tools(response, arm)
+
+
+def check(url, token, arm):
+ return asyncio.run(_probe(url, token, arm))
diff --git a/04-data-agent/hf/runtime/service_policy.py b/04-data-agent/hf/runtime/service_policy.py
new file mode 100644
index 0000000..ee67861
--- /dev/null
+++ b/04-data-agent/hf/runtime/service_policy.py
@@ -0,0 +1,90 @@
+"""Per-rollout budgets and shared admission for the two environment Spaces."""
+from contextlib import asynccontextmanager
+import asyncio
+import os
+from pathlib import Path
+import threading
+import time
+
+
+def workload(dataset):
+ return "train" if Path(str(dataset).rstrip("/")).name == "train" else "eval"
+
+
+def output_limit(dataset):
+ role = workload(dataset)
+ return int(os.environ.get("OPENENV_" + role.upper() + "_OUTPUT_TOKENS",
+ "16384" if role == "train" else "4096"))
+
+
+class Admission:
+ """Bound active sandboxes, keeping slots that eval cannot consume for training.
+
+ Synchronous native whitebox tools and async Harbor rollouts use the same accounting.
+ Waiting async callers never occupy the shared thread pool.
+ """
+ def __init__(self, total, train_reserve):
+ if not 0 < train_reserve < total:
+ raise ValueError("Require 0 < training reservation < sandbox capacity")
+ self.total, self.train_reserve = total, train_reserve
+ self.active = {"train": 0, "eval": 0}
+ self.waiting = {"train": 0, "eval": 0}
+ self.condition = threading.Condition()
+
+ def _available(self, role):
+ return (sum(self.active.values()) < self.total and
+ (role == "train" or self.active["eval"] < self.total - self.train_reserve))
+
+ def acquire(self, role, timeout=900):
+ with self.condition:
+ self.waiting[role] += 1
+ try:
+ if not self.condition.wait_for(lambda: self._available(role), timeout):
+ raise TimeoutError("Shared sandbox capacity wait expired")
+ self.active[role] += 1
+ finally:
+ self.waiting[role] -= 1
+
+ def release(self, role):
+ with self.condition:
+ if self.active[role] <= 0:
+ raise RuntimeError("Sandbox reservation released twice")
+ self.active[role] -= 1
+ self.condition.notify_all()
+
+ @asynccontextmanager
+ async def slot(self, dataset, timeout=900):
+ role = workload(dataset)
+ with self.condition:
+ self.waiting[role] += 1
+ acquired = False
+ waiting = True
+ try:
+ deadline = time.monotonic() + timeout
+ while not acquired:
+ with self.condition:
+ if self._available(role):
+ self.active[role] += 1
+ self.waiting[role] -= 1
+ waiting = False
+ acquired = True
+ if not acquired:
+ if time.monotonic() >= deadline:
+ raise TimeoutError("Shared sandbox capacity wait expired")
+ await asyncio.sleep(0.1)
+ yield
+ finally:
+ if waiting:
+ with self.condition:
+ self.waiting[role] -= 1
+ if acquired:
+ self.release(role)
+
+ def snapshot(self):
+ with self.condition:
+ return {"capacity": self.total, "train_reserved": self.train_reserve,
+ "active": dict(self.active), "waiting": dict(self.waiting)}
+
+
+admission = Admission(int(os.environ.get("SANDBOX_CAPACITY", "128")),
+ int(os.environ.get("TRAIN_RESERVED_SANDBOXES", "24")))
diff --git a/04-data-agent/hf/runtime/setup_pipeline.py b/04-data-agent/hf/runtime/setup_pipeline.py
new file mode 100644
index 0000000..f1eaf21
--- /dev/null
+++ b/04-data-agent/hf/runtime/setup_pipeline.py
@@ -0,0 +1,191 @@
+"""Finish the two HF baselines, deploy the tested runtime, smoke, then start training."""
+import argparse
+import json
+import math
+import os
+from pathlib import Path
+import shutil
+import sys
+import time
+
+from common import ROOT, ENV_PY, start, write_json
+from checkpoint_store import download_json
+from coordinator import TERMINAL
+BUNDLE = Path("/bundle")
+
+
+def run(output):
+ from huggingface_hub import HfApi
+ import httpx
+ sys.path.insert(0, str(ROOT / "hf"))
+ import deploy
+ api = HfApi()
+ config = json.loads((ROOT / "hf/configs/deployment.json").read_text())
+ namespace = config["namespace"]
+ bundle = json.loads((BUNDLE / "bundle.json").read_text())
+ bundle.update(repo=os.environ["BUNDLE_REPO"], revision=os.environ["BUNDLE_REVISION"])
+ destination = ("hf://buckets/" + os.environ["ARTIFACT_BUCKET"] + "/" + os.environ["RUN_ID"] +
+ "/pipelines/" + bundle["sha256"])
+ output.mkdir(parents=True, exist_ok=True)
+ try:
+ state = download_json(destination, "pipeline.json", output / "pipeline.json", api)
+ except Exception as exc:
+ from huggingface_hub.errors import EntryNotFoundError
+ if not isinstance(exc, EntryNotFoundError):
+ raise
+ state = {"bundle_sha256": bundle["sha256"], "jobs": {}, "phase": "waiting_baselines"}
+
+ def persist():
+ state["updated_at"] = time.time()
+ write_json(output / "pipeline.json", state)
+ api.sync_bucket(str(output), destination, include=["pipeline.json", "jobs/*.json", "launch-proofs/**", "ui-smoke.json"], quiet=True)
+
+ def wait_until(check):
+ while not check():
+ persist()
+ time.sleep(60)
+ persist()
+
+ secrets = {k: os.environ[k] for k in ["HF_TOKEN", "DAYTONA_API_KEY", "DAYTONA_API_URL", "DAYTONA_TARGET", "OPENAI_API_KEY"] if os.environ.get(k)}
+ args_base = dict(dp=1, limit=0, resume_eval_owner=None, training_job=None, baseline_job=None, smoke_job=None)
+
+ def ensure_job(role, arm, phase, **options):
+ key = ":".join([role, arm, phase])
+ jobs = [j for j in api.list_jobs(namespace=namespace, labels={"role": role, "arm": arm, "phase": phase,
+ "run": config["run_id"]}) if j.environment.get("BUNDLE_SHA256") == bundle["sha256"]]
+ if len(jobs) > 1:
+ raise RuntimeError(f"Multiple matching {key} jobs require reconciliation")
+ if jobs:
+ state["jobs"][key] = {"id": jobs[0].id, "stage": jobs[0].status.stage}
+ persist()
+ return jobs[0].id
+ if key in state["jobs"]:
+ raise RuntimeError(f"Unresolved {key} submission intent; inspect HF Jobs before retrying")
+ state["jobs"][key] = {"stage": "submitting"}
+ persist()
+ args = argparse.Namespace(**{**args_base, "role": role, "arm": arm, "phase": phase, **options})
+ value = deploy.submit(api, config, secrets, output, args)
+ state["jobs"][key] = {"id": value["id"], "stage": value["stage"]}
+ persist()
+ return value["id"]
+
+ def completed(ids):
+ done = True
+ for arm, job_id in ids.items():
+ job = api.inspect_job(job_id=job_id, namespace=namespace)
+ state.setdefault("job_status", {})[job_id] = job.status.stage
+ if job.status.stage in TERMINAL and job.status.stage != "COMPLETED":
+ raise RuntimeError(f"{arm} job {job_id} ended as {job.status.stage}; artifacts require inspection")
+ done = done and job.status.stage == "COMPLETED"
+ return done
+
+ try:
+ peers = [j for j in api.list_jobs(namespace=namespace, labels={"role": "coordinator", "phase": "setup", "run": config["run_id"]})
+ if j.status.stage not in TERMINAL and j.environment.get("BUNDLE_SHA256") == bundle["sha256"]]
+ if peers and min(peers, key=lambda j: j.id).environment["RUN_OWNER"] != os.environ["RUN_OWNER"]:
+ raise RuntimeError("Another setup pipeline owns this bundle")
+ baselines = json.loads(os.environ["BASELINE_JOB_MAP"]) if os.environ.get("BASELINE_JOB_MAP") else config["pipeline"]["baseline_jobs"]
+ if set(baselines) != {"blackbox", "whitebox"}:
+ raise ValueError("Both baseline job IDs are required")
+ state["baseline_jobs"] = baselines
+ wait_until(lambda: completed(baselines))
+ for arm, job_id in baselines.items():
+ job = api.inspect_job(job_id=job_id, namespace=namespace)
+ source = "hf://buckets/" + os.environ["ARTIFACT_BUCKET"] + "/" + os.environ["RUN_ID"] + "/jobs/" + job.environment["RUN_OWNER"]
+ scores = download_json(source, "canonical_scores.json", output / f"baseline-{arm}.json", api)
+ if not scores["comparison_ready"] or scores["arm"] != arm:
+ raise ValueError("Baseline coverage, token or version gate failed")
+ state["phase"] = "deploying_training_runtime"
+ persist()
+ bundle_dir = output / "bundle"
+ bundle_dir.mkdir(exist_ok=True)
+ for name in ["bundle.tar.gz", "bundle.json"]:
+ shutil.copy2(BUNDLE / name, bundle_dir / name)
+ write_json(output / "bundle_uploaded.json", bundle)
+
+ def deployments():
+ result = {}
+ for arm, repo in config["resources"]["environment_spaces"].items():
+ host = api.space_info(repo).host.rstrip("/")
+ if not host.startswith("https://"):
+ host = "https://" + host
+ response = httpx.get(host + "/deployment", timeout=30)
+ response.raise_for_status()
+ result[arm] = response.json()
+ return result
+
+ if not state.get("deployed"):
+ wait_until(lambda: all(sum(d["admission"]["active"].values()) == 0 for d in deployments().values()))
+ current = deployments()
+ needed = {arm for arm, d in current.items() if d["bundle_sha256"] != bundle["sha256"]}
+ if needed:
+ deploy.spaces(api, config, secrets, output, needed)
+ def ready():
+ try:
+ return all(d["bundle_sha256"] == bundle["sha256"] for d in deployments().values())
+ except (httpx.HTTPError, ValueError):
+ return False
+ wait_until(ready)
+ state["deployed"] = True
+ if not state.get("ui_passed"):
+ proc = start([ENV_PY, ROOT / "hf/runtime/ui_smoke.py", "--out", output / "ui-smoke.json"], output / "ui-smoke.log")
+ if proc.wait(timeout=600) != 0:
+ raise RuntimeError("Deployed train/test UI isolation test failed")
+ state["ui_passed"] = True
+ state["phase"] = "optimizer_smokes"
+ persist()
+ smokes = {arm: ensure_job("train", arm, "smoke", flavor=config["compute"]["training_flavor"], timeout="2h") for arm in baselines}
+ wait_until(lambda: completed(smokes))
+ state["phase"] = "starting_training"
+ persist()
+ trains = {arm: ensure_job("train", arm, "long", flavor=config["compute"]["training_flavor"], timeout="24h", baseline_job=baselines[arm], smoke_job=smokes[arm])
+ for arm in baselines}
+ for arm, job_id in trains.items():
+ ensure_job("coordinator", arm, "long", flavor="cpu-upgrade", timeout="36h", training_job=job_id)
+ state.update(phase="training_submitted", training_jobs=trains, smoke_jobs=smokes, passed=False)
+ persist()
+ while True:
+ finished = True
+ progress = {}
+ for arm, job_id in trains.items():
+ job = api.inspect_job(job_id=job_id, namespace=namespace)
+ if job.status.stage in TERMINAL and job.status.stage != "COMPLETED":
+ raise RuntimeError(f"Training job {job_id} ended as {job.status.stage}")
+ finished = finished and job.status.stage == "COMPLETED"
+ name = "audit/metrics.jsonl" if arm == "blackbox" else "run/metrics.jsonl"
+ path = output / "metrics" / (arm + ".jsonl")
+ path.parent.mkdir(exist_ok=True)
+ prefix = config["run_id"] + "/jobs/" + job.environment["RUN_OWNER"]
+ from huggingface_hub.errors import EntryNotFoundError
+ try:
+ api.download_bucket_files(os.environ["ARTIFACT_BUCKET"],
+ files=[(prefix + "/" + name, str(path))], raise_on_missing_files=True)
+ except EntryNotFoundError:
+ progress[arm] = {"job_id": job_id, "stage": job.status.stage, "step": 0}
+ continue
+ rows = []
+ for line in path.read_text().splitlines():
+ try:
+ row = json.loads(line)
+ except json.JSONDecodeError:
+ continue
+ if "step" in row and "grad_norm" in row:
+ rows.append(row)
+ if any(not math.isfinite(float(r[k])) for r in rows for k in ["loss", "grad_norm"] if k in r):
+ raise RuntimeError(f"Non-finite optimizer metrics in training job {job_id}")
+ latest = rows[-1] if rows else {"step": 0}
+ rewards = [r["reward"] for r in rows[-20:] if isinstance(r.get("reward"), (float, int))]
+ progress[arm] = {"job_id": job_id, "stage": job.status.stage, "step": latest["step"],
+ "latest_reward": latest.get("reward"), "reward_mean_last20": sum(rewards)/len(rewards) if rewards else None,
+ "nonzero_gradient_updates": sum(r["grad_norm"] > 0 for r in rows)}
+ stable = len(progress) == 2 and all(p["step"] >= 10 and p.get("nonzero_gradient_updates", 0) > 0 for p in progress.values())
+ state.update(phase="completed" if finished else "training_active" if stable else "training_startup",
+ training_progress=progress, passed=stable)
+ persist()
+ if finished:
+ return
+ time.sleep(600 if stable else 60)
+ except Exception as exc:
+ state.update(phase="needs_attention", error_type=type(exc).__name__, error=str(exc), passed=False)
+ persist()
+ raise
diff --git a/04-data-agent/hf/runtime/smoke_opencode_backends.py b/04-data-agent/hf/runtime/smoke_opencode_backends.py
new file mode 100644
index 0000000..e7272f9
--- /dev/null
+++ b/04-data-agent/hf/runtime/smoke_opencode_backends.py
@@ -0,0 +1,41 @@
+"""Exercise the standalone sandbox protocol, deleting only sandboxes created by this invocation."""
+import argparse
+import concurrent.futures
+import json
+import time
+from common import configure, write_json
+configure()
+from data_agent_env.sandbox import build_backend, DEFAULT_IMAGE
+
+
+def check(name):
+ start=time.monotonic();sandbox=None;result={"backend":name}
+ try:
+ sandbox=build_backend(name,image=DEFAULT_IMAGE).create(timeout_s=600,metadata={"purpose":"protocol-smoke"})
+ result["sandbox_id"]=sandbox.sandbox_id
+ content='literal `text` $(echo example)\nhello'
+ sandbox.write_text('/tmp/protocol/test file.txt',content)
+ assert sandbox.exists('/tmp/protocol/test file.txt')
+ assert sandbox.read_text('/tmp/protocol/test file.txt')==content
+ command=sandbox.exec('printf "%s" "$PROTOCOL_TRANSIENT"',envs={'PROTOCOL_TRANSIENT':'fixture-value'})
+ assert command.exit_code==0 and command.stdout=='fixture-value'
+ assert sandbox.exec('test -z "$PROTOCOL_TRANSIENT"').exit_code==0
+ assert sandbox.start_bg('sleep 1; exit 7').wait(timeout=30)==7
+ result['passed']=True
+ except Exception as exc:
+ result.update(passed=False,error_type=type(exc).__name__)
+ finally:
+ if sandbox is not None:
+ try:sandbox.kill();result['deleted']=True
+ except Exception as exc:result.update(passed=False,cleanup_error_type=type(exc).__name__)
+ result['elapsed_s']=time.monotonic()-start
+ return result
+
+if __name__=='__main__':
+ parser=argparse.ArgumentParser(description=__doc__)
+ parser.add_argument('--backends',default='daytona,hf');parser.add_argument('--out',required=True)
+ args=parser.parse_args()
+ with concurrent.futures.ThreadPoolExecutor(max_workers=2) as pool:
+ records=list(pool.map(check,args.backends.split(',')))
+ write_json(args.out,records);print(json.dumps(records))
+ if not all(r['passed'] and r.get('deleted') for r in records):raise SystemExit(1)
diff --git a/04-data-agent/hf/runtime/space_app.py b/04-data-agent/hf/runtime/space_app.py
new file mode 100644
index 0000000..b860b48
--- /dev/null
+++ b/04-data-agent/hf/runtime/space_app.py
@@ -0,0 +1,104 @@
+"""Run a private environment Space with the frozen native OpenEnv services."""
+import os
+import hmac
+from common import ROOT, RUN, configure
+
+configure()
+arm = os.environ["COMPARISON_ARM"]
+mode = "shared"
+owner = os.environ.get("SPACE_ID", f"{arm}-{mode}").replace("/", "-")
+os.environ["RUN_OWNER"] = owner
+trials = ROOT / "space-trials" / owner
+trials.mkdir(parents=True, exist_ok=True)
+os.environ["OPENENV_HARBOR_TRIALS_DIR"] = str(trials)
+os.environ["DAYTONA_WHITEBOX_TRIALS"] = str(trials)
+os.environ["WHITE_BOX_BASH_TASK_SOURCE"] = "harbor-frozen"
+os.environ["OPENENV_CAPTURE_TRANSPORT"] = "tunnel"
+os.environ["OPENENV_DATASETS"] = ",".join(str(RUN / "datasets" / s) for s in ["train", "test"])
+os.environ.setdefault("OPENENV_MODEL", "Qwen/Qwen3.5-2B")
+os.environ.setdefault("OPENENV_MAX_OUTPUT_TOKENS", "16384")
+os.environ.setdefault("OPENENV_EXPOSE", "gradio")
+os.environ.setdefault("MAX_CONCURRENT_ENVS", "1024")
+os.environ.setdefault("WHITE_BOX_BASH_MAX_CONCURRENT_ENVS", os.environ["MAX_CONCURRENT_ENVS"])
+os.environ.setdefault("WHITE_BOX_BASH_MAX_SESSIONS", os.environ.get("SANDBOX_CAPACITY", "128"))
+os.environ["ENABLE_WEB_INTERFACE"] = "false"
+
+if arm == "blackbox":
+ from harbor_service import install
+ install()
+ from harbor_env.server.app import app
+elif arm == "whitebox":
+ from whitebox_bash.server.app import app
+else:
+ raise ValueError("Unknown comparison arm")
+
+
+@app.middleware("http")
+async def protect_run_artifacts(request, call_next):
+ # A protected HF Space has a public app. Keep cross-session run artifacts
+ # behind the same bearer credential already sent by the training/eval bridge.
+ if request.url.path == "/diagnostics" or request.url.path.startswith("/trial/"):
+ from fastapi.responses import JSONResponse
+ secret = os.environ.get("HF_TOKEN", "")
+ provided = request.headers.get("Authorization", "")
+ if not secret or not hmac.compare_digest(provided, "Bearer " + secret):
+ return JSONResponse({"detail": "Authentication required"}, status_code=401)
+ return await call_next(request)
+
+
+@app.get("/deployment")
+def deployment():
+ from service_policy import admission
+ return {"arm": arm, "mode": mode, "owner": owner, "train_tasks": 1000, "test_tasks": 250,
+ "sandbox": "daytona", "bundle_sha256": os.environ.get("BUNDLE_SHA256"),
+ "max_concurrent_envs": int(os.environ["MAX_CONCURRENT_ENVS"]),
+ "admission": admission.snapshot(), "output_tokens": {"train": 16384, "eval": 4096},
+ "interactive_ui": True, "trackio": False}
+
+
+@app.get("/trial/{name}/result")
+def trial_result(name: str):
+ # Bearer authentication is enforced above. Expose only the native result metadata
+ # needed to verify harness versions; arbitrary filesystem access is intentionally absent.
+ import json
+ from pathlib import Path
+ from fastapi import HTTPException
+ if Path(name).name != name or name in {".", ".."}:
+ raise HTTPException(400)
+ path = trials / name / "result.json"
+ if not path.is_file():
+ raise HTTPException(404)
+ return json.loads(path.read_text())
+
+
+@app.get("/diagnostics")
+async def diagnostics():
+ # No frame locals, prompts, credentials, or answer files. Useful for distinguishing
+ # remote execution from response delivery when a long RPC stops making progress.
+ import asyncio
+ from service_policy import admission
+ tasks = []
+ for task in asyncio.all_tasks():
+ tasks.append({"name": task.get_name(), "stack": [f"{f.f_code.co_name}:{f.f_lineno}" for f in task.get_stack(limit=4)]})
+ rows = []
+ for path in trials.iterdir():
+ if path.is_dir():
+ rows.append({"trial": path.name, "result_ready": (path / "result.json").exists(),
+ "cleanup_ready": (path / "cleanup.json").exists()})
+ return {"admission": admission.snapshot(), "async_tasks": tasks, "trials": rows[-100:]}
+
+
+@app.on_event("startup")
+async def configure_thread_capacity():
+ import anyio.to_thread
+ anyio.to_thread.current_default_thread_limiter().total_tokens = 512
+
+
+from environment_ui import mount_ui
+app = mount_ui(app, arm)
+
+
+if __name__ == "__main__":
+ import uvicorn
+ uvicorn.run(app, host="0.0.0.0", port=7860, ws_ping_interval=20, ws_ping_timeout=None,
+ timeout_keep_alive=120, log_level="info")
diff --git a/04-data-agent/hf/runtime/space_smoke.py b/04-data-agent/hf/runtime/space_smoke.py
new file mode 100644
index 0000000..52f2159
--- /dev/null
+++ b/04-data-agent/hf/runtime/space_smoke.py
@@ -0,0 +1,57 @@
+"""Real private-Space task discovery and Daytona tool/verifier checks, without a model."""
+from common import RUN, ROOT, ENV_PY, configure, ready, start, write_json
+import argparse
+import concurrent.futures
+import json
+import os
+from pathlib import Path
+import signal
+
+
+def main():
+ p = argparse.ArgumentParser(description=__doc__)
+ p.add_argument("--arm", choices=["blackbox", "whitebox"], required=True)
+ p.add_argument("--out", type=Path, required=True)
+ a = p.parse_args()
+ configure()
+ a.out.mkdir(parents=True, exist_ok=True)
+ process = start([ENV_PY, ROOT / "hf/runtime/auth_bridge.py"], a.out / "bridge.log")
+ try:
+ ready("http://127.0.0.1:8100/health", process, seconds=60)
+ if a.arm == "whitebox":
+ from native_tool_smoke import one
+ with concurrent.futures.ThreadPoolExecutor(max_workers=4) as pool:
+ results = list(pool.map(lambda i: one("http://127.0.0.1:8100", i // 2, bool(i % 2), RUN), range(8)))
+ report = {"passed": all(r["passed"] for r in results), "cases": results}
+ else:
+ from openenv.harbor.client import HarborEnv
+ from openenv.harbor.tasks import read_instruction
+ client = HarborEnv(base_url="http://127.0.0.1:8100")
+ checks = []
+ try:
+ for split, expected in [("train", 1000), ("test", 250)]:
+ spec = "/workspace/repro/experiments/daytona_harness_comparison/logs/20260915/datasets/" + split
+ assert client.num_tasks(spec) == expected
+ manifest = json.loads((RUN / f"{split}_manifest.json").read_text())
+ catalog = sorted(manifest["tasks"], key=lambda row: row["name"])
+ for index in [0, expected - 1]:
+ remote = client.get_task(spec, index)
+ actual = remote.model_dump()
+ local = RUN / "datasets" / split / "tasks" / catalog[index]["name"]
+ assert actual["task_name"] == catalog[index]["name"]
+ assert actual["instruction"] == read_instruction(local)
+ checks.append({"split": split, "tasks": expected, "edge_instructions_match": True})
+ finally:
+ client.close()
+ report = {"passed": True, "cases": checks}
+ write_json(a.out / "result.json", report)
+ print(json.dumps(report), flush=True)
+ if not report["passed"]:
+ raise SystemExit(2)
+ finally:
+ if process.poll() is None:
+ os.killpg(process.pid, signal.SIGTERM)
+
+
+if __name__ == "__main__":
+ main()
diff --git a/04-data-agent/hf/runtime/telemetry.py b/04-data-agent/hf/runtime/telemetry.py
new file mode 100644
index 0000000..f18188b
--- /dev/null
+++ b/04-data-agent/hf/runtime/telemetry.py
@@ -0,0 +1,48 @@
+"""Lightweight inference/GPU samples outside the evaluator and optimizer loops."""
+import json
+import re
+import subprocess
+import threading
+import time
+
+METRICS = ("num_requests_running", "num_requests_waiting", "kv_cache_usage_perc",
+ "prompt_tokens_total", "generation_tokens_total", "request_success_total",
+ "request_queue_time_seconds_sum", "request_queue_time_seconds_count",
+ "time_to_first_token_seconds_sum", "time_to_first_token_seconds_count")
+
+
+class Telemetry:
+ def __init__(self, output, base_url="http://127.0.0.1:8000"):
+ self.path = output / "inference_metrics.jsonl"
+ self.base_url = base_url.rstrip("/")
+ self.stop_event = threading.Event()
+ self.thread = threading.Thread(target=self.run, daemon=True, name="inference-telemetry")
+
+ def run(self):
+ import httpx
+ with httpx.Client(timeout=5) as client:
+ while not self.stop_event.is_set():
+ row = {"time": time.time()}
+ try:
+ response = client.get(self.base_url + "/metrics").raise_for_status()
+ samples = {}
+ for line in response.text.splitlines():
+ match = re.match(r"vllm:([^ {]+)(?:\{[^}]*\})? ([^ ]+)", line)
+ if match and match[1] in METRICS:
+ samples[match[1]] = samples.get(match[1], 0) + float(match[2])
+ row["vllm"] = samples
+ gpu = subprocess.run(["nvidia-smi", "--query-gpu=index,utilization.gpu,memory.used,memory.total,power.draw",
+ "--format=csv,noheader,nounits"], capture_output=True, text=True, timeout=5)
+ row["gpu_csv"] = gpu.stdout.strip().splitlines() if gpu.returncode == 0 else []
+ except Exception as exc:
+ row["error_type"] = type(exc).__name__
+ with self.path.open("a") as stream:
+ stream.write(json.dumps(row) + "\n")
+ self.stop_event.wait(30)
+
+ def start(self):
+ self.thread.start()
+
+ def finish(self):
+ self.stop_event.set()
+ self.thread.join(timeout=15)
diff --git a/04-data-agent/hf/runtime/training_capture_audit.py b/04-data-agent/hf/runtime/training_capture_audit.py
new file mode 100644
index 0000000..1c523d8
--- /dev/null
+++ b/04-data-agent/hf/runtime/training_capture_audit.py
@@ -0,0 +1,73 @@
+"""Reconcile exact supervised positions with native TRL rows and optimizer receipts."""
+import collections
+import hashlib
+import json
+from pathlib import Path
+import struct
+
+from common import write_json
+
+
+def positions(ids, masks, logprobs):
+ result = collections.Counter()
+ context = hashlib.sha256()
+ for token, mask, lp in zip(ids, masks, logprobs, strict=True):
+ if mask:
+ result[(context.hexdigest(), token, float(lp).hex())] += 1
+ context.update(struct.pack(">q", token))
+ return result
+
+
+def audit_async(directory, arm):
+ from trl.experimental.async_grpo.openenv_harness import _turns_from_trace
+ from trl.experimental.async_grpo.async_rollout_worker import _chain_to_sequences
+ from openenv.core.harness.capture.validate import validate_training_turn
+ directory = Path(directory)
+ reports = {}
+ for path in sorted((directory / "rollouts").glob("*.json")):
+ record = json.loads(path.read_text())
+ raw = record.get("result")
+ if not raw:
+ continue
+ if arm == "opencode":
+ from data_agent_env import opencode_agent_turns, to_trace_entries
+ from data_agent_env.models import DataAgentRolloutResult
+ result = DataAgentRolloutResult.model_validate(raw)
+ entries = opencode_agent_turns(to_trace_entries(result))
+ else:
+ from openenv.harbor.models import HarborRolloutResult
+ from harbor_env.harness import to_trace_entries
+ result = HarborRolloutResult.model_validate(raw)
+ entries = to_trace_entries(result)
+ if not entries:
+ continue
+ expected = collections.Counter()
+ for entry in entries:
+ p, c, lp, mask = (entry[k] for k in ("prompt_token_ids", "completion_token_ids", "per_token_logps", "loss_mask"))
+ validate_training_turn(p, c, lp, mask)
+ expected.update(positions(p + c, mask, [0.] * len(p) + lp))
+ rows, _ = _chain_to_sequences(_turns_from_trace(entries), record["episode_id"], fork_threshold=0)
+ retained = collections.Counter()
+ for row in rows:
+ retained.update(positions(row.input_ids, row.completion_mask, row.old_log_probs))
+ passed = bool(expected) and retained == expected and any(lp < 0 for e in entries for lp in e["per_token_logps"])
+ assert passed, f"Exact supervised positions were not retained: {path.name}"
+ reports[record["episode_id"]] = {"rows": len(rows), "eligible_tokens": sum(expected.values()),
+ "retained_tokens": sum(retained.values()), "rows_over_token_budget": sum(len(r.input_ids) > 131072 for r in rows),
+ "tito_pass": passed}
+ receipts = [json.loads(line) for line in (directory / "optimizer_rollouts.jsonl").read_text().splitlines() if line.strip()]
+ consumed = 0
+ for receipt in receipts:
+ for row in receipt["rollouts"]:
+ expected = reports[row["rollout_id"]]
+ assert row["rows"] == expected["rows"]
+ assert row["supervised_tokens"] == expected["retained_tokens"]
+ consumed += 1
+ assert consumed and reports, "No audited rollouts reached the optimizer"
+ summary = {arm: {"completed_results": len(reports), "tito_pass": sum(r["tito_pass"] for r in reports.values()),
+ "eligible_tokens": sum(r["eligible_tokens"] for r in reports.values()),
+ "retained_tokens": sum(r["retained_tokens"] for r in reports.values()),
+ "rows_over_token_budget": sum(r["rows_over_token_budget"] for r in reports.values()),
+ "optimizer_rollouts_verified": consumed}}
+ write_json(directory / "tito_summary.json", summary)
+ return summary
diff --git a/04-data-agent/hf/runtime/training_smoke.py b/04-data-agent/hf/runtime/training_smoke.py
new file mode 100644
index 0000000..6c498ef
--- /dev/null
+++ b/04-data-agent/hf/runtime/training_smoke.py
@@ -0,0 +1,57 @@
+"""Validate real native optimizer/save/remote-resume evidence before an HF long run."""
+import json
+import math
+import os
+from pathlib import Path
+
+from common import configure, write_json
+
+
+def validate(output, arm):
+ configure()
+ from checkpoint_store import verify
+ from checkpoint_artifacts import resume_info
+ output = Path(output)
+ run = output / "run"
+ markers = {step: verify(run / f"checkpoint-{step}") for step in (2, 4)}
+ for step, marker in markers.items():
+ assert marker["step"] == step and marker["arm"] == arm
+ assert marker["bundle_sha256"] == os.environ["BUNDLE_SHA256"]
+ origin = json.loads((output / "remote-resume/checkpoint-2.remote-origin.json").read_text())
+ assert origin == markers[2], "Resume must come from the remotely verified step-2 checkpoint"
+ metric_file = output / "audit/metrics.jsonl" if arm in {"blackbox", "opencode"} else run / "metrics.jsonl"
+ rows = [json.loads(line) for line in metric_file.read_text().splitlines() if line.strip()]
+ updates = [row for row in rows if "grad_norm" in row]
+ assert {int(row["step"]) for row in updates} >= {1, 2, 3, 4}
+ assert all(math.isfinite(v) for row in updates for v in row.values() if isinstance(v, float))
+ assert any(row["grad_norm"] > 0 for row in updates), "No learning signal observed"
+ if arm in {"blackbox", "opencode"}:
+ restored = output / "remote-resume/checkpoint-2"
+ info = resume_info(restored, origin["base_model"], origin["base_revision"])
+ expected = f"resume checkpoint step=2, next schedule group={info['group_offset']}"
+ assert expected in (output / "train-resumed.log").read_text()
+ from training_capture_audit import audit_async
+ audit = audit_async(output / "audit", arm)
+ assert audit and all(v["tito_pass"] == v["completed_results"] and
+ v["retained_tokens"] == v["eligible_tokens"] and
+ v["rows_over_token_budget"] == 0 for v in audit.values())
+ weights = [name for name in markers[2]["files"] if name.endswith(".safetensors")]
+ assert weights and any(markers[2]["files"][name] != markers[4]["files"][name] for name in weights), "Weights did not change after resume"
+ else:
+ initial = json.loads((run / "optimizer_evidence_from_0.json").read_text())
+ resumed = json.loads((run / "optimizer_evidence_from_2.json").read_text())
+ assert initial["final_step"] == 2 and resumed["final_step"] == 4
+ assert initial["weights_changed"] or resumed["weights_changed"]
+ assert initial["final_weight_digest"] == resumed["initial_weight_digest"]
+ audit = [json.loads(line) for line in (run / "token_audit.jsonl").read_text().splitlines() if line.strip()]
+ assert {row["step"] for row in audit} >= {0, 1, 2, 3}
+ assert all(row["tito_pass"] and row["supervised"] > 0 for record in audit for row in record["rows"])
+ import torch
+ optimizer = torch.load(run / "checkpoint-4/optimizer.pt", map_location="cpu", weights_only=False)
+ assert optimizer["state"] and optimizer["param_groups"]
+ report = {"arm": arm, "passed": True, "bundle_sha256": os.environ["BUNDLE_SHA256"],
+ "optimizer_steps": [1, 2, 3, 4], "native_optimizer_state_verified": True,
+ "remote_restore_verified": True, "tito_pass": True, "weights_updated": True,
+ "nonzero_gradient_updates": sum(row["grad_norm"] > 0 for row in updates)}
+ write_json(output / "training_smoke_verified.json", report)
+ return report
diff --git a/04-data-agent/hf/runtime/ui_smoke.py b/04-data-agent/hf/runtime/ui_smoke.py
new file mode 100644
index 0000000..92c53a0
--- /dev/null
+++ b/04-data-agent/hf/runtime/ui_smoke.py
@@ -0,0 +1,56 @@
+"""Exercise the deployed UIs and two isolated Daytona sessions using Gradio's client."""
+from concurrent.futures import ThreadPoolExecutor
+import json
+import os
+from pathlib import Path
+import time
+
+from gradio_client import Client
+
+
+def main():
+ import argparse
+ p = argparse.ArgumentParser(description=__doc__)
+ p.add_argument("--env-file")
+ p.add_argument("--out", type=Path, required=True)
+ a = p.parse_args()
+ from dotenv import dotenv_values
+ values = dotenv_values(a.env_file) if a.env_file else {}
+ token = values.get("HF_API_KEY") or os.environ["HF_TOKEN"]
+ repo = "HuggingEnvs/data-agent-seta-whitebox-env"
+ clients = [Client(repo, token=token, verbose=False) for _ in range(2)]
+
+ def one(i):
+ c = clients[i]
+ split, index = ("train", 895) if i == 0 else ("test", 166)
+ instruction, _ = c.predict(split, index, api_name="/preview_task")
+ try:
+ prompt, _, state = c.predict(split, index, api_name="/start_task")
+ assert prompt == instruction and state == "Active"
+ content = f"ui-isolation-{i}"
+ out = c.predict("write", "", "/workdir/ui-isolation.txt", content, "", api_name="/run_tool")
+ assert "[error]" not in out
+ out = c.predict("bash", "cat /workdir/ui-isolation.txt", "/workdir", "", "", api_name="/run_tool")
+ assert content in out and f"ui-isolation-{1-i}" not in out
+ grade, state = c.predict("__known_wrong_ui_smoke__", api_name="/grade_task")
+ assert grade == "Reward: 0.0" and state == "Finished"
+ return {"split": split, "index": index, "passed": True, "isolated_tools": True,
+ "graded_zero": True, "closed": True}
+ finally:
+ c.predict(api_name="/close_task")
+
+ with ThreadPoolExecutor(max_workers=2) as pool:
+ results = list(pool.map(one, range(2)))
+ for arm in ["opencode-blackbox", "seta-whitebox"]:
+ c = Client("HuggingEnvs/data-agent-" + arm + "-env", token=token, verbose=False)
+ for split, index in [("train", 895), ("test", 249)]:
+ instruction, _ = c.predict(split, index, api_name="/preview_task")
+ assert len(instruction) > 50
+ results.append({"arm": arm, "split": split, "index": index, "preview_passed": True})
+ a.out.parent.mkdir(parents=True, exist_ok=True)
+ a.out.write_text(json.dumps({"passed": True, "checked_at": time.time(), "results": results}, indent=2) + "\n")
+ print(a.out.read_text())
+
+
+if __name__ == "__main__":
+ main()
diff --git a/04-data-agent/hf/runtime/whitebox_tito.py b/04-data-agent/hf/runtime/whitebox_tito.py
new file mode 100644
index 0000000..c2aedc8
--- /dev/null
+++ b/04-data-agent/hf/runtime/whitebox_tito.py
@@ -0,0 +1,45 @@
+"""Match retained supervised spans to distinct engine call occurrences exactly."""
+
+
+def audit_rows(prompt_ids, completions, masks, logprobs, calls):
+ candidates, checks = [], []
+ for row_index, (root, completion, mask, row_logprobs) in enumerate(zip(prompt_ids, completions, masks, logprobs, strict=True)):
+ assert len(completion) == len(mask) == len(row_logprobs)
+ assert set(mask) <= {0, 1}
+ full = root + completion
+ assert len(full) <= 131072
+ offset = 0
+ while offset < len(mask):
+ if not mask[offset]:
+ offset += 1
+ continue
+ end = offset + 1
+ while end < len(mask) and mask[end]:
+ end += 1
+ keep, start = end - offset, len(root) + offset
+ matches = [index for index, call in enumerate(calls)
+ if call['prompt_ids'] == full[:start]
+ and len(call['completion_ids']) >= keep
+ and (len(call['completion_ids']) == keep or end == len(mask))
+ and call['completion_ids'][:keep] == completion[offset:end]
+ and call['logprobs'][:keep] == row_logprobs[offset:end]]
+ assert matches, f'row {row_index} supervision lacks exact engine provenance at {offset}'
+ candidates.append(matches)
+ offset = end
+ assert all(p == 0 for p, m in zip(row_logprobs, mask, strict=True) if not m)
+ checks.append({'supervised': sum(mask), 'context': len(mask) - sum(mask), 'tito_pass': True})
+ # A truncated span may match several otherwise distinct calls. Find a complete
+ # one-to-one assignment rather than consuming the first match greedily.
+ assignments = {}
+ def assign(span, visited):
+ for call in candidates[span]:
+ if call in visited:
+ continue
+ visited.add(call)
+ if call not in assignments or assign(assignments[call], visited):
+ assignments[call] = span
+ return True
+ return False
+ for span in range(len(candidates)):
+ assert assign(span, set()), f'supervised span {span} has no distinct engine call occurrence'
+ return checks
diff --git a/04-data-agent/hf/status.py b/04-data-agent/hf/status.py
new file mode 100644
index 0000000..f161439
--- /dev/null
+++ b/04-data-agent/hf/status.py
@@ -0,0 +1,92 @@
+"""Read HF job state and small durable progress artifacts without streaming logs."""
+import argparse
+from concurrent.futures import ThreadPoolExecutor
+import json
+from pathlib import Path
+
+from deploy import credentials
+
+
+def main():
+ p = argparse.ArgumentParser(description=__doc__)
+ p.add_argument("jobs", nargs="+")
+ p.add_argument("--env-file", required=True)
+ p.add_argument("--out", type=Path, required=True)
+ p.add_argument("--logs", action="store_true")
+ a = p.parse_args()
+ from huggingface_hub import HfApi
+ secret = credentials(a.env_file)
+ api = HfApi(token=secret["HF_TOKEN"])
+ files = ["status.json", "services.json", "eval_progress.json", "canonical_scores.json",
+ "scalability.json", "upload_status.json", "upload_error.json", "training_recipe.json",
+ "training_smoke_verified.json", "trackio_verified.json"]
+ if a.logs:
+ files += ["serving.log", "vllm.log", "bridge.log", "eval-stage0-c8.log",
+ "smoke-opencode.log", "eval-opencode.log", "train-first.log", "train-resumed.log", "trackio-sync.log",
+ "eval-stage1-c32.log", "eval-stage2-c48.log", "eval-stage2-c53.log"]
+
+ def inspect(job_id):
+ j = api.inspect_job(job_id=job_id, namespace="HuggingEnvs")
+ owner = j.environment["RUN_OWNER"]
+ dest = a.out / "downloads" / owner
+ dest.mkdir(parents=True, exist_ok=True)
+ prefix = j.environment["RUN_ID"] + "/jobs/" + owner + "/"
+ requested = list(files)
+ if j.labels.get("role") == "coordinator":
+ if j.labels.get("phase") == "qualify":
+ requested = ["qualification.json"]
+ elif j.labels.get("phase") == "setup":
+ prefix = j.environment["RUN_ID"] + "/pipelines/" + j.environment["BUNDLE_SHA256"] + "/"
+ requested = ["pipeline.json"]
+ else:
+ parent = api.inspect_job(job_id=j.environment["TRAINING_JOB"], namespace="HuggingEnvs")
+ prefix = j.environment["RUN_ID"] + "/coordination/" + parent.environment["RUN_OWNER"] + "/"
+ requested = ["monitor.json", "state.json"]
+ from huggingface_hub.errors import EntryNotFoundError
+ try:
+ available = {item.path for item in api.list_bucket_tree(j.environment["ARTIFACT_BUCKET"],
+ prefix=prefix, recursive=False)}
+ except EntryNotFoundError:
+ available = set()
+ if j.labels.get("arm") == "opencode":
+ for backend in ("daytona", "hf"):
+ for sub in (backend, "smoke/" + backend):
+ try:
+ nested = {item.path for item in api.list_bucket_tree(j.environment["ARTIFACT_BUCKET"], prefix=prefix+sub+"/", recursive=False)}
+ except EntryNotFoundError:
+ nested = set()
+ available.update(nested)
+ requested += [sub + "/scores.json", sub + "/scalability.json", sub + "/configuration.json", sub + "/progress.json"]
+ (dest / sub).mkdir(parents=True, exist_ok=True)
+ if a.logs and j.labels.get("role") != "coordinator":
+ requested += sorted(Path(name).name for name in available
+ if Path(name).name.startswith("eval-stage")
+ and name.endswith(".log") and Path(name).name not in requested)
+ downloads = [(prefix + name, str(dest / name)) for name in requested if prefix + name in available]
+ if downloads:
+ api.download_bucket_files(j.environment["ARTIFACT_BUCKET"], files=downloads, raise_on_missing_files=False)
+ result = {"id": job_id, "owner": owner, "stage": j.status.stage}
+ for name in requested:
+ path = dest / name
+ if not path.exists():
+ continue
+ if name.endswith(".json"):
+ value = json.loads(path.read_text())
+ if name == "canonical_scores.json" and j.labels.get("arm") != "opencode":
+ value = {k: value.get(k) for k in ["graded_cells", "expected_cells", "complete",
+ "average_pass_at_1", "comparison_ready", "ungraded_attempts", "tito_pass"]}
+ result[name] = value
+ else:
+ content = "\n".join(path.read_text(errors="replace").splitlines()[-8:])
+ for value in secret.values():
+ content = content.replace(value, "[REDACTED]")
+ result[name] = content[-2000:]
+ return result
+
+ with ThreadPoolExecutor(max_workers=min(4, len(a.jobs))) as pool:
+ for result in pool.map(inspect, a.jobs):
+ print(json.dumps(result), flush=True)
+
+
+if __name__ == "__main__":
+ main()
diff --git a/04-data-agent/hf/stop_hf_after_checkpoint.py b/04-data-agent/hf/stop_hf_after_checkpoint.py
new file mode 100644
index 0000000..ee753d3
--- /dev/null
+++ b/04-data-agent/hf/stop_hf_after_checkpoint.py
@@ -0,0 +1,218 @@
+"""Stop an HF trainer only after verifying its next full checkpoint, then evaluate it.
+
+Uses the existing checkpoint controller and immutable GPU bundle. It never changes
+the running trainer, and it preserves an explicit planned-cancellation receipt.
+"""
+import argparse
+from datetime import datetime, timezone
+import fcntl
+import hashlib
+import json
+import os
+from pathlib import Path
+import shutil
+import subprocess
+import sys
+import time
+
+TERMINAL = {"COMPLETED", "ERROR", "CANCELED", "CANCELLED", "DELETED"}
+READY = "checkpoint.hf.ready.json"
+
+
+def now():
+ return datetime.now(timezone.utc).isoformat()
+
+
+def read(path):
+ return json.loads(Path(path).read_text())
+
+
+def save(path, value):
+ path = Path(path)
+ path.parent.mkdir(parents=True, exist_ok=True)
+ tmp = path.with_suffix(".tmp")
+ tmp.write_text(json.dumps(value, indent=2) + "\n")
+ tmp.replace(path)
+
+
+def sha(path):
+ with Path(path).open("rb") as f:
+ return hashlib.file_digest(f, "sha256").hexdigest()
+
+
+def verify_checkpoint(directory, *, job, step):
+ manifest = read(directory / READY)
+ if (manifest["arm"] != "whitebox" or manifest["step"] != step
+ or manifest["bundle_sha256"] != job.environment["BUNDLE_SHA256"]
+ or manifest["base_model"] != "Qwen/Qwen3.5-2B"
+ or manifest["base_revision"] != "15852e8c16360a2fea060d615a32b45270f8a8fc"):
+ raise ValueError("Checkpoint provenance mismatch")
+ required = {"optimizer.pt", "scheduler.pt", "rng_state.pth", "trainer_state.json"}
+ if not required.issubset(manifest["files"]) or not any(n.endswith(".safetensors") for n in manifest["files"]):
+ raise ValueError("Full resumable training state is missing")
+ for name, expected in manifest["files"].items():
+ if Path(name).name != name or sha(directory / name) != expected:
+ raise ValueError("Checkpoint content hash mismatch: " + name)
+ if read(directory / "trainer_state.json")["global_step"] != step:
+ raise ValueError("Checkpoint optimizer step mismatch")
+ return manifest
+
+
+def render(out, state):
+ lines = ["# SETA planned stop and final evaluation", "", f"Updated: {now()}", "",
+ f"- Training job: `{state['training_job']}`.",
+ f"- Target checkpoint: **{state['target_step']}**; phase: **{state['phase']}**.",
+ "- User requested the next regular save, then stop and evaluate the synchronous SETA run.",
+ "- Training stops only after a local readback verifies every checkpoint file hash, including optimizer/RNG state.",
+ "- The provider records CANCELED for this intentional stop; it is not a successful 1,000-step completion.",
+ "- Evaluation: the unchanged 250 fixed tests through native bash/SETA, pass@1, concurrency 50, one A100.",
+ "- All three environment Spaces use CPU Basic after SETA stops; limits remain 1024 transport sessions and 64/100/61 sandbox slots (Harbor/native OpenCode/SETA).",
+ "- CPU Basic preserves configured limits; no claim of equivalent measured peak throughput is made.", ""]
+ if state.get("checkpoint_verified"):
+ lines += [f"Verified checkpoint manifest: `{state['checkpoint_verified']['manifest_sha256']}`.", ""]
+ if state.get("evaluation"):
+ result = state["evaluation"]
+ score = result["scores"]
+ lines += [f"Evaluation job: `{result['job_id']}`. Final pass@1: **{100 * score['average_pass_at_1']:.1f}%**.",
+ f"Complete graded coverage: {score['graded_cells']}/{score['expected_cells']}; TiTO: {score['tito_pass']}.", "",
+ "```json", json.dumps(score, indent=2), "```", ""]
+ (out / "REPORT.md").write_text("\n".join(lines))
+
+
+def main():
+ p = argparse.ArgumentParser(description=__doc__)
+ p.add_argument("--request", type=Path, required=True)
+ args = p.parse_args()
+ request = read(args.request)
+ out = args.request.parent
+ lock = (out / "operation.lock").open("a")
+ fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
+ from dotenv import dotenv_values
+ from huggingface_hub import HfApi
+ from huggingface_hub.errors import EntryNotFoundError
+ import httpx
+ creds = dotenv_values(request["env_file"])
+ os.environ["HF_TOKEN"] = creds.get("HF_API_KEY") or creds["HF_TOKEN"]
+ api = HfApi(token=os.environ["HF_TOKEN"])
+ state = read(out / "status.json") if (out / "status.json").exists() else {
+ "training_job": request["training_job"], "target_step": request["target_step"],
+ "user_requested_stop": True, "started_at": now(), "phase": "waiting_for_checkpoint"}
+
+ def progress(phase, **values):
+ state.update(phase=phase, checked_at=now(), **values)
+ save(out / "status.json", state)
+ render(out, state)
+ print(json.dumps({k: state[k] for k in ["phase", "checked_at", "target_step"]}), flush=True)
+
+ try:
+ job = api.inspect_job(job_id=request["training_job"], namespace="HuggingEnvs")
+ assert job.labels["role"] == "train" and job.labels["arm"] == "whitebox"
+ bucket = job.environment["ARTIFACT_BUCKET"]
+ prefix = job.environment["RUN_ID"] + "/jobs/" + job.environment["RUN_OWNER"]
+ cp_prefix = prefix + f"/run/checkpoint-{request['target_step']}"
+ checkpoint = out / f"checkpoint-{request['target_step']}"
+ checkpoint.mkdir(exist_ok=True)
+ deadline = time.monotonic() + 4 * 3600
+ if not state.get("checkpoint_verified"):
+ while True:
+ try:
+ api.download_bucket_files(bucket, [(cp_prefix + "/" + READY, checkpoint / READY)], raise_on_missing_files=True)
+ break
+ except EntryNotFoundError:
+ job = api.inspect_job(job_id=request["training_job"], namespace="HuggingEnvs")
+ if job.status.stage in TERMINAL or time.monotonic() > deadline:
+ raise RuntimeError("Trainer ended or checkpoint publication timed out before safe stop")
+ progress("waiting_for_checkpoint", training_stage=job.status.stage)
+ time.sleep(30)
+ manifest = read(checkpoint / READY)
+ if any(Path(n).name != n for n in manifest["files"]):
+ raise ValueError("Invalid checkpoint member")
+ progress("verifying_full_checkpoint")
+ api.download_bucket_files(bucket, [(cp_prefix + "/" + n, checkpoint / n) for n in manifest["files"]], raise_on_missing_files=True)
+ verify_checkpoint(checkpoint, job=job, step=request["target_step"])
+ proof = {"training_job": job.id, "step": request["target_step"],
+ "checkpoint": "hf://buckets/" + bucket + "/" + cp_prefix,
+ "manifest_sha256": sha(checkpoint / READY), "full_checkpoint_verified": True,
+ "user_requested_stop": True, "verified_at": now()}
+ save(out / "checkpoint-verified.json", proof)
+ progress("checkpoint_verified", checkpoint_verified=proof)
+
+ # Freeze and test the final-eval controller before canceling the trainer.
+ controller = out / "controller"
+ if not (controller / "plan.json").exists():
+ old = Path(request["previous_controller_plan"]).parent
+ shutil.copytree(old, controller, dirs_exist_ok=True,
+ ignore=shutil.ignore_patterns("output", "*.out", "*.err", "__pycache__", "plan.json"))
+ plan = read(old / "plan.json")
+ for key in ["root", "logger_root"]:
+ plan[key] = str(controller / Path(plan[key]).name)
+ plan["final_checkpoint"] = state["checkpoint_verified"]
+ coordinator = Path(plan["root"]) / "hf/runtime/coordinator.py"
+ shutil.copy2(out / "source/coordinator.py", coordinator)
+ shutil.copy2(out / "source/late_hf_logging.py", controller / "late_hf_logging.py")
+ followup = controller / "hf_followup.py"
+ code = followup.read_text()
+ old_call = 'run(out / "output", "whitebox", admission=admission)'
+ assert old_call in code
+ followup.write_text(code.replace(old_call, 'run(out / "output", "whitebox", admission=admission, final_checkpoint=plan.get("final_checkpoint"))'))
+ (controller / "controller.slurm").write_text((out / "stop.slurm").read_text())
+ plan["files"] = {n: sha(controller / n) for n in plan["files"]}
+ save(controller / "plan.json", plan)
+ if not state.get("training_stopped"):
+ # Shut down only the old CPU coordinator; its evaluated checkpoint100 persists.
+ subprocess.run(["scancel", str(request["previous_controller_job"])], check=True)
+ launch = Path(request["launch_state"])
+ metadata = read(launch)
+ metadata.update(controller_job=os.environ["SLURM_JOB_ID"], controller_plan=str(controller / "plan.json"),
+ planned_stop={"target_step": request["target_step"], "request": str(args.request), "user_requested": True})
+ save(launch, metadata)
+ job = api.inspect_job(job_id=job.id, namespace="HuggingEnvs")
+ if job.status.stage not in TERMINAL:
+ api.cancel_job(job_id=job.id, namespace="HuggingEnvs")
+ for _ in range(60):
+ job = api.inspect_job(job_id=job.id, namespace="HuggingEnvs")
+ if job.status.stage in TERMINAL:
+ break
+ time.sleep(5)
+ if job.status.stage not in TERMINAL:
+ raise RuntimeError("Cancellation did not become terminal")
+ progress("training_stopped", training_stopped=True, training_stage=job.status.stage,
+ stopped_at=now(), final_saved_step=request["target_step"])
+
+ receipt = read(out / "space-hardware.json")
+ for repo, row in receipt["spaces"].items():
+ if not row.get("request_sent"):
+ api.request_space_hardware(repo, hardware="cpu-basic")
+ row.update(request_sent=True, requested_at=now())
+ save(out / "space-hardware.json", receipt)
+ for _ in range(60):
+ runtime = api.get_space_runtime(repo)
+ if runtime.hardware == "cpu-basic" and runtime.stage == "RUNNING":
+ url = "https://" + repo.replace("/", "-").lower() + ".hf.space"
+ response = httpx.get(url + "/deployment", timeout=30)
+ if response.status_code == 200:
+ current = response.json()
+ variables = api.get_space_variables(repo)
+ assert all(variables[k].value == v for k, v in row["concurrency_variables"].items())
+ assert current["bundle_sha256"] == row["bundle_sha256"]
+ row.update(verified_at=now(), hardware=runtime.hardware, stage=runtime.stage, after_deployment=current,
+ concurrency_unchanged=True)
+ save(out / "space-hardware.json", receipt)
+ break
+ time.sleep(10)
+ else:
+ raise RuntimeError("Space did not become healthy on CPU Basic: " + repo)
+ progress("final_evaluation_controller_running", spaces_cpu_basic=True)
+ subprocess.run([sys.executable, "-u", str(controller / "hf_followup.py"), "watch", "--plan", str(controller / "plan.json")], check=True)
+ score_path = controller / "output/decisions/scores" / f"step-{request['target_step']:06d}.json"
+ result = read(score_path)
+ if not result["scores"].get("comparison_ready"):
+ raise RuntimeError("Final evaluation did not pass its gates")
+ progress("complete", evaluation=result, completed_at=now())
+ except Exception as exc:
+ progress("needs_attention", error_type=type(exc).__name__, error=str(exc)[:500])
+ raise
+
+
+if __name__ == "__main__":
+ main()
diff --git a/04-data-agent/hf/tests/browser_smoke.py b/04-data-agent/hf/tests/browser_smoke.py
new file mode 100644
index 0000000..34c5598
--- /dev/null
+++ b/04-data-agent/hf/tests/browser_smoke.py
@@ -0,0 +1,65 @@
+"""Exercise public browser controls and verify two isolated whitebox workspaces."""
+import argparse
+import asyncio
+import json
+from pathlib import Path
+import time
+
+from playwright.async_api import async_playwright, expect
+
+
+async def check(browser, out, split, index, tag):
+ context = await browser.new_context(viewport={"width": 1440, "height": 1080})
+ page = await context.new_page()
+ errors = []
+ page.on("pageerror", lambda error: errors.append(str(error)))
+ await page.goto("https://huggingenvs-data-agent-seta-whitebox-env.hf.space",
+ wait_until="domcontentloaded", timeout=60000)
+ await page.get_by_role("button", name="Load task", exact=True).wait_for(timeout=30000)
+ if split == "train":
+ await page.get_by_role("combobox", name="Dataset", exact=True).click()
+ await page.get_by_role("option", name="Train · 1,000 tasks", exact=True).click()
+ await expect(page.get_by_role("spinbutton")).to_have_attribute("max", "999")
+ await page.get_by_role("spinbutton").fill(str(index))
+ await page.get_by_role("button", name="Load task", exact=True).click()
+ status = page.get_by_role("textbox", name="Workspace status", exact=True)
+ try:
+ await page.get_by_role("button", name="Start workspace", exact=True).click()
+ await expect(status).to_have_value("Active", timeout=180000)
+ # Distinct browser sessions write the same path and see only their own bytes.
+ await page.get_by_role("textbox", name="Shell command", exact=True).fill(
+ f"printf '{tag}' > /workdir/public-ui-isolation.txt; cat /workdir/public-ui-isolation.txt")
+ await page.get_by_role("button", name="Run tool", exact=True).click()
+ await expect(page.locator("#tool-console")).to_contain_text(tag, timeout=90000)
+ await page.get_by_role("textbox", name="Final answer", exact=True).fill("__known_wrong_ui_smoke__")
+ await page.get_by_role("button", name="Submit and grade", exact=True).click()
+ await expect(status).to_have_value("Finished", timeout=180000)
+ await expect(page.locator("#tool-console")).to_contain_text("Reward: 0.0")
+ await page.screenshot(path=str(out / f"whitebox-{split}-graded.png"), full_page=True)
+ assert not errors, errors
+ return {"split": split, "index": index, "public_browser": True, "graded_zero": True,
+ "tools_passed": True, "javascript_errors": errors}
+ finally:
+ await page.get_by_role("button", name="Close workspace", exact=True).click()
+ await expect(status).to_have_value("Closed", timeout=60000)
+ await context.close()
+
+
+async def main(out):
+ out.mkdir(parents=True, exist_ok=True)
+ async with async_playwright() as p:
+ browser = await p.chromium.launch(headless=True, args=["--no-sandbox"])
+ try:
+ results = await asyncio.gather(check(browser, out, "train", 895, "browser-train-895"),
+ check(browser, out, "test", 166, "browser-test-166"))
+ finally:
+ await browser.close()
+ report = {"passed": True, "checked_at": time.time(), "results": results}
+ (out / "browser-smoke.json").write_text(json.dumps(report, indent=2) + "\n")
+ print(json.dumps(report), flush=True)
+
+
+if __name__ == "__main__":
+ p = argparse.ArgumentParser(description=__doc__)
+ p.add_argument("--out", type=Path, required=True)
+ asyncio.run(main(p.parse_args().out))
diff --git a/04-data-agent/hf/tests/test_auth_bridge.py b/04-data-agent/hf/tests/test_auth_bridge.py
new file mode 100644
index 0000000..e1c315e
--- /dev/null
+++ b/04-data-agent/hf/tests/test_auth_bridge.py
@@ -0,0 +1,152 @@
+"""Exercise real HTTP streaming and WebSocket boundaries used by private Spaces."""
+import json
+import asyncio
+from pathlib import Path
+import socket
+import sys
+import threading
+import time
+import unittest
+
+import httpx
+from fastapi import FastAPI, Request, WebSocket
+from starlette.responses import StreamingResponse
+import uvicorn
+from websockets.sync.client import connect
+
+sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "runtime"))
+from auth_bridge import make_app
+
+
+def launch(app):
+ listener = socket.socket()
+ listener.bind(("127.0.0.1", 0))
+ port = listener.getsockname()[1]
+ server = uvicorn.Server(uvicorn.Config(app, log_level="error", ws_ping_interval=None))
+ thread = threading.Thread(target=server.run, kwargs={"sockets": [listener]}, daemon=True)
+ thread.start()
+ deadline = time.monotonic() + 10
+ while not server.started and time.monotonic() < deadline:
+ time.sleep(0.02)
+ assert server.started
+ return server, thread, port
+
+
+class BridgeTest(unittest.TestCase):
+ @classmethod
+ def setUpClass(cls):
+ upstream = FastAPI()
+
+ @upstream.post("/mcp")
+ async def mcp(req: Request):
+ assert req.headers["authorization"] == "Bearer integration-test-token"
+ body = await req.body()
+ async def chunks():
+ yield b"data: "
+ yield body
+ yield b"\n\n"
+ return StreamingResponse(chunks(), media_type="text/event-stream", headers={"x-test": "preserved"})
+
+ @upstream.websocket("/ws")
+ async def websocket(ws: WebSocket):
+ assert ws.headers["authorization"] == "Bearer integration-test-token"
+ await ws.accept()
+ for _ in range(2):
+ value = await ws.receive()
+ if value.get("text") is not None:
+ await ws.send_text(value["text"])
+ else:
+ await ws.send_bytes(value["bytes"])
+ await ws.close()
+
+ cls.remote, cls.remote_thread, port = launch(upstream)
+ cls.bridge, cls.bridge_thread, cls.port = launch(make_app(f"http://127.0.0.1:{port}", "integration-test-token"))
+
+ @classmethod
+ def tearDownClass(cls):
+ cls.bridge.should_exit = True
+ cls.remote.should_exit = True
+ cls.bridge_thread.join(5)
+ cls.remote_thread.join(5)
+
+ def test_streamed_mcp_body_and_headers_survive(self):
+ body = json.dumps({"jsonrpc": "2.0", "id": 1, "params": {"text": "tool output α\n"}}).encode()
+ with httpx.Client() as client:
+ response = client.post(f"http://127.0.0.1:{self.port}/mcp", content=body,
+ headers={"Authorization": "Bearer caller-value"})
+ self.assertEqual(response.status_code, 200)
+ self.assertEqual(response.content, b"data: " + body + b"\n\n")
+ self.assertEqual(response.headers["x-test"], "preserved")
+ self.assertNotIn("integration-test-token", response.text)
+
+ def test_websocket_text_and_binary_survive(self):
+ with connect(f"ws://127.0.0.1:{self.port}/ws", ping_interval=None) as ws:
+ ws.send('{"completion_token_ids":[1,4,8]}')
+ self.assertEqual(ws.recv(), '{"completion_token_ids":[1,4,8]}')
+ ws.send(b"\x00\xff\x01")
+ self.assertEqual(ws.recv(), b"\x00\xff\x01")
+
+
+class IdleProxyTest(unittest.IsolatedAsyncioTestCase):
+ async def test_delayed_result_survives_idle_proxy_with_keepalive(self):
+ from websockets.asyncio.client import connect as async_connect
+ from websockets.exceptions import ConnectionClosed
+ upstream = FastAPI()
+ @upstream.websocket("/delayed")
+ async def delayed(ws: WebSocket):
+ await ws.accept()
+ value = await ws.receive_text()
+ await asyncio.sleep(1.2)
+ try:
+ await ws.send_text(value)
+ except Exception:
+ pass # The no-keepalive control intentionally loses its connection.
+ remote, remote_thread, port = launch(upstream)
+ closed_idle = []
+ async def proxy(reader, writer):
+ other_reader, other_writer = await asyncio.open_connection("127.0.0.1", port)
+ last = [time.monotonic()]
+ async def relay(src, dest):
+ while chunk := await src.read(65536):
+ last[0] = time.monotonic()
+ dest.write(chunk)
+ await dest.drain()
+ async def expire():
+ while time.monotonic() - last[0] < 0.4:
+ await asyncio.sleep(0.03)
+ closed_idle.append(True)
+ tasks = [asyncio.create_task(relay(reader, other_writer)),
+ asyncio.create_task(relay(other_reader, writer)), asyncio.create_task(expire())]
+ try:
+ await asyncio.wait(tasks, return_when=asyncio.FIRST_COMPLETED)
+ finally:
+ for task in tasks: task.cancel()
+ await asyncio.gather(*tasks, return_exceptions=True)
+ writer.close()
+ other_writer.close()
+ proxy_server = await asyncio.start_server(proxy, "127.0.0.1", 0)
+ proxy_port = proxy_server.sockets[0].getsockname()[1]
+ try:
+ for interval, expected in [(None, False), (0.1, True)]:
+ bridge, thread, bridge_port = launch(make_app(f"http://127.0.0.1:{proxy_port}", "test", ping_interval=interval))
+ try:
+ async with async_connect(f"ws://127.0.0.1:{bridge_port}/delayed", ping_interval=None) as ws:
+ await ws.send('{"completion_token_ids":[1,2,3]}')
+ if expected:
+ self.assertEqual(await asyncio.wait_for(ws.recv(), 3), '{"completion_token_ids":[1,2,3]}')
+ else:
+ with self.assertRaises(ConnectionClosed):
+ await asyncio.wait_for(ws.recv(), 3)
+ finally:
+ bridge.should_exit = True
+ await asyncio.to_thread(thread.join, 5)
+ self.assertEqual(len(closed_idle), 1)
+ finally:
+ proxy_server.close()
+ await proxy_server.wait_closed()
+ remote.should_exit = True
+ await asyncio.to_thread(remote_thread.join, 5)
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/04-data-agent/hf/tests/test_bucket_publication.py b/04-data-agent/hf/tests/test_bucket_publication.py
new file mode 100644
index 0000000..ebd2c33
--- /dev/null
+++ b/04-data-agent/hf/tests/test_bucket_publication.py
@@ -0,0 +1,84 @@
+"""An upload timeout must not lose a checkpoint or publish its marker early."""
+import json
+from unittest.mock import Mock, patch
+
+import httpx
+import pytest
+from huggingface_hub.errors import HfHubHTTPError
+
+from artifacts import Publisher
+from bucket_io import sync_with_retry
+from checkpoint_store import READY
+
+
+def http_error(code):
+ return HfHubHTTPError("failure", response=httpx.Response(
+ code, request=httpx.Request("POST", "https://example.test/bucket")))
+
+
+@pytest.mark.parametrize("error", [TimeoutError("response decoding"),
+ httpx.ReadError("connection interrupted"), http_error(503)])
+def test_transfer_timeout_recovers_without_changing_filter(error):
+ api = Mock()
+ api.sync_bucket.side_effect = [error, None]
+ with patch("bucket_io.time.sleep") as sleep:
+ sync_with_retry(api, "checkpoint", "destination", include=[READY])
+ assert api.sync_bucket.call_args_list[0] == api.sync_bucket.call_args_list[1]
+ sleep.assert_called_once_with(30)
+
+
+@pytest.mark.parametrize("error,attempts", [(TimeoutError(), 3), (http_error(403), 1),
+ (ValueError("invalid data"), 1)])
+def test_transfer_retries_are_bounded_and_do_not_hide_invalid_requests(error, attempts):
+ api = Mock()
+ api.sync_bucket.side_effect = error
+ with patch("bucket_io.time.sleep"), pytest.raises(type(error)):
+ sync_with_retry(api, "checkpoint", "destination")
+ assert api.sync_bucket.call_count == attempts
+
+
+def test_completion_marker_follows_retried_data_upload(tmp_path, monkeypatch):
+ for key, value in {"ARTIFACT_BUCKET": "org/bucket", "RUN_ID": "run", "RUN_OWNER": "job",
+ "COMPARISON_ARM": "blackbox", "BUNDLE_SHA256": "bundle"}.items():
+ monkeypatch.setenv(key, value)
+ checkpoint = tmp_path / "run/checkpoint-2"
+ checkpoint.mkdir(parents=True)
+ (checkpoint / "checkpoint.saved.json").write_text("{}")
+ publisher = Publisher(tmp_path)
+ api = Mock()
+ operations = []
+ failed = False
+
+ def transfer(source, destination, **options):
+ nonlocal failed
+ assert not publisher.published
+ operations.append((source, options))
+ if source == str(checkpoint) and options.get("exclude") == [READY] and not failed:
+ failed = True
+ raise TimeoutError("committed data; lost response")
+
+ api.sync_bucket.side_effect = transfer
+ with patch("huggingface_hub.HfApi", return_value=api), patch("checkpoint_store.seal"), \
+ patch("bucket_io.time.sleep"):
+ publisher.sync()
+ assert [options.get("include") for _, options in operations] == [None, None, None, [READY]]
+ assert publisher.published == {"checkpoint-2"}
+ assert json.loads((tmp_path / "upload_status.json").read_text())["published_checkpoints"] == ["checkpoint-2"]
+
+
+def test_failed_data_upload_never_publishes_completion_marker(tmp_path, monkeypatch):
+ for key, value in {"ARTIFACT_BUCKET": "org/bucket", "RUN_ID": "run", "RUN_OWNER": "job",
+ "COMPARISON_ARM": "blackbox", "BUNDLE_SHA256": "bundle"}.items():
+ monkeypatch.setenv(key, value)
+ checkpoint = tmp_path / "run/checkpoint-2"
+ checkpoint.mkdir(parents=True)
+ (checkpoint / "checkpoint.saved.json").write_text("{}")
+ publisher = Publisher(tmp_path)
+ api = Mock()
+ api.sync_bucket.side_effect = [None, TimeoutError(), TimeoutError(), TimeoutError()]
+ with patch("huggingface_hub.HfApi", return_value=api), patch("checkpoint_store.seal"), \
+ patch("bucket_io.time.sleep"), pytest.raises(TimeoutError):
+ publisher.sync()
+ assert not publisher.published
+ assert not any("include" in call.kwargs for call in api.sync_bucket.call_args_list)
+ assert not (tmp_path / "upload_status.json").exists()
diff --git a/04-data-agent/hf/tests/test_consolidate_async_runs.py b/04-data-agent/hf/tests/test_consolidate_async_runs.py
new file mode 100644
index 0000000..910cf91
--- /dev/null
+++ b/04-data-agent/hf/tests/test_consolidate_async_runs.py
@@ -0,0 +1,45 @@
+import copy
+from pathlib import Path
+import sys
+import unittest
+
+sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
+import consolidate_async_runs as c
+
+
+class ComparisonTests(unittest.TestCase):
+ def setUp(self):
+ self.score = {"complete": True, "comparison_ready": True, "tito_pass": True,
+ "harness_versions_match_baseline": True, "graded_cells": 1000,
+ "average_pass_at_1": 33 / 250, "harnesses": {
+ h: {"graded": 250, "pass_at_1": 33 / 250, "difficulty": {
+ d: {"graded": n, "correct": n if d == 'easy' else 0}
+ for d, n in c.COUNTS.items()}} for h in c.HARNESSES}}
+
+ def test_weighting_uses_counts_not_mean_of_difficulty_rates(self):
+ c.validated_score(self.score)
+ m = c.score_metrics(self.score, self.score)
+ self.assertEqual(m['eval/pass_at_1'], .132)
+ self.assertEqual(m['eval/difficulty/easy/pass_at_1'], 1)
+ self.assertEqual(m['eval/difficulty/medium/pass_at_1'], 0)
+ self.assertEqual(len([k for k in m if k.startswith('eval/harness_difficulty/')]), 12)
+
+ def test_partial_or_bad_audit_cannot_be_published(self):
+ for field, value in [('graded_cells',999),('tito_pass',False),('comparison_ready',False)]:
+ s=copy.deepcopy(self.score);s[field]=value
+ with self.assertRaises(ValueError):c.validated_score(s)
+ s=copy.deepcopy(self.score);s['harnesses']['codex']['difficulty']['hard']['graded']=98
+ with self.assertRaises(ValueError):c.validated_score(s)
+
+ def test_training_and_eval_share_run_but_have_distinct_replay_ids(self):
+ train=c.native.event(c.PROJECT,c.ARMS[0],100,{'train/reward':.2},{},identity='training')
+ again=c.native.event(c.PROJECT,c.ARMS[0],100,{'train/reward':.2},{},identity='training')
+ evaluation=c.native.event(c.PROJECT,c.ARMS[0],100,{'eval/pass_at_1':.248},{},identity='evaluation')
+ other=c.native.event(c.PROJECT,c.ARMS[1],100,{'train/reward':.2},{},identity='training')
+ self.assertEqual(train['log_id'],again['log_id'])
+ self.assertNotEqual(train['log_id'],evaluation['log_id'])
+ self.assertEqual(train['run_id'],evaluation['run_id'])
+ self.assertNotEqual(train['run_id'],other['run_id'])
+
+
+if __name__=='__main__':unittest.main()
diff --git a/04-data-agent/hf/tests/test_coordinator_recovery.py b/04-data-agent/hf/tests/test_coordinator_recovery.py
new file mode 100644
index 0000000..d133a0c
--- /dev/null
+++ b/04-data-agent/hf/tests/test_coordinator_recovery.py
@@ -0,0 +1,68 @@
+import copy
+import sys
+import unittest
+from pathlib import Path
+from unittest.mock import Mock, patch
+
+import httpx
+from huggingface_hub.errors import HfHubHTTPError
+
+sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "runtime"))
+from coordinator import sync_decisions, verified_terminal_result
+
+
+class RecoveryTests(unittest.TestCase):
+ def setUp(self):
+ self.args = {
+ "stage": "ERROR",
+ "score": {"comparison_ready": True, "complete": True, "tito_pass": True,
+ "graded_cells": 250, "expected_cells": 250},
+ "evidence": {"manifest_sha256": "hash", "step": 100, "bundle_sha256": "bundle", "source": "checkpoint"},
+ "manifest": {"arm": "whitebox", "step": 100, "bundle_sha256": "bundle"},
+ "sha": "hash", "source": "checkpoint",
+ "status": {"passed": True, "finished_at": 123, "arm": "whitebox", "phase": "checkpoint"},
+ }
+
+ def test_uploaded_complete_result_survives_provider_failure(self):
+ verified_terminal_result(**self.args)
+
+ def test_failed_partial_wrong_checkpoint_and_unvalidated_results_rejected(self):
+ changes = [("score", "graded_cells", 249), ("score", "tito_pass", False),
+ ("score", "comparison_ready", False), ("status", "passed", False),
+ ("status", "phase", "baseline"), ("status", "finished_at", None),
+ ("evidence", "manifest_sha256", "other"), ("evidence", "step", 200),
+ ("evidence", "source", "other"), ("evidence", "bundle_sha256", "other")]
+ for group, field, value in changes:
+ with self.subTest(field=field):
+ args = copy.deepcopy(self.args)
+ args[group][field] = value
+ with self.assertRaises(ValueError):
+ verified_terminal_result(**args)
+ for stage in ("RUNNING", "CANCELED", "DELETED"):
+ with self.assertRaises(ValueError):
+ verified_terminal_result(**{**self.args, "stage": stage})
+
+ def error(self, code):
+ return HfHubHTTPError("test failure", response=httpx.Response(
+ code, request=httpx.Request("POST", "https://example.test/bucket")))
+
+ @patch("coordinator.time.sleep")
+ def test_transient_upload_retried(self, sleep):
+ api = Mock()
+ api.sync_bucket.side_effect = [self.error(500), None]
+ sync_decisions(api, Path("decisions"), "destination")
+ self.assertEqual(api.sync_bucket.call_count, 2)
+ sleep.assert_called_once_with(30)
+
+ @patch("coordinator.time.sleep")
+ def test_retries_bounded_and_permission_errors_fail(self, sleep):
+ for code, expected in ((503, 3), (403, 1)):
+ api = Mock()
+ api.sync_bucket.side_effect = self.error(code)
+ with self.assertRaises(HfHubHTTPError):
+ sync_decisions(api, Path("decisions"), "destination")
+ self.assertEqual(api.sync_bucket.call_count, expected)
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/04-data-agent/hf/tests/test_inference_providers.py b/04-data-agent/hf/tests/test_inference_providers.py
new file mode 100644
index 0000000..74a7de9
--- /dev/null
+++ b/04-data-agent/hf/tests/test_inference_providers.py
@@ -0,0 +1,99 @@
+"""Visitor credentials, provider selection, and relay boundaries."""
+import json
+from pathlib import Path
+import sys
+import time
+from types import SimpleNamespace as NS
+import unittest
+from unittest.mock import patch
+
+import httpx
+from fastapi import FastAPI, HTTPException
+
+sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "runtime"))
+import inference_providers as ip
+
+
+def oauth(token="visitor-a", **kwargs):
+ return NS(token=token, scope=kwargs.get("scope", "openid profile inference-api"),
+ expires_at=kwargs.get("expires_at", time.time()+3600))
+
+
+class ProviderTests(unittest.TestCase):
+ def test_only_live_tool_models_and_no_arbitrary_selection(self):
+ rows = {"data": [{"id": "org/model", "providers": [
+ {"provider": "live", "status": "live", "supports_tools": True},
+ {"provider": "offline", "status": "error", "supports_tools": True},
+ {"provider": "unknown", "status": "live"}]}]}
+ catalog = ip.Catalog()
+ with patch("inference_providers.httpx.get", return_value=NS(raise_for_status=lambda: None, json=lambda: rows)) as get:
+ self.assertEqual(catalog.select("live", "org/model"), "org/model:live")
+ self.assertEqual(len(catalog.rows()), 1)
+ self.assertEqual(get.call_count, 1)
+ with self.assertRaises(ValueError): catalog.select("offline", "org/model")
+ with self.assertRaises(ValueError): catalog.select("live", "https://attacker/model")
+
+ def test_requires_current_visitor_inference_permission(self):
+ with patch.dict("os.environ", {"HF_TOKEN": "team-secret"}):
+ for value in [None, oauth(expires_at=0), oauth(scope="openid profile"), oauth(token="")]:
+ with self.assertRaises(ValueError): ip.visitor_token(value)
+ self.assertEqual(ip.visitor_token(oauth()), "visitor-a")
+
+ def test_leases_are_isolated_revoked_and_bounded(self):
+ registry = ip.VisitorCredentials()
+ with registry.issue(oauth(), "a:model") as a, registry.issue(oauth("visitor-b"), "b:model") as b:
+ self.assertNotEqual(a, b)
+ self.assertEqual(registry.get(a).token, "visitor-a")
+ self.assertEqual(registry.get(b).token, "visitor-b")
+ self.assertNotIn("visitor-a", repr(registry.get(a)))
+ for _ in range(32): registry.get(a, consume=True)
+ with self.assertRaises(HTTPException) as error: registry.get(a, consume=True)
+ self.assertEqual(error.exception.status_code, 429)
+ registry.get(b).expires = 0
+ with self.assertRaises(HTTPException): registry.get(b)
+ with self.assertRaises(HTTPException): registry.get(a)
+ self.assertEqual(registry._leases, {})
+
+
+class RelayTests(unittest.IsolatedAsyncioTestCase):
+ async def test_only_selected_model_and_visitor_credential_forwarded(self):
+ app = ip.mount_provider_relay(FastAPI())
+ outgoing = []
+ def upstream(request):
+ outgoing.append(request)
+ return httpx.Response(200, json={"choices": [{"message": {"content": "ok", "role": "assistant"}}]})
+ original = httpx.AsyncClient
+ with ip.credentials.issue(oauth(), "org/model:provider") as key:
+ headers = {"authorization": "Bearer " + key}
+ async with original(transport=httpx.ASGITransport(app), base_url="http://app") as client:
+ self.assertEqual((await client.get("/hf-inference/v1/models")).status_code, 401)
+ with patch("inference_providers.httpx.AsyncClient", side_effect=lambda **kw: original(transport=httpx.MockTransport(upstream), **kw)):
+ bad = await client.post("/hf-inference/v1/chat/completions", headers=headers, json={"model": "other"})
+ self.assertEqual(bad.status_code, 400)
+ self.assertEqual(outgoing, [])
+ response = await client.post("/hf-inference/v1/chat/completions", headers=headers,
+ json={"model": "org/model:provider", "messages": [{"role": "user", "content": "hello"}], "max_tokens": 99999})
+ self.assertEqual(response.status_code, 200)
+ self.assertEqual(outgoing[0].headers["authorization"], "Bearer visitor-a")
+ self.assertEqual(str(outgoing[0].url), ip.ROUTER + "/chat/completions")
+ self.assertEqual(json.loads(outgoing[0].content)["max_tokens"], 4096)
+ self.assertNotIn("visitor-a", response.text)
+ async with original(transport=httpx.ASGITransport(app), base_url="http://app") as client:
+ self.assertEqual((await client.get("/hf-inference/v1/models", headers=headers)).status_code, 401)
+
+ async def test_provider_errors_cannot_echo_secrets(self):
+ app = ip.mount_provider_relay(FastAPI())
+ original = httpx.AsyncClient
+ with ip.credentials.issue(oauth(), "org/model:provider") as key:
+ async with original(transport=httpx.ASGITransport(app), base_url="http://app") as client:
+ with patch("inference_providers.httpx.AsyncClient", side_effect=lambda **kw: original(
+ transport=httpx.MockTransport(lambda r: httpx.Response(402, text="visitor-a raw secret")), **kw)):
+ response = await client.post("/hf-inference/v1/chat/completions", headers={"authorization": "Bearer " + key},
+ json={"model": "org/model:provider", "messages": []})
+ self.assertEqual(response.status_code, 402)
+ self.assertNotIn("visitor-a", response.text)
+ self.assertIn("credits", response.text)
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/04-data-agent/hf/tests/test_joint_eval_admission.py b/04-data-agent/hf/tests/test_joint_eval_admission.py
new file mode 100644
index 0000000..99a94e0
--- /dev/null
+++ b/04-data-agent/hf/tests/test_joint_eval_admission.py
@@ -0,0 +1,27 @@
+import importlib.util
+from pathlib import Path
+from types import SimpleNamespace
+import pytest
+
+spec = importlib.util.spec_from_file_location('local_watch', Path(__file__).parents[1] / 'local_watch.py')
+module = importlib.util.module_from_spec(spec)
+spec.loader.exec_module(module)
+
+
+def test_hf_eval_startup_also_reserves_capacity():
+ api = SimpleNamespace(list_jobs=lambda **_: [SimpleNamespace(status=SimpleNamespace(stage='STARTING'))])
+ assert not module.admit_with_hf(lambda _: pytest.fail('local GPU allocated during HF eval startup'), {}, api)
+
+
+def test_idle_hf_still_checks_local_and_sandbox_capacity():
+ api = SimpleNamespace(list_jobs=lambda **_: [SimpleNamespace(status=SimpleNamespace(stage='COMPLETED'))])
+ assert not module.admit_with_hf(lambda _: False, {}, api)
+ assert module.admit_with_hf(lambda _: True, {}, api)
+
+
+def test_unknown_hf_capacity_never_allocates():
+ def unavailable(**kwargs):
+ raise TimeoutError('HF unavailable')
+ api = SimpleNamespace(list_jobs=unavailable)
+ with pytest.raises(TimeoutError):
+ module.admit_with_hf(lambda _: pytest.fail('unknown capacity'), {}, api)
diff --git a/04-data-agent/hf/tests/test_late_hf_logging.py b/04-data-agent/hf/tests/test_late_hf_logging.py
new file mode 100644
index 0000000..2c7ff64
--- /dev/null
+++ b/04-data-agent/hf/tests/test_late_hf_logging.py
@@ -0,0 +1,76 @@
+"""A late logger must never race a trainer or publish scores from other weights."""
+import json
+from pathlib import Path
+import sys
+import tempfile
+from types import SimpleNamespace
+import unittest
+from unittest.mock import Mock, patch
+
+sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
+from late_hf_logging import replay, planned_stop_matches
+
+
+class LateLoggingTest(unittest.TestCase):
+ def setUp(self):
+ self.temporary = tempfile.TemporaryDirectory()
+ self.root = Path(self.temporary.name)
+ self.env = self.root / "env"
+ self.env.write_text("HF_API_KEY=unused-test-token\n")
+ self.plan = self.root / "plan.json"
+ self.plan.write_text(json.dumps({"files": {}, "env_file": str(self.env),
+ "training_job": "train", "namespace": "org", "bundle_sha256": "bundle"}))
+ self.api = Mock()
+ self.job = SimpleNamespace(status=SimpleNamespace(stage="COMPLETED"),
+ environment={"BUNDLE_SHA256": "bundle", "ARTIFACT_BUCKET": "org/bucket",
+ "RUN_ID": "run", "RUN_OWNER": "owner"})
+ self.api.inspect_job.return_value = self.job
+
+ def tearDown(self):
+ self.temporary.cleanup()
+
+ def run_replay(self):
+ with patch("huggingface_hub.HfApi", return_value=self.api), patch.dict("os.environ"):
+ replay(self.plan)
+
+ def test_active_trainer_rejected_before_any_bucket_sync(self):
+ self.job.status.stage = "RUNNING"
+ with self.assertRaisesRegex(ValueError, "completed trainer"):
+ self.run_replay()
+ self.api.sync_bucket.assert_not_called()
+
+ def test_only_explicit_bound_stop_is_eligible_for_late_logging(self):
+ self.job.status.stage = "CANCELED"
+ plan = json.loads(self.plan.read_text())
+ self.assertFalse(planned_stop_matches(self.job, plan))
+ plan["final_checkpoint"] = {"training_job": "train", "step": 150,
+ "checkpoint": "hf://buckets/org/bucket/run/jobs/owner/run/checkpoint-150",
+ "manifest_sha256": "a" * 64, "user_requested_stop": True, "full_checkpoint_verified": True}
+ self.assertTrue(planned_stop_matches(self.job, plan))
+ for key, value in [("training_job", "other"), ("step", 100), ("manifest_sha256", ""),
+ ("full_checkpoint_verified", False), ("user_requested_stop", False)]:
+ changed = {**plan, "final_checkpoint": {**plan["final_checkpoint"], key: value}}
+ with self.subTest(key=key):
+ self.assertFalse(planned_stop_matches(self.job, changed))
+ self.job.status.stage = "RUNNING"
+ self.assertFalse(planned_stop_matches(self.job, plan))
+
+ def test_other_weights_cannot_be_logged_under_this_trainer(self):
+ logs = self.root / "late-training-logs"
+ logs.mkdir()
+ (logs / "status.json").write_text(json.dumps({"passed": True, "finished_at": 123}))
+ scores = self.root / "output/decisions/scores"
+ scores.mkdir(parents=True)
+ (scores / "step-000100.json").write_text(json.dumps({"step": 100,
+ "source": {"bundle_sha256": "bundle", "step": 100,
+ "source": "hf://buckets/org/bucket/run/jobs/other-trainer/run/checkpoint-100"},
+ "scores": {"comparison_ready": True, "tito_pass": True, "arm": "whitebox", "graded_cells": 250}}))
+ with self.assertRaisesRegex(ValueError, "Unverified checkpoint"):
+ self.run_replay()
+ self.api.sync_bucket.assert_called_once()
+ # The only operation was a remote-to-local read, never a bucket upload.
+ self.assertEqual(self.api.sync_bucket.call_args.args[0], "hf://buckets/org/bucket/run/jobs/owner")
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/04-data-agent/hf/tests/test_launch_qualified.py b/04-data-agent/hf/tests/test_launch_qualified.py
new file mode 100644
index 0000000..07e9ffc
--- /dev/null
+++ b/04-data-agent/hf/tests/test_launch_qualified.py
@@ -0,0 +1,36 @@
+import importlib.util
+import json
+from pathlib import Path
+import pytest
+
+spec = importlib.util.spec_from_file_location('launch_qualified', Path(__file__).parents[1] / 'launch_qualified.py')
+module = importlib.util.module_from_spec(spec)
+spec.loader.exec_module(module)
+
+
+def test_ambiguous_hf_submission_cannot_allocate_twice(tmp_path):
+ class API:
+ calls = 0
+
+ def run_job(self, **kwargs):
+ self.calls += 1
+ assert json.loads((tmp_path / 'intent.json').read_text())['owner'] == 'unique-owner'
+ raise TimeoutError('Response lost after allocation')
+
+ api = API()
+ wrapped = module.RecordedAPI(api, tmp_path / 'intent.json')
+ kwargs = {'env': {'RUN_OWNER': 'unique-owner', 'BUNDLE_SHA256': 'abc'},
+ 'namespace': 'test', 'labels': {'role': 'train'}}
+ with pytest.raises(TimeoutError):
+ wrapped.run_job(**kwargs)
+ with pytest.raises(RuntimeError, match='intent already exists'):
+ wrapped.run_job(**kwargs)
+ assert api.calls == 1
+
+
+def test_unqualified_native_plan_does_not_launch(tmp_path, monkeypatch):
+ from argparse import Namespace
+ (tmp_path / 'plan.json').write_text('{"checkpoint_eval_gpu_validation": "pending"}')
+ monkeypatch.setattr(module, 'run', lambda _: pytest.fail('unqualified allocation'))
+ with pytest.raises(ValueError, match='not complete'):
+ module.opencode(Namespace(ready=tmp_path), {})
diff --git a/04-data-agent/hf/tests/test_local_followup.py b/04-data-agent/hf/tests/test_local_followup.py
new file mode 100644
index 0000000..4c3bff1
--- /dev/null
+++ b/04-data-agent/hf/tests/test_local_followup.py
@@ -0,0 +1,149 @@
+import importlib.util
+import json
+from pathlib import Path
+import tempfile
+import unittest
+
+HF = Path(__file__).resolve().parents[1]
+
+
+def module(name):
+ spec = importlib.util.spec_from_file_location(name, HF / (name + ".py"))
+ value = importlib.util.module_from_spec(spec)
+ spec.loader.exec_module(value)
+ return value
+
+
+followup = module("local_followup")
+long = module("local_long")
+gates = module("launch_gates")
+
+
+class LocalQualificationTests(unittest.TestCase):
+ def test_eval_admission_preserves_fifty_slots_with_valid_reservation(self):
+ spec = importlib.util.spec_from_file_location("tested_service_policy", HF / "runtime/service_policy.py")
+ policy = importlib.util.module_from_spec(spec)
+ spec.loader.exec_module(policy)
+ admission = policy.Admission(int(followup.EVAL_ADMISSION["SANDBOX_CAPACITY"]),
+ int(followup.EVAL_ADMISSION["TRAIN_RESERVED_SANDBOXES"]))
+ for _ in range(50):
+ admission.acquire("eval", timeout=0)
+ with self.assertRaises(TimeoutError):
+ admission.acquire("eval", timeout=0)
+ admission.acquire("train", timeout=0)
+ for _ in range(50):
+ admission.release("eval")
+ admission.release("train")
+ self.assertEqual(admission.active, {"train": 0, "eval": 0})
+
+ def test_checkpoint_smoke_requires_each_task_harness_and_pinned_versions(self):
+ with tempfile.TemporaryDirectory() as temporary:
+ root = Path(temporary)
+ config = json.loads((HF / "configs/deployment.json").read_text())
+ pins = config["harness_pins"]
+ (root / "hf/configs").mkdir(parents=True)
+ (root / "hf/configs/deployment.json").write_text(json.dumps(config))
+ tasks = root / "experiments/daytona_harness_comparison/logs/20260915"
+ tasks.mkdir(parents=True)
+ (tasks / "test_indices.txt").write_text("17,42,8")
+ output = root / "eval"
+ output.mkdir()
+ scores = {"graded_cells": 8, "comparison_ready": False,
+ "harness_versions": {h: {v: 2} for h, v in pins.items()}}
+ (output / "canonical_scores.json").write_text(json.dumps(scores))
+ reports = [{"harness": h, "index": i, "tito_pass": True} for h in pins for i in (17, 42)]
+ audit = output / "final_tito.json"
+ audit.write_text(json.dumps({"reports": reports}))
+ plan = {"root": str(root), "arm": "opencode", "checkpoint_eval_qualification": True,
+ "bundle_sha256": "runtime", "controller_sha256": "controller"}
+ record = {"step": 4, "manifest_sha256": "checkpoint"}
+ proof = followup.validate_evaluation(plan, output, record)
+ self.assertTrue(proof["qualification_only"])
+ audit.write_text(json.dumps({"reports": reports[:-1]}))
+ with self.assertRaises(ValueError):
+ followup.validate_evaluation(plan, output, record)
+ audit.write_text(json.dumps({"reports": reports}))
+ scores["harness_versions"]["opencode"] = {"unknown": 2}
+ (output / "canonical_scores.json").write_text(json.dumps(scores))
+ with self.assertRaises(ValueError):
+ followup.validate_evaluation(plan, output, record)
+ plan["checkpoint_eval_qualification"] = False
+ with self.assertRaises(ValueError):
+ followup.validate_evaluation(plan, output, record)
+
+ def test_admission_changes_do_not_hide_score_protocol_changes(self):
+ import copy
+ config = json.loads((HF / "configs/deployment.json").read_text())
+ changed = copy.deepcopy(config)
+ changed["evaluation"]["concurrency_per_arm"]["opencode"] = 50
+ self.assertEqual(gates.protocol_identity(config), gates.protocol_identity(changed))
+ changed["evaluation"]["max_output_tokens_per_call"] = 8192
+ self.assertNotEqual(gates.protocol_identity(config), gates.protocol_identity(changed))
+
+ def test_only_hundred_intervals_and_final_are_evaluated(self):
+ manifests = [{"step": n} for n in [0, 50, 100, 150, 200, 250]]
+ self.assertEqual(followup.eligible_steps(manifests, False), {100, 200})
+ self.assertEqual(followup.eligible_steps(manifests, True), {100, 200, 250})
+ manifests[-1]["final"] = True
+ self.assertEqual(followup.eligible_steps(manifests, False), {100, 200, 250})
+ self.assertEqual(followup.eligible_steps([{"step": 2, "final": True}], True, qualification=True), set())
+ self.assertEqual(followup.eligible_steps([{"step": 2, "final": True}, {"step": 4}], True,
+ qualification=True), {4})
+
+ def test_partial_or_wrong_implementation_cannot_qualify_native_training(self):
+ with tempfile.TemporaryDirectory() as temporary:
+ path = Path(temporary) / "scores.json"
+ valid = {"comparison_ready": True, "graded_cells": 250, "tito_pass": True,
+ "implementation": "standalone-opencode"}
+ path.write_text(json.dumps({"daytona": valid}))
+ long.validate_baseline(path, "opencode")
+ for key, value in [("graded_cells", 249), ("tito_pass", False),
+ ("implementation", "harbor")]:
+ path.write_text(json.dumps({**valid, key: value}))
+ with self.assertRaises(ValueError):
+ long.validate_baseline(path, "opencode")
+
+ def test_passing_proof_cannot_hide_source_changes(self):
+ import hashlib
+ with tempfile.TemporaryDirectory() as temporary:
+ root = Path(temporary)
+ smoke = root / "outputs/local-train-whitebox-123"
+ smoke.mkdir(parents=True)
+ source = root / "trainer.py"
+ source.write_text("original trainer")
+ encoded = json.dumps({"files": {"trainer.py": hashlib.sha256(source.read_bytes()).hexdigest()}}).encode()
+ sha = hashlib.sha256(encoded).hexdigest()
+ (root / "bundle_manifest.json").write_bytes(encoded)
+ (root / "local_manifest.json").write_text(json.dumps({"sha256": sha}))
+ proof = {"arm": "whitebox", "bundle_sha256": sha, "passed": True,
+ "remote_restore_verified": True, "tito_pass": True, "weights_updated": True,
+ "native_optimizer_state_verified": True}
+ (smoke / "training_smoke_verified.json").write_text(json.dumps(proof))
+ self.assertEqual(long.proof_for(smoke, "whitebox")[0], root)
+ source.write_text("changed trainer")
+ with self.assertRaises(ValueError):
+ long.proof_for(smoke, "whitebox")
+
+ def test_native_grading_gate_rejects_the_old_zero_tolerance_fallback(self):
+ import hashlib
+ with tempfile.TemporaryDirectory() as temporary:
+ root = Path(temporary)
+ package = root / "experiments/daytona_harness_comparison/logs/20260915/source/packages/data_agent_env"
+ package.mkdir(parents=True)
+ hashes = {}
+ for name in ("task.py", "tasks.py", "verifier.py", "grader.py"):
+ path = package / name
+ path.write_text("explicit zero tolerance preserved")
+ hashes[name] = hashlib.sha256(path.read_bytes()).hexdigest()
+ baseline = root / "canonical_scores.json"
+ (root / "verification.json").write_text(json.dumps({"passed": True,
+ "parameters_verified": 1250, "original_graded_records_preserved": True,
+ "reports": [{}] * 250, "runtime_files": hashes}))
+ long.validate_native_grading(root, baseline)
+ (package / "verifier.py").write_text("task.atol or 1e-3")
+ with self.assertRaisesRegex(ValueError, "Native grader differs"):
+ long.validate_native_grading(root, baseline)
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/04-data-agent/hf/tests/test_local_score_forwarding.py b/04-data-agent/hf/tests/test_local_score_forwarding.py
new file mode 100644
index 0000000..784d15a
--- /dev/null
+++ b/04-data-agent/hf/tests/test_local_score_forwarding.py
@@ -0,0 +1,70 @@
+import importlib.util
+import json
+from pathlib import Path
+import tempfile
+import unittest
+
+spec = importlib.util.spec_from_file_location("local_watch", Path(__file__).resolve().parents[1] / "local_watch.py")
+watch = importlib.util.module_from_spec(spec)
+spec.loader.exec_module(watch)
+
+
+class ScoreForwardingTest(unittest.TestCase):
+ def test_matching_baseline_and_checkpoint_forward_for_every_recipe(self):
+ import hashlib
+ for arm, cells in (("blackbox", 1000), ("opencode", 1000), ("whitebox", 250)):
+ with self.subTest(arm=arm), tempfile.TemporaryDirectory() as temporary:
+ root = Path(temporary)
+ baseline = root / "baseline.json"
+ baseline.write_text(json.dumps({"graded_cells": cells, "comparison_ready": True}))
+ plan = root / "plan.json"
+ plan.write_text(json.dumps({"root": str(root), "arm": arm, "training_job": "123",
+ "bundle_sha256": "bundle", "baseline_score": str(baseline),
+ "baseline_sha256": hashlib.sha256(baseline.read_bytes()).hexdigest()}))
+ directory = root / "checkpoint-evals"
+ directory.mkdir()
+ (directory / "scores-000100.json").write_text(json.dumps({
+ "proof": {"passed": True, "training_arm": arm, "graded_cells": cells,
+ "bundle_sha256": "bundle", "manifest_sha256": "model", "step": 100},
+ "evaluation": {"manifest_sha256": "model", "step": 100, "job_id": "456"},
+ "scores": {"comparison_ready": True, "graded_cells": cells}}))
+ _, training, changed = watch.forward_scores(plan)
+ self.assertTrue(changed)
+ for step in (0, 100):
+ record = json.loads((training / f"checkpoint-scores/step-{step:06d}.json").read_text())
+ self.assertEqual(record["scores"]["graded_cells"], cells)
+ baseline.write_text('{}')
+ with self.assertRaisesRegex(ValueError, "baseline score changed"):
+ watch.forward_scores(plan)
+
+ def test_only_verified_full_scores_enter_training_curve_and_conflicts_fail(self):
+ with tempfile.TemporaryDirectory() as temporary:
+ root = Path(temporary)
+ plan = root / "plan.json"
+ plan.write_text(json.dumps({"root": str(root), "arm": "opencode", "training_job": "123",
+ "bundle_sha256": "bundle"}))
+ (root / "checkpoint-evals").mkdir()
+ source = root / "checkpoint-evals/scores-000100.json"
+ record = {"proof": {"passed": True, "qualification_only": True, "training_arm": "opencode",
+ "graded_cells": 1000, "bundle_sha256": "bundle", "manifest_sha256": "model", "step": 100},
+ "evaluation": {"manifest_sha256": "model", "step": 100, "job_id": "456"},
+ "scores": {"comparison_ready": True, "average_pass_at_1": 0.2}}
+ source.write_text(json.dumps(record))
+ self.assertFalse(watch.forward_scores(plan)[2])
+ record["proof"]["qualification_only"] = False
+ source.write_text(json.dumps(record))
+ self.assertTrue(watch.forward_scores(plan)[2])
+ self.assertFalse(watch.forward_scores(plan)[2])
+ target = root / "outputs/local-train-opencode-123/checkpoint-scores/step-000100.json"
+ event = json.loads(target.read_text())
+ self.assertEqual(event["step"], 100)
+ self.assertEqual(event["source"]["job_id"], "456")
+ record["scores"]["average_pass_at_1"] = 0.4
+ source.write_text(json.dumps(record))
+ with self.assertRaisesRegex(ValueError, "Conflicting"):
+ watch.forward_scores(plan)
+ self.assertEqual(json.loads(target.read_text()), event)
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/04-data-agent/hf/tests/test_monitor_three_runs.py b/04-data-agent/hf/tests/test_monitor_three_runs.py
new file mode 100644
index 0000000..cbd367a
--- /dev/null
+++ b/04-data-agent/hf/tests/test_monitor_three_runs.py
@@ -0,0 +1,79 @@
+import importlib.util
+from pathlib import Path
+
+spec = importlib.util.spec_from_file_location('monitor_three_runs', Path(__file__).parents[1] / 'monitor_three_runs.py')
+module = importlib.util.module_from_spec(spec)
+spec.loader.exec_module(module)
+
+
+def test_summary_not_a_second_optimizer_update():
+ rows = [{'step': 1, 'reward': .25, 'grad_norm': 2}, {'step': 1, 'train_runtime': 4},
+ {'step': 2, 'reward': .5, 'grad_norm': 0}]
+ r = module.training_metrics(rows)
+ assert r['updates'] == 2 and r['reward_last20'] == .375
+ assert r['nonzero_gradients_last20'] == 1
+
+
+def test_timeout_observation_does_not_imply_training_failure():
+ prior = {'job': 'a', 'step': 12, 'last_optimizer_progress_at': 100}
+ current = {'job': 'a', 'step': 12, 'stage': 'UNKNOWN'}
+ assert not module.diagnose(current, prior, 2100)
+ current['stage'] = 'RUNNING'
+ assert 'no_optimizer_progress_over_30_minutes' in module.diagnose(current, prior, 2100)
+
+
+def test_resume_progress_and_tito_failure():
+ prior = {'job': 'a', 'step': 100, 'last_optimizer_progress_at': 100}
+ current = {'job': 'b', 'step': 100, 'stage': 'RUNNING', 'checkpoints': [100],
+ 'tito': {'rows': 8, 'passed': 7}}
+ alerts = module.diagnose(current, prior, 2200)
+ assert alerts == ['tito_failed']
+ assert current['last_optimizer_progress_at'] == 2200
+
+
+def test_enabled_online_logging_is_not_successful_sync():
+ row = {'trackio': {'online': True, 'local_ok': True, 'sync': {'ok': False}},
+ 'evaluations': [{'checkpoint': 'checkpoint-600', 'stage': 'FAILED', 'complete': False}]}
+ assert module.diagnose(row, {}, 1) == ['trackio_online_sync_failed', 'evaluation_failed_checkpoint-600']
+ row['trackio']['sync']['ok'] = True
+ row['evaluations'][0]['stage'] = 'RUNNING'
+ assert module.diagnose(row, {}, 2) == []
+
+
+def test_parent_evaluations_remain_visible_after_resume(tmp_path, monkeypatch):
+ import json
+ parent, child = tmp_path / 'parent', tmp_path / 'child'
+ child.mkdir(); parent.mkdir()
+ (parent / 'run_config.json').write_text('{}')
+ (child / 'run_config.json').write_text(json.dumps({'resume_state': {
+ 'step': 150, 'checkpoint': str(parent / 'job-1/run/checkpoint-150')}}))
+ for root, steps in [(parent, [100, 200]), (child, [300])]:
+ for step in steps:
+ d = root / f'checkpoint-evals/step-{step:06d}'
+ d.mkdir(parents=True)
+ (d / 'submission.json').write_text(json.dumps({'job_id': str(step)}))
+ (d / 'scores.json').write_text(json.dumps({'comparison_ready': step == 100,
+ 'average_pass_at_1': .25, 'graded_cells': 1000}))
+ monkeypatch.setattr(module, 'states', lambda jobs: {str(j): 'COMPLETED' for j in jobs})
+ evaluations = module.original_evaluations(child)
+ assert [e['checkpoint'] for e in evaluations] == ['checkpoint-100', 'checkpoint-300']
+ assert evaluations[0]['complete'] is True
+
+
+def test_restore_startup_retains_parent_metrics_and_checkpoints(tmp_path):
+ import json
+ parent, child = tmp_path / 'parent', tmp_path / 'child'
+ checkpoint = parent / 'job-1/run/checkpoint-2'
+ checkpoint.mkdir(parents=True)
+ (checkpoint / 'checkpoint.saved.json').write_text('{}')
+ (parent / 'submission.json').write_text('{"training": "1"}')
+ (parent / 'run_config.json').write_text('{}')
+ audit = parent / 'job-1/audit'
+ audit.mkdir()
+ (audit / 'metrics.jsonl').write_text(''.join(json.dumps({'step': s, 'reward': .5, 'grad_norm': 1}) + '\n' for s in [1,2,3]))
+ child.mkdir()
+ (child / 'submission.json').write_text('{"training": "2"}')
+ (child / 'run_config.json').write_text(json.dumps({'resume_state': {'step': 2, 'checkpoint': str(checkpoint)}}))
+ rows, saves = module.original_history(child)
+ assert [r['step'] for r in rows] == [1,2]
+ assert saves == [2]
diff --git a/04-data-agent/hf/tests/test_native_tolerances.py b/04-data-agent/hf/tests/test_native_tolerances.py
new file mode 100644
index 0000000..8458658
--- /dev/null
+++ b/04-data-agent/hf/tests/test_native_tolerances.py
@@ -0,0 +1,39 @@
+"""Explicit zero tolerances must survive task parsing and actual native grading."""
+import importlib
+from pathlib import Path
+import sys
+from types import ModuleType
+import unittest
+
+PACKAGE = "comparison_native_tolerance"
+package = ModuleType(PACKAGE)
+package.__path__ = [str(Path(__file__).resolve().parents[2] / "envs/blackbox-opencode")]
+sys.modules[PACKAGE] = package
+Task = importlib.import_module(PACKAGE + ".task").DataAgentTask
+grade_rollout = importlib.import_module(PACKAGE + ".verifier").grade_rollout
+
+
+class NativeToleranceTest(unittest.TestCase):
+ def row(self, **values):
+ return {"instruction": "Calculate the value", "answer": "1", "reward_mode": "numeric",
+ "hf_bucket": "org/test", "bucket_prefix": "task", **values}
+
+ def test_explicit_zero_is_strict_but_omitted_tolerance_defaults(self):
+ strict = Task.from_row(self.row(atol=0.0, rtol="0.0"))
+ default = Task.from_row(self.row(atol=None, rtol=""))
+ self.assertEqual((strict.atol, strict.rtol), (0.0, 0.0))
+ self.assertEqual((default.atol, default.rtol), (1e-3, 1e-3))
+ read = lambda _: "1.0005"
+ self.assertEqual(grade_rollout(strict, read, ("/answer",)).correctness, 0.0)
+ self.assertEqual(grade_rollout(default, read, ("/answer",)).correctness, 1.0)
+
+ def test_absolute_and_relative_tolerances_are_preserved_independently(self):
+ narrow = Task.from_row(self.row(atol=1e-5, rtol=0))
+ relative = Task.from_row(self.row(atol=0, rtol=1e-3))
+ read = lambda _: "1.0005"
+ self.assertEqual(grade_rollout(narrow, read, ("/answer",)).correctness, 0.0)
+ self.assertEqual(grade_rollout(relative, read, ("/answer",)).correctness, 1.0)
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/04-data-agent/hf/tests/test_portable_reproduction.py b/04-data-agent/hf/tests/test_portable_reproduction.py
new file mode 100644
index 0000000..a55ab0d
--- /dev/null
+++ b/04-data-agent/hf/tests/test_portable_reproduction.py
@@ -0,0 +1,83 @@
+"""Portable packaging and shared-service isolation at the actual rollout boundary."""
+import asyncio
+import json
+from pathlib import Path
+from types import SimpleNamespace
+from unittest.mock import patch
+
+import pytest
+
+import build
+import harbor_service
+from service_policy import Admission
+
+
+def test_harbor_policy_is_per_session_and_releases_failed_rollouts(monkeypatch):
+ from openenv.harbor import rollout, serving
+ gate = Admission(4, 1)
+ sessions = {}
+ observed = []
+ class Registry:
+ def get(self, key): return sessions.get(key)
+ async def native(**kwargs):
+ key = kwargs['dataset']
+ sessions[key] = SimpleNamespace(metadata={})
+ kwargs['on_session_created'](key)
+ await asyncio.sleep(.01)
+ if key == 'test': raise RuntimeError('trial failure')
+ return sessions[key].metadata['max_output_tokens']
+ monkeypatch.setattr(rollout, 'run_rollout', native)
+ monkeypatch.setattr(serving, 'space_public_url', lambda: 'https://example.hf.space')
+ monkeypatch.setattr(harbor_service, 'admission', gate)
+ harbor_service.install()
+ async def exercise():
+ return await asyncio.gather(*(rollout.run_rollout(dataset=d, registry=Registry(),
+ on_session_created=observed.append) for d in ('train', 'test')), return_exceptions=True)
+ values = asyncio.run(exercise())
+ assert values[0] == 16384 and isinstance(values[1], RuntimeError)
+ assert sessions['test'].metadata['max_output_tokens'] == 4096
+ assert set(observed) == {'train', 'test'}
+ assert gate.snapshot()['active'] == {'train': 0, 'eval': 0}
+
+
+def test_replaced_outputs_are_archived_without_data_loss(tmp_path):
+ old = tmp_path / 'stage'
+ old.mkdir()
+ (old / 'capture.json').write_text('original')
+ archive = tmp_path / 'archive'
+ build.preserve(old, archive)
+ assert not old.exists()
+ assert next(archive.glob('stage-*/capture.json')).read_text() == 'original'
+
+
+def test_packaging_excludes_local_credentials_and_caches(tmp_path):
+ source = tmp_path / 'source'
+ source.mkdir()
+ (source / '.env').write_text('secret')
+ (source / 'app.py').write_text('print(1)')
+ (source / 'temp').mkdir()
+ (source / 'temp' / 'token').write_text('secret')
+ target = tmp_path / 'packed'
+ build._copy(source, target)
+ assert sorted(p.name for p in target.iterdir()) == ['app.py']
+
+
+def test_hub_eval_capacity_is_bounded_and_source_pins_are_immutable():
+ root = Path(__file__).resolve().parents[1]
+ config = json.loads((root / 'configs/deployment.json').read_text())
+ assert set(config['evaluation']['concurrency_per_arm'].values()) == {35}
+ sources = json.loads((root / 'configs/sources.json').read_text())
+ for source in sources['repositories']:
+ assert len(source['revision']) == 40
+ assert set(source['revision']) <= set('0123456789abcdef')
+ assert len(sources['task_bundle']['sha256']) == 64
+
+
+def test_training_preflight_rejects_legacy_server_missing_sampling():
+ from service_contract import validate_tools
+ properties = {key: {} for key in ("llm_url", "model", "require_tokens", "agent_timeout_s")}
+ response = {"data": {"observation": {"tools": [{"name": "run_rollout", "input_schema": {"properties": properties}}]}}}
+ with pytest.raises(ValueError, match="lacks training arguments: sampling"):
+ validate_tools(response, "opencode")
+ properties["sampling"] = {}
+ assert validate_tools(response, "opencode")["passed"]
diff --git a/04-data-agent/hf/tests/test_setup_pipeline.py b/04-data-agent/hf/tests/test_setup_pipeline.py
new file mode 100644
index 0000000..ad1eca9
--- /dev/null
+++ b/04-data-agent/hf/tests/test_setup_pipeline.py
@@ -0,0 +1,95 @@
+"""Verify deployment/training ordering and replay without duplicate allocations."""
+import json
+import os
+from pathlib import Path
+import sys
+import tempfile
+from types import SimpleNamespace as NS
+import unittest
+from unittest.mock import patch
+
+HF = Path(__file__).resolve().parents[1]
+sys.path[:0] = [str(HF), str(HF / "runtime")]
+import setup_pipeline
+
+
+class PipelineTest(unittest.TestCase):
+ def execute(self, fail_baseline=False):
+ from huggingface_hub.errors import EntryNotFoundError
+ with tempfile.TemporaryDirectory() as tmp:
+ root = Path(tmp)
+ config = json.loads((HF / "configs/deployment.json").read_text())
+ config["run_id"] = "test-run"
+ config["pipeline"]["baseline_jobs"] = {"blackbox": "bb", "whitebox": "wb"}
+ (root / "hf/configs").mkdir(parents=True)
+ (root / "hf/configs/deployment.json").write_text(json.dumps(config))
+ bundle = root / "bundle"
+ bundle.mkdir()
+ (bundle / "bundle.json").write_text(json.dumps({"sha256": "new"}))
+ (bundle / "bundle.tar.gz").write_bytes(b"fixture bundle")
+ jobs = []
+ baselines = {job_id: NS(id=job_id, status=NS(stage="ERROR" if fail_baseline and arm == "whitebox" else "COMPLETED"),
+ environment={"RUN_OWNER": "base-" + arm}) for arm, job_id in config["pipeline"]["baseline_jobs"].items()}
+ remote = {f"test-run/jobs/base-{arm}/canonical_scores.json": json.dumps({"arm": arm, "comparison_ready": True})
+ for arm in ["blackbox", "whitebox"]}
+ events = []
+ deployed = [False]
+ class API:
+ def inspect_job(self, *, job_id, **kwargs):
+ return baselines.get(job_id) or next(j for j in jobs if j.id == job_id)
+ def list_jobs(self, *, labels, **kwargs):
+ return [j for j in jobs if all(j.labels.get(k) == v for k, v in labels.items())]
+ def download_bucket_files(self, bucket, *, files, **kwargs):
+ for name, dest in files:
+ if name not in remote: raise EntryNotFoundError(name)
+ Path(dest).write_text(remote[name])
+ def sync_bucket(self, source, dest, **kwargs):
+ remote[dest.removeprefix("hf://buckets/org/bucket/") + "/pipeline.json"] = (Path(source) / "pipeline.json").read_text()
+ def space_info(self, repo): return NS(host="https://" + repo.replace("/", "-"))
+ def spaces(*args):
+ events.append("deploy")
+ deployed[0] = True
+ def get(*args, **kwargs):
+ return NS(raise_for_status=lambda: None, json=lambda: {"admission": {"active": {"train": 0, "eval": 0}},
+ "bundle_sha256": "new" if deployed[0] else "old"})
+ def start(*args):
+ events.append("ui")
+ return NS(wait=lambda **kwargs: 0)
+ def submit(api, config, secret, out, args):
+ events.append(f"{args.role}:{args.arm}:{args.phase}")
+ job_id = "created" + str(len(jobs))
+ labels = {"role": args.role, "arm": args.arm, "phase": args.phase, "run": "test-run"}
+ job = NS(id=job_id, labels=labels, status=NS(stage="COMPLETED"),
+ environment={"BUNDLE_SHA256": "new", "RUN_OWNER": job_id})
+ jobs.append(job)
+ remote[f"test-run/jobs/{job_id}/" + ("audit/metrics.jsonl" if args.arm == "blackbox" else "run/metrics.jsonl")] = json.dumps(
+ {"step": 12, "loss": 0.0, "grad_norm": 1.0, "reward": 0.5}) + "\n"
+ return {"id": job_id, "stage": "COMPLETED"}
+ env = {"ARTIFACT_BUCKET": "org/bucket", "RUN_ID": "test-run", "BUNDLE_SHA256": "new",
+ "BUNDLE_REPO": "org/repro", "BUNDLE_REVISION": "revision", "RUN_OWNER": "setup-owner"}
+ with patch.dict(os.environ, env), patch("setup_pipeline.ROOT", root), patch("setup_pipeline.BUNDLE", bundle), \
+ patch("huggingface_hub.HfApi", API), patch("httpx.get", get), patch("deploy.spaces", spaces), \
+ patch("deploy.submit", submit), patch("setup_pipeline.start", start):
+ if fail_baseline:
+ with self.assertRaisesRegex(RuntimeError, "ERROR"):
+ setup_pipeline.run(root / "output")
+ self.assertEqual(events, [])
+ self.assertEqual(jobs, [])
+ return
+ setup_pipeline.run(root / "output")
+ self.assertEqual(events[:2], ["deploy", "ui"])
+ self.assertEqual(events[2:6], ["train:blackbox:smoke", "train:whitebox:smoke", "train:blackbox:long", "train:whitebox:long"])
+ self.assertEqual(len(jobs), 6)
+ setup_pipeline.run(root / "replayed")
+ self.assertEqual(len(jobs), 6, "Replay must adopt all existing jobs")
+ self.assertEqual(len(events), 8, "Replay must not rebuild Spaces or rerun the UI")
+ state = json.loads((root / "replayed/pipeline.json").read_text())
+ self.assertEqual(state["phase"], "completed")
+ self.assertTrue(state["passed"])
+
+ def test_complete_baselines_smoke_before_long_and_replay(self): self.execute()
+ def test_failed_baseline_cannot_deploy_or_allocate_training(self): self.execute(fail_baseline=True)
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/04-data-agent/hf/tests/test_shared_service.py b/04-data-agent/hf/tests/test_shared_service.py
new file mode 100644
index 0000000..aa6958c
--- /dev/null
+++ b/04-data-agent/hf/tests/test_shared_service.py
@@ -0,0 +1,63 @@
+"""Regression checks for simultaneous train/eval admission and capture budgets."""
+import asyncio
+from concurrent.futures import ThreadPoolExecutor
+import unittest
+
+from common import configure
+configure()
+from service_policy import Admission, output_limit
+
+
+class SharedServiceTest(unittest.TestCase):
+ def test_reserved_train_slot_and_cancelled_waiter(self):
+ async def run():
+ gate = Admission(3, 1)
+ async with gate.slot("test"), gate.slot("test"):
+ blocked = asyncio.create_task(enter(gate, "test"))
+ await asyncio.sleep(0.15)
+ self.assertFalse(blocked.done())
+ async with gate.slot("train"):
+ self.assertEqual(gate.snapshot()["active"], {"train": 1, "eval": 2})
+ blocked.cancel()
+ with self.assertRaises(asyncio.CancelledError):
+ await blocked
+ self.assertEqual(gate.snapshot()["active"], {"train": 0, "eval": 0})
+ self.assertEqual(gate.snapshot()["waiting"], {"train": 0, "eval": 0})
+ async with gate.slot("test"):
+ pass
+ async def enter(gate, role):
+ async with gate.slot(role):
+ pass
+ asyncio.run(run())
+
+ def test_simultaneous_split_caps_reach_engine_unchanged(self):
+ from fastapi.testclient import TestClient
+ from openenv.core.harness.capture.server import create_app
+ class Engine:
+ served_model = "test-model"
+ param_fixes = {}
+ capture_level = "text"
+ async def completion(self, request):
+ return {"id": "cap-test", "object": "chat.completion", "model": self.served_model,
+ "choices": [{"index": 0, "message": {"role": "assistant", "content": str(request['max_tokens'])},
+ "finish_reason": "stop"}],
+ "usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2}}
+ app = create_app(llm_url="http://unused.invalid/v1", model="test-model", capture_level="text", max_output_tokens=16384)
+ engine = Engine()
+ app.state.inference = engine
+ app.state.upstreams._default = (engine, "text")
+ sessions = {split: app.state.registry.create(dataset=split, max_output_tokens=output_limit(split))
+ for split in ("train", "test")}
+ with TestClient(app) as client:
+ def call(split):
+ r = client.post("/v1/chat/completions", headers={"Authorization": "Bearer " + sessions[split].session_id},
+ json={"model": "test-model", "messages": [{"role": "user", "content": split}], "max_tokens": 32768})
+ self.assertEqual(r.status_code, 200, r.text)
+ return split, int(r.json()["choices"][0]["message"]["content"])
+ with ThreadPoolExecutor(max_workers=2) as pool:
+ for split, cap in pool.map(call, ["train", "test"] * 4):
+ self.assertEqual(cap, 16384 if split == "train" else 4096)
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/04-data-agent/hf/tests/test_standalone_opencode.py b/04-data-agent/hf/tests/test_standalone_opencode.py
new file mode 100644
index 0000000..78d294d
--- /dev/null
+++ b/04-data-agent/hf/tests/test_standalone_opencode.py
@@ -0,0 +1,68 @@
+"""Regression checks for the standalone arm's token and task boundaries."""
+import importlib.util
+import json
+from pathlib import Path
+import sys
+import unittest
+from unittest.mock import patch
+
+ROOT=Path(__file__).resolve().parents[4]
+NATIVE=Path(__file__).resolve().parents[2]/'envs/blackbox-opencode'
+sys.path.insert(0,str(ROOT/'OpenEnv/src'))
+spec=importlib.util.spec_from_file_location('data_agent_env',NATIVE/'__init__.py',submodule_search_locations=[str(NATIVE)])
+module=importlib.util.module_from_spec(spec);sys.modules[spec.name]=module;spec.loader.exec_module(module)
+from data_agent_env.models import DataAgentRolloutResult
+from data_agent_env.server.rollout import turns_from_capture, _stage_inputs, _run_agent
+from data_agent_env.harness import to_trace_entries
+from data_agent_env.config import DataAgentConfig
+from data_agent_env.tasks import _frozen_rows
+from data_agent_env.task import DataAgentTask
+
+class StandaloneTests(unittest.TestCase):
+ def test_agent_deadline_terminates_background_process(self):
+ from unittest.mock import Mock
+ sandbox=Mock()
+ sandbox.start_bg.return_value.wait.side_effect=TimeoutError('agent budget expired')
+ code=_run_agent(sandbox,'http://capture','session','model',DataAgentConfig(),'solve this')
+ self.assertEqual(code,124)
+ sandbox.start_bg.return_value.kill.assert_called_once()
+ sandbox.exec.assert_not_called()
+ def test_background_transport_failure_remains_ungraded(self):
+ from unittest.mock import Mock
+ sandbox=Mock()
+ sandbox.start_bg.return_value.wait.side_effect=ConnectionError('transport unavailable')
+ with self.assertRaises(ConnectionError):
+ _run_agent(sandbox,'http://capture','session','model',DataAgentConfig(),'solve this')
+ def test_partial_and_zero_masks_survive_wire_boundary(self):
+ for mask in ([0,0,1,0],[0,0,0,0]):
+ raw={'prompt_token_ids':[1,2],'completion_token_ids':[3,4],
+ 'per_token_logps':[-.1,-.2],'loss_mask':mask}
+ result=DataAgentRolloutResult.model_validate_json(DataAgentRolloutResult(
+ rollout_type='train',turns=turns_from_capture([raw])).model_dump_json())
+ entries=to_trace_entries(result)
+ self.assertEqual(entries[0]['loss_mask'] if entries else [],mask if any(mask) else [])
+ def test_bad_logprobs_cannot_become_trainable(self):
+ raw={'prompt_token_ids':[1,2],'completion_token_ids':[3,4],
+ 'per_token_logps':[-.1],'loss_mask':[0,0,1,1]}
+ with self.assertRaises(ValueError):
+ to_trace_entries(DataAgentRolloutResult(turns=turns_from_capture([raw])))
+ def test_stage_token_is_transient_and_absent_from_shell(self):
+ from unittest.mock import Mock
+ task=DataAgentTask(task_id='fixture',instruction='question',answer='a',hf_bucket='owner/bucket',bucket_prefix='data')
+ sandbox=Mock();sandbox.exec.return_value.exit_code=0
+ _stage_inputs(sandbox,task,'fixture-secret',DataAgentConfig())
+ call=sandbox.exec.call_args
+ self.assertNotIn('fixture-secret',call.args[0])
+ self.assertEqual(call.kwargs['envs']['HF_TOKEN'],'fixture-secret')
+ self.assertNotIn('HF_TOKEN',task.env(None))
+ def test_fixed_catalog_preserves_identity_and_difficulty(self):
+ root=ROOT/'experiments/daytona_harness_comparison/logs/20260915/datasets'
+ if not root.exists():self.skipTest('frozen comparison dataset is unavailable')
+ from collections import Counter
+ rows=_frozen_rows(str(root),'test')
+ self.assertEqual(len(rows),250)
+ self.assertEqual(Counter(r['difficulty_tier'] for r in rows),{'easy':33,'medium':118,'hard':99})
+ manifest=json.loads((root.parent/'test_manifest.json').read_text())
+ self.assertEqual([r['task_id'] for r in rows],sorted(t['name'] for t in manifest['tasks']))
+
+if __name__=='__main__':unittest.main()
diff --git a/04-data-agent/hf/tests/test_standalone_training.py b/04-data-agent/hf/tests/test_standalone_training.py
new file mode 100644
index 0000000..4ce7c30
--- /dev/null
+++ b/04-data-agent/hf/tests/test_standalone_training.py
@@ -0,0 +1,87 @@
+"""Exercise native scheduling, reward semantics and the exact capture admission audit."""
+import importlib.util
+import json
+from pathlib import Path
+import pickle
+import sys
+import unittest
+from unittest.mock import Mock, patch
+
+ROOT = Path(__file__).resolve().parents[2]
+NATIVE = ROOT / "envs/blackbox-opencode"
+sys.path.insert(0, str(ROOT / "train"))
+spec = importlib.util.spec_from_file_location("data_agent_env", NATIVE / "__init__.py", submodule_search_locations=[str(NATIVE)])
+module = importlib.util.module_from_spec(spec)
+sys.modules[spec.name] = module
+spec.loader.exec_module(module)
+from data_agent_env.models import DataAgentRolloutResult
+from data_agent_env.tasks import DataAgentTaskProvider
+from standalone_comparison import ScheduledOpenCodeFactory, ComparisonSession
+from data_agent_env.client import DataAgentEnv
+
+
+class StandaloneTrainingTests(unittest.TestCase):
+ def test_sampling_policy_reaches_the_wire_and_native_capture_registry(self):
+ from data_agent_env.server.capture import mint_session
+ from openenv.core.harness.capture.sessions import SessionRegistry
+ from openenv.core.harness.capture.upstream import training_sampling
+ policy = {"temperature": .8, "top_p": 1., "top_k": 0}
+ factory = self.factory()
+ client = object.__new__(DataAgentEnv)
+ calls = []
+ def call(name, **kwargs):
+ calls.append((name, kwargs))
+ return DataAgentRolloutResult().model_dump_json()
+ client._call = call
+ session = ComparisonSession(client, "train", 0, "fixture", **factory._rollout_kwargs)
+ session.wait_for_completion(timeout_s=123)
+ self.assertEqual(calls[0][1]["sampling"], policy)
+ self.assertEqual(calls[0][1]["_timeout_s"], 123)
+ server = Mock(registry=SessionRegistry())
+ sid, _ = mint_session(server, llm_url="http://inference", model="fixture",
+ rollout_id="fixture", capture_level="tokens", sampling=calls[0][1]["sampling"])
+ captured = server.registry.get(sid)
+ self.assertEqual(captured.sampling, training_sampling(policy))
+ self.assertEqual(captured.sampling["top_k"], -1)
+
+ def factory(self):
+ from harness_schedule import make_schedule
+ self.schedule = make_schedule([{ "name": f"fixture-{i}", "task_index": i, "difficulty": "easy"} for i in range(12)], ["opencode"], easy_start=4)
+ return ScheduledOpenCodeFactory("http://fixture", harnesses=["opencode"], schedule=self.schedule,
+ llm_url="http://inference", model="Qwen/Qwen3.5-2B", sampling={"temperature": .8, "top_p": 1., "top_k": 0})
+
+ def test_all_scheduled_groups_and_resume_preserve_task_identity(self):
+ factory = self.factory()
+ tasks = [{"index": t["task_index"], "task_id": t["name"], "instruction": t["name"]} for t in self.schedule["tasks"]]
+ client = Mock(); client.get_task_range.return_value = tasks
+ with patch.object(ScheduledOpenCodeFactory, "_new_client", return_value=client):
+ rows = factory.prompt_rows()
+ restored = pickle.loads(pickle.dumps(factory))
+ for offset in (0, 997, 3999):
+ restored.group_offset = offset
+ for group_id in range(20):
+ expected = self.schedule["groups"][(offset + group_id) % len(self.schedule["groups"])]
+ row = rows[expected["task_row"]]
+ for generation in range(8):
+ session = restored.create(row, seed=group_id, episode_id=str(generation))
+ self.assertEqual(session._task_index, expected["task_index"])
+ restored.group_offset = 0
+ with self.assertRaises(ValueError):
+ restored.create({"prompt": [{"role": "user", "content": "unknown task"}]}, seed=0)
+
+ def test_reward_matches_binary_comparison_and_preserves_ungraded(self):
+ for correctness, raw_reward, expected in [(None, None, None), (0., 0., 0.), (.3, .3, 0.), (1., 1.1, 1.)]:
+ session = ComparisonSession(Mock(), "train", 0, "task")
+ session._result = DataAgentRolloutResult(correctness=correctness, reward=raw_reward)
+ self.assertEqual(session.verify([]).env_reward, expected)
+ self.assertEqual(session.result.reward, raw_reward)
+
+ def test_changed_server_task_identity_is_rejected(self):
+ factory = self.factory()
+ tasks = [{"index": t["task_index"], "task_id": t["name"], "instruction": t["name"]} for t in self.schedule["tasks"]]
+ tasks[0]["task_id"] = "wrong"
+ client = Mock(); client.get_task_range.return_value = tasks
+ with patch.object(ScheduledOpenCodeFactory, "_new_client", return_value=client):
+ with self.assertRaises(ValueError): factory.prompt_rows()
+
+if __name__ == "__main__": unittest.main()
diff --git a/04-data-agent/hf/tests/test_stop_hf_after_checkpoint.py b/04-data-agent/hf/tests/test_stop_hf_after_checkpoint.py
new file mode 100644
index 0000000..20ba321
--- /dev/null
+++ b/04-data-agent/hf/tests/test_stop_hf_after_checkpoint.py
@@ -0,0 +1,45 @@
+"""A stop requires exact full-state checkpoint readback, including optimizer state."""
+import hashlib
+import json
+from pathlib import Path
+import sys
+from types import SimpleNamespace
+
+import pytest
+
+sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
+from stop_hf_after_checkpoint import READY, verify_checkpoint
+
+
+def make_checkpoint(path):
+ files = {"model.safetensors": b"weights", "optimizer.pt": b"optimizer",
+ "scheduler.pt": b"scheduler", "rng_state.pth": b"rng",
+ "trainer_state.json": b'{"global_step":150}'}
+ for name, data in files.items():
+ (path / name).write_bytes(data)
+ manifest = {"arm": "whitebox", "step": 150, "bundle_sha256": "bundle",
+ "base_model": "Qwen/Qwen3.5-2B", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc",
+ "files": {n: hashlib.sha256(v).hexdigest() for n, v in files.items()}}
+ (path / READY).write_text(json.dumps(manifest))
+ return SimpleNamespace(environment={"BUNDLE_SHA256": "bundle"})
+
+
+def test_complete_state_can_authorize_stop(tmp_path):
+ job = make_checkpoint(tmp_path)
+ assert verify_checkpoint(tmp_path, job=job, step=150)["step"] == 150
+
+
+def test_corrupt_optimizer_prevents_stop(tmp_path):
+ job = make_checkpoint(tmp_path)
+ (tmp_path / "optimizer.pt").write_bytes(b"partial upload")
+ with pytest.raises(ValueError, match="hash mismatch"):
+ verify_checkpoint(tmp_path, job=job, step=150)
+
+
+def test_wrong_checkpoint_or_bundle_prevents_stop(tmp_path):
+ job = make_checkpoint(tmp_path)
+ with pytest.raises(ValueError, match="provenance"):
+ verify_checkpoint(tmp_path, job=job, step=200)
+ job.environment["BUNDLE_SHA256"] = "other"
+ with pytest.raises(ValueError, match="provenance"):
+ verify_checkpoint(tmp_path, job=job, step=150)
diff --git a/04-data-agent/hf/tests/test_training_runtime.py b/04-data-agent/hf/tests/test_training_runtime.py
new file mode 100644
index 0000000..5366e6f
--- /dev/null
+++ b/04-data-agent/hf/tests/test_training_runtime.py
@@ -0,0 +1,308 @@
+"""Failure-boundary checks for HF checkpoint dispatch, model loading and launch gates."""
+import copy
+import json
+import os
+from pathlib import Path
+import shutil
+import sys
+import tempfile
+from types import SimpleNamespace
+import unittest
+from unittest.mock import patch
+
+HF = Path(__file__).resolve().parents[1]
+sys.path[:0] = [str(HF), str(HF / "runtime"), str(HF.parent / "train")]
+from checkpoint_store import READY, digest, restore_model
+from common import MODEL, REVISION
+from coordinator import eligible, evaluation_key, submit_once
+from launch_gates import validate_proofs, validate_checkpoint_eval
+
+
+class DispatchTest(unittest.TestCase):
+ def test_ready_steps_and_final(self):
+ self.assertFalse(eligible({"step": 50}, 100, True))
+ self.assertTrue(eligible({"step": 100}, 100, True))
+ self.assertTrue(eligible({"step": 151, "final": True}, 100, True))
+ self.assertFalse(eligible({"step": 0, "final": True}, 100, True))
+
+ def test_manifest_and_protocol_bind_eval_identity(self):
+ key = evaluation_key("train1", "hash1", {"temperature": 0.8})
+ self.assertNotEqual(key, evaluation_key("train1", "hash2", {"temperature": 0.8}))
+ self.assertNotEqual(key, evaluation_key("train1", "hash1", {"temperature": 0.7}))
+
+ def test_ambiguous_submission_is_not_repeated(self):
+ state, snapshots, calls = {}, [], []
+ def persist(): snapshots.append(copy.deepcopy(state))
+ def launch():
+ calls.append(1)
+ raise TimeoutError("server may already have accepted the job")
+ with self.assertRaises(TimeoutError):
+ submit_once(state, "key", [], persist, launch)
+ self.assertEqual(snapshots[0]["key"]["status"], "submitting")
+ with self.assertRaisesRegex(RuntimeError, "Unresolved"):
+ submit_once(state, "key", [], persist, launch)
+ self.assertEqual(len(calls), 1)
+
+ def test_adopt_job_after_ambiguous_response(self):
+ state = {"key": {"status": "submitting"}}
+ job = SimpleNamespace(id="accepted", labels={"evaluation_key": "key"})
+ with patch("builtins.print"):
+ actual = submit_once(state, "key", [job], lambda: None,
+ lambda: self.fail("must adopt, not submit"))
+ self.assertIs(actual, job)
+ self.assertEqual(state["key"]["job_id"], "accepted")
+
+ def test_failed_intent_persistence_never_submits(self):
+ def persist(): raise OSError("remote storage unavailable")
+ with self.assertRaises(OSError):
+ submit_once({}, "key", [], persist, lambda: self.fail("unsafe submission"))
+
+
+class LaunchGateTest(unittest.TestCase):
+ def setUp(self):
+ self.config = json.loads((HF / "configs/deployment.json").read_text())
+ self.baseline = {"arm": "blackbox", "comparison_ready": True, "graded_cells": 1000, "tito_pass": True}
+ self.smoke = {"arm": "blackbox", "passed": True, "bundle_sha256": "new", "remote_restore_verified": True,
+ "tito_pass": True, "weights_updated": True, "native_optimizer_state_verified": True}
+
+ def check(self, config=None, baseline=None, smoke=None):
+ validate_proofs(self.config, {"sha256": "new"}, "blackbox", config or self.config,
+ baseline or self.baseline, smoke or self.smoke)
+
+ def test_matching_evidence(self): self.check()
+
+ def test_native_diagnostic_cannot_be_a_four_harness_curve_baseline(self):
+ from launch_gates import validate_comparison_baseline
+ native = {"comparison_ready": True, "graded_cells": 250, "tito_pass": True,
+ "implementation": "standalone-opencode"}
+ with self.assertRaisesRegex(ValueError, "four-harness"):
+ validate_comparison_baseline(native)
+ matching = {"comparison_ready": True, "graded_cells": 1000, "tito_pass": True,
+ "harnesses": {name: {"graded": 250} for name in self.config["harness_pins"]}}
+ validate_comparison_baseline(matching)
+ matching["harnesses"]["opencode"]["graded"] = 249
+ with self.assertRaises(ValueError):
+ validate_comparison_baseline(matching)
+
+ def test_dataset_change_rejected(self):
+ old = copy.deepcopy(self.config)
+ old["data"]["manifest_sha256"]["test_manifest.json"] = "different"
+ with self.assertRaises(ValueError): self.check(config=old)
+
+ def test_incomplete_baseline_rejected(self):
+ with self.assertRaises(ValueError): self.check(baseline={**self.baseline, "graded_cells": 999})
+
+ def test_old_smoke_bundle_rejected(self):
+ with self.assertRaises(ValueError): self.check(smoke={**self.smoke, "bundle_sha256": "old"})
+
+ def test_checkpoint_gate_binds_complete_scores_to_actual_smoke_weights(self):
+ job = SimpleNamespace(status=SimpleNamespace(stage="COMPLETED"),
+ labels={"role": "eval", "phase": "checkpoint", "arm": "whitebox", "training_job": "smoke"},
+ environment={"BUNDLE_SHA256": "runtime", "CHECKPOINT_MANIFEST_SHA": "manifest",
+ "CHECKPOINT_STEP": "4", "CHECKPOINT_PREFIX": "bucket/smoke/checkpoint-4"})
+ score = {"comparison_ready": True, "tito_pass": True, "graded_cells": 250, "arm": "whitebox"}
+ evidence = {"step": 4, "bundle_sha256": "runtime", "manifest_sha256": "manifest",
+ "source": "bucket/smoke/checkpoint-4"}
+ validate_checkpoint_eval({"sha256": "runtime"}, "whitebox", "smoke", job, score, evidence)
+ for changed in [{**score, "graded_cells": 249}, {**score, "tito_pass": False}]:
+ with self.assertRaises(ValueError):
+ validate_checkpoint_eval({"sha256": "runtime"}, "whitebox", "smoke", job, changed, evidence)
+ with self.assertRaises(ValueError):
+ validate_checkpoint_eval({"sha256": "runtime"}, "whitebox", "other-smoke", job, score, evidence)
+ with self.assertRaises(ValueError):
+ validate_checkpoint_eval({"sha256": "runtime"}, "whitebox", "smoke", job, score,
+ {**evidence, "manifest_sha256": "other-weights"})
+
+ def test_dry_run_validates_without_submitting_any_job(self):
+ from deploy import submit
+ from unittest.mock import Mock
+ with tempfile.TemporaryDirectory() as temporary:
+ out = Path(temporary)
+ (out / "bundle_uploaded.json").write_text(json.dumps({"sha256": "runtime", "repo": "org/repro", "revision": "pin"}))
+ config = copy.deepcopy(self.config)
+ config.pop("training_launch_hold", None)
+ args = SimpleNamespace(role="train", phase="long", arm="whitebox", baseline_job="baseline",
+ smoke_job="smoke", checkpoint_eval_job="eval", flavor="h200x2", training_job=None,
+ external_checkpoint_coordinator=True, dp=1, limit=0, resume_eval_owner=None,
+ timeout="24h", dry_run=True)
+ api = Mock()
+ api.space_info.return_value = SimpleNamespace(host="https://example.hf.space")
+ proof = {"smoke_prefix": "smoke", "baseline_prefix": "base", "space_bundle_sha256": "space",
+ "space_url": "https://example.hf.space"}
+ with patch("launch_gates.verify", return_value=proof) as smoke_gate, \
+ patch("launch_gates.verify_checkpoint_eval", return_value={"passed": True}) as eval_gate, \
+ patch("builtins.print"):
+ preview = submit(api, config, {"HF_TOKEN": "never-record-this"}, out, args)
+ smoke_gate.assert_called_once()
+ eval_gate.assert_called_once()
+ api.run_job.assert_not_called()
+ self.assertFalse(preview["submitted"])
+ self.assertNotIn("never-record-this", (out / "launch-preview.json").read_text())
+
+ def test_native_launch_logs_comparison_baseline_and_retains_diagnostic(self):
+ from deploy import submit
+ from unittest.mock import Mock
+ with tempfile.TemporaryDirectory() as temporary:
+ out = Path(temporary)
+ (out / "bundle_uploaded.json").write_text(json.dumps({"sha256": "runtime", "repo": "org/repro", "revision": "pin"}))
+ args = SimpleNamespace(role="train", phase="long", arm="opencode", baseline_job="native",
+ comparison_baseline_job="four-harness", smoke_job="smoke", flavor="a100x4",
+ training_job=None, external_checkpoint_coordinator=False, dp=1, limit=0,
+ resume_eval_owner=None, timeout="24h", dry_run=True)
+ api = Mock()
+ api.space_info.return_value = SimpleNamespace(host="https://example.hf.space")
+ proof = {"smoke_prefix": "smoke", "baseline_prefix": "native-prefix", "space_bundle_sha256": "space",
+ "space_url": "https://example.hf.space"}
+ response = Mock()
+ response.raise_for_status.return_value.json.return_value = {
+ "arm": "blackbox", "test_tasks": 250, "bundle_sha256": "space"}
+ with patch("launch_gates.verify", return_value=proof), \
+ patch("launch_gates.verify_comparison_baseline", return_value="comparison-prefix") as gate, \
+ patch("runtime.service_contract.check", return_value={"passed": True}), \
+ patch("httpx.get", return_value=response), patch("builtins.print"):
+ result = submit(api, self.config, {"HF_TOKEN": "test-only"}, out, args)
+ gate.assert_called_once_with(api, self.config, "four-harness", out)
+ self.assertEqual(result["environment"]["BASELINE_PREFIX"], "comparison-prefix")
+ self.assertEqual(result["environment"]["BASELINE_JOB"], "four-harness")
+ self.assertEqual(result["environment"]["NATIVE_BASELINE_PREFIX"], "native-prefix")
+ api.run_job.assert_not_called()
+
+
+class ModelRestoreTest(unittest.TestCase):
+ def setUp(self):
+ self.tmp = tempfile.TemporaryDirectory()
+ self.root = Path(self.tmp.name)
+ self.source = self.root / "remote"
+ self.source.mkdir()
+ import torch
+ from safetensors.torch import save_file
+ save_file({"weight": torch.tensor([1.0, 2.0])}, self.source / "model.safetensors")
+ for name in ["config.json", "tokenizer.json", "tokenizer_config.json"]:
+ (self.source / name).write_text("{}")
+ (self.source / "optimizer.pt").write_bytes(b"optimizer must not be fetched for inference")
+ manifest = {"arm": "blackbox", "bundle_sha256": "bundle", "step": 100,
+ "base_model": MODEL, "base_revision": REVISION,
+ "files": {p.name: digest(p) for p in self.source.iterdir()}}
+ (self.source / READY).write_text(json.dumps(manifest))
+ self.sha = digest(self.source / READY)
+ self.downloaded = []
+ case = self
+ class API:
+ def download_bucket_files(self, bucket, *, files, **kwargs):
+ for name, target in files:
+ case.downloaded.append(Path(name).name)
+ shutil.copy2(case.source / Path(name).name, target)
+ self.api = API
+
+ def tearDown(self): self.tmp.cleanup()
+
+ def run_restore(self):
+ with patch("huggingface_hub.HfApi", self.api):
+ return restore_model("hf://buckets/org/bucket/run/checkpoint-100", self.root / "model",
+ arm="blackbox", bundle_sha256="bundle", manifest_sha256=self.sha)
+
+ def test_model_only_restore_has_exact_hashes(self):
+ self.assertEqual(self.run_restore()["step"], 100)
+ self.assertNotIn("optimizer.pt", self.downloaded)
+ self.assertEqual(digest(self.root / "model/model.safetensors"), digest(self.source / "model.safetensors"))
+
+ def test_changed_manifest_is_rejected_before_weights(self):
+ (self.source / READY).write_text((self.source / READY).read_text() + " ")
+ with self.assertRaisesRegex(ValueError, "manifest changed"): self.run_restore()
+ self.assertNotIn("model.safetensors", self.downloaded)
+
+ def test_changed_model_is_rejected(self):
+ (self.source / "model.safetensors").write_bytes(b"tampered")
+ with self.assertRaisesRegex(ValueError, "model hash mismatch"): self.run_restore()
+
+
+class CoordinatorLifecycleTest(unittest.TestCase):
+ def test_saved50_eval100_and_off_interval_final150(self):
+ self.exercise_lifecycle()
+
+ def test_user_stopped_job_evaluates_only_verified_final_checkpoint(self):
+ self.exercise_lifecycle(stage="CANCELED", requested=True, expected=[150])
+
+ def test_normal_cancellation_does_not_launch_an_evaluation(self):
+ self.exercise_lifecycle(stage="CANCELED", expected=[])
+
+ def exercise_lifecycle(self, stage="RUNNING", requested=False, expected=None):
+ from huggingface_hub.errors import EntryNotFoundError
+ import coordinator
+ with tempfile.TemporaryDirectory() as tmp:
+ root = Path(tmp)
+ config = json.loads((HF / "configs/deployment.json").read_text())
+ (root / "hf/configs").mkdir(parents=True)
+ (root / "hf/configs/deployment.json").write_text(json.dumps(config))
+ remote = {}
+ prefix = "run/jobs/train-owner/run"
+ folders = [prefix + f"/checkpoint-{n}" for n in [50, 100, 150]]
+ for n, folder in zip([50, 100, 150], folders):
+ remote[folder + "/" + READY] = json.dumps({"arm": "blackbox", "bundle_sha256": "bundle", "step": n})
+ submitted, snapshots = [], []
+ trainer = SimpleNamespace(labels={"role": "train", "arm": "blackbox"},
+ environment={"RUN_OWNER": "train-owner", "BUNDLE_SHA256": "bundle", "SPACE_URL": "https://environment.invalid",
+ "SPACE_BUNDLE_SHA256": "qualified-environment-bundle"},
+ status=SimpleNamespace(stage=stage))
+ peer = SimpleNamespace(id="coord1", status=SimpleNamespace(stage="RUNNING"), environment={"RUN_OWNER": "coord-owner"})
+ class API:
+ def inspect_job(self, **kwargs):
+ if submitted and stage == "RUNNING": trainer.status.stage = "COMPLETED"
+ return trainer
+ def list_jobs(self, **kwargs):
+ return [peer] if kwargs["labels"]["role"] == "coordinator" else submitted
+ def list_bucket_tree(self, bucket, **kwargs):
+ return [SimpleNamespace(path=f) for f in folders]
+ def download_bucket_files(self, bucket, *, files, **kwargs):
+ for source, dest in files:
+ if source not in remote: raise EntryNotFoundError(source)
+ Path(dest).write_text(remote[source])
+ def sync_bucket(self, source, dest, **kwargs):
+ snapshots.append(json.loads((Path(source) / "state.json").read_text()))
+ def run_job(self, **kwargs):
+ self_case.assertTrue(snapshots[-1][kwargs["labels"]["evaluation_key"]]["status"] == "submitting")
+ self_case.assertEqual(kwargs["flavor"], "a100-large")
+ self_case.assertEqual(kwargs["env"]["SPACE_BUNDLE_SHA256"], "qualified-environment-bundle")
+ job = SimpleNamespace(id=f"eval{len(submitted)}", labels=kwargs["labels"],
+ environment=kwargs["env"], status=SimpleNamespace(stage="COMPLETED"))
+ owner = kwargs["env"]["RUN_OWNER"]
+ remote[f"run/jobs/{owner}/canonical_scores.json"] = json.dumps({"comparison_ready": True})
+ remote[f"run/jobs/{owner}/checkpoint_evaluation.json"] = json.dumps({"manifest_sha256": kwargs["env"]["CHECKPOINT_MANIFEST_SHA"]})
+ submitted.append(job)
+ return job
+ self_case = self
+ clock = [0]
+ def sleep(seconds):
+ clock[0] += seconds
+ if clock[0] > 300: self.fail("coordinator failed to drain")
+ env = {"TRAINING_JOB": "train1", "ARTIFACT_BUCKET": "org/bucket", "RUN_ID": "run",
+ "RUN_OWNER": "coord-owner", "BUNDLE_SHA256": "bundle", "BUNDLE_REPO": "org/bundle",
+ "BUNDLE_REVISION": "revision", "HF_TOKEN": "test-token"}
+ request = None
+ if requested:
+ import hashlib
+ request = {"training_job": "train1", "step": 150,
+ "checkpoint": "hf://buckets/org/bucket/" + folders[-1],
+ "manifest_sha256": hashlib.sha256(remote[folders[-1] + "/" + READY].encode()).hexdigest(),
+ "user_requested_stop": True, "full_checkpoint_verified": True}
+ with patch.dict(os.environ, env), patch("coordinator.ROOT", root), patch("huggingface_hub.HfApi", API), \
+ patch("coordinator.time.sleep", sleep), patch("coordinator.time.monotonic", lambda: clock[0]):
+ coordinator.run(root / "output", "blackbox", final_checkpoint=request)
+ expected = [100, 150] if expected is None else expected
+ self.assertEqual([int(j.environment["CHECKPOINT_STEP"]) for j in submitted], expected)
+ self.assertEqual(len(list((root / "output/decisions/scores").glob("*.json"))), len(expected))
+
+ def test_stop_authorization_rejects_changed_checkpoint_or_job(self):
+ from coordinator import validate_requested_final
+ request = {"training_job": "train1", "step": 150, "checkpoint": "bucket/cp150",
+ "manifest_sha256": "hash", "user_requested_stop": True, "full_checkpoint_verified": True}
+ self.assertTrue(validate_requested_final(request, "train1", "bucket/cp150", {"step": 150}, "hash"))
+ for key, value in [("training_job", "train2"), ("step", 100), ("checkpoint", "bucket/cp100"),
+ ("manifest_sha256", "changed"), ("full_checkpoint_verified", False),
+ ("user_requested_stop", False)]:
+ with self.subTest(key=key), self.assertRaises(ValueError):
+ validate_requested_final({**request, key: value}, "train1", "bucket/cp150", {"step": 150}, "hash")
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/04-data-agent/hf/tests/test_vllm_control_http.py b/04-data-agent/hf/tests/test_vllm_control_http.py
new file mode 100644
index 0000000..a0a96d3
--- /dev/null
+++ b/04-data-agent/hf/tests/test_vllm_control_http.py
@@ -0,0 +1,69 @@
+"""Real HTTP response-contract checks without importing the GPU inference stack."""
+import ast
+from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
+from pathlib import Path
+import threading
+import unittest
+
+import requests
+
+WORKSPACE = Path(__file__).resolve().parents[4]
+
+
+class ControlHTTPTest(unittest.TestCase):
+ @classmethod
+ def setUpClass(cls):
+ import importlib.util
+ source = Path(importlib.util.find_spec("trl").origin).parent / "generation/vllm_client.py"
+ tree = ast.parse(source.read_text())
+ client = next(node for node in tree.body if isinstance(node, ast.ClassDef) and node.name == "VLLMClient")
+ # Execute the production HTTP methods verbatim; constructors require GPUs.
+ client.body = [node for node in client.body if isinstance(node, ast.FunctionDef)
+ and node.name in {"_post", "reset_prefix_cache"}]
+ namespace = {}
+ exec(compile(ast.Module(body=[client], type_ignores=[]), str(source), "exec"), namespace)
+ cls.client_type = namespace["VLLMClient"]
+
+ def setUp(self):
+ self.status, self.body, self.paths = 200, b"", []
+ case = self
+ class Handler(BaseHTTPRequestHandler):
+ def do_POST(self):
+ case.paths.append(self.path)
+ self.send_response(case.status)
+ self.send_header("Content-Length", str(len(case.body)))
+ self.end_headers()
+ self.wfile.write(case.body)
+ def log_message(self, *args):
+ pass
+ self.server = ThreadingHTTPServer(("127.0.0.1", 0), Handler)
+ self.thread = threading.Thread(target=self.server.serve_forever, daemon=True)
+ self.thread.start()
+ self.client = self.client_type()
+ self.client.base_url = f"http://127.0.0.1:{self.server.server_port}"
+ self.client.session = requests.Session()
+
+ def tearDown(self):
+ self.client.session.close()
+ self.server.shutdown()
+ self.server.server_close()
+ self.thread.join()
+
+ def test_empty_reset_success(self):
+ self.assertIsNone(self.client.reset_prefix_cache())
+ self.assertEqual(self.paths, ["/reset_prefix_cache"])
+
+ def test_reset_failure_is_not_suppressed(self):
+ self.status, self.body = 500, b"cache reset failed"
+ with self.assertRaisesRegex(Exception, "500, cache reset failed"):
+ self.client.reset_prefix_cache()
+
+ def test_structured_response_still_requires_json(self):
+ with self.assertRaises(requests.exceptions.JSONDecodeError):
+ self.client._post(self.client.base_url + "/v1/completions")
+ self.body = b'{"choices": []}'
+ self.assertEqual(self.client._post(self.client.base_url + "/v1/completions"), {"choices": []})
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/04-data-agent/hf/tests/test_whitebox_tito.py b/04-data-agent/hf/tests/test_whitebox_tito.py
new file mode 100644
index 0000000..b670dc0
--- /dev/null
+++ b/04-data-agent/hf/tests/test_whitebox_tito.py
@@ -0,0 +1,40 @@
+import sys
+from pathlib import Path
+import unittest
+
+sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'runtime'))
+from whitebox_tito import audit_rows
+
+
+class WhiteboxProvenanceTest(unittest.TestCase):
+ def test_identical_tokens_with_distinct_observed_logprobs(self):
+ # Repeated GRPO samples can emit identical tokens with slightly different
+ # floating-point logprobs. Matching the first token-identical call is wrong.
+ calls = [{'prompt_ids': [1], 'completion_ids': [2], 'logprobs': [p]} for p in [-.1, -.10001]]
+ result = audit_rows([[1], [1]], [[2], [2]], [[1], [1]], [[-.10001], [-.1]], calls)
+ self.assertTrue(all(row['tito_pass'] for row in result))
+
+ def test_one_call_cannot_prove_two_occurrences(self):
+ calls = [{'prompt_ids': [1], 'completion_ids': [2], 'logprobs': [-.1]}]
+ with self.assertRaisesRegex(AssertionError, 'distinct engine call'):
+ audit_rows([[1], [1]], [[2], [2]], [[1], [1]], [[-.1], [-.1]], calls)
+
+ def test_changed_probability_is_rejected(self):
+ calls = [{'prompt_ids': [1], 'completion_ids': [2], 'logprobs': [-.1]}]
+ with self.assertRaisesRegex(AssertionError, 'provenance'):
+ audit_rows([[1]], [[2]], [[1]], [[-.2]], calls)
+
+ def test_tool_context_and_budget_trim_preserve_exact_prefix(self):
+ calls = [{'prompt_ids': [1], 'completion_ids': [2], 'logprobs': [-.1]},
+ {'prompt_ids': [1, 2, 3], 'completion_ids': [4, 5], 'logprobs': [-.2, -.3]}]
+ result = audit_rows([[1]], [[2, 3, 4]], [[1, 0, 1]], [[-.1, 0., -.2]], calls)
+ self.assertEqual(result[0]['supervised'], 2)
+
+ def test_ambiguous_truncation_preserves_occurrences(self):
+ calls = [{'prompt_ids': [1], 'completion_ids': [2, 3], 'logprobs': [-.1, -.2]},
+ {'prompt_ids': [1], 'completion_ids': [2], 'logprobs': [-.1]}]
+ self.assertEqual(len(audit_rows([[1], [1]], [[2], [2, 3]], [[1], [1, 1]],
+ [[-.1], [-.1, -.2]], calls)), 2)
+
+
+if __name__ == '__main__': unittest.main()
diff --git a/04-data-agent/hf/tests/ui_smoke.py b/04-data-agent/hf/tests/ui_smoke.py
new file mode 100644
index 0000000..b8ead2d
--- /dev/null
+++ b/04-data-agent/hf/tests/ui_smoke.py
@@ -0,0 +1,8 @@
+"""CLI wrapper for the packaged deployment smoke."""
+from pathlib import Path
+import sys
+sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "runtime"))
+from ui_smoke import main
+
+if __name__ == "__main__":
+ main()
diff --git a/04-data-agent/hf/tests/ws_idle_probe.py b/04-data-agent/hf/tests/ws_idle_probe.py
new file mode 100644
index 0000000..47bb068
--- /dev/null
+++ b/04-data-agent/hf/tests/ws_idle_probe.py
@@ -0,0 +1,46 @@
+"""Compare idle HF Space WebSockets with/without keepalive; creates no sandboxes."""
+import argparse
+import asyncio
+import json
+from pathlib import Path
+import time
+
+from dotenv import dotenv_values
+from websockets.asyncio.client import connect
+
+
+async def main(args):
+ token = dotenv_values(args.env_file)["HF_API_KEY"]
+ url = args.url.replace("https://", "wss://").rstrip("/") + "/ws"
+ async def one(interval):
+ row = {"ping_interval_s": interval, "idle_seconds": args.seconds, "started_at": time.time()}
+ try:
+ async with connect(url, additional_headers={"Authorization": "Bearer " + token},
+ max_size=104857600, ping_interval=interval, ping_timeout=None,
+ open_timeout=30) as ws:
+ await ws.send('{"type":"state"}')
+ initial = json.loads(await asyncio.wait_for(ws.recv(), 30))
+ assert initial.get("type") != "error", "Native state request rejected"
+ print(json.dumps({"connected": True, "ping_interval_s": interval}), flush=True)
+ await asyncio.sleep(args.seconds)
+ await ws.send('{"type":"state"}')
+ after = json.loads(await asyncio.wait_for(ws.recv(), 30))
+ assert after.get("type") != "error", "State request rejected after idle"
+ row["passed"] = True
+ except Exception as exc:
+ row.update(passed=False, error_type=type(exc).__name__)
+ row["finished_at"] = time.time()
+ print(json.dumps(row), flush=True)
+ return row
+ rows = await asyncio.gather(one(None), one(20))
+ args.out.parent.mkdir(parents=True, exist_ok=True)
+ args.out.write_text(json.dumps({"url": args.url, "results": rows}, indent=2) + "\n")
+
+
+if __name__ == "__main__":
+ p = argparse.ArgumentParser(description=__doc__)
+ p.add_argument("--url", required=True)
+ p.add_argument("--env-file", required=True)
+ p.add_argument("--out", type=Path, required=True)
+ p.add_argument("--seconds", type=int, default=720)
+ asyncio.run(main(p.parse_args()))
diff --git a/04-data-agent/hf/trackio_app.py b/04-data-agent/hf/trackio_app.py
new file mode 100644
index 0000000..1c32055
--- /dev/null
+++ b/04-data-agent/hf/trackio_app.py
@@ -0,0 +1,24 @@
+"""Open the current training project first in the pinned Trackio 0.33.0 UI."""
+import os
+
+import trackio
+from trackio import server
+
+
+DEFAULT_PROJECT = os.environ.get(
+ "TRACKIO_DEFAULT_PROJECT", "multi4-qwen35-2b-prod-20260915"
+)
+_get_all_projects = server.get_all_projects
+
+
+def get_all_projects() -> list[str]:
+ # The browser selects the first project at the bare Space URL. The project
+ # argument to show() only changes its printed/browser-launch URL.
+ # Keep every historical project available in the normal project picker.
+ return sorted(_get_all_projects(), key=lambda name: name != DEFAULT_PROJECT)
+
+
+server.get_all_projects = get_all_projects
+
+if __name__ == "__main__":
+ trackio.show()
diff --git a/04-data-agent/project.yaml b/04-data-agent/project.yaml
new file mode 100644
index 0000000..2b4a6ba
--- /dev/null
+++ b/04-data-agent/project.yaml
@@ -0,0 +1,43 @@
+name: data-agent
+title: Data Agent
+tagline: Three agent-loop implementations with exact-token training, fixed pass@1 evaluations, and local or HF Jobs
+ reproduction.
+order: 4
+status: trained
+hub:
+ org: HuggingEnvs
+ datasets:
+ - HuggingEnvs/data-agent
+ - HuggingEnvs/data-agent-harbor-train
+ - HuggingEnvs/data-agent-harbor-test
+ - HuggingEnvs/data-agent-harbor-eval
+envs:
+- name: blackbox-opencode
+ summary: Native OpenCode owns the sandbox agent loop; capture retains exact engine tokens, log probabilities and
+ loss masks.
+ backend: e2b / hf / daytona
+ tools: agent-owned
+ frameworks:
+ openenv:
+ transport: http-mcp
+ verified: true
+ space: HuggingEnvs/data-agent-blackbox-opencode-env
+- name: blackbox-harbor
+ summary: Harbor serves the fixed tasks through multiple harnesses, with exact capture for asynchronous training.
+ backend: Harbor / Daytona / E2B
+ tools: agent-owned
+ frameworks:
+ openenv:
+ transport: http-mcp
+ verified: true
+ space: HuggingEnvs/data-agent-blackbox-harbor-env
+- name: whitebox-bash
+ summary: TRL owns the synchronous bash/SETA tool loop; the frozen-task adapter preserves the shared grading contract.
+ backend: e2b / daytona
+ tools: bash, read, write, edit, grep, glob, ls, submit_solution
+ trainer: sync
+ frameworks:
+ openenv:
+ transport: http-mcp
+ verified: true
+ space: HuggingEnvs/data-agent-seta-whitebox-env
diff --git a/04-data-agent/reports/async-comparison-20260916/REPORT.md b/04-data-agent/reports/async-comparison-20260916/REPORT.md
new file mode 100644
index 0000000..c21c133
--- /dev/null
+++ b/04-data-agent/reports/async-comparison-20260916/REPORT.md
@@ -0,0 +1,317 @@
+# Harbor and OpenCode — consolidated training and pass@1
+
+Updated: 2026-09-17T10:25:46.725435+00:00
+
+[Live Trackio dashboard](https://huggingface.co/spaces/HuggingEnvs/data-agent-training-comparison-trackio) · [Overview image](comparison.png) · [Accepted checkpoint counts](checkpoint_scores.csv)
+
+Qwen3.5-2B; 1,000 optimizer-step target per run. Recorded training: Harbor multi-harness: 1000 steps; Native OpenCode: 1000 steps; Harbor OpenCode-only: 1000 steps. Every accepted checkpoint has 250 fixed tasks × four harnesses = 1,000 grades. Task difficulty: 33 easy, 118 medium, 99 hard (13.2% / 47.2% / 39.6%). Scores retain first graded attempts; incomplete and failed-audit evaluations are excluded. Missing scores are not estimated.
+
+Baselines are separate measured cohorts: Harbor/E2B 14.6%; Harbor/Daytona 15.9% for the native OpenCode checkpoint evaluator. The standalone native OpenCode 8.4% baseline uses a different harness protocol and is excluded here. Infrastructure and training recipe histories differ; this is an observational comparison, not a controlled causal experiment.
+
+## Overall checkpoint curve
+
+| Checkpoint | Harbor multi-harness | Native OpenCode | Harbor OpenCode-only |
+| --- | ---: | ---: | ---: |
+| 0 (baseline) | 14.6% | 15.9% | 14.6% |
+| 100 | 24.8% | 19.7% | 24.5% |
+| 200 | 26.3% | 22.1% | 27.1% |
+| 300 | 28.6% | 21.6% | 28.5% |
+| 400 | 33.3% | 26.4% | 32.8% |
+| 500 | 37.0% | 23.1% | 33.0% |
+| 600 | 31.8% | 25.1% | 33.0% |
+| 684 (recovery) | 32.1% | Not scheduled | Not scheduled |
+| 700 | 28.8% | 23.2% | 39.5% |
+| 800 | 27.0% | 25.6% | 33.1% |
+| 900 | 22.7% | 25.3% | 29.6% |
+| 1000 | 26.3% | 29.8% | 26.4% |
+
+## Harbor multi-harness
+
+### Overall and difficulty
+
+| Checkpoint | Overall | Easy (132 cells) | Medium (472) | Hard (396) |
+| --- | ---: | ---: | ---: | ---: |
+| 0 | 14.6% | 40.2% | 14.4% | 6.3% |
+| 100 | 24.8% | 53.0% | 30.5% | 8.6% |
+| 200 | 26.3% | 58.3% | 30.1% | 11.1% |
+| 300 | 28.6% | 64.4% | 32.8% | 11.6% |
+| 400 | 33.3% | 73.5% | 39.6% | 12.4% |
+| 500 | 37.0% | 72.7% | 44.3% | 16.4% |
+| 600 | 31.8% | 72.7% | 37.3% | 11.6% |
+| 684 | 32.1% | 75.8% | 35.8% | 13.1% |
+| 700 | 28.8% | 60.6% | 33.9% | 12.1% |
+| 800 | 27.0% | 59.1% | 32.6% | 9.6% |
+| 900 | 22.7% | 46.2% | 26.1% | 10.9% |
+| 1000 | 26.3% | 42.4% | 32.6% | 13.4% |
+
+### Harness × difficulty at every checkpoint
+
+| Checkpoint | Harness | Overall (250) | Easy (33) | Medium (118) | Hard (99) |
+| --- | --- | ---: | ---: | ---: | ---: |
+| 0 | opencode | 10.8% | 33.3% (11/33) | 8.5% (10/118) | 6.1% (6/99) |
+| 0 | claude-code | 16.8% | 42.4% (14/33) | 18.6% (22/118) | 6.1% (6/99) |
+| 0 | codex | 16.4% | 42.4% (14/33) | 16.9% (20/118) | 7.1% (7/99) |
+| 0 | mini-swe-agent | 14.4% | 42.4% (14/33) | 13.6% (16/118) | 6.1% (6/99) |
+| 100 | opencode | 24.4% | 51.5% (17/33) | 31.4% (37/118) | 7.1% (7/99) |
+| 100 | claude-code | 27.6% | 60.6% (20/33) | 32.2% (38/118) | 11.1% (11/99) |
+| 100 | codex | 28.0% | 57.6% (19/33) | 35.6% (42/118) | 9.1% (9/99) |
+| 100 | mini-swe-agent | 19.2% | 42.4% (14/33) | 22.9% (27/118) | 7.1% (7/99) |
+| 200 | opencode | 30.4% | 51.5% (17/33) | 34.7% (41/118) | 18.2% (18/99) |
+| 200 | claude-code | 30.0% | 63.6% (21/33) | 33.1% (39/118) | 15.2% (15/99) |
+| 200 | codex | 26.4% | 63.6% (21/33) | 29.7% (35/118) | 10.1% (10/99) |
+| 200 | mini-swe-agent | 18.4% | 54.5% (18/33) | 22.9% (27/118) | 1.0% (1/99) |
+| 300 | opencode | 29.6% | 60.6% (20/33) | 33.1% (39/118) | 15.2% (15/99) |
+| 300 | claude-code | 33.2% | 66.7% (22/33) | 39.0% (46/118) | 15.2% (15/99) |
+| 300 | codex | 29.6% | 69.7% (23/33) | 33.1% (39/118) | 12.1% (12/99) |
+| 300 | mini-swe-agent | 22.0% | 60.6% (20/33) | 26.3% (31/118) | 4.0% (4/99) |
+| 400 | opencode | 32.8% | 72.7% (24/33) | 38.1% (45/118) | 13.1% (13/99) |
+| 400 | claude-code | 36.8% | 75.8% (25/33) | 44.9% (53/118) | 14.1% (14/99) |
+| 400 | codex | 34.8% | 72.7% (24/33) | 42.4% (50/118) | 13.1% (13/99) |
+| 400 | mini-swe-agent | 28.8% | 72.7% (24/33) | 33.1% (39/118) | 9.1% (9/99) |
+| 500 | opencode | 32.8% | 69.7% (23/33) | 40.7% (48/118) | 11.1% (11/99) |
+| 500 | claude-code | 44.8% | 75.8% (25/33) | 52.5% (62/118) | 25.3% (25/99) |
+| 500 | codex | 39.2% | 75.8% (25/33) | 46.6% (55/118) | 18.2% (18/99) |
+| 500 | mini-swe-agent | 31.2% | 69.7% (23/33) | 37.3% (44/118) | 11.1% (11/99) |
+| 600 | opencode | 34.0% | 66.7% (22/33) | 41.5% (49/118) | 14.1% (14/99) |
+| 600 | claude-code | 30.0% | 72.7% (24/33) | 33.9% (40/118) | 11.1% (11/99) |
+| 600 | codex | 32.4% | 66.7% (22/33) | 39.8% (47/118) | 12.1% (12/99) |
+| 600 | mini-swe-agent | 30.8% | 84.8% (28/33) | 33.9% (40/118) | 9.1% (9/99) |
+| 684 | opencode | 29.2% | 72.7% (24/33) | 30.5% (36/118) | 13.1% (13/99) |
+| 684 | claude-code | 35.6% | 72.7% (24/33) | 41.5% (49/118) | 16.2% (16/99) |
+| 684 | codex | 33.2% | 84.8% (28/33) | 36.4% (43/118) | 12.1% (12/99) |
+| 684 | mini-swe-agent | 30.4% | 72.7% (24/33) | 34.7% (41/118) | 11.1% (11/99) |
+| 700 | opencode | 21.2% | 30.3% (10/33) | 28.0% (33/118) | 10.1% (10/99) |
+| 700 | claude-code | 31.6% | 78.8% (26/33) | 34.7% (41/118) | 12.1% (12/99) |
+| 700 | codex | 32.0% | 72.7% (24/33) | 34.7% (41/118) | 15.2% (15/99) |
+| 700 | mini-swe-agent | 30.4% | 60.6% (20/33) | 38.1% (45/118) | 11.1% (11/99) |
+| 800 | opencode | 23.2% | 36.4% (12/33) | 33.1% (39/118) | 7.1% (7/99) |
+| 800 | claude-code | 32.0% | 66.7% (22/33) | 37.3% (44/118) | 14.1% (14/99) |
+| 800 | codex | 26.8% | 66.7% (22/33) | 31.4% (37/118) | 8.1% (8/99) |
+| 800 | mini-swe-agent | 26.0% | 66.7% (22/33) | 28.8% (34/118) | 9.1% (9/99) |
+| 900 | opencode | 2.4% | 0.0% (0/33) | 3.4% (4/118) | 2.0% (2/99) |
+| 900 | claude-code | 36.4% | 72.7% (24/33) | 43.2% (51/118) | 16.2% (16/99) |
+| 900 | codex | 19.2% | 48.5% (16/33) | 20.3% (24/118) | 8.1% (8/99) |
+| 900 | mini-swe-agent | 32.8% | 63.6% (21/33) | 37.3% (44/118) | 17.2% (17/99) |
+| 1000 | opencode | 5.6% | 6.1% (2/33) | 8.5% (10/118) | 2.0% (2/99) |
+| 1000 | claude-code | 38.8% | 72.7% (24/33) | 44.9% (53/118) | 20.2% (20/99) |
+| 1000 | codex | 24.8% | 45.5% (15/33) | 31.4% (37/118) | 10.1% (10/99) |
+| 1000 | mini-swe-agent | 36.0% | 45.5% (15/33) | 45.8% (54/118) | 21.2% (21/99) |
+
+### Training history
+
+| Allocation | First optimizer step | Last optimizer step |
+| --- | ---: | ---: |
+| 78647 | 1 | 17 |
+| 78681 | 18 | 25 |
+| 78767 | 26 | 30 |
+| 78831 | 31 | 53 |
+| 78956 | 54 | 196 |
+| 79083 | 197 | 684 |
+| 80608 | 685 | 1000 |
+
+### Score provenance
+
+- Step 0: `experiments/async_grpo_harbor_data_agent/logs/multi4-baseline-20260914/job-78215/canonical_results.json`; SHA256 `8c4f5bced4eff04b0c2e5f41806da9ae1b8a4c0fe356ddf926e1f781c6bb9ac6`.
+- Step 100: `experiments/async_grpo_harbor_data_agent/logs/multi4-long-bounded-20260915/checkpoint-evals/step-000100/scores.json`; SHA256 `1e4f42f54526c9b8b71156f5b1f3673529519201401bc88f88ccff805f291181`.
+- Step 200: `experiments/async_grpo_harbor_data_agent/logs/multi4-long-prod-20260915/checkpoint-evals/step-000200/scores.json`; SHA256 `699dce549d1002e879c675555ac06142448b0cbeef534a0816616faf669862af`.
+- Step 300: `experiments/async_grpo_harbor_data_agent/logs/multi4-long-prod-20260915/checkpoint-evals/step-000300/scores.json`; SHA256 `299e5ed3528922d9912a591f3cff0c4d85070fef4de732d37723967934bfd691`.
+- Step 400: `experiments/async_grpo_harbor_data_agent/logs/multi4-long-prod-20260915/checkpoint-evals/step-000400/scores.json`; SHA256 `f8909af81669f4ec092317c5f9889ef8735754c0dcbc826da82a270d2d92dc7a`.
+- Step 500: `experiments/async_grpo_harbor_data_agent/logs/multi4-long-prod-20260915/checkpoint-evals/step-000500/scores.json`; SHA256 `86d56b65edbdc2f5a3f5d54151d0888dae83ca2b81753a7a9dbc2e80ee4f6130`.
+- Step 600: `experiments/async_grpo_harbor_data_agent/logs/multi4-long-prod-20260915/checkpoint-evals/step-000600/scores.json`; SHA256 `02fc5a5e84978c198dc880c143fe223507c30485563b3545cb64b2794e3e60b0`.
+- Step 684: `experiments/async_grpo_harbor_data_agent/logs/multi4-long-prod-20260915/checkpoint-evals/step-000684/scores.json`; SHA256 `fb54bd52f352463e405bee8067ef24b27147bfd02d41429561f440a916f5d776`.
+- Step 700: `experiments/async_grpo_harbor_data_agent/logs/multi4-long-prod-cont-20260915/checkpoint-evals/step-000700/scores.json`; SHA256 `39c2ead8681847c6c398eb6b22ae8919aabaff3ad6b668d81b5961564139f8be`.
+- Step 800: `experiments/async_grpo_harbor_data_agent/logs/multi4-long-prod-cont-20260915/checkpoint-evals/step-000800/scores.json`; SHA256 `65130786707d07f68cfa145fcd5ad7308890cb89a305b65382dfbec36ef7a150`.
+- Step 900: `experiments/async_grpo_harbor_data_agent/logs/multi4-long-prod-cont-20260915/checkpoint-evals/step-000900/scores.json`; SHA256 `2276bc4d8ad85c2913b0bed921013951ec2ca968bfbbece737681ce13df9b250`.
+- Step 1000: `experiments/async_grpo_harbor_data_agent/logs/multi4-long-prod-cont-20260915/checkpoint-evals/step-001000/scores.json`; SHA256 `3172c293f6386c7732267fa46c9718403b3fa3462f88e9299a01139ad410c99b`.
+
+## Native OpenCode
+
+### Overall and difficulty
+
+| Checkpoint | Overall | Easy (132 cells) | Medium (472) | Hard (396) |
+| --- | ---: | ---: | ---: | ---: |
+| 0 | 15.9% | 37.9% | 18.0% | 6.1% |
+| 100 | 19.7% | 44.7% | 22.5% | 8.1% |
+| 200 | 22.1% | 59.8% | 24.6% | 6.6% |
+| 300 | 21.6% | 51.5% | 25.6% | 6.8% |
+| 400 | 26.4% | 59.1% | 29.4% | 11.9% |
+| 500 | 23.1% | 53.0% | 27.3% | 8.1% |
+| 600 | 25.1% | 56.1% | 28.8% | 10.4% |
+| 700 | 23.2% | 49.2% | 28.8% | 7.8% |
+| 800 | 25.6% | 52.3% | 29.7% | 11.9% |
+| 900 | 25.3% | 50.0% | 30.3% | 11.1% |
+| 1000 | 29.8% | 59.8% | 35.8% | 12.6% |
+
+### Harness × difficulty at every checkpoint
+
+| Checkpoint | Harness | Overall (250) | Easy (33) | Medium (118) | Hard (99) |
+| --- | --- | ---: | ---: | ---: | ---: |
+| 0 | opencode | 12.8% | 24.2% (8/33) | 16.9% (20/118) | 4.0% (4/99) |
+| 0 | claude-code | 16.8% | 42.4% (14/33) | 18.6% (22/118) | 6.1% (6/99) |
+| 0 | codex | 15.2% | 33.3% (11/33) | 16.9% (20/118) | 7.1% (7/99) |
+| 0 | mini-swe-agent | 18.8% | 51.5% (17/33) | 19.5% (23/118) | 7.1% (7/99) |
+| 100 | opencode | 19.6% | 42.4% (14/33) | 24.6% (29/118) | 6.1% (6/99) |
+| 100 | claude-code | 20.4% | 42.4% (14/33) | 22.9% (27/118) | 10.1% (10/99) |
+| 100 | codex | 18.0% | 27.3% (9/33) | 21.2% (25/118) | 11.1% (11/99) |
+| 100 | mini-swe-agent | 20.8% | 66.7% (22/33) | 21.2% (25/118) | 5.1% (5/99) |
+| 200 | opencode | 17.2% | 48.5% (16/33) | 21.2% (25/118) | 2.0% (2/99) |
+| 200 | claude-code | 24.0% | 60.6% (20/33) | 26.3% (31/118) | 9.1% (9/99) |
+| 200 | codex | 25.6% | 63.6% (21/33) | 28.8% (34/118) | 9.1% (9/99) |
+| 200 | mini-swe-agent | 21.6% | 66.7% (22/33) | 22.0% (26/118) | 6.1% (6/99) |
+| 300 | opencode | 20.8% | 48.5% (16/33) | 28.0% (33/118) | 3.0% (3/99) |
+| 300 | claude-code | 29.2% | 66.7% (22/33) | 30.5% (36/118) | 15.2% (15/99) |
+| 300 | codex | 16.8% | 39.4% (13/33) | 20.3% (24/118) | 5.1% (5/99) |
+| 300 | mini-swe-agent | 19.6% | 51.5% (17/33) | 23.7% (28/118) | 4.0% (4/99) |
+| 400 | opencode | 20.4% | 48.5% (16/33) | 24.6% (29/118) | 6.1% (6/99) |
+| 400 | claude-code | 32.4% | 60.6% (20/33) | 35.6% (42/118) | 19.2% (19/99) |
+| 400 | codex | 25.6% | 57.6% (19/33) | 28.0% (33/118) | 12.1% (12/99) |
+| 400 | mini-swe-agent | 27.2% | 69.7% (23/33) | 29.7% (35/118) | 10.1% (10/99) |
+| 500 | opencode | 18.0% | 48.5% (16/33) | 19.5% (23/118) | 6.1% (6/99) |
+| 500 | claude-code | 26.4% | 51.5% (17/33) | 33.1% (39/118) | 10.1% (10/99) |
+| 500 | codex | 18.8% | 48.5% (16/33) | 22.0% (26/118) | 5.1% (5/99) |
+| 500 | mini-swe-agent | 29.2% | 63.6% (21/33) | 34.7% (41/118) | 11.1% (11/99) |
+| 600 | opencode | 16.8% | 42.4% (14/33) | 18.6% (22/118) | 6.1% (6/99) |
+| 600 | claude-code | 32.4% | 63.6% (21/33) | 37.3% (44/118) | 16.2% (16/99) |
+| 600 | codex | 18.4% | 45.5% (15/33) | 23.7% (28/118) | 3.0% (3/99) |
+| 600 | mini-swe-agent | 32.8% | 72.7% (24/33) | 35.6% (42/118) | 16.2% (16/99) |
+| 700 | opencode | 18.4% | 42.4% (14/33) | 23.7% (28/118) | 4.0% (4/99) |
+| 700 | claude-code | 29.6% | 63.6% (21/33) | 33.1% (39/118) | 14.1% (14/99) |
+| 700 | codex | 14.4% | 24.2% (8/33) | 22.9% (27/118) | 1.0% (1/99) |
+| 700 | mini-swe-agent | 30.4% | 66.7% (22/33) | 35.6% (42/118) | 12.1% (12/99) |
+| 800 | opencode | 20.0% | 36.4% (12/33) | 26.3% (31/118) | 7.1% (7/99) |
+| 800 | claude-code | 32.4% | 72.7% (24/33) | 35.6% (42/118) | 15.2% (15/99) |
+| 800 | codex | 18.0% | 30.3% (10/33) | 21.2% (25/118) | 10.1% (10/99) |
+| 800 | mini-swe-agent | 32.0% | 69.7% (23/33) | 35.6% (42/118) | 15.2% (15/99) |
+| 900 | opencode | 18.8% | 33.3% (11/33) | 23.7% (28/118) | 8.1% (8/99) |
+| 900 | claude-code | 34.0% | 60.6% (20/33) | 42.4% (50/118) | 15.2% (15/99) |
+| 900 | codex | 14.4% | 27.3% (9/33) | 17.8% (21/118) | 6.1% (6/99) |
+| 900 | mini-swe-agent | 34.0% | 78.8% (26/33) | 37.3% (44/118) | 15.2% (15/99) |
+| 1000 | opencode | 20.4% | 42.4% (14/33) | 25.4% (30/118) | 7.1% (7/99) |
+| 1000 | claude-code | 33.2% | 66.7% (22/33) | 37.3% (44/118) | 17.2% (17/99) |
+| 1000 | codex | 29.6% | 57.6% (19/33) | 35.6% (42/118) | 13.1% (13/99) |
+| 1000 | mini-swe-agent | 36.0% | 72.7% (24/33) | 44.9% (53/118) | 13.1% (13/99) |
+
+### Training history
+
+| Allocation | First optimizer step | Last optimizer step |
+| --- | ---: | ---: |
+| 80626 | 1 | 1000 |
+
+### Score provenance
+
+- Step 0: `experiments/daytona_harness_comparison/logs/20260915/blackbox/canonical_scores.json`; SHA256 `ece2b0e03c7c7e0e54988eaaa473ba6b53bd6028315b1e99ec39df5137c7632e`.
+- Step 100: `experiments/daytona_harness_comparison/logs/hf-20260915/local-opencode-smoke-v4/repro/outputs/local-eval-opencode-80657/canonical_scores.json`; SHA256 `37eab8fe9e97fda23e9803846f965b08c3de2a6e4142363219a4467af41c6f5c`.
+- Step 200: `experiments/daytona_harness_comparison/logs/hf-20260915/local-opencode-smoke-v4/repro/outputs/local-eval-opencode-80675/canonical_scores.json`; SHA256 `fa67547d19c7f4e63166fc7c1518ca15eb5e2c28ec8f1f3739445029006617ad`.
+- Step 300: `experiments/daytona_harness_comparison/logs/hf-20260915/local-opencode-smoke-v4/repro/outputs/local-eval-opencode-80748/canonical_scores.json`; SHA256 `a0e0cc489e3195827d5ac035945277bfd9ca67e985a015148e5b3c94ca938b0f`.
+- Step 400: `experiments/daytona_harness_comparison/logs/hf-20260915/local-opencode-smoke-v4/repro/outputs/local-eval-opencode-80807/canonical_scores.json`; SHA256 `f30bb541e207a2e8b83b2aabd05bf2e3d96eeac40c2fd8226ff1985606397a7b`.
+- Step 500: `experiments/daytona_harness_comparison/logs/hf-20260915/local-opencode-smoke-v4/repro/outputs/local-eval-opencode-80861/canonical_scores.json`; SHA256 `b6df2570695c2a15ba43f185719b647dc22319eb82ca1494d56e705572e3f1a2`.
+- Step 600: `experiments/daytona_harness_comparison/logs/hf-20260915/local-opencode-smoke-v4/repro/outputs/local-eval-opencode-80902/canonical_scores.json`; SHA256 `d86128c2cae4813a7ae8b99f06d1cd8ca109cefade9944c1cdbebf2b1550e1d9`.
+- Step 700: `experiments/daytona_harness_comparison/logs/hf-20260915/local-opencode-smoke-v4/repro/outputs/local-eval-opencode-80956/canonical_scores.json`; SHA256 `7c60cfab333948e63bf44bd01ed4ce3f0a788f76de84c4ceda2177144e9bc9ba`.
+- Step 800: `experiments/daytona_harness_comparison/logs/hf-20260915/local-opencode-smoke-v4/repro/outputs/local-eval-opencode-80993/canonical_scores.json`; SHA256 `b689c9dd76c3e2230bfea49eea393f2d5842fe8f3630f04f59380a131497ae98`.
+- Step 900: `experiments/daytona_harness_comparison/logs/hf-20260915/local-opencode-smoke-v4/repro/outputs/local-eval-opencode-81034/canonical_scores.json`; SHA256 `996f5ea262419b9639fa8f33c1b33fef9b49959c1cbe61e62ba922c0d642985f`.
+- Step 1000: `experiments/daytona_harness_comparison/logs/hf-20260915/local-opencode-smoke-v4/repro/outputs/local-eval-opencode-81098/canonical_scores.json`; SHA256 `1355a9a2ecc1ec165cf413120dacfc672e5d8d59ef2807b28bcf02322dca142b`.
+
+## Harbor OpenCode-only
+
+### Overall and difficulty
+
+| Checkpoint | Overall | Easy (132 cells) | Medium (472) | Hard (396) |
+| --- | ---: | ---: | ---: | ---: |
+| 0 | 14.6% | 40.2% | 14.4% | 6.3% |
+| 100 | 24.5% | 50.0% | 29.2% | 10.4% |
+| 200 | 27.1% | 62.1% | 32.2% | 9.3% |
+| 300 | 28.5% | 64.4% | 32.2% | 12.1% |
+| 400 | 32.8% | 70.5% | 39.0% | 12.9% |
+| 500 | 33.0% | 71.2% | 37.9% | 14.4% |
+| 600 | 33.0% | 66.7% | 39.8% | 13.6% |
+| 700 | 39.5% | 71.2% | 44.5% | 23.0% |
+| 800 | 33.1% | 65.9% | 37.1% | 17.4% |
+| 900 | 29.6% | 63.6% | 32.2% | 15.2% |
+| 1000 | 26.4% | 59.1% | 30.3% | 10.9% |
+
+### Harness × difficulty at every checkpoint
+
+| Checkpoint | Harness | Overall (250) | Easy (33) | Medium (118) | Hard (99) |
+| --- | --- | ---: | ---: | ---: | ---: |
+| 0 | opencode | 10.8% | 33.3% (11/33) | 8.5% (10/118) | 6.1% (6/99) |
+| 0 | claude-code | 16.8% | 42.4% (14/33) | 18.6% (22/118) | 6.1% (6/99) |
+| 0 | codex | 16.4% | 42.4% (14/33) | 16.9% (20/118) | 7.1% (7/99) |
+| 0 | mini-swe-agent | 14.4% | 42.4% (14/33) | 13.6% (16/118) | 6.1% (6/99) |
+| 100 | opencode | 22.0% | 33.3% (11/33) | 28.8% (34/118) | 10.1% (10/99) |
+| 100 | claude-code | 25.2% | 48.5% (16/33) | 29.7% (35/118) | 12.1% (12/99) |
+| 100 | codex | 30.4% | 48.5% (16/33) | 36.4% (43/118) | 17.2% (17/99) |
+| 100 | mini-swe-agent | 20.4% | 69.7% (23/33) | 22.0% (26/118) | 2.0% (2/99) |
+| 200 | opencode | 29.2% | 57.6% (19/33) | 35.6% (42/118) | 12.1% (12/99) |
+| 200 | claude-code | 28.4% | 57.6% (19/33) | 37.3% (44/118) | 8.1% (8/99) |
+| 200 | codex | 30.8% | 66.7% (22/33) | 37.3% (44/118) | 11.1% (11/99) |
+| 200 | mini-swe-agent | 20.0% | 66.7% (22/33) | 18.6% (22/118) | 6.1% (6/99) |
+| 300 | opencode | 28.0% | 69.7% (23/33) | 30.5% (36/118) | 11.1% (11/99) |
+| 300 | claude-code | 30.4% | 66.7% (22/33) | 33.9% (40/118) | 14.1% (14/99) |
+| 300 | codex | 32.0% | 63.6% (21/33) | 36.4% (43/118) | 16.2% (16/99) |
+| 300 | mini-swe-agent | 23.6% | 57.6% (19/33) | 28.0% (33/118) | 7.1% (7/99) |
+| 400 | opencode | 35.6% | 75.8% (25/33) | 43.2% (51/118) | 13.1% (13/99) |
+| 400 | claude-code | 33.6% | 72.7% (24/33) | 42.4% (50/118) | 10.1% (10/99) |
+| 400 | codex | 33.2% | 69.7% (23/33) | 34.7% (41/118) | 19.2% (19/99) |
+| 400 | mini-swe-agent | 28.8% | 63.6% (21/33) | 35.6% (42/118) | 9.1% (9/99) |
+| 500 | opencode | 32.8% | 69.7% (23/33) | 39.8% (47/118) | 12.1% (12/99) |
+| 500 | claude-code | 32.4% | 66.7% (22/33) | 38.1% (45/118) | 14.1% (14/99) |
+| 500 | codex | 37.2% | 75.8% (25/33) | 42.4% (50/118) | 18.2% (18/99) |
+| 500 | mini-swe-agent | 29.6% | 72.7% (24/33) | 31.4% (37/118) | 13.1% (13/99) |
+| 600 | opencode | 35.6% | 78.8% (26/33) | 43.2% (51/118) | 12.1% (12/99) |
+| 600 | claude-code | 31.2% | 63.6% (21/33) | 36.4% (43/118) | 14.1% (14/99) |
+| 600 | codex | 34.4% | 63.6% (21/33) | 42.4% (50/118) | 15.2% (15/99) |
+| 600 | mini-swe-agent | 30.8% | 60.6% (20/33) | 37.3% (44/118) | 13.1% (13/99) |
+| 700 | opencode | 40.0% | 69.7% (23/33) | 44.1% (52/118) | 25.3% (25/99) |
+| 700 | claude-code | 46.4% | 78.8% (26/33) | 53.4% (63/118) | 27.3% (27/99) |
+| 700 | codex | 37.2% | 69.7% (23/33) | 42.4% (50/118) | 20.2% (20/99) |
+| 700 | mini-swe-agent | 34.4% | 66.7% (22/33) | 38.1% (45/118) | 19.2% (19/99) |
+| 800 | opencode | 26.4% | 45.5% (15/33) | 30.5% (36/118) | 15.2% (15/99) |
+| 800 | claude-code | 38.8% | 78.8% (26/33) | 44.1% (52/118) | 19.2% (19/99) |
+| 800 | codex | 35.6% | 63.6% (21/33) | 39.8% (47/118) | 21.2% (21/99) |
+| 800 | mini-swe-agent | 31.6% | 75.8% (25/33) | 33.9% (40/118) | 14.1% (14/99) |
+| 900 | opencode | 30.0% | 66.7% (22/33) | 31.4% (37/118) | 16.2% (16/99) |
+| 900 | claude-code | 35.2% | 72.7% (24/33) | 39.0% (46/118) | 18.2% (18/99) |
+| 900 | codex | 27.2% | 54.5% (18/33) | 29.7% (35/118) | 15.2% (15/99) |
+| 900 | mini-swe-agent | 26.0% | 60.6% (20/33) | 28.8% (34/118) | 11.1% (11/99) |
+| 1000 | opencode | 22.0% | 51.5% (17/33) | 25.4% (30/118) | 8.1% (8/99) |
+| 1000 | claude-code | 30.8% | 57.6% (19/33) | 34.7% (41/118) | 17.2% (17/99) |
+| 1000 | codex | 32.4% | 60.6% (20/33) | 39.0% (46/118) | 15.2% (15/99) |
+| 1000 | mini-swe-agent | 20.4% | 66.7% (22/33) | 22.0% (26/118) | 3.0% (3/99) |
+
+### Training history
+
+| Allocation | First optimizer step | Last optimizer step |
+| --- | ---: | ---: |
+| 81075 | 1 | 1000 |
+
+### Score provenance
+
+- Step 0: `experiments/async_grpo_harbor_data_agent/logs/multi4-baseline-20260914/job-78215/canonical_results.json`; SHA256 `8c4f5bced4eff04b0c2e5f41806da9ae1b8a4c0fe356ddf926e1f781c6bb9ac6`.
+- Step 100: `experiments/async_grpo_harbor_data_agent/logs/harbor-opencode-only-20260916/checkpoint-evals/step-000100/scores.json`; SHA256 `08278ea43f152961c9fd3b0b8cc598a0cf5d891b658ef7f5314f95c62ade44d8`.
+- Step 200: `experiments/async_grpo_harbor_data_agent/logs/harbor-opencode-only-20260916/checkpoint-evals/step-000200/scores.json`; SHA256 `718d706a3b61a8a171e567e99084c30f0b4e532d55d9cbe1f38b0a0f88e9741b`.
+- Step 300: `experiments/async_grpo_harbor_data_agent/logs/harbor-opencode-only-20260916/checkpoint-evals/step-000300/scores.json`; SHA256 `58ccb317e9c50e09afe5211744049ccac6cf8aeb690e9021307591798733118c`.
+- Step 400: `experiments/async_grpo_harbor_data_agent/logs/harbor-opencode-only-20260916/checkpoint-evals/step-000400/scores.json`; SHA256 `f473d7a6595f5b461da11eeab86b0265fc0d5a903dd50a7f07c13fe591841071`.
+- Step 500: `experiments/async_grpo_harbor_data_agent/logs/harbor-opencode-only-20260916/checkpoint-evals/step-000500/scores.json`; SHA256 `3ea7d52077764f3e662f95eb7d2606942f5a8a5745f9527de1b08409ecf2849d`.
+- Step 600: `experiments/async_grpo_harbor_data_agent/logs/harbor-opencode-only-20260916/checkpoint-evals/step-000600/scores.json`; SHA256 `304f219743ea44713603e6e36f6b2ace485ea0e8caf504d5fee6509ca8081c61`.
+- Step 700: `experiments/async_grpo_harbor_data_agent/logs/harbor-opencode-only-20260916/checkpoint-evals/step-000700/scores.json`; SHA256 `eff27df90ecb2960693a86592246ba27ed9065b0f391c8dc339a2e14d85d6b1f`.
+- Step 800: `experiments/async_grpo_harbor_data_agent/logs/harbor-opencode-only-20260916/checkpoint-evals/step-000800/scores.json`; SHA256 `64bd9dd7395cf0105535abddf60f44b79af65ff60815ad6eb755cba461636845`.
+- Step 900: `experiments/async_grpo_harbor_data_agent/logs/harbor-opencode-only-20260916/checkpoint-evals/step-000900/scores.json`; SHA256 `d71b7033e8317d204be052663d2b121c2d32b24714bdbd796a941f4f3fec20d5`.
+- Step 1000: `experiments/async_grpo_harbor_data_agent/logs/harbor-opencode-only-20260916/checkpoint-evals/step-001000/scores.json`; SHA256 `c4af433eebf56b98842150a88c467711b62475177a7626f541565392dfd74ea8`.
+
+## Dashboard metric guide
+
+Both runs use identical metric names and optimizer-step axes. `eval/pass_at_1` is the overall score; `eval/difficulty/*` aggregates each difficulty; `eval/harness/*` compares each harness; `eval/harness_difficulty/*` contains all twelve intersections. `train/*` preserves recorded loss, reward, learning rate, gradient norm, entropy, KL, staleness, throughput, token, batching and rollout metrics where observed. Missing metrics are not filled with zeros. `train/reward_rolling20` and `train/nonzero_gradient_rolling20` are explicitly derived trailing windows. Raw metrics remain available. Use zero dashboard smoothing for exact checkpoint values.
+
+The independent CPU publisher refreshes every 60 seconds and admits new evaluations only after their full comparison gates pass. It never changes trainer state. Local SQLite backup, event ledger and remote exact-content verification receipts are kept alongside this report.
+
+Storage and deployment follow the [Trackio guide](https://huggingface.co/docs/trackio/quickstart) and [environment configuration](https://huggingface.co/docs/trackio/environment_variables).
+
+- [Overview](https://huggingenvs-data-agent-training-comparison-trackio.hf.space/?project=qwen35-2b-harbor-vs-opencode-20260916&run_ids=3ae29a23763093285702b71a1f76805a%2Cbeb2604c8f737da4262b1ab19b8b0cbd%2Cc5b445fa337b56b139c1e6b34ae35409&smoothing=0&metric_filter=%5E%28eval%2Fpass_at_1%7Ctrain%2Freward_rolling20%29%24)
+- [Difficulty](https://huggingenvs-data-agent-training-comparison-trackio.hf.space/?project=qwen35-2b-harbor-vs-opencode-20260916&run_ids=3ae29a23763093285702b71a1f76805a%2Cbeb2604c8f737da4262b1ab19b8b0cbd%2Cc5b445fa337b56b139c1e6b34ae35409&smoothing=0&metric_filter=%5Eeval%2Fdifficulty%2F)
+- [Harness](https://huggingenvs-data-agent-training-comparison-trackio.hf.space/?project=qwen35-2b-harbor-vs-opencode-20260916&run_ids=3ae29a23763093285702b71a1f76805a%2Cbeb2604c8f737da4262b1ab19b8b0cbd%2Cc5b445fa337b56b139c1e6b34ae35409&smoothing=0&metric_filter=%5Eeval%2Fharness%2F)
+- [Harness × difficulty](https://huggingenvs-data-agent-training-comparison-trackio.hf.space/?project=qwen35-2b-harbor-vs-opencode-20260916&run_ids=3ae29a23763093285702b71a1f76805a%2Cbeb2604c8f737da4262b1ab19b8b0cbd%2Cc5b445fa337b56b139c1e6b34ae35409&smoothing=0&metric_filter=%5Eeval%2Fharness_difficulty%2F)
+- [Optimizer diagnostics](https://huggingenvs-data-agent-training-comparison-trackio.hf.space/?project=qwen35-2b-harbor-vs-opencode-20260916&run_ids=3ae29a23763093285702b71a1f76805a%2Cbeb2604c8f737da4262b1ab19b8b0cbd%2Cc5b445fa337b56b139c1e6b34ae35409&smoothing=0&metric_filter=%5Etrain%2F%28loss%7Cgrad_norm%7Centropy%7Ckl%7Clearning_rate%7Cnonzero_gradient_rolling20%29%24)
+- [Throughput and rollout diagnostics](https://huggingenvs-data-agent-training-comparison-trackio.hf.space/?project=qwen35-2b-harbor-vs-opencode-20260916&run_ids=3ae29a23763093285702b71a1f76805a%2Cbeb2604c8f737da4262b1ab19b8b0cbd%2Cc5b445fa337b56b139c1e6b34ae35409&smoothing=0&metric_filter=%5Etrain%2F%28perf%7Crollout%7Csample%7Cbatch%29%2F)
+- [All metrics](https://huggingenvs-data-agent-training-comparison-trackio.hf.space/?project=qwen35-2b-harbor-vs-opencode-20260916&run_ids=3ae29a23763093285702b71a1f76805a%2Cbeb2604c8f737da4262b1ab19b8b0cbd%2Cc5b445fa337b56b139c1e6b34ae35409&smoothing=0&metric_filter=)
+
+[Download checkpoint scores as CSV](checkpoint_scores.csv)
diff --git a/04-data-agent/reports/async-comparison-20260916/checkpoint_scores.csv b/04-data-agent/reports/async-comparison-20260916/checkpoint_scores.csv
new file mode 100644
index 0000000..9daf258
--- /dev/null
+++ b/04-data-agent/reports/async-comparison-20260916/checkpoint_scores.csv
@@ -0,0 +1,409 @@
+run,checkpoint,harness,difficulty,correct,graded,pass_at_1
+Harbor multi-harness,0,opencode,easy,11,33,0.3333333333333333
+Harbor multi-harness,0,opencode,medium,10,118,0.0847457627118644
+Harbor multi-harness,0,opencode,hard,6,99,0.06060606060606061
+Harbor multi-harness,0,claude-code,easy,14,33,0.42424242424242425
+Harbor multi-harness,0,claude-code,medium,22,118,0.1864406779661017
+Harbor multi-harness,0,claude-code,hard,6,99,0.06060606060606061
+Harbor multi-harness,0,codex,easy,14,33,0.42424242424242425
+Harbor multi-harness,0,codex,medium,20,118,0.1694915254237288
+Harbor multi-harness,0,codex,hard,7,99,0.0707070707070707
+Harbor multi-harness,0,mini-swe-agent,easy,14,33,0.42424242424242425
+Harbor multi-harness,0,mini-swe-agent,medium,16,118,0.13559322033898305
+Harbor multi-harness,0,mini-swe-agent,hard,6,99,0.06060606060606061
+Harbor multi-harness,100,opencode,easy,17,33,0.5151515151515151
+Harbor multi-harness,100,opencode,medium,37,118,0.3135593220338983
+Harbor multi-harness,100,opencode,hard,7,99,0.0707070707070707
+Harbor multi-harness,100,claude-code,easy,20,33,0.6060606060606061
+Harbor multi-harness,100,claude-code,medium,38,118,0.3220338983050847
+Harbor multi-harness,100,claude-code,hard,11,99,0.1111111111111111
+Harbor multi-harness,100,codex,easy,19,33,0.5757575757575758
+Harbor multi-harness,100,codex,medium,42,118,0.3559322033898305
+Harbor multi-harness,100,codex,hard,9,99,0.09090909090909091
+Harbor multi-harness,100,mini-swe-agent,easy,14,33,0.42424242424242425
+Harbor multi-harness,100,mini-swe-agent,medium,27,118,0.2288135593220339
+Harbor multi-harness,100,mini-swe-agent,hard,7,99,0.0707070707070707
+Harbor multi-harness,200,opencode,easy,17,33,0.5151515151515151
+Harbor multi-harness,200,opencode,medium,41,118,0.3474576271186441
+Harbor multi-harness,200,opencode,hard,18,99,0.18181818181818182
+Harbor multi-harness,200,claude-code,easy,21,33,0.6363636363636364
+Harbor multi-harness,200,claude-code,medium,39,118,0.3305084745762712
+Harbor multi-harness,200,claude-code,hard,15,99,0.15151515151515152
+Harbor multi-harness,200,codex,easy,21,33,0.6363636363636364
+Harbor multi-harness,200,codex,medium,35,118,0.2966101694915254
+Harbor multi-harness,200,codex,hard,10,99,0.10101010101010101
+Harbor multi-harness,200,mini-swe-agent,easy,18,33,0.5454545454545454
+Harbor multi-harness,200,mini-swe-agent,medium,27,118,0.2288135593220339
+Harbor multi-harness,200,mini-swe-agent,hard,1,99,0.010101010101010102
+Harbor multi-harness,300,opencode,easy,20,33,0.6060606060606061
+Harbor multi-harness,300,opencode,medium,39,118,0.3305084745762712
+Harbor multi-harness,300,opencode,hard,15,99,0.15151515151515152
+Harbor multi-harness,300,claude-code,easy,22,33,0.6666666666666666
+Harbor multi-harness,300,claude-code,medium,46,118,0.3898305084745763
+Harbor multi-harness,300,claude-code,hard,15,99,0.15151515151515152
+Harbor multi-harness,300,codex,easy,23,33,0.696969696969697
+Harbor multi-harness,300,codex,medium,39,118,0.3305084745762712
+Harbor multi-harness,300,codex,hard,12,99,0.12121212121212122
+Harbor multi-harness,300,mini-swe-agent,easy,20,33,0.6060606060606061
+Harbor multi-harness,300,mini-swe-agent,medium,31,118,0.2627118644067797
+Harbor multi-harness,300,mini-swe-agent,hard,4,99,0.04040404040404041
+Harbor multi-harness,400,opencode,easy,24,33,0.7272727272727273
+Harbor multi-harness,400,opencode,medium,45,118,0.3813559322033898
+Harbor multi-harness,400,opencode,hard,13,99,0.13131313131313133
+Harbor multi-harness,400,claude-code,easy,25,33,0.7575757575757576
+Harbor multi-harness,400,claude-code,medium,53,118,0.4491525423728814
+Harbor multi-harness,400,claude-code,hard,14,99,0.1414141414141414
+Harbor multi-harness,400,codex,easy,24,33,0.7272727272727273
+Harbor multi-harness,400,codex,medium,50,118,0.423728813559322
+Harbor multi-harness,400,codex,hard,13,99,0.13131313131313133
+Harbor multi-harness,400,mini-swe-agent,easy,24,33,0.7272727272727273
+Harbor multi-harness,400,mini-swe-agent,medium,39,118,0.3305084745762712
+Harbor multi-harness,400,mini-swe-agent,hard,9,99,0.09090909090909091
+Harbor multi-harness,500,opencode,easy,23,33,0.696969696969697
+Harbor multi-harness,500,opencode,medium,48,118,0.4067796610169492
+Harbor multi-harness,500,opencode,hard,11,99,0.1111111111111111
+Harbor multi-harness,500,claude-code,easy,25,33,0.7575757575757576
+Harbor multi-harness,500,claude-code,medium,62,118,0.5254237288135594
+Harbor multi-harness,500,claude-code,hard,25,99,0.25252525252525254
+Harbor multi-harness,500,codex,easy,25,33,0.7575757575757576
+Harbor multi-harness,500,codex,medium,55,118,0.4661016949152542
+Harbor multi-harness,500,codex,hard,18,99,0.18181818181818182
+Harbor multi-harness,500,mini-swe-agent,easy,23,33,0.696969696969697
+Harbor multi-harness,500,mini-swe-agent,medium,44,118,0.3728813559322034
+Harbor multi-harness,500,mini-swe-agent,hard,11,99,0.1111111111111111
+Harbor multi-harness,600,opencode,easy,22,33,0.6666666666666666
+Harbor multi-harness,600,opencode,medium,49,118,0.4152542372881356
+Harbor multi-harness,600,opencode,hard,14,99,0.1414141414141414
+Harbor multi-harness,600,claude-code,easy,24,33,0.7272727272727273
+Harbor multi-harness,600,claude-code,medium,40,118,0.3389830508474576
+Harbor multi-harness,600,claude-code,hard,11,99,0.1111111111111111
+Harbor multi-harness,600,codex,easy,22,33,0.6666666666666666
+Harbor multi-harness,600,codex,medium,47,118,0.3983050847457627
+Harbor multi-harness,600,codex,hard,12,99,0.12121212121212122
+Harbor multi-harness,600,mini-swe-agent,easy,28,33,0.8484848484848485
+Harbor multi-harness,600,mini-swe-agent,medium,40,118,0.3389830508474576
+Harbor multi-harness,600,mini-swe-agent,hard,9,99,0.09090909090909091
+Harbor multi-harness,684,opencode,easy,24,33,0.7272727272727273
+Harbor multi-harness,684,opencode,medium,36,118,0.3050847457627119
+Harbor multi-harness,684,opencode,hard,13,99,0.13131313131313133
+Harbor multi-harness,684,claude-code,easy,24,33,0.7272727272727273
+Harbor multi-harness,684,claude-code,medium,49,118,0.4152542372881356
+Harbor multi-harness,684,claude-code,hard,16,99,0.16161616161616163
+Harbor multi-harness,684,codex,easy,28,33,0.8484848484848485
+Harbor multi-harness,684,codex,medium,43,118,0.3644067796610169
+Harbor multi-harness,684,codex,hard,12,99,0.12121212121212122
+Harbor multi-harness,684,mini-swe-agent,easy,24,33,0.7272727272727273
+Harbor multi-harness,684,mini-swe-agent,medium,41,118,0.3474576271186441
+Harbor multi-harness,684,mini-swe-agent,hard,11,99,0.1111111111111111
+Harbor multi-harness,700,opencode,easy,10,33,0.30303030303030304
+Harbor multi-harness,700,opencode,medium,33,118,0.2796610169491525
+Harbor multi-harness,700,opencode,hard,10,99,0.10101010101010101
+Harbor multi-harness,700,claude-code,easy,26,33,0.7878787878787878
+Harbor multi-harness,700,claude-code,medium,41,118,0.3474576271186441
+Harbor multi-harness,700,claude-code,hard,12,99,0.12121212121212122
+Harbor multi-harness,700,codex,easy,24,33,0.7272727272727273
+Harbor multi-harness,700,codex,medium,41,118,0.3474576271186441
+Harbor multi-harness,700,codex,hard,15,99,0.15151515151515152
+Harbor multi-harness,700,mini-swe-agent,easy,20,33,0.6060606060606061
+Harbor multi-harness,700,mini-swe-agent,medium,45,118,0.3813559322033898
+Harbor multi-harness,700,mini-swe-agent,hard,11,99,0.1111111111111111
+Harbor multi-harness,800,opencode,easy,12,33,0.36363636363636365
+Harbor multi-harness,800,opencode,medium,39,118,0.3305084745762712
+Harbor multi-harness,800,opencode,hard,7,99,0.0707070707070707
+Harbor multi-harness,800,claude-code,easy,22,33,0.6666666666666666
+Harbor multi-harness,800,claude-code,medium,44,118,0.3728813559322034
+Harbor multi-harness,800,claude-code,hard,14,99,0.1414141414141414
+Harbor multi-harness,800,codex,easy,22,33,0.6666666666666666
+Harbor multi-harness,800,codex,medium,37,118,0.3135593220338983
+Harbor multi-harness,800,codex,hard,8,99,0.08080808080808081
+Harbor multi-harness,800,mini-swe-agent,easy,22,33,0.6666666666666666
+Harbor multi-harness,800,mini-swe-agent,medium,34,118,0.288135593220339
+Harbor multi-harness,800,mini-swe-agent,hard,9,99,0.09090909090909091
+Harbor multi-harness,900,opencode,easy,0,33,0.0
+Harbor multi-harness,900,opencode,medium,4,118,0.03389830508474576
+Harbor multi-harness,900,opencode,hard,2,99,0.020202020202020204
+Harbor multi-harness,900,claude-code,easy,24,33,0.7272727272727273
+Harbor multi-harness,900,claude-code,medium,51,118,0.4322033898305085
+Harbor multi-harness,900,claude-code,hard,16,99,0.16161616161616163
+Harbor multi-harness,900,codex,easy,16,33,0.48484848484848486
+Harbor multi-harness,900,codex,medium,24,118,0.2033898305084746
+Harbor multi-harness,900,codex,hard,8,99,0.08080808080808081
+Harbor multi-harness,900,mini-swe-agent,easy,21,33,0.6363636363636364
+Harbor multi-harness,900,mini-swe-agent,medium,44,118,0.3728813559322034
+Harbor multi-harness,900,mini-swe-agent,hard,17,99,0.1717171717171717
+Harbor multi-harness,1000,opencode,easy,2,33,0.06060606060606061
+Harbor multi-harness,1000,opencode,medium,10,118,0.0847457627118644
+Harbor multi-harness,1000,opencode,hard,2,99,0.020202020202020204
+Harbor multi-harness,1000,claude-code,easy,24,33,0.7272727272727273
+Harbor multi-harness,1000,claude-code,medium,53,118,0.4491525423728814
+Harbor multi-harness,1000,claude-code,hard,20,99,0.20202020202020202
+Harbor multi-harness,1000,codex,easy,15,33,0.45454545454545453
+Harbor multi-harness,1000,codex,medium,37,118,0.3135593220338983
+Harbor multi-harness,1000,codex,hard,10,99,0.10101010101010101
+Harbor multi-harness,1000,mini-swe-agent,easy,15,33,0.45454545454545453
+Harbor multi-harness,1000,mini-swe-agent,medium,54,118,0.4576271186440678
+Harbor multi-harness,1000,mini-swe-agent,hard,21,99,0.21212121212121213
+Native OpenCode,0,opencode,easy,8,33,0.24242424242424243
+Native OpenCode,0,opencode,medium,20,118,0.1694915254237288
+Native OpenCode,0,opencode,hard,4,99,0.04040404040404041
+Native OpenCode,0,claude-code,easy,14,33,0.42424242424242425
+Native OpenCode,0,claude-code,medium,22,118,0.1864406779661017
+Native OpenCode,0,claude-code,hard,6,99,0.06060606060606061
+Native OpenCode,0,codex,easy,11,33,0.3333333333333333
+Native OpenCode,0,codex,medium,20,118,0.1694915254237288
+Native OpenCode,0,codex,hard,7,99,0.0707070707070707
+Native OpenCode,0,mini-swe-agent,easy,17,33,0.5151515151515151
+Native OpenCode,0,mini-swe-agent,medium,23,118,0.19491525423728814
+Native OpenCode,0,mini-swe-agent,hard,7,99,0.0707070707070707
+Native OpenCode,100,opencode,easy,14,33,0.42424242424242425
+Native OpenCode,100,opencode,medium,29,118,0.2457627118644068
+Native OpenCode,100,opencode,hard,6,99,0.06060606060606061
+Native OpenCode,100,claude-code,easy,14,33,0.42424242424242425
+Native OpenCode,100,claude-code,medium,27,118,0.2288135593220339
+Native OpenCode,100,claude-code,hard,10,99,0.10101010101010101
+Native OpenCode,100,codex,easy,9,33,0.2727272727272727
+Native OpenCode,100,codex,medium,25,118,0.211864406779661
+Native OpenCode,100,codex,hard,11,99,0.1111111111111111
+Native OpenCode,100,mini-swe-agent,easy,22,33,0.6666666666666666
+Native OpenCode,100,mini-swe-agent,medium,25,118,0.211864406779661
+Native OpenCode,100,mini-swe-agent,hard,5,99,0.050505050505050504
+Native OpenCode,200,opencode,easy,16,33,0.48484848484848486
+Native OpenCode,200,opencode,medium,25,118,0.211864406779661
+Native OpenCode,200,opencode,hard,2,99,0.020202020202020204
+Native OpenCode,200,claude-code,easy,20,33,0.6060606060606061
+Native OpenCode,200,claude-code,medium,31,118,0.2627118644067797
+Native OpenCode,200,claude-code,hard,9,99,0.09090909090909091
+Native OpenCode,200,codex,easy,21,33,0.6363636363636364
+Native OpenCode,200,codex,medium,34,118,0.288135593220339
+Native OpenCode,200,codex,hard,9,99,0.09090909090909091
+Native OpenCode,200,mini-swe-agent,easy,22,33,0.6666666666666666
+Native OpenCode,200,mini-swe-agent,medium,26,118,0.22033898305084745
+Native OpenCode,200,mini-swe-agent,hard,6,99,0.06060606060606061
+Native OpenCode,300,opencode,easy,16,33,0.48484848484848486
+Native OpenCode,300,opencode,medium,33,118,0.2796610169491525
+Native OpenCode,300,opencode,hard,3,99,0.030303030303030304
+Native OpenCode,300,claude-code,easy,22,33,0.6666666666666666
+Native OpenCode,300,claude-code,medium,36,118,0.3050847457627119
+Native OpenCode,300,claude-code,hard,15,99,0.15151515151515152
+Native OpenCode,300,codex,easy,13,33,0.3939393939393939
+Native OpenCode,300,codex,medium,24,118,0.2033898305084746
+Native OpenCode,300,codex,hard,5,99,0.050505050505050504
+Native OpenCode,300,mini-swe-agent,easy,17,33,0.5151515151515151
+Native OpenCode,300,mini-swe-agent,medium,28,118,0.23728813559322035
+Native OpenCode,300,mini-swe-agent,hard,4,99,0.04040404040404041
+Native OpenCode,400,opencode,easy,16,33,0.48484848484848486
+Native OpenCode,400,opencode,medium,29,118,0.2457627118644068
+Native OpenCode,400,opencode,hard,6,99,0.06060606060606061
+Native OpenCode,400,claude-code,easy,20,33,0.6060606060606061
+Native OpenCode,400,claude-code,medium,42,118,0.3559322033898305
+Native OpenCode,400,claude-code,hard,19,99,0.1919191919191919
+Native OpenCode,400,codex,easy,19,33,0.5757575757575758
+Native OpenCode,400,codex,medium,33,118,0.2796610169491525
+Native OpenCode,400,codex,hard,12,99,0.12121212121212122
+Native OpenCode,400,mini-swe-agent,easy,23,33,0.696969696969697
+Native OpenCode,400,mini-swe-agent,medium,35,118,0.2966101694915254
+Native OpenCode,400,mini-swe-agent,hard,10,99,0.10101010101010101
+Native OpenCode,500,opencode,easy,16,33,0.48484848484848486
+Native OpenCode,500,opencode,medium,23,118,0.19491525423728814
+Native OpenCode,500,opencode,hard,6,99,0.06060606060606061
+Native OpenCode,500,claude-code,easy,17,33,0.5151515151515151
+Native OpenCode,500,claude-code,medium,39,118,0.3305084745762712
+Native OpenCode,500,claude-code,hard,10,99,0.10101010101010101
+Native OpenCode,500,codex,easy,16,33,0.48484848484848486
+Native OpenCode,500,codex,medium,26,118,0.22033898305084745
+Native OpenCode,500,codex,hard,5,99,0.050505050505050504
+Native OpenCode,500,mini-swe-agent,easy,21,33,0.6363636363636364
+Native OpenCode,500,mini-swe-agent,medium,41,118,0.3474576271186441
+Native OpenCode,500,mini-swe-agent,hard,11,99,0.1111111111111111
+Native OpenCode,600,opencode,easy,14,33,0.42424242424242425
+Native OpenCode,600,opencode,medium,22,118,0.1864406779661017
+Native OpenCode,600,opencode,hard,6,99,0.06060606060606061
+Native OpenCode,600,claude-code,easy,21,33,0.6363636363636364
+Native OpenCode,600,claude-code,medium,44,118,0.3728813559322034
+Native OpenCode,600,claude-code,hard,16,99,0.16161616161616163
+Native OpenCode,600,codex,easy,15,33,0.45454545454545453
+Native OpenCode,600,codex,medium,28,118,0.23728813559322035
+Native OpenCode,600,codex,hard,3,99,0.030303030303030304
+Native OpenCode,600,mini-swe-agent,easy,24,33,0.7272727272727273
+Native OpenCode,600,mini-swe-agent,medium,42,118,0.3559322033898305
+Native OpenCode,600,mini-swe-agent,hard,16,99,0.16161616161616163
+Native OpenCode,700,opencode,easy,14,33,0.42424242424242425
+Native OpenCode,700,opencode,medium,28,118,0.23728813559322035
+Native OpenCode,700,opencode,hard,4,99,0.04040404040404041
+Native OpenCode,700,claude-code,easy,21,33,0.6363636363636364
+Native OpenCode,700,claude-code,medium,39,118,0.3305084745762712
+Native OpenCode,700,claude-code,hard,14,99,0.1414141414141414
+Native OpenCode,700,codex,easy,8,33,0.24242424242424243
+Native OpenCode,700,codex,medium,27,118,0.2288135593220339
+Native OpenCode,700,codex,hard,1,99,0.010101010101010102
+Native OpenCode,700,mini-swe-agent,easy,22,33,0.6666666666666666
+Native OpenCode,700,mini-swe-agent,medium,42,118,0.3559322033898305
+Native OpenCode,700,mini-swe-agent,hard,12,99,0.12121212121212122
+Native OpenCode,800,opencode,easy,12,33,0.36363636363636365
+Native OpenCode,800,opencode,medium,31,118,0.2627118644067797
+Native OpenCode,800,opencode,hard,7,99,0.0707070707070707
+Native OpenCode,800,claude-code,easy,24,33,0.7272727272727273
+Native OpenCode,800,claude-code,medium,42,118,0.3559322033898305
+Native OpenCode,800,claude-code,hard,15,99,0.15151515151515152
+Native OpenCode,800,codex,easy,10,33,0.30303030303030304
+Native OpenCode,800,codex,medium,25,118,0.211864406779661
+Native OpenCode,800,codex,hard,10,99,0.10101010101010101
+Native OpenCode,800,mini-swe-agent,easy,23,33,0.696969696969697
+Native OpenCode,800,mini-swe-agent,medium,42,118,0.3559322033898305
+Native OpenCode,800,mini-swe-agent,hard,15,99,0.15151515151515152
+Native OpenCode,900,opencode,easy,11,33,0.3333333333333333
+Native OpenCode,900,opencode,medium,28,118,0.23728813559322035
+Native OpenCode,900,opencode,hard,8,99,0.08080808080808081
+Native OpenCode,900,claude-code,easy,20,33,0.6060606060606061
+Native OpenCode,900,claude-code,medium,50,118,0.423728813559322
+Native OpenCode,900,claude-code,hard,15,99,0.15151515151515152
+Native OpenCode,900,codex,easy,9,33,0.2727272727272727
+Native OpenCode,900,codex,medium,21,118,0.17796610169491525
+Native OpenCode,900,codex,hard,6,99,0.06060606060606061
+Native OpenCode,900,mini-swe-agent,easy,26,33,0.7878787878787878
+Native OpenCode,900,mini-swe-agent,medium,44,118,0.3728813559322034
+Native OpenCode,900,mini-swe-agent,hard,15,99,0.15151515151515152
+Native OpenCode,1000,opencode,easy,14,33,0.42424242424242425
+Native OpenCode,1000,opencode,medium,30,118,0.2542372881355932
+Native OpenCode,1000,opencode,hard,7,99,0.0707070707070707
+Native OpenCode,1000,claude-code,easy,22,33,0.6666666666666666
+Native OpenCode,1000,claude-code,medium,44,118,0.3728813559322034
+Native OpenCode,1000,claude-code,hard,17,99,0.1717171717171717
+Native OpenCode,1000,codex,easy,19,33,0.5757575757575758
+Native OpenCode,1000,codex,medium,42,118,0.3559322033898305
+Native OpenCode,1000,codex,hard,13,99,0.13131313131313133
+Native OpenCode,1000,mini-swe-agent,easy,24,33,0.7272727272727273
+Native OpenCode,1000,mini-swe-agent,medium,53,118,0.4491525423728814
+Native OpenCode,1000,mini-swe-agent,hard,13,99,0.13131313131313133
+Harbor OpenCode-only,0,opencode,easy,11,33,0.3333333333333333
+Harbor OpenCode-only,0,opencode,medium,10,118,0.0847457627118644
+Harbor OpenCode-only,0,opencode,hard,6,99,0.06060606060606061
+Harbor OpenCode-only,0,claude-code,easy,14,33,0.42424242424242425
+Harbor OpenCode-only,0,claude-code,medium,22,118,0.1864406779661017
+Harbor OpenCode-only,0,claude-code,hard,6,99,0.06060606060606061
+Harbor OpenCode-only,0,codex,easy,14,33,0.42424242424242425
+Harbor OpenCode-only,0,codex,medium,20,118,0.1694915254237288
+Harbor OpenCode-only,0,codex,hard,7,99,0.0707070707070707
+Harbor OpenCode-only,0,mini-swe-agent,easy,14,33,0.42424242424242425
+Harbor OpenCode-only,0,mini-swe-agent,medium,16,118,0.13559322033898305
+Harbor OpenCode-only,0,mini-swe-agent,hard,6,99,0.06060606060606061
+Harbor OpenCode-only,100,opencode,easy,11,33,0.3333333333333333
+Harbor OpenCode-only,100,opencode,medium,34,118,0.288135593220339
+Harbor OpenCode-only,100,opencode,hard,10,99,0.10101010101010101
+Harbor OpenCode-only,100,claude-code,easy,16,33,0.48484848484848486
+Harbor OpenCode-only,100,claude-code,medium,35,118,0.2966101694915254
+Harbor OpenCode-only,100,claude-code,hard,12,99,0.12121212121212122
+Harbor OpenCode-only,100,codex,easy,16,33,0.48484848484848486
+Harbor OpenCode-only,100,codex,medium,43,118,0.3644067796610169
+Harbor OpenCode-only,100,codex,hard,17,99,0.1717171717171717
+Harbor OpenCode-only,100,mini-swe-agent,easy,23,33,0.696969696969697
+Harbor OpenCode-only,100,mini-swe-agent,medium,26,118,0.22033898305084745
+Harbor OpenCode-only,100,mini-swe-agent,hard,2,99,0.020202020202020204
+Harbor OpenCode-only,200,opencode,easy,19,33,0.5757575757575758
+Harbor OpenCode-only,200,opencode,medium,42,118,0.3559322033898305
+Harbor OpenCode-only,200,opencode,hard,12,99,0.12121212121212122
+Harbor OpenCode-only,200,claude-code,easy,19,33,0.5757575757575758
+Harbor OpenCode-only,200,claude-code,medium,44,118,0.3728813559322034
+Harbor OpenCode-only,200,claude-code,hard,8,99,0.08080808080808081
+Harbor OpenCode-only,200,codex,easy,22,33,0.6666666666666666
+Harbor OpenCode-only,200,codex,medium,44,118,0.3728813559322034
+Harbor OpenCode-only,200,codex,hard,11,99,0.1111111111111111
+Harbor OpenCode-only,200,mini-swe-agent,easy,22,33,0.6666666666666666
+Harbor OpenCode-only,200,mini-swe-agent,medium,22,118,0.1864406779661017
+Harbor OpenCode-only,200,mini-swe-agent,hard,6,99,0.06060606060606061
+Harbor OpenCode-only,300,opencode,easy,23,33,0.696969696969697
+Harbor OpenCode-only,300,opencode,medium,36,118,0.3050847457627119
+Harbor OpenCode-only,300,opencode,hard,11,99,0.1111111111111111
+Harbor OpenCode-only,300,claude-code,easy,22,33,0.6666666666666666
+Harbor OpenCode-only,300,claude-code,medium,40,118,0.3389830508474576
+Harbor OpenCode-only,300,claude-code,hard,14,99,0.1414141414141414
+Harbor OpenCode-only,300,codex,easy,21,33,0.6363636363636364
+Harbor OpenCode-only,300,codex,medium,43,118,0.3644067796610169
+Harbor OpenCode-only,300,codex,hard,16,99,0.16161616161616163
+Harbor OpenCode-only,300,mini-swe-agent,easy,19,33,0.5757575757575758
+Harbor OpenCode-only,300,mini-swe-agent,medium,33,118,0.2796610169491525
+Harbor OpenCode-only,300,mini-swe-agent,hard,7,99,0.0707070707070707
+Harbor OpenCode-only,400,opencode,easy,25,33,0.7575757575757576
+Harbor OpenCode-only,400,opencode,medium,51,118,0.4322033898305085
+Harbor OpenCode-only,400,opencode,hard,13,99,0.13131313131313133
+Harbor OpenCode-only,400,claude-code,easy,24,33,0.7272727272727273
+Harbor OpenCode-only,400,claude-code,medium,50,118,0.423728813559322
+Harbor OpenCode-only,400,claude-code,hard,10,99,0.10101010101010101
+Harbor OpenCode-only,400,codex,easy,23,33,0.696969696969697
+Harbor OpenCode-only,400,codex,medium,41,118,0.3474576271186441
+Harbor OpenCode-only,400,codex,hard,19,99,0.1919191919191919
+Harbor OpenCode-only,400,mini-swe-agent,easy,21,33,0.6363636363636364
+Harbor OpenCode-only,400,mini-swe-agent,medium,42,118,0.3559322033898305
+Harbor OpenCode-only,400,mini-swe-agent,hard,9,99,0.09090909090909091
+Harbor OpenCode-only,500,opencode,easy,23,33,0.696969696969697
+Harbor OpenCode-only,500,opencode,medium,47,118,0.3983050847457627
+Harbor OpenCode-only,500,opencode,hard,12,99,0.12121212121212122
+Harbor OpenCode-only,500,claude-code,easy,22,33,0.6666666666666666
+Harbor OpenCode-only,500,claude-code,medium,45,118,0.3813559322033898
+Harbor OpenCode-only,500,claude-code,hard,14,99,0.1414141414141414
+Harbor OpenCode-only,500,codex,easy,25,33,0.7575757575757576
+Harbor OpenCode-only,500,codex,medium,50,118,0.423728813559322
+Harbor OpenCode-only,500,codex,hard,18,99,0.18181818181818182
+Harbor OpenCode-only,500,mini-swe-agent,easy,24,33,0.7272727272727273
+Harbor OpenCode-only,500,mini-swe-agent,medium,37,118,0.3135593220338983
+Harbor OpenCode-only,500,mini-swe-agent,hard,13,99,0.13131313131313133
+Harbor OpenCode-only,600,opencode,easy,26,33,0.7878787878787878
+Harbor OpenCode-only,600,opencode,medium,51,118,0.4322033898305085
+Harbor OpenCode-only,600,opencode,hard,12,99,0.12121212121212122
+Harbor OpenCode-only,600,claude-code,easy,21,33,0.6363636363636364
+Harbor OpenCode-only,600,claude-code,medium,43,118,0.3644067796610169
+Harbor OpenCode-only,600,claude-code,hard,14,99,0.1414141414141414
+Harbor OpenCode-only,600,codex,easy,21,33,0.6363636363636364
+Harbor OpenCode-only,600,codex,medium,50,118,0.423728813559322
+Harbor OpenCode-only,600,codex,hard,15,99,0.15151515151515152
+Harbor OpenCode-only,600,mini-swe-agent,easy,20,33,0.6060606060606061
+Harbor OpenCode-only,600,mini-swe-agent,medium,44,118,0.3728813559322034
+Harbor OpenCode-only,600,mini-swe-agent,hard,13,99,0.13131313131313133
+Harbor OpenCode-only,700,opencode,easy,23,33,0.696969696969697
+Harbor OpenCode-only,700,opencode,medium,52,118,0.4406779661016949
+Harbor OpenCode-only,700,opencode,hard,25,99,0.25252525252525254
+Harbor OpenCode-only,700,claude-code,easy,26,33,0.7878787878787878
+Harbor OpenCode-only,700,claude-code,medium,63,118,0.5338983050847458
+Harbor OpenCode-only,700,claude-code,hard,27,99,0.2727272727272727
+Harbor OpenCode-only,700,codex,easy,23,33,0.696969696969697
+Harbor OpenCode-only,700,codex,medium,50,118,0.423728813559322
+Harbor OpenCode-only,700,codex,hard,20,99,0.20202020202020202
+Harbor OpenCode-only,700,mini-swe-agent,easy,22,33,0.6666666666666666
+Harbor OpenCode-only,700,mini-swe-agent,medium,45,118,0.3813559322033898
+Harbor OpenCode-only,700,mini-swe-agent,hard,19,99,0.1919191919191919
+Harbor OpenCode-only,800,opencode,easy,15,33,0.45454545454545453
+Harbor OpenCode-only,800,opencode,medium,36,118,0.3050847457627119
+Harbor OpenCode-only,800,opencode,hard,15,99,0.15151515151515152
+Harbor OpenCode-only,800,claude-code,easy,26,33,0.7878787878787878
+Harbor OpenCode-only,800,claude-code,medium,52,118,0.4406779661016949
+Harbor OpenCode-only,800,claude-code,hard,19,99,0.1919191919191919
+Harbor OpenCode-only,800,codex,easy,21,33,0.6363636363636364
+Harbor OpenCode-only,800,codex,medium,47,118,0.3983050847457627
+Harbor OpenCode-only,800,codex,hard,21,99,0.21212121212121213
+Harbor OpenCode-only,800,mini-swe-agent,easy,25,33,0.7575757575757576
+Harbor OpenCode-only,800,mini-swe-agent,medium,40,118,0.3389830508474576
+Harbor OpenCode-only,800,mini-swe-agent,hard,14,99,0.1414141414141414
+Harbor OpenCode-only,900,opencode,easy,22,33,0.6666666666666666
+Harbor OpenCode-only,900,opencode,medium,37,118,0.3135593220338983
+Harbor OpenCode-only,900,opencode,hard,16,99,0.16161616161616163
+Harbor OpenCode-only,900,claude-code,easy,24,33,0.7272727272727273
+Harbor OpenCode-only,900,claude-code,medium,46,118,0.3898305084745763
+Harbor OpenCode-only,900,claude-code,hard,18,99,0.18181818181818182
+Harbor OpenCode-only,900,codex,easy,18,33,0.5454545454545454
+Harbor OpenCode-only,900,codex,medium,35,118,0.2966101694915254
+Harbor OpenCode-only,900,codex,hard,15,99,0.15151515151515152
+Harbor OpenCode-only,900,mini-swe-agent,easy,20,33,0.6060606060606061
+Harbor OpenCode-only,900,mini-swe-agent,medium,34,118,0.288135593220339
+Harbor OpenCode-only,900,mini-swe-agent,hard,11,99,0.1111111111111111
+Harbor OpenCode-only,1000,opencode,easy,17,33,0.5151515151515151
+Harbor OpenCode-only,1000,opencode,medium,30,118,0.2542372881355932
+Harbor OpenCode-only,1000,opencode,hard,8,99,0.08080808080808081
+Harbor OpenCode-only,1000,claude-code,easy,19,33,0.5757575757575758
+Harbor OpenCode-only,1000,claude-code,medium,41,118,0.3474576271186441
+Harbor OpenCode-only,1000,claude-code,hard,17,99,0.1717171717171717
+Harbor OpenCode-only,1000,codex,easy,20,33,0.6060606060606061
+Harbor OpenCode-only,1000,codex,medium,46,118,0.3898305084745763
+Harbor OpenCode-only,1000,codex,hard,15,99,0.15151515151515152
+Harbor OpenCode-only,1000,mini-swe-agent,easy,22,33,0.6666666666666666
+Harbor OpenCode-only,1000,mini-swe-agent,medium,26,118,0.22033898305084745
+Harbor OpenCode-only,1000,mini-swe-agent,hard,3,99,0.030303030303030304
diff --git a/04-data-agent/reports/async-comparison-20260916/comparison.png b/04-data-agent/reports/async-comparison-20260916/comparison.png
new file mode 100644
index 0000000..f13a39b
Binary files /dev/null and b/04-data-agent/reports/async-comparison-20260916/comparison.png differ
diff --git a/04-data-agent/reports/three-run-analysis-20260917/REPORT.md b/04-data-agent/reports/three-run-analysis-20260917/REPORT.md
new file mode 100644
index 0000000..d81df6a
--- /dev/null
+++ b/04-data-agent/reports/three-run-analysis-20260917/REPORT.md
@@ -0,0 +1,170 @@
+# Three-run training and evaluation analysis
+
+September 17, 2026. Qwen3.5-2B; 250 fixed test tasks and four evaluation harnesses.
+
+**The two late declines have different signatures.** Multi-harness training develops long, often truncated responses and loses effective tool use, especially in OpenCode. Harbor OpenCode-only continues using tools but spends more calls, repeats more work, and submits fewer answers. Native OpenCode shows neither extreme and finishes at its best aggregate checkpoint.
+
+I also found a concrete resume problem: the multi-harness continuation largely revisited tasks already trained on. These observations identify useful diagnostics; they do not establish one causal explanation for all score changes.
+
+The [training companion](TRAINING.md) adds a census of 13,625 optimizer-admitted rollouts,
+with token counts checked against receipts, per-harness tool use, context overhead and
+zero-variance groups. See [the concise results overview](../../results.md) for the combined findings.
+
+
+
+## Evidence and scope
+
+The analysis reconstructs **33,000 unique accepted evaluation cells**, represented by 34 checkpoint cohorts because the two Harbor runs share the same 1,000-cell baseline. Every reconstructed harness score matches its accepted score file, and every task index matches the frozen manifest. Training analysis uses all 3,000 optimizer-step metric records, coverage reports, frozen schedules, and available per-rollout optimizer receipts.
+
+| Run | Baseline | Best measured checkpoint | Final checkpoint 1,000 |
+| --- | ---: | ---: | ---: |
+| Harbor multi-harness | 14.6% | 500: **37.0%** | 26.3% |
+| Native OpenCode | 15.9% | 1,000: **29.8%** | 29.8% |
+| Harbor OpenCode-only | 14.6% | 700: **39.5%** | 26.4% |
+
+“Tool calls” below means distinct call IDs in the longest captured agent transcript. This transcript can omit discarded branches; it is not a billable-request counter. “Model calls” comes from the capture graph and includes auxiliary calls. A model response can emit several tools, so the 17-model-call ceiling does not imply a 17-tool-call ceiling.
+
+## 1. Multi-harness regression is concentrated in OpenCode, with output truncation
+
+From checkpoint 500 to 1,000:
+
+| Evaluation harness | Pass@1 | Mean recorded tool calls | Rollouts with output truncation |
+| --- | ---: | ---: | ---: |
+| OpenCode | 32.8% → **5.6%** | 15.48 → **3.61** | 3/250 → **204/250** |
+| Claude Code | 44.8% → 38.8% | 16.06 → 12.22 | 5/250 → 132/250 |
+| Codex | 39.2% → 24.8% | 16.45 → 9.24 | 1/250 → 162/250 |
+| Mini-SWE-Agent | 31.2% → **36.0%** | 14.33 → 11.00 | 0/250 → 58/250 |
+
+OpenCode accounts for **68 of the 107 net lost successful cells**, about 64% of the aggregate decline. At checkpoint 900 it is worse: 2.4% success, 2.22 tool calls on average, and 87.2% of rollouts have at most two recorded tool calls. The final checkpoint has 74.0% in that category.
+
+Across all four harnesses, output truncation rises from **9/1,000 to 556/1,000 rollouts**. At the final checkpoint, truncated rollouts score 9.9%, versus 46.8% for those without a truncation warning. This is an association: hard or poorly handled tasks can cause both long outputs and failure.
+
+Directly inspected examples read a CSV once, then produce a long explanatory response that ends with `finish_reason="length"` at **4,096 output tokens**, without executing the calculation or writing the required answer. A few checkpoint-900 examples instead emit tool-like XML or JSON as plain text, with no parsed tool call. Thus the low call count should not be interpreted as successful efficiency.
+
+**Budget mismatch is a plausible contributor.** The evaluation manifest caps each response at 4,096 output tokens. Saved late training captures contain individual responses up to **16,384 tokens**. The complete admitted-rollout census finds responses longer than 4,096 in **182/485 (37.5%)** rollouts at steps 901–1,000, versus **10/512 (2.0%)** at steps 401–500. Whole-rollout completion tokens rise **3,521 → 9,480** over those windows. This replaces the earlier small-sample estimate. Worker completion-length telemetry averages 3,658 → 10,657 because it describes generated rollouts and uses different aggregation; [training metric definitions](TRAINING.md#what-is-counted) explain the distinction.
+
+This suggests a policy increasingly incompatible with the evaluation budget. It does **not** show that a larger budget would recover the score; some inspected responses repeat reasoning or invent facts rather than making progress.
+
+## 2. OpenCode-only declines through longer tool loops and missing submissions
+
+Harbor OpenCode-only, checkpoint 700 → 1,000:
+
+| Measure | Checkpoint 700 | Checkpoint 1,000 |
+| --- | ---: | ---: |
+| Overall pass@1 | 39.5% | 26.4% |
+| Mean recorded tool calls, all harnesses | 16.62 | 20.97 |
+| Mean exact repeated calls | 2.09 | 3.03 |
+| Rollouts reaching at least 17 captured model calls | 56.4% | 70.2% |
+| Answer submission, instrumented subset | 68.9% | 40.7% |
+| Output-truncated rollouts | 7/1,000 | 9/1,000 |
+
+Submission is observable for **86 of the 250 tasks per harness**, or 344 cells. It must not be reported as a whole-test-set rate. Of 73 previously correct cells in this subset that become incorrect, **64 no longer submit an answer**.
+
+On matched task/harness cells that regress, tool calls rise from **15.26 to 22.48** on average. OpenCode itself goes from 20.16 to 25.80 calls while its score falls from 40.0% to 22.0%. Mini-SWE-Agent reaches at least 17 captured model calls on **95.6%** of final rollouts.
+
+The signature is continued work without reliable completion, rather than the widespread output truncation in the multi-harness model. Context-budget exhaustion increases too, from 7 to 25 cells, but is too rare to account for the entire decline. Exact repetition is only a proxy for wasted work: repeating a command can sometimes be useful.
+
+
+
+## 3. The multi-harness resume replayed nearly the entire continuation's task set
+
+The continuation at step 684 resumed with schedule offset **230**. The previous allocation had already trained many later groups, but group 230 was the first hole in its completed-group set.
+
+The saved trainer code writes only `dataset_start_index + first_untrained` to `rollout_state.json`; it does not persist the later completed-group set. Resuming therefore schedules those later tasks again.
+
+Evidence: frozen checkpoint-saving code (`experiments/async_grpo_harbor_data_agent/logs/multi4-long-prod-cont-20260915/source-snapshot/trl/trl/experimental/async_grpo/async_grpo_trainer.py:1798`, local evidence), continuation resume audit (`experiments/async_grpo_harbor_data_agent/logs/multi4-long-prod-cont-20260915/job-80608/audit/resume.json`, local evidence), and optimizer receipts (`experiments/analysis-three-runs-20260917/optimizer_rollouts.csv`, local evidence).
+
+Optimizer receipts confirm that steps 685–1,000 admitted **1,579 fresh rollouts**, of which **1,575 used previously seen tasks**. About **99.8% of supervised tokens** in that segment came from already-seen tasks. Its 203 task IDs add only one new task to the previous detailed optimizer receipts. Coverage reports count 202 stable tasks for the segment because their callback waits for an additional optimizer boundary.
+
+This is task replay, not replay of cached model responses. It materially changes data exposure and is a credible contributor to later over-specialization. **It cannot explain the initial 500 → 600 drop**, which occurred before this restart. OpenCode-only also declines without a restart, so resume replay is not a universal explanation.
+
+Before another resumed run, preserve completed absolute group IDs and explicitly handle unfinished groups; simply advancing to the largest group ID would silently skip gaps.
+
+## 4. Training on one harness transferred to other harnesses
+
+Harbor OpenCode-only reaches **46.4% on Claude Code** at checkpoint 700, above its 40.0% on OpenCode. All four harnesses improve substantially from the shared baseline.
+
+Native OpenCode's final checkpoint scores:
+
+| Evaluation harness | Baseline | Final | Gain |
+| --- | ---: | ---: | ---: |
+| OpenCode | 12.8% | 20.4% | +7.6 pp |
+| Claude Code | 16.8% | 33.2% | +16.4 pp |
+| Codex | 15.2% | 29.6% | +14.4 pp |
+| Mini-SWE-Agent | 18.8% | 36.0% | +17.2 pp |
+
+This contradicts the simple expectation that single-harness training mainly improves that same harness. It is consistent with transfer of task-solving behavior combined with different harness prompting and tool interfaces. These runs do not isolate the marginal benefit of multi-harness training, because exposure, backends and training histories differ.
+
+## 5. Equal optimizer steps were not equal data or compute budgets
+
+| Run | Unique training tasks in coverage logs | Supervised tokens | Forwarded tokens |
+| --- | ---: | ---: | ---: |
+| Harbor multi-harness | 482/1,000 | 22.15M | 1,001.32M |
+| Native OpenCode | 566/1,000 | 5.33M | 228.41M |
+| Harbor OpenCode-only | 523/1,000 | 14.63M | 419.43M |
+
+Native OpenCode used about **one quarter of the multi-harness supervised-token count**. It also finishes with lower recorded tool use: 11.97 calls per evaluation rollout, versus 20.97 for Harbor OpenCode-only. This is a useful efficiency observation, but wall-time comparisons are confounded by E2B versus Daytona, serving allocations and retries.
+
+The 1,000-step cap stopped all three before full task coverage. A future controlled comparison should specify both the task exposure target and supervised-token budget. “1,000 steps on the same 1,000-task dataset” is insufficient.
+
+
+
+## 6. Balanced harness rollouts produced unequal token weighting
+
+Detailed optimizer receipts are available from multi-harness step 31 onward:
+
+| Harness | Share of admitted rollouts | Share of training rows | Share of supervised tokens |
+| --- | ---: | ---: | ---: |
+| OpenCode | 25.2% | 11.4% | 18.3% |
+| Claude Code | 24.8% | **77.3%** | **35.3%** |
+| Codex | 24.6% | 5.3% | 31.1% |
+| Mini-SWE-Agent | 25.4% | 5.9% | 15.3% |
+
+The harness schedule was balanced in rollout count. Claude's prompt forks greatly increased row count and repeated context, but **77.3% of rows does not mean 77.3% of the gradient**: these runs use supervised-token normalization. Token shares better describe loss exposure, though actual gradient contributions also depend on advantages, clipping and token gradients.
+
+This supports measuring rollout, row, context-token and supervised-token shares separately. It does not establish that Claude's row count caused the decline.
+
+## 7. Difficulty and harness agreement reveal more than the average
+
+- Multi-harness easy-task pass@1 drops **72.7% → 42.4%** from peak to final, while hard-task pass@1 drops 16.4% → 13.4%. This is not merely a loss on difficult tasks. There are only 33 distinct easy tasks, evaluated under four harnesses.
+- OpenCode-only declines across all three difficulty levels: easy 71.2% → 59.1%, medium 44.5% → 30.3%, hard 23.0% → 10.9%.
+- At the multi-harness peak, **45 tasks succeed under all four harnesses**; at the final checkpoint, only **7** do. Tasks solved by at least one harness fall less sharply, from 141 to 129. Harness consistency erodes more than the set of tasks solvable by any harness.
+- OpenCode-only checkpoint 700 solves 149/250 tasks under at least one harness (59.6%); native final solves 134/250 (53.6%). These are retrospective four-harness oracle rates, **not pass@1** and not deployable routing policies.
+
+## 8. The declines are clear; the ranking of the two peaks is not
+
+Paired bootstrap intervals resample the 250 task IDs, keeping each task's four harness outcomes together:
+
+| Comparison | Score change | Conditional 95% interval |
+| --- | ---: | ---: |
+| Multi-harness 500 → 1,000 | −10.7 pp | −14.1 to −7.4 pp |
+| OpenCode-only 700 → 1,000 | −13.1 pp | −16.6 to −9.7 pp |
+| Multi-harness 500 → OpenCode-only 700 | +2.5 pp | −0.6 to +5.7 pp |
+
+The late declines are larger than the task-level variability captured by this calculation. The best-score difference does not clearly separate the runs. These intervals condition on observed checkpoints and attempts: they do not account for selecting the best of many checkpoints, another sampling seed, another training seed, or protocol differences.
+
+## What is ruled out, and what remains uncertain
+
+- Recorded gradient norms are finite throughout; observed maximum staleness never exceeds four. There is no obvious numerical blow-up or staleness-limit violation in these metrics. Neither check proves healthy policy learning.
+- All accepted eval cohorts pass their existing TiTO and version gates. TiTO validates token identity and alignment; it cannot guarantee useful tool behavior or improving rewards.
+- **All three comparison trainers use binary correctness.** Native raw rollout artifacts retain an efficiency bonus, but `ComparisonSession.verify()` removes it before training. The preliminary suspicion of different reward objectives was rejected after inspecting the frozen adapter.
+- The native run uses Daytona and the Harbor runs use E2B. Baselines are separate measured cohorts. Multi-harness also has several resumed allocations and early recipe changes.
+- The published metric retains the experiment's “pass@1” name, with first graded attempt selection and retries for ungraded failures. Native baseline logs show preceding ungraded attempts for 830 cells; multi-harness checkpoint 500 has 65 such cells and final has 205. These counts exclude skip markers and are not necessarily unique infrastructure outages. Multi-harness 900/1,000 also have documented verifier-budget reconciliation. Uniform first-attempt accounting is needed for a stronger controlled comparison.
+
+## Recommended next work
+
+1. **Fix and test resume task accounting first.** Preserve completed groups and unfinished work across restarts. Add a test with an early missing group and later completed groups; verify the latter are not rescheduled unintentionally.
+2. **Run a small, separately labeled output-budget diagnostic.** Compare multi-harness checkpoints 500 and 1,000 on the same tasks and harness with 4,096 versus 16,384 output tokens. Keep the canonical evaluation scores unchanged. Measure submission, truncation and actual executed tool results, not just reward. Prefer a development split for subsequent tuning.
+3. **Investigate completion discipline for OpenCode-only.** Compare peak/final trajectories on lost tasks: repeated calls, unbounded file reads, context exhaustion and failure to write the answer. Increasing the tool budget alone is not supported by these results.
+4. **Track behavior during training and evaluation.** Add per-harness output-truncation rate, submission rate, recorded/executed tool calls, repeated-call fraction, completion tokens, task coverage and cumulative supervised tokens. Keep them separate from the headline reward.
+5. **Make the next ablation controlled.** Same backend, output budgets, task exposure, capture/agent filters and loss normalization; compare multiple seeds and choose checkpoints on validation data. Measure token-weighted harness exposure before changing weighting.
+
+## Artifacts and reproduction
+
+- [Harness/checkpoint metrics](by_harness.csv), [difficulty metrics](by_difficulty.csv), [outcome-conditioned metrics](by_outcome.csv), [output warnings](capture_warnings.csv).
+- [Paired score differences](paired_differences.csv), [harness agreement](harness_complementarity.csv), [training windows](training_windows.csv), [training exposure](evaluation_vs_training_exposure.csv).
+- [Training analysis](TRAINING.md), [admitted-rollout metrics](training_behavior_windows.csv), [training harness breakdown](training_behavior_by_harness.csv), [optimizer accounting](training_accounting.csv), [training figure](training_diagnostics.png).
+- Local analysis code and trace hashes: `experiments/analysis-three-runs-20260917/` (`extract.py`, `training.py`, `analyze.py`, `provenance.json`).
+- `sampled_training_lengths.json` records the fixed training sample. `training_lineage.csv` and `optimizer_rollouts.csv` support the resume and weighting calculations. Raw captures and accepted scores were not changed.
+
+Run the three scripts in order with `.venv312/bin/python`. Outputs remain local; no new inference or training jobs were launched for this analysis.
diff --git a/04-data-agent/reports/three-run-analysis-20260917/TLDR.md b/04-data-agent/reports/three-run-analysis-20260917/TLDR.md
new file mode 100644
index 0000000..5425204
--- /dev/null
+++ b/04-data-agent/reports/three-run-analysis-20260917/TLDR.md
@@ -0,0 +1,8 @@
+Qwen3.5-2B comparison is complete: three 1,000-step runs, evaluated on 250 fixed tasks × four harnesses, pass@1.
+
+- Best scores: **Harbor multi-harness 37.0% @500**, **Harbor OpenCode-only 39.5% @700**, **native OpenCode 29.8% @1,000**. The Harbor runs finish lower, at 26.3% and 26.4%.
+- Multi-harness training outputs grow **3.5k → 9.5k tokens/rollout**, while tool calls fall **15.9 → 11.2**. In the final window, 37.5% of admitted rollouts contain a response exceeding the eval's 4k cap; eval truncation rises sharply.
+- Harbor OpenCode-only makes more calls but submits fewer answers. Native keeps shorter outputs and uses fewer training tokens, although late context duplication increases its compute cost.
+- We found a resume-accounting bug: **1,575/1,579 post-resume rollouts revisit seen tasks**. Also, **35–58% of steps have zero fresh gradient** because groups have no reward contrast.
+
+Next: fix resume accounting, test matched output budgets, and track completion, tokens, tool use and task coverage. Equal steps were not equal exposure; these results do not establish that multi-harness training is worse.
diff --git a/04-data-agent/reports/three-run-analysis-20260917/TRAINING.md b/04-data-agent/reports/three-run-analysis-20260917/TRAINING.md
new file mode 100644
index 0000000..757d1d6
--- /dev/null
+++ b/04-data-agent/reports/three-run-analysis-20260917/TRAINING.md
@@ -0,0 +1,122 @@
+# Training behavior and its relation to evaluation
+
+September 17, 2026. Companion to the [evaluation analysis](REPORT.md).
+
+The training-side evidence supports different failure modes in the two Harbor runs: multi-harness generates much longer responses with fewer tool calls, while OpenCode-only increases work without reliably finishing. Native OpenCode uses substantially fewer supervised tokens. These are observations from the completed runs, not an isolated comparison of training algorithms.
+
+## What is counted
+
+- **Optimizer telemetry:** all 3,000 step records, including supervised/forwarded tokens, gradient norms and timing.
+- **Admitted rollouts:** 13,625 saved captures matched to optimizer receipts: 5,013 multi-harness, 4,493 native OpenCode and 4,119 Harbor OpenCode-only. Multi-harness receipts begin at step 31; its first 30 steps have telemetry but are excluded from capture-level comparisons.
+- **Tool calls:** parsed tool-call emissions across retained agent turns. They are requests, not proof of successful execution. Auxiliary and discarded turns are excluded. This differs from the evaluation report's longest-transcript counter, which may omit earlier branches.
+- **Completion tokens:** captured output IDs across those turns. **Supervised tokens** are positions selected by the loss mask. **Forwarded tokens** include context and completion processed by the trainer, including repeated context across forked rows; they are neither unique tokens nor serving-token bills.
+
+Every capture's masked token count is checked against its optimizer receipt. All 13,625 agree. Per-rollout averages below count each admitted rollout once. The older `tools/call_frequency` and reward dashboard metrics average over training rows inside microbatches, then logging windows; they can overweight rollouts with more rows. Worker completion-length telemetry also includes generated rollouts independently of their later admission. The datasets and denominators must remain distinct.
+
+## Token growth in multi-harness training
+
+Comparing optimizer steps **401–500** with **901–1,000**:
+
+| Measure, admitted rollouts | Earlier window | Final window |
+| --- | ---: | ---: |
+| Rollouts | 512 | 485 |
+| Mean completion tokens | 3,521 | 9,480 |
+| Median completion tokens | 2,359 | 6,776 |
+| Mean emitted tool calls | 15.86 | 11.24 |
+| At least one response longer than 4,096 tokens | 2.0% | 37.5% |
+| Completion tokens in turns without a tool call | 0.9% | 12.4% |
+| Mean binary training reward | 37.5% | 39.4% |
+| Fixed-test pass@1 at window end | 37.0% | 26.3% |
+
+The median grows too, so this is not just a few enormous outliers. More tokens accompany fewer actions, and slightly higher reward on the changing training cohort accompanies lower held-out performance. Task replay after the step-684 resume further complicates the training reward.
+
+The worker's previously reported mean completion length was 3,658 → 10,657. That remains a valid telemetry summary, but the table uses the cleaner once-per-admitted-rollout measure. The full capture census supersedes the earlier small sample for estimating how frequently training responses exceed the eval limit.
+
+Most late completion tokens still occur in turns that eventually emit a tool call. Inspected successful training examples contain a long explanation followed by a short tool request. Under a shorter output limit, that request may never be emitted. This is a testable explanation for the evaluation's long text responses, low tool-call count and high truncation rate; it does not prove that increasing the eval cap would restore accuracy.
+
+Per-harness token inflation is broad, despite a balanced admitted-rollout mix:
+
+| Training harness | Completion tokens, mean | Tool calls, mean | Final rollouts with a response >4,096 tokens |
+| --- | ---: | ---: | ---: |
+| OpenCode | 2,583 → 5,602 | 14.82 → 7.79 | 38.1% |
+| Claude Code | 5,924 → 14,148 | 17.22 → 12.68 | 51.3% |
+| Codex | 4,199 → 11,193 | 16.44 → 14.13 | 37.5% |
+| Mini-SWE-Agent | 1,603 → 7,110 | 15.11 → 10.37 | 24.2% |
+
+These compare the same two step windows, not matched training tasks. In particular, task replay and changing difficulty prevent interpreting the training-reward changes as generalization gains.
+
+Length growth also occurs among **successful** admitted rollouts: their mean completion
+length rises **2,453 → 9,084 tokens**. It is not confined to failed attempts. This makes
+train/eval budget compatibility worth testing even when training reward appears healthy.
+
+## Harbor OpenCode-only: more output and actions near the end
+
+Comparing steps **601–700**, ending at the best evaluated checkpoint, with **901–1,000**:
+
+| Measure, admitted rollouts | Peak window | Final window |
+| --- | ---: | ---: |
+| Rollouts | 420 | 404 |
+| Mean completion tokens | 2,122 | 3,631 |
+| Median completion tokens | 1,552 | 2,102 |
+| Mean emitted tool calls | 14.63 | 17.43 |
+| Mean retained agent turns | 11.70 | 11.49 |
+| At least one response longer than 4,096 tokens | 1.2% | 6.4% |
+| Mean binary training reward | 46.2% | 39.4% |
+| Fixed-test pass@1 at window end | 39.5% | 26.4% |
+
+Tool calls rise without more model turns: a response can request multiple tools. The
+same broad increase in work appears on the fixed evaluation set, where average tool
+calls rise 16.62 → 20.97 and submission declines sharply. Training and evaluation
+do not agree on every diagnostic: mean exact repetitions fall 3.82 → 3.03 in these
+training windows, while rising in eval. Different tasks and harness mixtures matter.
+
+Training length is also non-monotonic: steps 101–200 average 7,750 tokens per admitted
+rollout, higher than the final window, and the model later recovers. Length alone is
+therefore not a sufficient explanation or stopping rule.
+
+## Native OpenCode: short outputs, but growing context overhead
+
+From steps 401–500 to 901–1,000, admitted native OpenCode rollouts average **1,145 → 1,075 completion tokens** and **7.72 → 6.05 emitted tool calls**. None of the final window's 402 admitted rollouts contains a response longer than 4,096 tokens. Its final held-out score is its highest measured, 29.8%.
+
+There is nevertheless a late efficiency regression: **rows per admitted rollout grow 1.91 → 4.56**, and forwarded tokens per supervised token grow **22.3× → 99.4×**. Mean forward/backward time rises **4.69 → 15.96 seconds per step**. Short outputs therefore do not guarantee cheap training when history forks into more context-bearing rows. Overall step time stays approximately flat because recorded rollout waiting time falls; this is not evidence of equal GPU work.
+
+## Compute exposure and zero-variance groups
+
+| Full run, 1,000 steps | Supervised tokens | Forwarded tokens | Forwarded / supervised | Zero-fresh-gradient steps |
+| --- | ---: | ---: | ---: | ---: |
+| Harbor multi-harness | 22.15M | 1,001.32M | 45.2× | 349/1,000 |
+| Native OpenCode | 5.33M | 228.41M | 42.9× | 583/1,000 |
+| Harbor OpenCode-only | 14.63M | 419.43M | 28.7× | 380/1,000 |
+
+The context overhead is real, but context is required for correct conditional training; these ratios do not mean that all masked tokens are avoidable waste. Prompt forks can increase repeated context substantially.
+
+Every recorded zero-gradient step coincides with zero within-group reward standard deviation. With binary rewards, a group whose scorable rollouts all agree has zero GRPO advantage and supplies no fresh learning signal. Different uniform groups can share a step, so a zero-gradient step can still have an intermediate average reward.
+
+Those steps consume **35.0%, 58.8% and 39.1% of forwarded tokens**, respectively, for multi-harness, native and Harbor OpenCode-only. Native's final 100 steps include 72 zero-gradient steps. This suggests investigating rejection of zero-advantage groups before expensive training forwards, while preserving coverage, scheduling and checkpoint semantics. Generation and grading costs would remain. **Zero fresh gradient does not mean unchanged weights:** optimizer momentum and weight decay can still act.
+
+## Measurement cautions
+
+- Training rows are not independent task examples. Claude's 77.3% row share corresponds to 35.3% of supervised tokens in the receipt-covered multi-harness segment, not 77.3% of the gradient.
+- `completions/clipped_ratio` in the frozen worker checks whether the **last** output token is EOS/pad. It does not directly measure whether **any** earlier response was truncated, or whether a response exceeds the evaluation's 4,096-token cap. The new capture metrics record those separately.
+- Tool-failure telemetry infers errors from result text; it is not a structured count of sandbox failures. Emitted calls and successful executions should be logged separately in future runs.
+- A token can be selected by the loss mask yet have zero advantage. “Supervised tokens” is accounting for eligible positions, not a count of nonzero gradient contributions.
+- Task mix changes throughout training. Training rewards and per-window rollout statistics are not repeated measurements on a fixed dataset. The evaluation cohorts are fixed, subject to the [documented historical retry policy](REPORT.md#what-is-ruled-out-and-what-remains-uncertain).
+
+## Reproduction
+
+
+
+[Per-window capture metrics](training_behavior_windows.csv) ·
+[Harness breakdown](training_behavior_by_harness.csv) ·
+[Success/failure breakdown](training_behavior_by_outcome.csv) ·
+[Optimizer accounting](training_accounting.csv) ·
+[Original step-metric aggregation](training_step_diagnostics.csv)
+
+Use Python with `orjson`, `pandas`, `numpy` and `matplotlib`. The evidence directory must contain the frozen `optimizer_rollouts.csv`, `training_lineage.csv`, `training_metrics.csv` and the saved captures referenced by the lineage. No model loading or GPU jobs are required.
+
+```bash
+python training_behavior.py --evidence /path/to/analysis-three-runs-20260917 --workers 2
+python summarize_training.py --evidence /path/to/analysis-three-runs-20260917
+```
+
+The extractor saves per-rollout counts and source hashes in the evidence directory. The summarizer writes aggregate CSVs and `training_diagnostics.png` beside this document. Local raw evidence is preserved under `experiments/analysis-three-runs-20260917/`; it is not copied into the repository.
diff --git a/04-data-agent/reports/three-run-analysis-20260917/behavior.png b/04-data-agent/reports/three-run-analysis-20260917/behavior.png
new file mode 100644
index 0000000..01bc8f6
Binary files /dev/null and b/04-data-agent/reports/three-run-analysis-20260917/behavior.png differ
diff --git a/04-data-agent/reports/three-run-analysis-20260917/by_difficulty.csv b/04-data-agent/reports/three-run-analysis-20260917/by_difficulty.csv
new file mode 100644
index 0000000..71b130d
--- /dev/null
+++ b/04-data-agent/reports/three-run-analysis-20260917/by_difficulty.csv
@@ -0,0 +1,103 @@
+run,step,difficulty,cells,score,model_calls,tool_calls,tool_calls_median,tool_calls_p90,repeats,repeat_fraction,repeat_rollout_fraction,short_tool_fraction,budget_fraction,python_fraction,answer_path_fraction,tool_error_text_fraction,submission_fraction,submission_observations,trainable_tokens,wall_s,retry_cells,training_rows,packed_tokens
+Harbor OpenCode-only,0,easy,132,0.4015151515151515,11.43939393939394,10.037878787878787,9.0,17.0,0.5378787878787878,0.04280062258003434,0.2803030303030303,0.10606060606060606,0.29545454545454547,0.7272727272727273,0.6287878787878788,0.3939393939393939,0.7777777777777778,36,1347.689393939394,197.37515151515152,0,3.462121212121212,106132.51515151515
+Harbor OpenCode-only,0,hard,396,0.06313131313131314,14.926767676767676,13.972222222222221,16.0,18.0,0.5277777777777778,0.03536039196441673,0.3106060606060606,0.05303030303030303,0.6818181818181818,0.7727272727272727,0.31565656565656564,0.6565656565656566,0.3515625,128,3117.5353535353534,300.3289898989899,5,4.292929292929293,158614.65404040404
+Harbor OpenCode-only,0,medium,472,0.1440677966101695,14.254237288135593,13.572033898305085,16.0,18.0,0.5127118644067796,0.034147748041146085,0.2648305084745763,0.06567796610169492,0.6207627118644068,0.798728813559322,0.3644067796610169,0.6483050847457628,0.3611111111111111,180,2774.4639830508477,273.17495762711866,6,4.163135593220339,141250.06991525425
+Harbor OpenCode-only,100,easy,132,0.5,13.840909090909092,13.93939393939394,17.0,18.0,3.696969696969697,0.23140387778516355,0.6742424242424242,0.0,0.6287878787878788,0.9696969696969697,0.8484848484848485,0.3939393939393939,0.7777777777777778,36,1520.2727272727273,160.64651515151516,0,4.424242424242424,135707.05303030304
+Harbor OpenCode-only,100,hard,396,0.10353535353535354,16.373737373737374,16.946969696969695,17.0,19.0,1.893939393939394,0.11147086153407622,0.5580808080808081,0.0,0.8888888888888888,0.9646464646464646,0.40404040404040403,0.7727272727272727,0.421875,128,3529.4570707070707,302.15959595959595,10,5.108585858585859,203949.1590909091
+Harbor OpenCode-only,100,medium,472,0.2923728813559322,15.754237288135593,16.059322033898304,17.0,18.0,2.582627118644068,0.15455961387261238,0.6567796610169492,0.0,0.815677966101695,0.9682203389830508,0.5614406779661016,0.7542372881355932,0.55,180,3074.0254237288136,266.08033898305086,6,5.033898305084746,167436.80508474575
+Harbor OpenCode-only,200,easy,132,0.6212121212121212,14.780303030303031,14.931818181818182,17.0,18.0,2.803030303030303,0.17419060180107185,0.6893939393939394,0.0,0.6666666666666666,0.9696969696969697,0.803030303030303,0.45454545454545453,0.8055555555555556,36,2004.0151515151515,182.47007575757576,1,4.886363636363637,150108.42424242425
+Harbor OpenCode-only,200,hard,396,0.09343434343434344,16.38131313131313,17.23989898989899,17.0,19.0,1.3232323232323233,0.07787139040835915,0.5126262626262627,0.0025252525252525255,0.9015151515151515,0.9292929292929293,0.30808080808080807,0.7297979797979798,0.3515625,128,3795.3055555555557,302.7638383838384,3,5.116161616161616,205488.398989899
+Harbor OpenCode-only,200,medium,472,0.3220338983050847,16.11864406779661,16.593220338983052,17.0,19.0,1.7055084745762712,0.10145528461083442,0.565677966101695,0.0,0.8283898305084746,0.972457627118644,0.5211864406779662,0.7690677966101694,0.5777777777777777,180,3543.3072033898306,281.685593220339,4,5.165254237288136,172365.45974576272
+Harbor OpenCode-only,300,easy,132,0.6439393939393939,15.515151515151516,16.477272727272727,17.0,20.0,3.007575757575758,0.17679510585681427,0.75,0.0,0.7272727272727273,0.9848484848484849,0.8257575757575758,0.49242424242424243,0.8333333333333334,36,2195.8939393939395,209.41825757575756,1,4.96969696969697,161420.0909090909
+Harbor OpenCode-only,300,hard,396,0.12121212121212122,16.43686868686869,18.785353535353536,18.0,22.0,1.1868686868686869,0.06538184289339485,0.5025252525252525,0.007575757575757576,0.9116161616161617,0.9696969696969697,0.2398989898989899,0.7449494949494949,0.2734375,128,3653.313131313131,313.2989141414141,1,5.075757575757576,207065.547979798
+Harbor OpenCode-only,300,medium,472,0.3220338983050847,16.203389830508474,17.76271186440678,17.0,20.0,1.625,0.09379305345879678,0.5550847457627118,0.0,0.8622881355932204,0.9703389830508474,0.4597457627118644,0.7182203389830508,0.4888888888888889,180,3486.5360169491523,300.88974576271187,2,5.25635593220339,183729.25423728814
+Harbor OpenCode-only,400,easy,132,0.7045454545454546,15.340909090909092,16.916666666666668,17.0,20.0,3.6363636363636362,0.20632059927544402,0.803030303030303,0.0,0.7272727272727273,0.9696969696969697,0.8560606060606061,0.38636363636363635,0.8611111111111112,36,1676.3863636363637,180.5655303030303,0,5.128787878787879,149934.71212121213
+Harbor OpenCode-only,400,hard,396,0.12878787878787878,16.414141414141415,19.532828282828284,18.0,25.0,1.505050505050505,0.07031318895954543,0.5025252525252525,0.005050505050505051,0.9166666666666666,0.9823232323232324,0.2727272727272727,0.7348484848484849,0.3125,128,3457.2550505050503,313.50217171717173,5,5.207070707070707,201173.5883838384
+Harbor OpenCode-only,400,medium,472,0.3898305084745763,16.180084745762713,17.372881355932204,18.0,20.0,2.01271186440678,0.11542638188868691,0.6186440677966102,0.0,0.8538135593220338,0.9851694915254238,0.5741525423728814,0.6927966101694916,0.6111111111111112,180,2831.968220338983,271.41398305084743,0,5.298728813559322,180928.74788135593
+Harbor OpenCode-only,500,easy,132,0.7121212121212122,14.734848484848484,15.083333333333334,17.0,18.0,2.992424242424242,0.18970042854436325,0.7727272727272727,0.0,0.6439393939393939,1.0,0.8560606060606061,0.3333333333333333,0.7777777777777778,36,1893.530303030303,183.36272727272728,1,4.795454545454546,142063.48484848486
+Harbor OpenCode-only,500,hard,396,0.14393939393939395,16.38131313131313,17.42929292929293,17.0,20.0,0.9494949494949495,0.055124705453869986,0.4015151515151515,0.005050505050505051,0.8964646464646465,0.9848484848484849,0.2803030303030303,0.7575757575757576,0.328125,128,3985.656565656566,333.39484848484847,2,5.202020202020202,205821.90151515152
+Harbor OpenCode-only,500,medium,472,0.3792372881355932,15.921610169491526,16.569915254237287,17.0,19.0,1.6864406779661016,0.1010791847098716,0.5508474576271186,0.0,0.809322033898305,0.9936440677966102,0.5720338983050848,0.722457627118644,0.6,180,3351.6271186440677,287.58572033898304,2,5.156779661016949,172525.14194915254
+Harbor OpenCode-only,600,easy,132,0.6666666666666666,14.901515151515152,15.696969696969697,17.0,19.900000000000006,3.659090909090909,0.21904873611844314,0.803030303030303,0.0,0.6060606060606061,0.9848484848484849,0.8409090909090909,0.3409090909090909,0.8055555555555556,36,1570.2348484848485,179.39204545454547,0,4.9393939393939394,147863.96212121213
+Harbor OpenCode-only,600,hard,396,0.13636363636363635,16.17676767676768,17.94191919191919,17.0,21.0,1.22979797979798,0.06901818387853778,0.4595959595959596,0.0,0.8686868686868687,0.9696969696969697,0.35353535353535354,0.6944444444444444,0.375,128,3186.2954545454545,319.99123737373736,0,5.361111111111111,212746.1792929293
+Harbor OpenCode-only,600,medium,472,0.3983050847457627,15.766949152542374,16.54449152542373,17.0,19.0,2.315677966101695,0.13637707702339463,0.6292372881355932,0.0,0.7648305084745762,0.972457627118644,0.6292372881355932,0.6546610169491526,0.6444444444444445,180,2482.741525423729,266.1048516949153,1,5.173728813559322,168450.36016949153
+Harbor OpenCode-only,700,easy,132,0.7121212121212122,12.757575757575758,14.977272727272727,13.0,21.0,2.606060606060606,0.17263439056620186,0.7272727272727273,0.0,0.3409090909090909,0.9772727272727273,0.8787878787878788,0.32575757575757575,0.9166666666666666,36,1731.909090909091,172.15318181818182,0,4.583333333333333,123597.16666666667
+Harbor OpenCode-only,700,hard,396,0.2297979797979798,15.242424242424242,18.343434343434343,17.0,25.0,1.8333333333333333,0.09107917556386938,0.5858585858585859,0.0025252525252525255,0.6944444444444444,0.9772727272727273,0.4595959595959596,0.7424242424242424,0.5234375,128,3642.497474747475,332.80845959595956,1,6.05050505050505,232909.63636363635
+Harbor OpenCode-only,700,medium,472,0.4449152542372881,14.260593220338983,15.633474576271187,17.0,20.0,2.1652542372881354,0.1307367474754911,0.635593220338983,0.0,0.5169491525423728,0.9936440677966102,0.7288135593220338,0.652542372881356,0.7611111111111111,180,2623.4004237288136,258.2209957627119,3,5.538135593220339,165609.76059322033
+Harbor OpenCode-only,800,easy,132,0.6590909090909091,11.303030303030303,10.659090909090908,10.0,17.0,1.4545454545454546,0.1265268468019664,0.6590909090909091,0.022727272727272728,0.19696969696969696,0.8257575757575758,0.8181818181818182,0.26515151515151514,0.8055555555555556,36,2056.9772727272725,177.76227272727272,5,4.234848484848484,105009.15151515152
+Harbor OpenCode-only,800,hard,396,0.17424242424242425,15.166666666666666,15.522727272727273,17.0,18.5,1.0176767676767677,0.07053617680409899,0.5404040404040404,0.005050505050505051,0.6843434343434344,0.946969696969697,0.4015151515151515,0.7424242424242424,0.453125,128,4003.3358585858587,325.8557323232323,13,6.898989898989899,246732.48484848486
+Harbor OpenCode-only,800,medium,472,0.3707627118644068,13.542372881355933,13.59322033898305,15.0,18.0,1.0169491525423728,0.07452813333401831,0.527542372881356,0.012711864406779662,0.4576271186440678,0.934322033898305,0.652542372881356,0.5783898305084746,0.6944444444444444,180,2984.476694915254,253.6126906779661,2,6.095338983050848,174970.31779661018
+Harbor OpenCode-only,900,easy,132,0.6363636363636364,13.265151515151516,14.045454545454545,14.0,19.900000000000006,2.5606060606060606,0.18899229005741108,0.7954545454545454,0.0,0.38636363636363635,0.8863636363636364,0.7954545454545454,0.38636363636363635,0.8055555555555556,36,2139.780303030303,212.3985606060606,1,5.0227272727272725,134276.99242424243
+Harbor OpenCode-only,900,hard,396,0.15151515151515152,15.502525252525253,17.80808080808081,17.0,23.0,1.5833333333333333,0.09042994213003926,0.6237373737373737,0.0,0.7474747474747475,0.946969696969697,0.3939393939393939,0.7171717171717171,0.3125,128,3725.429292929293,331.87939393939394,1,6.709595959595959,248034.23484848486
+Harbor OpenCode-only,900,medium,472,0.3220338983050847,14.273305084745763,15.694915254237289,17.0,21.0,1.951271186440678,0.12628037266993286,0.6949152542372882,0.00211864406779661,0.5614406779661016,0.9216101694915254,0.6101694915254238,0.6652542372881356,0.5,180,2919.612288135593,283.6933686440678,1,5.930084745762712,189920.5529661017
+Harbor OpenCode-only,1000,easy,132,0.5909090909090909,13.795454545454545,18.939393939393938,17.0,28.900000000000006,3.5681818181818183,0.16730068526042882,0.8106060606060606,0.0,0.5151515151515151,0.8863636363636364,0.7954545454545454,0.5303030303030303,0.8055555555555556,36,2469.8257575757575,220.0061363636364,0,4.75,134870.32575757575
+Harbor OpenCode-only,1000,hard,396,0.10858585858585859,15.914141414141413,21.994949494949495,21.0,31.0,2.6944444444444446,0.10768922316242634,0.73989898989899,0.0,0.803030303030303,0.9419191919191919,0.42676767676767674,0.7702020202020202,0.2265625,128,3927.467171717172,330.1743434343434,1,5.628787878787879,210299.43181818182
+Harbor OpenCode-only,1000,medium,472,0.3029661016949153,15.14406779661017,20.66949152542373,20.0,29.0,3.1504237288135593,0.13107595833889285,0.7754237288135594,0.0,0.6694915254237288,0.9639830508474576,0.6101694915254238,0.7139830508474576,0.45555555555555555,180,3224.561440677966,283.37841101694914,1,5.351694915254237,172530.56355932204
+Harbor multi-harness,0,easy,132,0.4015151515151515,11.43939393939394,10.037878787878787,9.0,17.0,0.5378787878787878,0.04280062258003434,0.2803030303030303,0.10606060606060606,0.29545454545454547,0.7272727272727273,0.6287878787878788,0.3939393939393939,0.7777777777777778,36,1347.689393939394,197.37515151515152,0,3.462121212121212,106132.51515151515
+Harbor multi-harness,0,hard,396,0.06313131313131314,14.926767676767676,13.972222222222221,16.0,18.0,0.5277777777777778,0.03536039196441673,0.3106060606060606,0.05303030303030303,0.6818181818181818,0.7727272727272727,0.31565656565656564,0.6565656565656566,0.3515625,128,3117.5353535353534,300.3289898989899,5,4.292929292929293,158614.65404040404
+Harbor multi-harness,0,medium,472,0.1440677966101695,14.254237288135593,13.572033898305085,16.0,18.0,0.5127118644067796,0.034147748041146085,0.2648305084745763,0.06567796610169492,0.6207627118644068,0.798728813559322,0.3644067796610169,0.6483050847457628,0.3611111111111111,180,2774.4639830508477,273.17495762711866,6,4.163135593220339,141250.06991525425
+Harbor multi-harness,100,easy,132,0.5303030303030303,15.803030303030303,15.393939393939394,16.0,17.0,4.803030303030303,0.3027108631495693,0.8106060606060606,0.007575757575757576,0.7803030303030303,0.8560606060606061,0.803030303030303,0.4393939393939394,0.7777777777777778,36,1823.6363636363637,200.79045454545454,5,4.871212121212121,157866.45454545456
+Harbor multi-harness,100,hard,396,0.08585858585858586,16.515151515151516,16.11111111111111,17.0,17.0,1.7575757575757576,0.1072761546011851,0.5151515151515151,0.007575757575757576,0.9065656565656566,0.8282828282828283,0.2474747474747475,0.6818181818181818,0.2734375,128,3185.2954545454545,301.60459595959594,15,5.005050505050505,198126.00252525252
+Harbor multi-harness,100,medium,472,0.3050847457627119,16.345338983050848,15.944915254237289,16.5,17.0,3.0487288135593222,0.18845578778813613,0.6207627118644068,0.0,0.8707627118644068,0.9046610169491526,0.5063559322033898,0.6610169491525424,0.5444444444444444,180,2867.959745762712,279.280593220339,24,5.059322033898305,181082.50635593222
+Harbor multi-harness,200,easy,132,0.5833333333333334,16.022727272727273,15.651515151515152,17.0,17.0,5.409090909090909,0.3318122101210337,0.8333333333333334,0.007575757575757576,0.8333333333333334,0.8939393939393939,0.8106060606060606,0.4318181818181818,0.8333333333333334,36,1668.3181818181818,189.73189393939396,5,5.136363636363637,163443.44696969696
+Harbor multi-harness,200,hard,396,0.1111111111111111,16.70959595959596,16.232323232323232,17.0,17.0,2.073232323232323,0.12704229163123362,0.5984848484848485,0.0025252525252525255,0.9318181818181818,0.8661616161616161,0.27525252525252525,0.6666666666666666,0.34375,128,3032.2676767676767,285.23492424242426,38,4.9646464646464645,197455.54545454544
+Harbor multi-harness,200,medium,472,0.3008474576271186,16.665254237288135,16.194915254237287,17.0,17.0,3.2245762711864407,0.19659043545689528,0.6122881355932204,0.00211864406779661,0.9216101694915254,0.934322033898305,0.4978813559322034,0.6991525423728814,0.5277777777777778,180,2803.1228813559323,274.7260805084746,33,5.0826271186440675,168070.72033898305
+Harbor multi-harness,300,easy,132,0.6439393939393939,15.659090909090908,15.272727272727273,17.0,17.0,6.045454545454546,0.3819660496465309,0.8560606060606061,0.0,0.7878787878787878,0.7803030303030303,0.8560606060606061,0.3333333333333333,0.8333333333333334,36,1396.8333333333333,175.7655303030303,3,4.96969696969697,155371.10606060605
+Harbor multi-harness,300,hard,396,0.11616161616161616,16.535353535353536,16.106060606060606,17.0,17.0,2.6313131313131315,0.16167301953726573,0.6666666666666666,0.007575757575757576,0.9141414141414141,0.8106060606060606,0.3282828282828283,0.6616161616161617,0.3671875,128,2608.436868686869,279.7158585858586,13,4.843434343434343,188384.80555555556
+Harbor multi-harness,300,medium,472,0.3283898305084746,16.48728813559322,16.08686440677966,17.0,17.0,3.8262711864406778,0.235899959494308,0.711864406779661,0.0,0.8877118644067796,0.8919491525423728,0.5360169491525424,0.6419491525423728,0.5666666666666667,180,2191.4300847457625,243.57158898305084,13,5.091101694915254,168238.16313559323
+Harbor multi-harness,400,easy,132,0.7348484848484849,16.386363636363637,15.992424242424242,17.0,17.0,7.424242424242424,0.45267999417397814,0.9090909090909091,0.0,0.8939393939393939,0.8787878787878788,0.8863636363636364,0.2878787878787879,0.8611111111111112,36,1567.7045454545455,176.5731818181818,4,5.068181818181818,155643.80303030304
+Harbor multi-harness,400,hard,396,0.12373737373737374,16.613636363636363,16.11868686868687,17.0,17.0,2.121212121212121,0.13048737587566275,0.6212121212121212,0.005050505050505051,0.9267676767676768,0.8131313131313131,0.2398989898989899,0.6666666666666666,0.3359375,128,3232.558080808081,306.61883838383835,27,4.946969696969697,208357.90151515152
+Harbor multi-harness,400,medium,472,0.3961864406779661,16.656779661016948,16.22669491525424,17.0,17.0,3.7436440677966103,0.22888539848597175,0.7033898305084746,0.0,0.9067796610169492,0.913135593220339,0.5550847457627118,0.7033898305084746,0.5888888888888889,180,2700.843220338983,269.51296610169493,13,5.120762711864407,175150.1843220339
+Harbor multi-harness,500,easy,132,0.7272727272727273,14.946969696969697,14.553030303030303,16.0,17.0,6.393939393939394,0.3937819378828737,0.7878787878787878,0.0,0.7424242424242424,0.7803030303030303,0.8787878787878788,0.3560606060606061,0.8888888888888888,36,1651.9015151515152,179.44037878787879,7,5.166666666666667,154823.67424242425
+Harbor multi-harness,500,hard,396,0.16414141414141414,16.454545454545453,15.909090909090908,16.0,17.0,2.1565656565656566,0.13296133201641458,0.6035353535353535,0.0,0.8838383838383839,0.8409090909090909,0.2904040404040404,0.7196969696969697,0.3515625,128,3444.967171717172,320.602601010101,23,5.005050505050505,203945.94191919192
+Harbor multi-harness,500,medium,472,0.4427966101694915,16.04237288135593,15.595338983050848,16.0,17.0,3.7415254237288136,0.22843953201933537,0.652542372881356,0.0,0.8283898305084746,0.8983050847457628,0.6016949152542372,0.6652542372881356,0.6222222222222222,180,2728.8792372881358,264.19088983050847,35,5.148305084745763,177085.14406779662
+Harbor multi-harness,600,easy,132,0.7272727272727273,15.106060606060606,14.659090909090908,16.0,17.0,5.598484848484849,0.3503047828582588,0.7954545454545454,0.0,0.6893939393939394,0.7803030303030303,0.8787878787878788,0.30303030303030304,0.8333333333333334,36,2073.5757575757575,198.59560606060606,7,5.128787878787879,150598.77272727274
+Harbor multi-harness,600,hard,396,0.11616161616161616,16.244949494949495,15.55050505050505,16.0,17.0,2.823232323232323,0.1767592715626654,0.75,0.0025252525252525255,0.8282828282828283,0.73989898989899,0.255050505050505,0.5883838383838383,0.3359375,128,3866.5833333333335,324.36267676767676,36,5.0353535353535355,197541.06313131313
+Harbor multi-harness,600,medium,472,0.3728813559322034,16.158898305084747,15.588983050847459,16.0,17.0,3.4978813559322033,0.21541417257886272,0.7182203389830508,0.00211864406779661,0.8199152542372882,0.8538135593220338,0.5423728813559322,0.6271186440677966,0.5888888888888889,180,3292.677966101695,285.78447033898306,40,5.2690677966101696,189113.47033898305
+Harbor multi-harness,684,easy,132,0.7575757575757576,14.545454545454545,13.931818181818182,16.0,17.0,5.5606060606060606,0.3535762100100336,0.7575757575757576,0.0,0.6515151515151515,0.7348484848484849,0.8560606060606061,0.25757575757575757,0.8333333333333334,36,1858.0984848484848,203.4105303030303,7,4.871212121212121,158956.40151515152
+Harbor multi-harness,684,hard,396,0.13131313131313133,15.691919191919192,15.030303030303031,16.0,17.0,3.275252525252525,0.21139097649118524,0.8131313131313131,0.007575757575757576,0.7803030303030303,0.7095959595959596,0.29797979797979796,0.6212121212121212,0.390625,128,3455.123737373737,312.76631313131315,29,4.896464646464646,211740.76515151514
+Harbor multi-harness,684,medium,472,0.3580508474576271,15.802966101694915,15.292372881355933,16.0,17.0,4.002118644067797,0.24882787577606083,0.7902542372881356,0.0,0.7648305084745762,0.8813559322033898,0.5635593220338984,0.6080508474576272,0.5722222222222222,180,2715.548728813559,265.1866525423729,29,5.129237288135593,190155.84533898305
+Harbor multi-harness,700,easy,132,0.6060606060606061,14.416666666666666,13.856060606060606,16.0,17.0,5.265151515151516,0.33007927591817965,0.7954545454545454,0.022727272727272728,0.6136363636363636,0.6742424242424242,0.7878787878787878,0.2803030303030303,0.8333333333333334,36,1970.878787878788,199.6037878787879,8,4.954545454545454,163062.37121212122
+Harbor multi-harness,700,hard,396,0.12121212121212122,15.79040404040404,15.244949494949495,16.0,17.0,3.111111111111111,0.19627702269923633,0.7929292929292929,0.007575757575757576,0.8181818181818182,0.6742424242424242,0.2702020202020202,0.5934343434343434,0.3359375,128,3476.1111111111113,308.2049494949495,24,4.909090909090909,216072.91919191918
+Harbor multi-harness,700,medium,472,0.3389830508474576,15.434322033898304,14.896186440677965,16.0,17.0,3.7012711864406778,0.23418002384663258,0.8029661016949152,0.006355932203389831,0.7266949152542372,0.8199152542372882,0.5254237288135594,0.559322033898305,0.55,180,2776.2669491525426,264.39697033898307,30,4.961864406779661,178243.32627118644
+Harbor multi-harness,800,easy,132,0.5909090909090909,15.545454545454545,14.795454545454545,16.0,17.0,5.242424242424242,0.32816583854351233,0.8181818181818182,0.007575757575757576,0.6893939393939394,0.7727272727272727,0.7348484848484849,0.32575757575757575,0.7777777777777778,36,2645.75,266.72719696969693,16,5.204545454545454,173656.93181818182
+Harbor multi-harness,800,hard,396,0.09595959595959595,16.01010101010101,15.26010101010101,16.0,17.0,3.101010101010101,0.2001530884666615,0.8560606060606061,0.007575757575757576,0.7727272727272727,0.6388888888888888,0.15656565656565657,0.4797979797979798,0.1953125,128,4195.964646464647,353.6538636363636,40,4.7272727272727275,204053.08333333334
+Harbor multi-harness,800,medium,472,0.326271186440678,15.96822033898305,15.28177966101695,16.0,17.0,3.722457627118644,0.23435742060787976,0.836864406779661,0.01059322033898305,0.7669491525423728,0.8177966101694916,0.4364406779661017,0.5911016949152542,0.45555555555555555,180,3643.9618644067796,319.04599576271187,51,5.065677966101695,183445.52966101695
+Harbor multi-harness,900,easy,132,0.4621212121212121,10.113636363636363,9.090909090909092,8.0,17.0,3.5757575757575757,0.2309408079945302,0.49242424242424243,0.26515151515151514,0.3333333333333333,0.5606060606060606,0.5909090909090909,0.1590909090909091,0.5833333333333334,36,4353.030303030303,268.02113636363634,12,5.151515151515151,168218.24242424243
+Harbor multi-harness,900,hard,396,0.10858585858585859,10.63888888888889,9.542929292929292,10.0,17.0,1.904040404040404,0.13378210546646188,0.48737373737373735,0.25252525252525254,0.3813131313131313,0.48484848484848486,0.1691919191919192,0.4065656565656566,0.25,128,5734.916666666667,351.60782828282834,82,4.694444444444445,192182.9116161616
+Harbor multi-harness,900,medium,472,0.2605932203389831,10.773305084745763,9.88771186440678,10.5,17.0,2.36228813559322,0.156900177786103,0.4894067796610169,0.2478813559322034,0.4173728813559322,0.6101694915254238,0.3686440677966102,0.3961864406779661,0.36666666666666664,180,5214.561440677966,315.79783898305084,55,5.088983050847458,186332.98093220338
+Harbor multi-harness,1000,easy,132,0.42424242424242425,8.946969696969697,7.863636363636363,7.0,16.0,2.7803030303030303,0.21633946057408626,0.5151515151515151,0.20454545454545456,0.19696969696969696,0.5833333333333334,0.6212121212121212,0.21212121212121213,0.5833333333333334,36,4066.7651515151515,234.94992424242423,10,4.151515151515151,128018.90151515152
+Harbor multi-harness,1000,hard,396,0.13383838383838384,10.234848484848484,8.977272727272727,9.0,17.0,1.803030303030303,0.1410369605606612,0.5176767676767676,0.2474747474747475,0.3005050505050505,0.5151515151515151,0.23737373737373738,0.41919191919191917,0.2578125,128,5776.79797979798,344.4250757575757,115,4.3232323232323235,165884.72474747474
+Harbor multi-harness,1000,medium,472,0.326271186440678,10.351694915254237,9.375,9.0,17.0,2.3792372881355934,0.169109094771335,0.4957627118644068,0.2033898305084746,0.2860169491525424,0.6673728813559322,0.4788135593220339,0.461864406779661,0.4888888888888889,180,5047.576271186441,290.01902542372886,80,4.6440677966101696,150476.43008474575
+Native OpenCode,0,easy,132,0.3787878787878788,11.878787878787879,11.409090909090908,11.0,17.0,0.7424242424242424,0.05489180321716207,0.3787878787878788,0.022727272727272728,0.32575757575757575,0.7727272727272727,0.6515151515151515,0.42424242424242425,0.6944444444444444,36,1456.3636363636363,189.78530303030303,108,4.151515151515151,130537.18939393939
+Native OpenCode,0,hard,396,0.06060606060606061,15.222222222222221,15.219696969696969,17.0,18.0,0.6262626262626263,0.03914278073038497,0.3484848484848485,0.005050505050505051,0.7222222222222222,0.8207070707070707,0.30808080808080807,0.7070707070707071,0.375,128,3037.590909090909,285.3653787878788,352,4.275252525252525,169716.2095959596
+Native OpenCode,0,medium,472,0.18008474576271186,14.114406779661017,13.845338983050848,16.0,17.0,0.6610169491525424,0.04324850105837735,0.3432203389830508,0.006355932203389831,0.6165254237288136,0.8516949152542372,0.4343220338983051,0.684322033898305,0.4388888888888889,180,2670.188559322034,257.00951271186443,370,4.213983050847458,140378.36652542374
+Native OpenCode,100,easy,132,0.44696969696969696,10.606060606060606,9.840909090909092,9.0,17.0,0.3409090909090909,0.02672873489261049,0.22727272727272727,0.022727272727272728,0.25757575757575757,0.8863636363636364,0.7803030303030303,0.3560606060606061,0.75,36,1582.1969696969697,112.94318181818181,1,3.5606060606060606,95629.91666666667
+Native OpenCode,100,hard,396,0.08080808080808081,14.883838383838384,14.717171717171718,17.0,18.0,0.6085858585858586,0.03662273736687111,0.32575757575757575,0.0025252525252525255,0.6994949494949495,0.8888888888888888,0.35353535353535354,0.7474747474747475,0.3984375,128,3246.669191919192,142.5600505050505,0,4.585858585858586,184907.19444444444
+Native OpenCode,100,medium,472,0.2245762711864407,13.788135593220339,13.328389830508474,16.0,17.0,0.4322033898305085,0.02840055160326255,0.2754237288135593,0.00423728813559322,0.5677966101694916,0.9491525423728814,0.5190677966101694,0.7266949152542372,0.5444444444444444,180,2862.802966101695,132.84074152542374,0,4.396186440677966,146513.50211864407
+Native OpenCode,200,easy,132,0.5984848484848485,9.727272727272727,8.727272727272727,7.0,16.0,0.2727272727272727,0.020509533544292906,0.16666666666666666,0.015151515151515152,0.1893939393939394,0.9015151515151515,0.9015151515151515,0.3712121212121212,0.8888888888888888,36,1205.1060606060605,104.05166666666666,0,3.5303030303030303,96414.14393939394
+Native OpenCode,200,hard,396,0.06565656565656566,14.906565656565656,14.666666666666666,17.0,17.0,0.6464646464646465,0.03929988557064268,0.3055555555555556,0.0025252525252525255,0.7247474747474747,0.9419191919191919,0.35858585858585856,0.8282828282828283,0.3515625,128,3278.257575757576,145.20815656565657,0,4.578282828282828,180455.54292929292
+Native OpenCode,200,medium,472,0.2457627118644068,12.595338983050848,12.025423728813559,13.0,17.0,0.461864406779661,0.031698056281479994,0.24152542372881355,0.00211864406779661,0.4576271186440678,0.9661016949152542,0.6101694915254238,0.7033898305084746,0.6666666666666666,180,2361.6228813559323,118.78567796610169,0,4.226694915254237,132614.3029661017
+Native OpenCode,300,easy,132,0.5151515151515151,10.219696969696969,9.083333333333334,7.0,17.0,0.5833333333333334,0.0475081013015505,0.30303030303030304,0.0,0.2196969696969697,0.9393939393939394,0.8257575757575758,0.3409090909090909,0.7777777777777778,36,1203.8484848484848,101.6919696969697,0,3.3636363636363638,79576.4696969697
+Native OpenCode,300,hard,396,0.06818181818181818,15.330808080808081,15.093434343434344,17.0,17.0,0.5353535353535354,0.03276041565837747,0.30303030303030304,0.0,0.76010101010101,0.952020202020202,0.2828282828282828,0.8080808080808081,0.296875,128,3543.3661616161617,142.15734848484848,3,4.646464646464646,184838.16666666666
+Native OpenCode,300,medium,472,0.2563559322033898,13.705508474576272,13.120762711864407,16.0,17.0,0.5190677966101694,0.034250818389192736,0.3156779661016949,0.00211864406779661,0.5677966101694916,0.9703389830508474,0.527542372881356,0.7563559322033898,0.5111111111111111,180,2723.3707627118642,122.8614406779661,3,4.122881355932203,126944.3813559322
+Native OpenCode,400,easy,132,0.5909090909090909,10.401515151515152,9.174242424242424,8.0,17.0,0.4696969696969697,0.03480055978384855,0.26515151515151514,0.0,0.26515151515151514,0.9696969696969697,0.8560606060606061,0.49242424242424243,0.8055555555555556,36,1231.7651515151515,100.40598484848485,0,3.492424242424242,70788.75757575757
+Native OpenCode,400,hard,396,0.11868686868686869,15.244949494949495,14.883838383838384,17.0,17.0,0.5833333333333334,0.035786410478865216,0.33080808080808083,0.0025252525252525255,0.7348484848484849,0.9494949494949495,0.3207070707070707,0.797979797979798,0.4140625,128,3534.520202020202,130.27315656565656,0,4.590909090909091,161968.89646464647
+Native OpenCode,400,medium,472,0.2944915254237288,13.264830508474576,12.629237288135593,15.0,17.0,0.5338983050847458,0.034969736066117973,0.2796610169491525,0.0,0.5127118644067796,0.9766949152542372,0.5783898305084746,0.7563559322033898,0.6,180,2593.9025423728813,116.06038135593221,0,4.188559322033898,118681.89406779662
+Native OpenCode,500,easy,132,0.5303030303030303,10.128787878787879,8.651515151515152,7.0,17.0,0.5681818181818182,0.04908322948095676,0.3106060606060606,0.022727272727272728,0.21212121212121213,0.9696969696969697,0.7878787878787878,0.3712121212121212,0.7222222222222222,36,1078.0984848484848,102.65560606060605,0,3.727272727272727,79580.84090909091
+Native OpenCode,500,hard,396,0.08080808080808081,14.828282828282829,14.282828282828282,17.0,17.0,0.9191919191919192,0.06007285763624304,0.3686868686868687,0.0,0.6868686868686869,0.9621212121212122,0.3409090909090909,0.7777777777777778,0.390625,128,3083.530303030303,138.30416666666665,2,4.792929292929293,176411.02777777778
+Native OpenCode,500,medium,472,0.2733050847457627,13.139830508474576,12.209745762711865,15.0,17.0,0.8283898305084746,0.05650738937064172,0.3538135593220339,0.00423728813559322,0.5148305084745762,0.9766949152542372,0.5677966101694916,0.6970338983050848,0.6,180,2329.8072033898306,121.81180084745762,1,4.546610169491525,122263.01694915254
+Native OpenCode,600,easy,132,0.5606060606060606,10.333333333333334,8.681818181818182,7.0,17.0,1.0,0.08159403047437806,0.3712121212121212,0.030303030303030304,0.24242424242424243,0.9318181818181818,0.8106060606060606,0.3560606060606061,0.75,36,1129.7424242424242,103.59560606060607,2,4.295454545454546,93592.34848484848
+Native OpenCode,600,hard,396,0.10353535353535354,14.224747474747474,13.416666666666666,16.0,17.0,1.2601010101010102,0.08298572439234086,0.44696969696969696,0.0025252525252525255,0.6287878787878788,0.9343434343434344,0.43686868686868685,0.7525252525252525,0.4453125,128,3258.3535353535353,134.60962121212123,6,5.0227272727272725,174275.86363636365
+Native OpenCode,600,medium,472,0.288135593220339,12.39406779661017,11.358050847457626,12.0,17.0,1.1716101694915255,0.08242081922307692,0.4067796610169492,0.006355932203389831,0.4258474576271186,0.9766949152542372,0.6377118644067796,0.6716101694915254,0.6611111111111111,180,2190.1991525423728,115.10735169491525,6,4.447033898305085,111872.16949152542
+Native OpenCode,700,easy,132,0.49242424242424243,10.25,8.893939393939394,7.0,17.0,0.7121212121212122,0.061389237742847366,0.3409090909090909,0.022727272727272728,0.2878787878787879,0.9545454545454546,0.8181818181818182,0.3560606060606061,0.75,36,1008.5,100.61128787878788,0,3.553030303030303,77474.76515151515
+Native OpenCode,700,hard,396,0.07828282828282829,14.462121212121213,13.717171717171718,16.0,17.0,0.8560606060606061,0.05696271237744406,0.36363636363636365,0.005050505050505051,0.6666666666666666,0.9671717171717171,0.4444444444444444,0.7954545454545454,0.40625,128,3196.368686868687,135.29641414141415,0,4.547979797979798,169050.83080808082
+Native OpenCode,700,medium,472,0.288135593220339,12.182203389830509,11.14406779661017,11.5,17.0,0.6419491525423728,0.045274267663924346,0.3093220338983051,0.00847457627118644,0.4385593220338983,0.9745762711864406,0.6779661016949152,0.6588983050847458,0.6944444444444444,180,2100.697033898305,114.62976694915254,0,4.252118644067797,113986.72881355933
+Native OpenCode,800,easy,132,0.5227272727272727,9.371212121212121,8.143939393939394,7.0,17.0,0.25757575757575757,0.026358094272532776,0.20454545454545456,0.015151515151515152,0.19696969696969696,0.9621212121212122,0.8636363636363636,0.4318181818181818,0.8055555555555556,36,1181.0984848484848,101.63969696969697,0,3.5757575757575757,78131.18939393939
+Native OpenCode,800,hard,396,0.11868686868686869,14.227272727272727,13.54040404040404,16.0,17.0,0.5353535353535354,0.03464554153086145,0.31565656565656564,0.005050505050505051,0.6186868686868687,0.9747474747474747,0.4671717171717172,0.7929292929292929,0.5234375,128,3752.9747474747473,140.07709595959597,0,4.51010101010101,178000.86363636365
+Native OpenCode,800,medium,472,0.2966101694915254,12.10593220338983,11.194915254237289,11.5,17.0,0.3559322033898305,0.02621620604292654,0.211864406779661,0.014830508474576272,0.4300847457627119,0.9766949152542372,0.6610169491525424,0.6800847457627118,0.6666666666666666,180,2456.8008474576272,117.00741525423729,0,4.1059322033898304,117600.02754237287
+Native OpenCode,900,easy,132,0.5,9.704545454545455,8.25,7.0,17.0,0.4696969696969697,0.0431211174693795,0.25,0.06060606060606061,0.21212121212121213,0.9015151515151515,0.8181818181818182,0.4015151515151515,0.75,36,1190.0378787878788,101.35371212121213,0,3.4318181818181817,83409.48484848485
+Native OpenCode,900,hard,396,0.1111111111111111,13.669191919191919,12.984848484848484,16.0,17.0,0.5025252525252525,0.03297950094290041,0.2727272727272727,0.015151515151515152,0.5984848484848485,0.9570707070707071,0.4595959595959596,0.7474747474747475,0.46875,128,3514.742424242424,134.26853535353536,2,4.290404040404041,170370.898989899
+Native OpenCode,900,medium,472,0.3029661016949153,12.171610169491526,11.226694915254237,12.0,17.0,0.5021186440677966,0.03788172352323649,0.288135593220339,0.019067796610169493,0.4110169491525424,0.9639830508474576,0.6504237288135594,0.690677966101695,0.7,180,2557.326271186441,118.18383474576271,1,4.120762711864407,122658.96398305085
+Native OpenCode,1000,easy,132,0.5984848484848485,10.113636363636363,8.666666666666666,7.0,17.0,0.696969696969697,0.06439452122941428,0.36363636363636365,0.0,0.21212121212121213,0.9772727272727273,0.9393939393939394,0.45454545454545453,0.8611111111111112,36,896.0833333333334,101.40204545454546,0,4.871212121212121,101680.84848484848
+Native OpenCode,1000,hard,396,0.12626262626262627,14.1489898989899,13.469696969696969,16.0,17.0,0.8207070707070707,0.06018674457845581,0.41414141414141414,0.007575757575757576,0.5909090909090909,0.9646464646464646,0.5505050505050505,0.8484848484848485,0.5859375,128,3492.89898989899,144.32358585858586,0,7.53030303030303,228936.2702020202
+Native OpenCode,1000,medium,472,0.3580508474576271,12.540254237288135,11.63135593220339,12.0,17.0,0.7521186440677966,0.059763516115020815,0.3665254237288136,0.00211864406779661,0.4173728813559322,0.9915254237288136,0.7669491525423728,0.7584745762711864,0.7722222222222223,180,2242.133474576271,119.11391949152542,0,6.923728813559322,172306.16313559323
diff --git a/04-data-agent/reports/three-run-analysis-20260917/by_harness.csv b/04-data-agent/reports/three-run-analysis-20260917/by_harness.csv
new file mode 100644
index 0000000..50de53b
--- /dev/null
+++ b/04-data-agent/reports/three-run-analysis-20260917/by_harness.csv
@@ -0,0 +1,137 @@
+run,step,harness,cells,score,model_calls,tool_calls,tool_calls_median,tool_calls_p90,repeats,repeat_fraction,repeat_rollout_fraction,short_tool_fraction,budget_fraction,python_fraction,answer_path_fraction,tool_error_text_fraction,submission_fraction,submission_observations,trainable_tokens,wall_s,retry_cells,training_rows,packed_tokens
+Harbor OpenCode-only,0,claude-code,250,0.168,13.248,12.696,15.0,18.0,0.552,0.03765119948518259,0.232,0.032,0.52,0.768,0.44,0.664,0.4883720930232558,86,3510.776,306.70408000000003,4,12.9,505615.996
+Harbor OpenCode-only,0,codex,250,0.164,14.56,14.244,17.0,19.099999999999994,0.644,0.04206620581633926,0.34,0.08,0.632,0.756,0.376,0.576,0.3953488372093023,86,3011.892,288.04967999999997,0,1.124,19415.248
+Harbor OpenCode-only,0,mini-swe-agent,250,0.144,16.028,15.152,17.0,17.0,0.404,0.026998038725979903,0.276,0.04,0.808,0.844,0.316,0.636,0.3372093023255814,86,2479.664,227.70248,4,1.248,17148.996
+Harbor OpenCode-only,0,opencode,250,0.108,12.76,10.964,12.5,17.099999999999994,0.488,0.03652784315559323,0.292,0.112,0.448,0.748,0.388,0.596,0.38372093023255816,86,1885.612,273.23328000000004,3,1.216,31783.472
+Harbor OpenCode-only,100,claude-code,250,0.252,15.408,15.812,17.0,19.0,2.724,0.16640826366412326,0.644,0.0,0.768,0.936,0.624,0.768,0.5930232558139535,86,3641.972,286.32432,1,15.348,608059.336
+Harbor OpenCode-only,100,codex,250,0.304,15.496,16.648,17.0,20.0,2.544,0.15182294032879173,0.636,0.0,0.796,0.96,0.536,0.704,0.5348837209302325,86,3298.592,285.90372,2,1.076,17268.192
+Harbor OpenCode-only,100,mini-swe-agent,250,0.204,15.156,15.084,17.0,17.0,0.824,0.05239198056845116,0.404,0.0,0.724,0.976,0.392,0.644,0.38372093023255816,86,2140.036,212.45007999999999,11,1.088,12346.168
+Harbor OpenCode-only,100,opencode,250,0.22,16.928,16.98,17.0,18.0,3.736,0.2199364585706691,0.796,0.0,0.992,0.996,0.596,0.74,0.5930232558139535,86,3116.524,281.12372000000005,2,2.42,73155.784
+Harbor OpenCode-only,200,claude-code,250,0.284,15.568,16.092,17.0,19.0,1.832,0.11433170750166054,0.54,0.004,0.776,0.924,0.508,0.748,0.5697674418604651,86,4204.184,290.2222,1,15.52,608079.264
+Harbor OpenCode-only,200,codex,250,0.308,15.924,17.268,18.0,20.0,2.092,0.12244325080925565,0.588,0.0,0.812,0.952,0.492,0.676,0.5116279069767442,86,3497.2,277.06620000000004,0,1.044,18567.156
+Harbor OpenCode-only,200,mini-swe-agent,250,0.2,15.84,15.768,17.0,17.0,0.752,0.047604835687188624,0.404,0.0,0.784,0.968,0.364,0.636,0.43023255813953487,86,2314.748,210.73584,6,1.08,14755.168
+Harbor OpenCode-only,200,opencode,250,0.292,16.852,17.392,17.0,19.0,2.12,0.12248870350495737,0.712,0.0,0.972,0.976,0.532,0.788,0.5581395348837209,86,3743.516,329.72027999999995,1,2.792,88775.272
+Harbor OpenCode-only,300,claude-code,250,0.304,15.784,16.72,17.0,19.0,1.672,0.10361074801180678,0.504,0.012,0.816,0.948,0.432,0.756,0.5,86,4454.876,327.57912,0,15.772,637861.168
+Harbor OpenCode-only,300,codex,250,0.32,16.276,18.028,18.0,21.0,2.116,0.12139348994119253,0.628,0.0,0.868,0.976,0.5,0.728,0.5116279069767442,86,3702.284,318.68788,2,1.084,19064.132
+Harbor OpenCode-only,300,mini-swe-agent,250,0.236,16.208,16.164,17.0,17.0,0.652,0.04040749838396897,0.416,0.0,0.824,0.984,0.328,0.64,0.3488372093023256,86,2457.928,244.22476,1,1.076,14375.756
+Harbor OpenCode-only,300,opencode,250,0.28,16.552,21.08,19.0,24.0,2.096,0.10858220362877542,0.692,0.0,0.948,0.98,0.424,0.672,0.4186046511627907,86,2913.772,284.4264,1,2.656,88801.412
+Harbor OpenCode-only,400,claude-code,250,0.336,15.956,17.244,17.0,20.0,2.224,0.1318628296618753,0.628,0.008,0.848,0.96,0.512,0.732,0.5232558139534884,86,3706.02,288.30676,1,15.952,616640.776
+Harbor OpenCode-only,400,codex,250,0.332,16.192,18.852,18.0,24.0,2.504,0.14037034449347674,0.672,0.0,0.84,0.98,0.5,0.66,0.5232558139534884,86,3238.676,300.32312,1,1.072,18137.108
+Harbor OpenCode-only,400,mini-swe-agent,250,0.288,15.636,15.632,17.0,17.0,0.64,0.04062847936377348,0.404,0.0,0.776,0.988,0.428,0.6,0.5,86,2049.448,231.96988000000002,2,1.12,13630.868
+Harbor OpenCode-only,400,opencode,250,0.356,16.864,20.944,19.0,23.099999999999994,2.736,0.12537672321606977,0.684,0.0,0.984,1.0,0.528,0.684,0.5581395348837209,86,2714.036,283.75588,1,2.816,91009.216
+Harbor OpenCode-only,500,claude-code,250,0.324,15.568,16.032,17.0,19.0,1.584,0.10352479582616418,0.556,0.008,0.764,0.976,0.496,0.78,0.5348837209302325,86,4486.608,320.51367999999997,2,15.56,607378.124
+Harbor OpenCode-only,500,codex,250,0.372,15.768,17.164,18.0,20.0,2.144,0.12828688929781673,0.62,0.0,0.804,0.996,0.524,0.672,0.5697674418604651,86,3544.256,311.3784,2,1.132,19307.52
+Harbor OpenCode-only,500,mini-swe-agent,250,0.296,15.712,15.688,17.0,17.0,0.464,0.03036280386280386,0.34,0.0,0.748,0.992,0.396,0.632,0.45348837209302323,86,2479.62,243.41188,1,1.124,13965.672
+Harbor OpenCode-only,500,opencode,250,0.328,16.74,17.972,18.0,19.0,2.076,0.11614237145580664,0.568,0.0,0.972,1.0,0.56,0.656,0.5116279069767442,86,3130.452,292.57084000000003,0,2.692,86107.564
+Harbor OpenCode-only,600,claude-code,250,0.312,15.46,16.128,17.0,19.0,2.1,0.1315176920271254,0.616,0.0,0.724,0.944,0.592,0.676,0.627906976744186,86,3963.312,291.50164,0,15.452,593529.86
+Harbor OpenCode-only,600,codex,250,0.344,15.844,17.608,18.0,21.0,2.616,0.1513637730010365,0.676,0.0,0.792,0.988,0.572,0.596,0.5465116279069767,86,3014.716,290.20484000000005,0,1.188,18723.596
+Harbor OpenCode-only,600,mini-swe-agent,250,0.308,15.204,15.2,17.0,17.0,0.492,0.03401022457075088,0.36,0.0,0.664,0.976,0.444,0.592,0.4883720930232558,86,1631.68,240.04868,0,1.408,18290.932
+Harbor OpenCode-only,600,opencode,250,0.356,16.752,19.008,19.0,21.0,3.044,0.16557076775539808,0.688,0.0,0.96,0.984,0.584,0.652,0.5813953488372093,86,1953.884,282.23591999999996,1,2.82,102552.012
+Harbor OpenCode-only,700,claude-code,250,0.464,13.644,14.408,16.0,19.099999999999994,2.016,0.13538080056961474,0.636,0.004,0.484,0.988,0.716,0.692,0.7325581395348837,86,4399.992,286.99879999999996,0,13.636,505591.964
+Harbor OpenCode-only,700,codex,250,0.372,13.428,16.54,17.0,23.0,1.772,0.1027915198845654,0.604,0.0,0.468,0.984,0.636,0.6,0.6976744186046512,86,2704.632,270.38576,1,1.484,21606.22
+Harbor OpenCode-only,700,mini-swe-agent,250,0.344,15.264,15.372,17.0,17.0,0.552,0.036614323295698165,0.372,0.0,0.676,0.992,0.524,0.628,0.5930232558139535,86,1786.416,249.399,2,2.528,29388.456
+Harbor OpenCode-only,700,opencode,250,0.4,15.468,20.16,19.0,26.0,4.028,0.2074647077959726,0.9,0.0,0.628,0.976,0.692,0.66,0.7325581395348837,86,2746.104,298.80316,1,4.812,190272.756
+Harbor OpenCode-only,800,claude-code,250,0.388,12.752,12.588,13.0,17.0,0.976,0.07193511025699746,0.444,0.0,0.388,0.92,0.672,0.648,0.7441860465116279,86,4724.304,288.79735999999997,4,12.74,474807.112
+Harbor OpenCode-only,800,codex,250,0.356,13.648,14.192,15.0,19.0,1.172,0.07992012584594628,0.556,0.004,0.432,0.96,0.584,0.62,0.6511627906976745,86,2506.392,264.28364,4,1.68,24492.112
+Harbor OpenCode-only,800,mini-swe-agent,250,0.316,15.924,15.4,17.0,17.0,0.76,0.05157312883195236,0.436,0.0,0.74,0.984,0.508,0.592,0.5581395348837209,86,2379.072,239.972,3,4.524,48372.472
+Harbor OpenCode-only,800,opencode,250,0.264,13.236,13.7,16.0,18.0,1.392,0.11581622996886155,0.764,0.04,0.492,0.836,0.536,0.548,0.5116279069767442,86,3452.292,295.78172,9,5.728,228941.352
+Harbor OpenCode-only,900,claude-code,250,0.352,13.796,14.408,16.5,19.099999999999994,1.324,0.09232925935881918,0.568,0.004,0.516,0.928,0.516,0.696,0.5581395348837209,86,4797.792,323.89572000000004,0,13.784,509683.264
+Harbor OpenCode-only,900,codex,250,0.272,14.192,16.332,17.0,22.0,1.532,0.0915357853779021,0.628,0.0,0.496,0.94,0.48,0.632,0.47674418604651164,86,2731.02,303.751,0,2.016,31778.196
+Harbor OpenCode-only,900,mini-swe-agent,250,0.26,16.752,16.116,17.0,17.0,1.016,0.06716071771365889,0.56,0.0,0.928,0.984,0.372,0.668,0.4186046511627907,86,2291.792,263.71732000000003,2,4.244,60279.032
+Harbor OpenCode-only,900,opencode,250,0.3,13.768,18.4,20.0,25.0,3.672,0.2304205386347483,0.964,0.0,0.508,0.856,0.828,0.6,0.3953488372093023,86,2722.508,282.09244,1,4.432,220613.992
+Harbor OpenCode-only,1000,claude-code,250,0.308,14.284,18.888,19.0,27.0,2.564,0.12207600811203947,0.708,0.0,0.588,0.908,0.556,0.748,0.5,86,4909.94,326.24172,0,14.28,546690.0
+Harbor OpenCode-only,1000,codex,250,0.324,14.164,20.392,20.0,32.0,3.024,0.1276550610578384,0.78,0.0,0.528,0.948,0.448,0.68,0.5116279069767442,86,2927.156,303.94488,0,2.136,35868.228
+Harbor OpenCode-only,1000,mini-swe-agent,250,0.204,16.852,18.788,17.0,24.099999999999994,1.88,0.09276147753946476,0.692,0.0,0.956,0.952,0.288,0.668,0.26744186046511625,86,2458.74,245.79756,0,2.152,38802.564
+Harbor OpenCode-only,1000,opencode,250,0.22,15.784,25.796,25.0,34.099999999999994,4.632,0.16389335394127683,0.884,0.0,0.736,0.972,0.956,0.752,0.3488372093023256,86,3317.312,298.19368,2,2.96,108702.744
+Harbor multi-harness,0,claude-code,250,0.168,13.248,12.696,15.0,18.0,0.552,0.03765119948518259,0.232,0.032,0.52,0.768,0.44,0.664,0.4883720930232558,86,3510.776,306.70408000000003,4,12.9,505615.996
+Harbor multi-harness,0,codex,250,0.164,14.56,14.244,17.0,19.099999999999994,0.644,0.04206620581633926,0.34,0.08,0.632,0.756,0.376,0.576,0.3953488372093023,86,3011.892,288.04967999999997,0,1.124,19415.248
+Harbor multi-harness,0,mini-swe-agent,250,0.144,16.028,15.152,17.0,17.0,0.404,0.026998038725979903,0.276,0.04,0.808,0.844,0.316,0.636,0.3372093023255814,86,2479.664,227.70248,4,1.248,17148.996
+Harbor multi-harness,0,opencode,250,0.108,12.76,10.964,12.5,17.099999999999994,0.488,0.03652784315559323,0.292,0.112,0.448,0.748,0.388,0.596,0.38372093023255816,86,1885.612,273.23328000000004,3,1.216,31783.472
+Harbor multi-harness,100,claude-code,250,0.276,16.26,15.808,17.0,17.0,3.0,0.18587976378626533,0.652,0.008,0.876,0.78,0.504,0.632,0.5232558139534884,86,3830.668,330.60032,13,15.588,640878.472
+Harbor multi-harness,100,codex,250,0.28,16.62,16.456,17.0,18.0,1.852,0.11074251513805289,0.56,0.0,0.904,0.896,0.388,0.688,0.46511627906976744,86,3349.404,320.15596,0,1.1,17822.008
+Harbor multi-harness,100,mini-swe-agent,250,0.192,15.86,15.804,17.0,17.0,0.984,0.06046919126625009,0.484,0.0,0.78,0.932,0.324,0.608,0.3372093023255814,86,1925.932,211.64244,9,1.064,14148.712
+Harbor multi-harness,100,opencode,250,0.244,16.624,15.684,16.0,16.0,5.24,0.3284698217846824,0.72,0.008,0.932,0.864,0.556,0.632,0.5465116279069767,86,2317.092,248.64208000000002,22,2.3,66219.656
+Harbor multi-harness,200,claude-code,250,0.3,16.616,16.076,17.0,17.0,3.616,0.21978729800732763,0.68,0.004,0.908,0.892,0.488,0.672,0.4883720930232558,86,3915.984,332.44284000000005,44,15.9,626332.568
+Harbor multi-harness,200,codex,250,0.264,16.828,16.548,17.0,17.0,2.108,0.12487952724949629,0.584,0.004,0.928,0.936,0.436,0.72,0.5,86,3338.936,312.86604,0,1.016,16807.012
+Harbor multi-harness,200,mini-swe-agent,250,0.184,16.176,16.144,17.0,17.0,1.264,0.07661575809811104,0.552,0.0,0.848,0.864,0.288,0.584,0.313953488372093,86,1583.696,185.23776,21,1.068,13119.784
+Harbor multi-harness,200,opencode,250,0.304,16.772,15.784,16.0,16.0,5.24,0.32631199567546315,0.728,0.004,0.972,0.916,0.592,0.628,0.6627906976744186,86,2137.664,240.12676000000002,11,2.188,60125.88
+Harbor multi-harness,300,claude-code,250,0.332,16.24,15.936,17.0,17.099999999999994,4.148,0.25688259150550175,0.768,0.012,0.876,0.844,0.504,0.652,0.5697674418604651,86,3264.88,309.29952000000003,17,15.692,612400.22
+Harbor multi-harness,300,codex,250,0.296,16.28,15.888,17.0,17.0,2.496,0.15552757522834923,0.604,0.0,0.856,0.932,0.472,0.716,0.46511627906976744,86,2760.52,294.50336,0,1.024,16410.9
+Harbor multi-harness,300,mini-swe-agent,250,0.22,16.336,16.312,17.0,17.0,2.652,0.15764385189973423,0.716,0.0,0.872,0.792,0.356,0.508,0.3372093023255814,86,1409.048,182.90048000000002,4,1.032,10683.584
+Harbor multi-harness,300,opencode,250,0.296,16.732,15.812,16.0,17.0,5.288,0.3330932420520656,0.764,0.0,0.936,0.812,0.652,0.56,0.7093023255813954,86,1572.264,209.03392000000002,8,2.16,58576.424
+Harbor multi-harness,400,claude-code,250,0.368,16.476,15.952,17.0,17.0,3.776,0.23215647616916968,0.78,0.008,0.884,0.832,0.5,0.628,0.5581395348837209,86,3781.936,331.85516,12,15.808,645411.508
+Harbor multi-harness,400,codex,250,0.348,16.764,16.5,17.0,17.0,3.636,0.21900172409290056,0.644,0.0,0.924,0.964,0.476,0.74,0.45348837209302323,86,3134.512,301.95732,0,1.032,17655.752
+Harbor multi-harness,400,mini-swe-agent,250,0.288,16.552,16.476,17.0,17.0,2.652,0.15686939302527536,0.704,0.0,0.924,0.844,0.4,0.572,0.4883720930232558,86,1856.436,206.72688,7,1.076,13024.596
+Harbor multi-harness,400,opencode,250,0.328,16.624,15.684,16.0,16.0,4.284,0.26981507936507937,0.664,0.0,0.92,0.836,0.52,0.596,0.5930232558139535,86,2274.428,247.216,25,2.264,66810.536
+Harbor multi-harness,500,claude-code,250,0.448,16.444,16.064,17.0,17.0,4.388,0.2668000983708414,0.8,0.0,0.888,0.852,0.548,0.616,0.627906976744186,86,3942.864,327.30096000000003,16,15.888,636345.004
+Harbor multi-harness,500,codex,250,0.392,16.86,16.452,17.0,17.0,3.644,0.2180356946974594,0.716,0.0,0.936,0.924,0.508,0.78,0.5581395348837209,86,3375.588,314.03576,0,1.02,17570.668
+Harbor multi-harness,500,mini-swe-agent,250,0.312,14.384,14.328,17.0,17.0,0.9,0.05523096282802165,0.388,0.0,0.612,0.864,0.452,0.596,0.45348837209302323,86,1786.76,199.01392,12,1.124,13975.28
+Harbor multi-harness,500,opencode,250,0.328,16.556,15.484,16.0,16.0,4.924,0.30975469367234076,0.7,0.0,0.92,0.8,0.552,0.592,0.5581395348837209,86,2375.944,261.0208,37,2.344,71243.072
+Harbor multi-harness,600,claude-code,250,0.3,16.392,16.1,17.0,18.0,3.848,0.23563521318722555,0.848,0.004,0.872,0.732,0.444,0.584,0.4883720930232558,86,4780.032,350.56232,5,15.836,642993.316
+Harbor multi-harness,600,codex,250,0.324,16.784,16.14,17.0,17.0,3.504,0.2106117257699301,0.736,0.0,0.872,0.928,0.472,0.672,0.5232558139534884,86,4121.492,349.44771999999995,0,1.06,18950.12
+Harbor multi-harness,600,mini-swe-agent,250,0.308,14.732,14.552,16.0,17.0,1.864,0.11646161550867433,0.644,0.0,0.624,0.804,0.436,0.472,0.4883720930232558,86,1926.224,196.08360000000002,13,1.2,11305.592
+Harbor multi-harness,600,opencode,250,0.34,16.308,15.012,16.0,16.0,4.816,0.3089410148674855,0.736,0.004,0.856,0.732,0.54,0.548,0.5813953488372093,86,2608.344,262.1164,65,2.536,76218.4
+Harbor multi-harness,684,claude-code,250,0.356,15.964,15.64,17.0,17.0,4.424,0.28016906517974627,0.892,0.008,0.828,0.768,0.52,0.608,0.6046511627906976,86,4017.62,326.16508,10,15.504,680171.832
+Harbor multi-harness,684,codex,250,0.332,16.788,16.196,17.0,17.0,4.176,0.2533938959389424,0.816,0.0,0.86,0.92,0.464,0.652,0.46511627906976744,86,3783.12,349.0168,0,1.04,18502.496
+Harbor multi-harness,684,mini-swe-agent,250,0.304,13.936,13.728,16.0,17.0,2.192,0.14189798077739255,0.688,0.0,0.528,0.796,0.5,0.484,0.5348837209302325,86,1764.008,194.05316,8,1.2,12895.984
+Harbor multi-harness,684,opencode,250,0.292,15.684,14.472,16.0,16.0,4.888,0.31585763321645677,0.784,0.004,0.808,0.692,0.504,0.524,0.5232558139534884,86,2016.2,234.25995999999998,47,2.268,66770.276
+Harbor multi-harness,700,claude-code,250,0.316,15.704,15.412,17.0,17.099999999999994,4.228,0.2679575435228686,0.892,0.008,0.792,0.72,0.464,0.584,0.5116279069767442,86,4169.628,335.68408,5,15.192,665029.128
+Harbor multi-harness,700,codex,250,0.32,16.784,16.356,17.0,17.0,4.284,0.2596398617722147,0.88,0.0,0.888,0.884,0.492,0.644,0.5348837209302325,86,3697.384,331.43440000000004,0,1.008,17985.792
+Harbor multi-harness,700,mini-swe-agent,250,0.304,14.388,14.116,16.0,17.0,2.12,0.1378408680071219,0.736,0.0,0.58,0.772,0.456,0.488,0.5348837209302325,86,1896.328,201.05236,11,1.292,13888.552
+Harbor multi-harness,700,opencode,250,0.212,14.888,13.704,16.0,16.0,4.064,0.2618782733606263,0.684,0.028,0.732,0.596,0.424,0.428,0.4186046511627907,86,2025.036,224.59807999999998,46,2.268,67976.364
+Harbor multi-harness,800,claude-code,250,0.32,15.736,15.392,17.0,17.0,4.896,0.3106986468811546,0.912,0.008,0.796,0.688,0.416,0.612,0.46511627906976744,86,4761.008,360.24136,9,15.132,649276.748
+Harbor multi-harness,800,codex,250,0.268,16.62,15.448,16.0,17.0,3.856,0.24653691700456407,0.9,0.0,0.688,0.872,0.384,0.524,0.38372093023255816,86,4708.132,423.90656,0,1.044,19406.28
+Harbor multi-harness,800,mini-swe-agent,250,0.26,15.344,15.228,17.0,17.0,2.22,0.1355879032731974,0.728,0.0,0.684,0.78,0.324,0.448,0.32558139534883723,86,2640.172,239.52272,25,1.116,15126.248
+Harbor multi-harness,800,opencode,250,0.232,16.016,14.768,16.0,16.0,3.736,0.23995739783092726,0.828,0.028,0.868,0.624,0.336,0.464,0.3953488372093023,86,2813.852,279.70788,73,2.508,77446.828
+Harbor multi-harness,900,claude-code,250,0.364,15.512,14.936,16.0,17.0,5.904,0.3726758302861345,0.888,0.02,0.82,0.688,0.492,0.548,0.5930232558139535,86,9519.716,419.02456,34,15.492,664811.652
+Harbor multi-harness,900,codex,250,0.192,9.212,8.444,7.0,17.0,1.884,0.14712137633608222,0.492,0.116,0.192,0.62,0.288,0.336,0.2441860465116279,86,3711.06,337.0398,2,1.1,14518.416
+Harbor multi-harness,900,mini-swe-agent,250,0.328,13.824,12.988,14.0,17.0,1.092,0.07299727102962397,0.508,0.0,0.496,0.852,0.448,0.544,0.5,86,3963.104,307.00915999999995,110,1.112,14503.728
+Harbor multi-harness,900,opencode,250,0.024,3.984,2.216,1.0,3.0,0.484,0.034528688524590165,0.068,0.872,0.06,0.056,0.048,0.048,0.046511627906976744,86,4033.72,231.61476000000002,3,2.06,51199.836
+Harbor multi-harness,1000,claude-code,250,0.388,13.172,12.216,14.0,17.0,3.784,0.2879195793542543,0.864,0.008,0.476,0.688,0.532,0.58,0.5581395348837209,86,9486.256,409.62556,73,13.168,529513.164
+Harbor multi-harness,1000,codex,250,0.248,10.044,9.244,9.0,17.0,2.784,0.21200627215921333,0.58,0.136,0.228,0.692,0.388,0.452,0.3372093023255814,86,4252.036,336.64468,8,1.056,14959.276
+Harbor multi-harness,1000,mini-swe-agent,250,0.36,11.948,11.004,11.0,17.0,1.124,0.08019051993757875,0.424,0.0,0.304,0.832,0.564,0.484,0.627906976744186,86,3199.972,268.63016,120,1.408,13323.572
+Harbor multi-harness,1000,opencode,250,0.056,5.316,3.608,1.0,16.0,1.124,0.076791380188439,0.16,0.74,0.112,0.172,0.124,0.132,0.12790697674418605,86,3889.26,202.2784,4,2.176,56658.872
+Native OpenCode,0,claude-code,250,0.168,13.564,13.444,16.0,18.0,0.624,0.04410033896959594,0.304,0.012,0.544,0.78,0.432,0.684,0.4418604651162791,86,3470.204,300.24328,207,13.332,531470.596
+Native OpenCode,0,codex,250,0.152,14.888,15.444,17.0,19.0,0.844,0.05171758901179685,0.444,0.0,0.684,0.756,0.372,0.592,0.4186046511627907,86,2925.184,298.26488,214,1.164,19845.676
+Native OpenCode,0,mini-swe-agent,250,0.188,15.88,15.62,17.0,17.0,0.632,0.042267335932041815,0.368,0.0,0.768,0.916,0.364,0.676,0.3953488372093023,86,2504.592,217.98556,204,1.244,17376.724
+Native OpenCode,0,opencode,250,0.128,12.7,11.764,14.0,17.0,0.532,0.0345529428603732,0.284,0.02,0.484,0.864,0.484,0.684,0.5116279069767442,86,1721.84,220.96564,205,1.18,34095.472
+Native OpenCode,100,claude-code,250,0.204,12.648,12.48,15.0,18.0,0.58,0.03509144543406221,0.284,0.016,0.5,0.872,0.536,0.692,0.5348837209302325,86,3698.748,143.16423999999998,0,12.584,505121.256
+Native OpenCode,100,codex,250,0.18,14.984,15.22,17.0,18.0,0.516,0.0335499714839573,0.352,0.004,0.66,0.884,0.404,0.692,0.4069767441860465,86,2993.728,158.66124,0,1.304,19672.46
+Native OpenCode,100,mini-swe-agent,250,0.208,15.376,15.172,17.0,17.0,0.416,0.028294982794982795,0.312,0.0,0.728,0.96,0.396,0.692,0.4186046511627907,86,2541.22,90.21448,1,1.292,15771.72
+Native OpenCode,100,opencode,250,0.196,12.2,10.8,12.0,16.0,0.448,0.028807029726379572,0.208,0.004,0.428,0.952,0.616,0.668,0.686046511627907,86,2149.4,144.21248,0,2.264,79437.648
+Native OpenCode,200,claude-code,250,0.24,12.36,12.156,16.0,17.0,0.688,0.04049570551567456,0.256,0.008,0.488,0.944,0.556,0.736,0.6046511627906976,86,3558.216,138.13363999999999,0,12.348,472189.48
+Native OpenCode,200,codex,250,0.256,13.516,13.576,17.0,18.0,0.464,0.029959747012533393,0.272,0.0,0.532,0.912,0.54,0.696,0.5697674418604651,86,2495.156,147.19407999999999,0,1.348,17961.452
+Native OpenCode,200,mini-swe-agent,250,0.216,15.116,14.708,17.0,17.0,0.4,0.03000764594882242,0.272,0.0,0.692,0.96,0.464,0.692,0.5,86,2482.148,83.50396,0,1.528,16233.036
+Native OpenCode,200,opencode,250,0.172,11.536,10.104,9.5,16.0,0.488,0.032465955835906224,0.228,0.008,0.4,0.976,0.636,0.712,0.6162790697674418,86,1752.28,140.38468,0,1.872,80740.084
+Native OpenCode,300,claude-code,250,0.292,12.476,12.104,14.0,17.0,0.392,0.02518707996509854,0.216,0.004,0.448,0.96,0.592,0.76,0.627906976744186,86,3980.676,140.191,0,12.444,487869.28
+Native OpenCode,300,codex,250,0.168,15.132,15.164,17.0,18.0,0.62,0.038108605979503815,0.392,0.0,0.72,0.912,0.372,0.696,0.29069767441860467,86,2723.488,148.16628,0,1.292,17298.944
+Native OpenCode,300,mini-swe-agent,250,0.196,15.668,15.244,17.0,17.0,0.576,0.041733681677799325,0.368,0.0,0.736,0.968,0.356,0.736,0.37209302325581395,86,2633.112,85.03076,4,1.5,16793.236
+Native OpenCode,300,opencode,250,0.208,12.28,10.964,14.0,16.0,0.548,0.0366129533864828,0.26,0.0,0.488,0.996,0.56,0.696,0.5465116279069767,86,2052.772,137.44495999999998,2,1.684,52509.564
+Native OpenCode,400,claude-code,250,0.324,12.264,11.9,15.0,18.0,0.552,0.03280133957889376,0.224,0.004,0.484,0.948,0.544,0.772,0.6046511627906976,86,3900.464,129.74188,0,12.236,446078.456
+Native OpenCode,400,codex,250,0.256,14.96,14.968,17.0,18.0,0.636,0.03993127256300631,0.368,0.0,0.684,0.96,0.42,0.716,0.4186046511627907,86,2660.132,142.42044,0,1.284,17106.884
+Native OpenCode,400,mini-swe-agent,250,0.272,15.504,14.892,17.0,17.0,0.504,0.03642990179754886,0.36,0.0,0.696,0.96,0.452,0.776,0.5116279069767442,86,2719.588,81.37939999999999,0,1.716,18353.556
+Native OpenCode,400,opencode,250,0.204,11.956,10.504,11.0,16.0,0.488,0.031920717517776344,0.24,0.0,0.408,0.992,0.636,0.688,0.6744186046511628,86,1866.156,124.94732,0,1.788,36467.716
+Native OpenCode,500,claude-code,250,0.264,12.184,11.512,12.0,17.0,0.744,0.051082755531362344,0.332,0.012,0.456,0.964,0.56,0.716,0.6046511627906976,86,3580.768,137.15052,0,12.152,460326.656
+Native OpenCode,500,codex,250,0.188,14.748,14.58,17.0,17.0,0.572,0.03571082699550192,0.344,0.004,0.656,0.96,0.384,0.64,0.3488372093023256,86,2350.368,149.06395999999998,0,1.352,17247.444
+Native OpenCode,500,mini-swe-agent,250,0.292,15.336,14.256,16.0,17.0,0.468,0.03840567341155577,0.328,0.0,0.696,0.972,0.456,0.732,0.5116279069767442,86,2713.324,89.19475999999999,3,2.372,21986.98
+Native OpenCode,500,opencode,250,0.18,11.376,9.896,9.0,16.0,1.536,0.10255804685510567,0.412,0.004,0.364,0.984,0.628,0.656,0.6744186046511628,86,1207.764,127.8474,0,2.268,52725.248
+Native OpenCode,600,claude-code,250,0.324,11.48,10.624,9.0,17.0,0.948,0.0612672097768073,0.344,0.016,0.384,0.952,0.628,0.676,0.5930232558139535,86,3209.396,128.68632,1,11.452,434024.612
+Native OpenCode,600,codex,250,0.184,14.524,14.364,17.0,17.0,1.136,0.07073384025675047,0.54,0.0,0.644,0.94,0.448,0.608,0.38372093023255816,86,2301.072,143.4634,0,1.44,17967.132
+Native OpenCode,600,mini-swe-agent,250,0.328,14.712,13.14,15.0,17.0,0.696,0.061164984362043186,0.424,0.0,0.58,0.988,0.568,0.764,0.6395348837209303,86,2964.56,89.3322,5,3.12,27073.072
+Native OpenCode,600,opencode,250,0.168,10.672,9.152,8.0,16.0,1.956,0.13697550782550783,0.364,0.016,0.32,0.936,0.68,0.6,0.7441860465116279,86,1417.804,123.76088,8,2.608,57619.568
+Native OpenCode,700,claude-code,250,0.296,11.404,10.44,9.0,17.0,0.82,0.05602791266181359,0.296,0.024,0.376,0.916,0.704,0.724,0.6511627906976745,86,3060.304,126.11748,0,11.388,444784.676
+Native OpenCode,700,codex,250,0.144,15.484,15.244,17.0,17.0,0.96,0.06026415414650709,0.444,0.0,0.764,0.988,0.384,0.62,0.32558139534883723,86,2521.796,143.21072,0,1.328,16509.94
+Native OpenCode,700,mini-swe-agent,250,0.304,14.548,13.608,16.0,17.0,0.756,0.0629743749714338,0.404,0.0,0.632,0.992,0.568,0.728,0.6046511627906976,86,2763.38,90.39956,0,2.444,23490.592
+Native OpenCode,700,opencode,250,0.184,9.884,8.172,6.0,16.0,0.408,0.028853829503829507,0.196,0.012,0.264,0.98,0.76,0.62,0.7906976744186046,86,1216.172,124.12552000000001,0,1.948,39104.928
+Native OpenCode,800,claude-code,250,0.324,11.424,10.528,9.5,17.0,0.436,0.02966107916850951,0.228,0.02,0.332,0.96,0.712,0.736,0.7325581395348837,86,3660.24,130.67832,0,11.4,451090.5
+Native OpenCode,800,codex,250,0.18,14.792,14.496,17.0,17.0,0.552,0.03512954953543188,0.32,0.008,0.688,0.976,0.412,0.588,0.3488372093023256,86,2570.196,144.53616,0,1.376,17461.052
+Native OpenCode,800,mini-swe-agent,250,0.32,14.352,13.776,17.0,17.0,0.464,0.03996033019268313,0.344,0.0,0.616,0.996,0.552,0.804,0.6046511627906976,86,3235.884,93.91984,0,2.104,23560.276
+Native OpenCode,800,opencode,250,0.2,9.772,8.084,5.5,16.0,0.204,0.013540849673202616,0.116,0.016,0.26,0.964,0.768,0.64,0.8255813953488372,86,1740.452,127.32356,0,1.904,53123.66
+Native OpenCode,900,claude-code,250,0.34,10.984,10.08,8.5,17.0,0.4,0.027455024060906413,0.192,0.032,0.324,0.888,0.7,0.688,0.7674418604651163,86,3712.704,133.01479999999998,0,10.972,444943.16
+Native OpenCode,900,codex,250,0.144,15.312,15.088,17.0,17.0,0.72,0.04548971003197009,0.396,0.004,0.74,0.984,0.308,0.604,0.313953488372093,86,2710.16,143.14632,0,1.296,16285.736
+Native OpenCode,900,mini-swe-agent,250,0.34,14.636,13.84,16.0,17.0,0.672,0.057749588320176556,0.388,0.0,0.608,1.0,0.596,0.78,0.5581395348837209,86,3245.656,91.29476,2,2.124,24792.192
+Native OpenCode,900,opencode,250,0.188,8.824,7.112,5.0,16.0,0.2,0.015833851116204056,0.132,0.056,0.164,0.94,0.784,0.628,0.8372093023255814,86,1355.404,121.87132000000001,1,1.996,59466.748
+Native OpenCode,1000,claude-code,250,0.332,11.224,10.528,10.0,17.0,0.496,0.03395395028088526,0.26,0.012,0.336,0.94,0.752,0.748,0.7558139534883721,86,3596.888,133.73032,0,11.224,449093.488
+Native OpenCode,1000,codex,250,0.296,14.788,14.46,17.0,17.0,0.512,0.03377497992203875,0.324,0.004,0.632,1.0,0.604,0.864,0.5232558139534884,86,2098.572,147.44584,0,4.988,66343.82
+Native OpenCode,1000,mini-swe-agent,250,0.36,14.684,13.736,15.0,17.0,0.836,0.06810367149190678,0.424,0.0,0.6,0.996,0.656,0.724,0.7790697674418605,86,2917.784,98.07616,0,5.208,63338.732
+Native OpenCode,1000,opencode,250,0.204,10.732,9.148,8.0,16.0,1.244,0.10633702735173323,0.532,0.0,0.268,0.98,0.804,0.68,0.7906976744186046,86,1625.788,127.7836,0,6.152,162860.536
diff --git a/04-data-agent/reports/three-run-analysis-20260917/by_outcome.csv b/04-data-agent/reports/three-run-analysis-20260917/by_outcome.csv
new file mode 100644
index 0000000..f791904
--- /dev/null
+++ b/04-data-agent/reports/three-run-analysis-20260917/by_outcome.csv
@@ -0,0 +1,273 @@
+run,step,harness,reward,cells,score,model_calls,tool_calls,tool_calls_median,tool_calls_p90,repeats,repeat_fraction,repeat_rollout_fraction,short_tool_fraction,budget_fraction,python_fraction,answer_path_fraction,tool_error_text_fraction,submission_fraction,submission_observations,trainable_tokens,wall_s,retry_cells,training_rows,packed_tokens
+Harbor OpenCode-only,0,claude-code,0,208,0.0,14.177884615384615,13.759615384615385,16.0,18.0,0.6009615384615384,0.03858411235582612,0.24519230769230768,0.028846153846153848,0.6105769230769231,0.7548076923076923,0.33653846153846156,0.7163461538461539,0.35294117647058826,68,3907.5673076923076,330.2196153846154,4,13.774038461538462,551784.2644230769
+Harbor OpenCode-only,0,claude-code,1,42,1.0,8.642857142857142,7.428571428571429,7.0,13.799999999999997,0.30952380952380953,0.032916666666666664,0.16666666666666666,0.047619047619047616,0.07142857142857142,0.8333333333333334,0.9523809523809523,0.40476190476190477,1.0,18,1545.7142857142858,190.2461904761905,0,8.571428571428571,276973.14285714284
+Harbor OpenCode-only,0,codex,0,209,0.0,15.167464114832535,15.014354066985646,17.0,20.0,0.5933014354066986,0.03703165718455073,0.3492822966507177,0.07655502392344497,0.7129186602870813,0.7416267942583732,0.2727272727272727,0.6172248803827751,0.23529411764705882,68,3264.622009569378,302.87047846889953,0,1.1483253588516746,20704.593301435405
+Harbor OpenCode-only,0,codex,1,41,1.0,11.463414634146341,10.317073170731707,9.0,17.0,0.9024390243902439,0.06859963779468424,0.2926829268292683,0.0975609756097561,0.21951219512195122,0.8292682926829268,0.9024390243902439,0.36585365853658536,1.0,18,1723.5853658536585,212.49975609756098,0,1.0,12842.731707317073
+Harbor OpenCode-only,0,mini-swe-agent,0,214,0.0,16.490654205607477,15.682242990654206,17.0,17.0,0.37850467289719625,0.024792598653963587,0.26635514018691586,0.037383177570093455,0.8878504672897196,0.8317757009345794,0.2102803738317757,0.677570093457944,0.20833333333333334,72,2659.411214953271,232.60500000000002,4,1.1682242990654206,17348.168224299065
+Harbor OpenCode-only,0,mini-swe-agent,1,36,1.0,13.277777777777779,12.0,12.0,16.0,0.5555555555555556,0.040360410927019925,0.3333333333333333,0.05555555555555555,0.3333333333333333,0.9166666666666666,0.9444444444444444,0.3888888888888889,1.0,14,1411.1666666666667,198.5597222222222,0,1.7222222222222223,15965.027777777777
+Harbor OpenCode-only,0,opencode,0,223,0.0,13.179372197309418,11.448430493273543,14.0,18.0,0.5246636771300448,0.038140999359143606,0.3094170403587444,0.10762331838565023,0.4977578475336323,0.7488789237668162,0.33183856502242154,0.6322869955156951,0.3291139240506329,79,2014.8161434977578,275.3364125560538,3,1.1973094170403586,31591.430493273543
+Harbor OpenCode-only,0,opencode,1,27,1.0,9.296296296296296,6.962962962962963,7.0,12.400000000000002,0.18518518518518517,0.022289986228605157,0.14814814814814814,0.14814814814814814,0.037037037037037035,0.7407407407407407,0.8518518518518519,0.2962962962962963,1.0,7,818.4814814814815,255.86296296296297,0,1.3703703703703705,33369.59259259259
+Harbor OpenCode-only,100,claude-code,0,187,0.0,15.737967914438503,16.27807486631016,17.0,19.0,2.1176470588235294,0.1291534628941908,0.5935828877005348,0.0,0.8074866310160428,0.9251336898395722,0.49732620320855614,0.8235294117647058,0.4067796610169492,59,4037.96256684492,313.77401069518714,1,15.657754010695188,648994.1336898396
+Harbor OpenCode-only,100,claude-code,1,63,1.0,14.428571428571429,14.428571428571429,17.0,18.0,4.523809523809524,0.2769899738859863,0.7936507936507936,0.0,0.6507936507936508,0.9682539682539683,1.0,0.6031746031746031,1.0,27,2466.5714285714284,204.84666666666666,0,14.428571428571429,486554.4603174603
+Harbor OpenCode-only,100,codex,0,174,0.0,16.028735632183906,17.339080459770116,17.0,21.0,1.528735632183908,0.09085112631089642,0.5229885057471264,0.0,0.8735632183908046,0.9482758620689655,0.3333333333333333,0.7816091954022989,0.2982456140350877,57,3815.8333333333335,325.60994252873564,1,1.0804597701149425,18601.24712643678
+Harbor OpenCode-only,100,codex,1,76,1.0,14.276315789473685,15.06578947368421,17.0,19.0,4.868421052631579,0.2914163040013415,0.8947368421052632,0.0,0.618421052631579,0.9868421052631579,1.0,0.5263157894736842,1.0,29,2114.3815789473683,194.9973684210526,1,1.0657894736842106,14216.197368421053
+Harbor OpenCode-only,100,mini-swe-agent,0,199,0.0,16.08542713567839,16.0,17.0,17.0,0.7437185929648241,0.04511599675655248,0.3969849246231156,0.0,0.8442211055276382,0.9698492462311558,0.23618090452261306,0.7236180904522613,0.208955223880597,67,2407.542713567839,232.3070854271357,8,1.0954773869346734,13758.788944723618
+Harbor OpenCode-only,100,mini-swe-agent,1,51,1.0,11.529411764705882,11.509803921568627,10.0,17.0,1.1372549019607843,0.08078258406978131,0.43137254901960786,0.0,0.2549019607843137,1.0,1.0,0.3333333333333333,1.0,19,1096.235294117647,134.96882352941176,3,1.0588235294117647,6834.176470588235
+Harbor OpenCode-only,100,opencode,0,195,0.0,16.907692307692308,16.984615384615385,17.0,18.0,3.082051282051282,0.1810856795996115,0.7435897435897436,0.0,0.9897435897435898,0.9948717948717949,0.48205128205128206,0.8,0.43548387096774194,62,3563.723076923077,307.79835897435896,1,2.471794871794872,79324.01025641026
+Harbor OpenCode-only,100,opencode,1,55,1.0,17.0,16.963636363636365,17.0,17.0,6.054545454545455,0.35768012946805516,0.9818181818181818,0.0,1.0,1.0,1.0,0.5272727272727272,1.0,24,1531.0,186.55,1,2.2363636363636363,51286.61818181818
+Harbor OpenCode-only,200,claude-code,0,179,0.0,15.843575418994414,16.54189944134078,17.0,19.0,1.223463687150838,0.07660509892269075,0.45251396648044695,0.00558659217877095,0.8435754189944135,0.9050279329608939,0.3128491620111732,0.7932960893854749,0.3392857142857143,56,4604.905027932961,313.4882122905028,0,15.776536312849162,641328.3743016759
+Harbor OpenCode-only,200,claude-code,1,71,1.0,14.873239436619718,14.95774647887324,17.0,18.0,3.3661971830985915,0.20944526997540122,0.7605633802816901,0.0,0.6056338028169014,0.971830985915493,1.0,0.6338028169014085,1.0,30,3193.9154929577467,231.5656338028169,1,14.873239436619718,524254.04225352115
+Harbor OpenCode-only,200,codex,0,173,0.0,16.67630057803468,18.36416184971098,18.0,21.0,1.4046242774566473,0.07753095167875124,0.4682080924855491,0.0,0.9248554913294798,0.9421965317919075,0.2658959537572254,0.7803468208092486,0.25,56,4099.884393063584,316.54907514450866,0,1.0578034682080926,20914.49710982659
+Harbor OpenCode-only,200,codex,1,77,1.0,14.233766233766234,14.805194805194805,17.0,18.400000000000006,3.6363636363636362,0.22335010469986946,0.8571428571428571,0.0,0.5584415584415584,0.974025974025974,1.0,0.44155844155844154,1.0,30,2143.116883116883,188.35792207792207,0,1.0129870129870129,13293.25974025974
+Harbor OpenCode-only,200,mini-swe-agent,0,200,0.0,16.525,16.47,17.0,17.0,0.67,0.040635387488328664,0.37,0.0,0.885,0.96,0.205,0.72,0.24615384615384617,65,2534.265,224.93630000000002,5,1.065,16091.62
+Harbor OpenCode-only,200,mini-swe-agent,1,50,1.0,13.1,12.96,13.0,17.0,1.08,0.07548262848262849,0.54,0.0,0.38,1.0,1.0,0.3,1.0,21,1436.68,153.934,1,1.14,9409.36
+Harbor OpenCode-only,200,opencode,0,177,0.0,16.819209039548024,17.43502824858757,17.0,19.0,1.5819209039548023,0.0906328115726237,0.615819209039548,0.0,0.9774011299435028,0.9717514124293786,0.3389830508474576,0.8192090395480226,0.3090909090909091,55,3841.7909604519773,338.96146892655366,1,2.7288135593220337,89303.62711864407
+Harbor OpenCode-only,200,opencode,1,73,1.0,16.931506849315067,17.28767123287671,17.0,18.0,3.4246575342465753,0.19972833188883488,0.9452054794520548,0.0,0.958904109589041,0.9863013698630136,1.0,0.7123287671232876,1.0,31,3505.2328767123286,307.3135616438356,0,2.9452054794520546,87494.19178082192
+Harbor OpenCode-only,300,claude-code,0,174,0.0,15.879310344827585,17.166666666666668,17.0,20.0,0.8735632183908046,0.05466958958801767,0.3505747126436782,0.017241379310344827,0.8735632183908046,0.9367816091954023,0.1839080459770115,0.8160919540229885,0.17307692307692307,52,4942.620689655172,354.0668390804598,0,15.862068965517242,642441.3448275862
+Harbor OpenCode-only,300,claude-code,1,76,1.0,15.56578947368421,15.697368421052632,17.0,18.0,3.5,0.21566024229785025,0.8552631578947368,0.0,0.6842105263157895,0.9736842105263158,1.0,0.618421052631579,1.0,34,3338.1973684210525,266.93618421052633,0,15.56578947368421,627374.9736842106
+Harbor OpenCode-only,300,codex,0,170,0.0,16.58823529411765,18.641176470588235,18.0,21.0,1.3058823529411765,0.07170575137739442,0.5,0.0,0.9176470588235294,0.9705882352941176,0.2647058823529412,0.8117647058823529,0.20754716981132076,53,4100.770588235294,347.1957058823529,2,1.0823529411764705,20818.035294117646
+Harbor OpenCode-only,300,codex,1,80,1.0,15.6125,16.725,18.0,19.10000000000001,3.8375,0.2269799343892635,0.9,0.0,0.7625,0.9875,1.0,0.55,1.0,33,2855.5,258.10875,0,1.0875,15337.0875
+Harbor OpenCode-only,300,mini-swe-agent,0,191,0.0,16.79581151832461,16.774869109947645,17.0,17.0,0.5287958115183246,0.031608664408171644,0.36649214659685864,0.0,0.9581151832460733,0.9842931937172775,0.12041884816753927,0.7225130890052356,0.1111111111111111,63,2663.8743455497383,258.0570680628272,0,1.0628272251308901,15376.921465968586
+Harbor OpenCode-only,300,mini-swe-agent,1,59,1.0,14.305084745762711,14.186440677966102,15.0,17.0,1.0508474576271187,0.06889185922087218,0.576271186440678,0.0,0.3898305084745763,0.9830508474576272,1.0,0.3728813559322034,1.0,23,1791.2203389830509,199.445593220339,1,1.11864406779661,11134.694915254237
+Harbor OpenCode-only,300,opencode,0,180,0.0,16.383333333333333,20.872222222222224,19.0,25.0,1.4111111111111112,0.07390547982431366,0.5944444444444444,0.0,0.9333333333333333,0.9722222222222222,0.2,0.7555555555555555,0.12280701754385964,57,3246.2555555555555,306.0449444444444,1,2.6277777777777778,91514.25
+Harbor OpenCode-only,300,opencode,1,70,1.0,16.985714285714284,21.614285714285714,19.0,21.0,3.857142857142857,0.1977509219831057,0.9428571428571428,0.0,0.9857142857142858,1.0,1.0,0.45714285714285713,1.0,29,2058.8142857142857,228.83585714285715,0,2.7285714285714286,81825.54285714286
+Harbor OpenCode-only,400,claude-code,0,166,0.0,16.066265060240966,17.746987951807228,17.0,21.0,1.2650602409638554,0.07470659022909078,0.4819277108433735,0.012048192771084338,0.8855421686746988,0.9457831325301205,0.26506024096385544,0.8072289156626506,0.2545454545454545,55,4236.331325301205,320.7774096385542,1,16.06024096385542,647455.2710843374
+Harbor OpenCode-only,400,claude-code,1,84,1.0,15.738095238095237,16.25,17.0,18.700000000000003,4.119047619047619,0.2448144456838066,0.9166666666666666,0.0,0.7738095238095238,0.9880952380952381,1.0,0.5833333333333334,1.0,31,2658.0238095238096,224.13857142857142,0,15.738095238095237,555745.4642857143
+Harbor OpenCode-only,400,codex,0,167,0.0,16.538922155688624,19.904191616766468,19.0,27.400000000000006,1.4610778443113772,0.07713188883852998,0.5568862275449101,0.0,0.8982035928143712,0.9760479041916168,0.25149700598802394,0.7485029940119761,0.24074074074074073,54,3711.1616766467064,333.80592814371255,1,1.0718562874251496,20052.305389221558
+Harbor OpenCode-only,400,codex,1,83,1.0,15.493975903614459,16.734939759036145,18.0,19.799999999999997,4.602409638554217,0.26760916490764675,0.9036144578313253,0.0,0.7228915662650602,0.9879518072289156,1.0,0.4819277108433735,1.0,32,2288.012048192771,232.95409638554216,0,1.072289156626506,14283.638554216868
+Harbor OpenCode-only,400,mini-swe-agent,0,178,0.0,16.3876404494382,16.382022471910112,17.0,17.0,0.47191011235955055,0.02903264301975472,0.3595505617977528,0.0,0.898876404494382,0.9887640449438202,0.19662921348314608,0.6741573033707865,0.18867924528301888,53,2280.5955056179773,247.88219101123596,2,1.146067415730337,15718.421348314607
+Harbor OpenCode-only,400,mini-swe-agent,1,72,1.0,13.777777777777779,13.777777777777779,16.0,17.0,1.0555555555555556,0.06929596365870877,0.5138888888888888,0.0,0.4722222222222222,0.9861111111111112,1.0,0.4166666666666667,1.0,33,1478.0,192.63111111111112,0,1.0555555555555556,8469.972222222223
+Harbor OpenCode-only,400,opencode,0,161,0.0,16.788819875776397,22.124223602484474,19.0,27.0,1.813664596273292,0.06485506419447218,0.515527950310559,0.0,0.9751552795031055,1.0,0.2670807453416149,0.8571428571428571,0.22448979591836735,49,3299.44099378882,331.341552795031,1,2.8260869565217392,97048.1552795031
+Harbor OpenCode-only,400,opencode,1,89,1.0,17.0,18.808988764044944,18.0,20.0,4.404494382022472,0.2348597243674991,0.9887640449438202,0.0,1.0,1.0,1.0,0.3707865168539326,1.0,37,1655.0449438202247,197.67393258426966,0,2.797752808988764,80084.84269662922
+Harbor OpenCode-only,500,claude-code,0,169,0.0,16.11242603550296,16.928994082840237,17.0,19.0,0.8284023668639053,0.0499699922790218,0.378698224852071,0.011834319526627219,0.8757396449704142,0.9644970414201184,0.25443786982248523,0.8579881656804734,0.25925925925925924,54,5185.940828402367,362.2280473372781,0,16.100591715976332,669836.5739644971
+Harbor OpenCode-only,500,claude-code,1,81,1.0,14.432098765432098,14.160493827160494,16.0,17.0,3.1604938271604937,0.2152625958195847,0.9259259259259259,0.0,0.5308641975308642,1.0,1.0,0.6172839506172839,1.0,32,3027.5061728395062,233.48000000000002,2,14.432098765432098,477063.5802469136
+Harbor OpenCode-only,500,codex,0,157,0.0,16.401273885350317,18.133757961783438,18.0,21.400000000000006,1.1656050955414012,0.06638657863160223,0.4585987261146497,0.0,0.9044585987261147,0.9936305732484076,0.24203821656050956,0.8089171974522293,0.24489795918367346,49,4227.649681528663,359.2949681528662,0,1.1401273885350318,21509.375796178345
+Harbor OpenCode-only,500,codex,1,93,1.0,14.698924731182796,15.526881720430108,17.0,19.0,3.795698924731183,0.2327852632182004,0.8924731182795699,0.0,0.6344086021505376,1.0,1.0,0.44086021505376344,1.0,37,2390.569892473118,230.48698924731184,2,1.118279569892473,15590.408602150537
+Harbor OpenCode-only,500,mini-swe-agent,0,176,0.0,16.619318181818183,16.613636363636363,17.0,17.0,0.35795454545454547,0.021529206088029617,0.29545454545454547,0.0,0.9147727272727273,0.9886363636363636,0.14204545454545456,0.7045454545454546,0.12962962962962962,54,2710.3125,259.2145454545455,0,1.1193181818181819,15618.602272727272
+Harbor OpenCode-only,500,mini-swe-agent,1,74,1.0,13.554054054054054,13.486486486486486,14.0,17.0,0.7162162162162162,0.05137244181361828,0.44594594594594594,0.0,0.35135135135135137,1.0,1.0,0.4594594594594595,1.0,32,1930.945945945946,205.82716216216218,1,1.135135135135135,10034.378378378378
+Harbor OpenCode-only,500,opencode,0,168,0.0,16.613095238095237,18.00595238095238,18.0,20.0,0.9702380952380952,0.05309178575358532,0.3630952380952381,0.0,0.9583333333333334,1.0,0.34523809523809523,0.7797619047619048,0.23636363636363636,55,3617.5476190476193,328.1172619047619,0,2.6488095238095237,91067.48809523809
+Harbor OpenCode-only,500,opencode,1,82,1.0,17.0,17.902439024390244,17.0,18.900000000000006,4.341463414634147,0.2453191811871869,0.9878048780487805,0.0,1.0,1.0,1.0,0.4024390243902439,1.0,31,2132.5,219.74402439024388,0,2.7804878048780486,75945.76829268293
+Harbor OpenCode-only,600,claude-code,0,172,0.0,15.965116279069768,16.906976744186046,17.0,20.0,1.3604651162790697,0.08308600729175007,0.5116279069767442,0.0,0.8197674418604651,0.9244186046511628,0.4069767441860465,0.7558139534883721,0.39622641509433965,53,4603.308139534884,329.1661046511628,0,15.953488372093023,650861.6569767442
+Harbor OpenCode-only,600,claude-code,1,78,1.0,14.346153846153847,14.41025641025641,17.0,18.0,3.730769230769231,0.23831576605897872,0.8461538461538461,0.0,0.5128205128205128,0.9871794871794872,1.0,0.5,1.0,33,2552.0384615384614,208.44666666666666,0,14.346153846153847,467105.89743589744
+Harbor OpenCode-only,600,codex,0,164,0.0,16.414634146341463,18.664634146341463,18.0,22.0,1.8048780487804879,0.09961361347511631,0.573170731707317,0.0,0.8841463414634146,0.9878048780487805,0.3475609756097561,0.7134146341463414,0.2641509433962264,53,3480.298780487805,326.8166463414634,0,1.2073170731707317,20979.896341463416
+Harbor OpenCode-only,600,codex,1,86,1.0,14.755813953488373,15.593023255813954,17.0,19.0,4.162790697674419,0.25005012372488433,0.872093023255814,0.0,0.6162790697674418,0.9883720930232558,1.0,0.37209302325581395,1.0,33,2126.860465116279,220.38697674418603,0,1.1511627906976745,14420.883720930233
+Harbor OpenCode-only,600,mini-swe-agent,0,173,0.0,16.404624277456648,16.404624277456648,17.0,17.0,0.36416184971098264,0.022052389410430893,0.27167630057803466,0.0,0.861271676300578,0.9710982658959537,0.19653179190751446,0.6705202312138728,0.21428571428571427,56,1886.8612716763005,262.41884393063583,0,1.4508670520231215,21945.52023121387
+Harbor OpenCode-only,600,mini-swe-agent,1,77,1.0,12.506493506493506,12.493506493506494,12.0,17.0,0.7792207792207793,0.06087652954133996,0.5584415584415584,0.0,0.22077922077922077,0.987012987012987,1.0,0.4155844155844156,1.0,30,1058.3506493506493,189.78844155844155,0,1.3116883116883118,10079.974025974027
+Harbor OpenCode-only,600,opencode,0,161,0.0,16.70186335403727,19.453416149068325,19.0,22.0,1.6583850931677018,0.08749843522181466,0.515527950310559,0.0,0.9627329192546584,0.9813664596273292,0.35403726708074534,0.8012422360248447,0.3076923076923077,52,2257.472049689441,318.1972049689441,1,3.0683229813664594,124388.36645962733
+Harbor OpenCode-only,600,opencode,1,89,1.0,16.84269662921348,18.202247191011235,18.0,19.0,5.550561797752809,0.3068027400914311,1.0,0.0,0.9550561797752809,0.9887640449438202,1.0,0.38202247191011235,1.0,34,1404.6966292134832,217.18235955056178,0,2.3707865168539324,63050.29213483146
+Harbor OpenCode-only,700,claude-code,0,134,0.0,14.955223880597014,16.32089552238806,17.0,20.700000000000003,1.6940298507462686,0.10036111108362263,0.5597014925373134,0.007462686567164179,0.6865671641791045,0.9776119402985075,0.4701492537313433,0.7910447761194029,0.43902439024390244,41,5475.335820895522,346.01283582089553,0,14.940298507462687,589280.671641791
+Harbor OpenCode-only,700,claude-code,1,116,1.0,12.129310344827585,12.198275862068966,12.0,18.0,2.3879310344827585,0.17583457980343323,0.7241379310344828,0.0,0.25,1.0,1.0,0.5775862068965517,1.0,45,3157.7844827586205,218.82741379310343,0,12.129310344827585,408917.0775862069
+Harbor OpenCode-only,700,codex,0,157,0.0,14.859872611464969,18.910828025477706,19.0,25.400000000000006,1.5923566878980893,0.08045271972899054,0.5668789808917197,0.0,0.6624203821656051,0.9808917197452229,0.42038216560509556,0.7133757961783439,0.46938775510204084,49,3415.5732484076434,328.84222929936305,1,1.6305732484076434,26271.261146496814
+Harbor OpenCode-only,700,codex,1,93,1.0,11.010752688172044,12.53763440860215,12.0,19.0,2.075268817204301,0.14050325778161113,0.6666666666666666,0.0,0.13978494623655913,0.989247311827957,1.0,0.40860215053763443,1.0,37,1504.4408602150538,171.7011827956989,0,1.2365591397849462,13730.827956989247
+Harbor OpenCode-only,700,mini-swe-agent,0,164,0.0,16.5,16.682926829268293,17.0,17.0,0.4573170731707317,0.026926464246577145,0.3170731707317073,0.0,0.8719512195121951,0.9878048780487805,0.27439024390243905,0.7682926829268293,0.3137254901960784,51,2083.548780487805,280.42603658536586,2,2.9207317073170733,37507.32317073171
+Harbor OpenCode-only,700,mini-swe-agent,1,86,1.0,12.906976744186046,12.872093023255815,13.0,17.0,0.7325581395348837,0.055088845203324285,0.47674418604651164,0.0,0.3023255813953488,1.0,1.0,0.36046511627906974,1.0,35,1219.7906976744187,190.23116279069765,0,1.7790697674418605,13905.96511627907
+Harbor OpenCode-only,700,opencode,0,150,0.0,16.053333333333335,21.273333333333333,20.0,27.0,3.4066666666666667,0.1529009440988349,0.8333333333333334,0.0,0.7866666666666666,0.98,0.4866666666666667,0.7866666666666666,0.5208333333333334,48,3196.12,342.27186666666665,0,5.54,238433.54
+Harbor OpenCode-only,700,opencode,1,100,1.0,14.59,18.49,17.0,24.0,4.96,0.28931035334167915,1.0,0.0,0.39,0.97,1.0,0.47,1.0,38,2071.08,233.6001,1,3.72,118031.58
+Harbor OpenCode-only,800,claude-code,0,153,0.0,14.18954248366013,14.30718954248366,17.0,18.0,0.9281045751633987,0.060710275289534106,0.46405228758169936,0.0,0.5620915032679739,0.8954248366013072,0.46405228758169936,0.7320261437908496,0.5686274509803921,51,5360.098039215686,321.1671895424837,3,14.169934640522875,530368.8692810457
+Harbor OpenCode-only,800,claude-code,1,97,1.0,10.484536082474227,9.876288659793815,9.0,16.400000000000006,1.0515463917525774,0.08964026231907883,0.41237113402061853,0.0,0.1134020618556701,0.9587628865979382,1.0,0.5154639175257731,1.0,35,3721.453608247423,237.73979381443297,1,10.484536082474227,387168.4639175258
+Harbor OpenCode-only,800,codex,0,161,0.0,14.503105590062113,15.322981366459627,17.0,19.0,1.0683229813664596,0.06989377183399643,0.5403726708074534,0.006211180124223602,0.5652173913043478,0.9503105590062112,0.35403726708074534,0.7267080745341615,0.4230769230769231,52,2914.1801242236024,298.87801242236026,2,1.9565217391304348,30247.006211180124
+Harbor OpenCode-only,800,codex,1,89,1.0,12.101123595505618,12.146067415730338,12.0,17.0,1.3595505617977528,0.09805768759790048,0.5842696629213483,0.0,0.19101123595505617,0.9775280898876404,1.0,0.42696629213483145,1.0,34,1768.7078651685392,201.70280898876405,2,1.1797752808988764,14081.573033707866
+Harbor OpenCode-only,800,mini-swe-agent,0,171,0.0,16.608187134502923,16.4093567251462,17.0,17.0,0.6257309941520468,0.03928024446600298,0.3684210526315789,0.0,0.9064327485380117,0.9766081871345029,0.2807017543859649,0.7368421052631579,0.2962962962962963,54,2913.12865497076,264.1375438596491,1,5.514619883040936,63107.6432748538
+Harbor OpenCode-only,800,mini-swe-agent,1,79,1.0,14.443037974683545,13.215189873417721,13.0,17.0,1.0506329113924051,0.07818177726964026,0.5822784810126582,0.0,0.379746835443038,1.0,1.0,0.27848101265822783,1.0,32,1223.0759493670887,187.66430379746834,2,2.3797468354430378,16477.354430379746
+Harbor OpenCode-only,800,opencode,0,184,0.0,13.434782608695652,14.065217391304348,17.0,18.0,1.125,0.09618213972391856,0.6793478260869565,0.05434782608695652,0.6086956521739131,0.7934782608695652,0.3695652173913043,0.6141304347826086,0.3114754098360656,61,4090.092391304348,322.4482065217391,7,6.559782608695652,274310.0706521739
+Harbor OpenCode-only,800,opencode,1,66,1.0,12.681818181818182,12.681818181818182,12.0,17.0,2.1363636363636362,0.1705536936820359,1.0,0.0,0.16666666666666666,0.9545454545454546,1.0,0.36363636363636365,1.0,25,1674.1818181818182,221.4387878787879,2,3.409090909090909,102458.86363636363
+Harbor OpenCode-only,900,claude-code,0,162,0.0,15.024691358024691,16.179012345679013,17.0,20.0,1.1049382716049383,0.06768826268428678,0.5185185185185185,0.006172839506172839,0.7345679012345679,0.9135802469135802,0.25308641975308643,0.8271604938271605,0.24,50,5573.407407407408,365.2506790123457,0,15.006172839506172,562142.2160493827
+Harbor OpenCode-only,900,claude-code,1,88,1.0,11.534090909090908,11.147727272727273,11.0,17.0,1.7272727272727273,0.1376910941460266,0.6590909090909091,0.0,0.11363636363636363,0.9545454545454546,1.0,0.45454545454545453,1.0,36,3369.9545454545455,247.765,0,11.534090909090908,413111.1022727273
+Harbor OpenCode-only,900,codex,0,182,0.0,14.95054945054945,17.516483516483518,18.0,23.0,1.5384615384615385,0.08547364529253798,0.6208791208791209,0.0,0.6428571428571429,0.9230769230769231,0.2857142857142857,0.6758241758241759,0.26229508196721313,61,3011.0604395604396,326.71725274725276,0,2.291208791208791,37625.153846153844
+Harbor OpenCode-only,900,codex,1,68,1.0,12.161764705882353,13.161764705882353,13.0,17.300000000000004,1.5147058823529411,0.10776092501814138,0.6470588235294118,0.0,0.10294117647058823,0.9852941176470589,1.0,0.5147058823529411,1.0,25,1981.5,242.2825,0,1.2794117647058822,16128.985294117647
+Harbor OpenCode-only,900,mini-swe-agent,0,185,0.0,16.935135135135134,16.72972972972973,17.0,17.0,0.8216216216216217,0.05056493321199204,0.4702702702702703,0.0,0.9783783783783784,0.9783783783783784,0.15135135135135136,0.7351351351351352,0.18032786885245902,61,2656.8,276.98718918918917,2,4.762162162162162,71131.2054054054
+Harbor OpenCode-only,900,mini-swe-agent,1,65,1.0,16.23076923076923,14.36923076923077,15.0,17.0,1.5692307692307692,0.11439487360301841,0.8153846153846154,0.0,0.7846153846153846,1.0,1.0,0.47692307692307695,1.0,25,1252.923076923077,225.94923076923075,0,2.769230769230769,29392.076923076922
+Harbor OpenCode-only,900,opencode,0,175,0.0,13.725714285714286,18.914285714285715,20.0,26.0,3.222857142857143,0.2060761657218522,0.9485714285714286,0.0,0.6342857142857142,0.8457142857142858,0.7542857142857143,0.6457142857142857,0.07142857142857142,56,3102.0057142857145,292.66497142857145,1,5.228571428571429,276230.77714285714
+Harbor OpenCode-only,900,opencode,1,75,1.0,13.866666666666667,17.2,18.0,21.0,4.72,0.28722407543150574,1.0,0.0,0.21333333333333335,0.88,1.0,0.49333333333333335,1.0,30,1837.0133333333333,257.4232,0,2.5733333333333333,90841.49333333333
+Harbor OpenCode-only,1000,claude-code,0,173,0.0,15.398843930635838,21.14450867052023,21.0,29.0,2.959537572254335,0.1300178697767679,0.7456647398843931,0.0,0.7687861271676301,0.8901734104046243,0.3583815028901734,0.815028901734104,0.21818181818181817,55,5566.254335260116,360.69156069364163,0,15.393063583815028,603630.0173410404
+Harbor OpenCode-only,1000,claude-code,1,77,1.0,11.779220779220779,13.818181818181818,14.0,20.0,1.6753246753246753,0.10423260463154578,0.6233766233766234,0.0,0.18181818181818182,0.948051948051948,1.0,0.5974025974025974,1.0,31,3435.3636363636365,248.8414285714286,0,11.779220779220779,418759.83116883115
+Harbor OpenCode-only,1000,codex,0,169,0.0,15.0,22.159763313609467,21.0,33.20000000000002,3.2071005917159763,0.1257243878254321,0.7928994082840237,0.0,0.6627218934911243,0.9585798816568047,0.1834319526627219,0.7100591715976331,0.20754716981132076,53,3303.2544378698226,331.3844970414201,0,2.42603550295858,41973.36686390533
+Harbor OpenCode-only,1000,codex,1,81,1.0,12.419753086419753,16.703703703703702,17.0,24.0,2.6419753086419755,0.13168325582668605,0.7530864197530864,0.0,0.24691358024691357,0.9259259259259259,1.0,0.6172839506172839,1.0,33,2142.456790123457,246.69432098765435,0,1.5308641975308641,23130.345679012345
+Harbor OpenCode-only,1000,mini-swe-agent,0,199,0.0,16.949748743718594,19.246231155778894,17.0,25.0,1.849246231155779,0.0862166297164493,0.6582914572864321,0.0,0.9849246231155779,0.9547738693467337,0.10552763819095477,0.7236180904522613,0.08695652173913043,69,2559.618090452261,246.21407035175878,0,2.030150753768844,39199.25125628141
+Harbor OpenCode-only,1000,mini-swe-agent,1,51,1.0,16.470588235294116,17.0,16.0,22.0,2.0,0.11829921708417217,0.8235294117647058,0.0,0.8431372549019608,0.9411764705882353,1.0,0.45098039215686275,1.0,17,2065.1176470588234,244.17235294117648,0,2.627450980392157,37254.705882352944
+Harbor OpenCode-only,1000,opencode,0,195,0.0,16.148717948717948,26.92820512820513,27.0,35.599999999999994,4.569230769230769,0.15155612586225803,0.8512820512820513,0.0,0.8564102564102564,0.9743589743589743,0.9435897435897436,0.8,0.0967741935483871,62,3554.9128205128204,314.7382051282051,2,3.0358974358974358,116502.93846153846
+Harbor OpenCode-only,1000,opencode,1,55,1.0,14.49090909090909,21.78181818181818,22.0,28.0,4.8545454545454545,0.2076344353123434,1.0,0.0,0.3090909090909091,0.9636363636363636,1.0,0.5818181818181818,1.0,24,2474.909090909091,239.53581818181817,0,2.690909090909091,81047.5090909091
+Harbor multi-harness,0,claude-code,0,208,0.0,14.177884615384615,13.759615384615385,16.0,18.0,0.6009615384615384,0.03858411235582612,0.24519230769230768,0.028846153846153848,0.6105769230769231,0.7548076923076923,0.33653846153846156,0.7163461538461539,0.35294117647058826,68,3907.5673076923076,330.2196153846154,4,13.774038461538462,551784.2644230769
+Harbor multi-harness,0,claude-code,1,42,1.0,8.642857142857142,7.428571428571429,7.0,13.799999999999997,0.30952380952380953,0.032916666666666664,0.16666666666666666,0.047619047619047616,0.07142857142857142,0.8333333333333334,0.9523809523809523,0.40476190476190477,1.0,18,1545.7142857142858,190.2461904761905,0,8.571428571428571,276973.14285714284
+Harbor multi-harness,0,codex,0,209,0.0,15.167464114832535,15.014354066985646,17.0,20.0,0.5933014354066986,0.03703165718455073,0.3492822966507177,0.07655502392344497,0.7129186602870813,0.7416267942583732,0.2727272727272727,0.6172248803827751,0.23529411764705882,68,3264.622009569378,302.87047846889953,0,1.1483253588516746,20704.593301435405
+Harbor multi-harness,0,codex,1,41,1.0,11.463414634146341,10.317073170731707,9.0,17.0,0.9024390243902439,0.06859963779468424,0.2926829268292683,0.0975609756097561,0.21951219512195122,0.8292682926829268,0.9024390243902439,0.36585365853658536,1.0,18,1723.5853658536585,212.49975609756098,0,1.0,12842.731707317073
+Harbor multi-harness,0,mini-swe-agent,0,214,0.0,16.490654205607477,15.682242990654206,17.0,17.0,0.37850467289719625,0.024792598653963587,0.26635514018691586,0.037383177570093455,0.8878504672897196,0.8317757009345794,0.2102803738317757,0.677570093457944,0.20833333333333334,72,2659.411214953271,232.60500000000002,4,1.1682242990654206,17348.168224299065
+Harbor multi-harness,0,mini-swe-agent,1,36,1.0,13.277777777777779,12.0,12.0,16.0,0.5555555555555556,0.040360410927019925,0.3333333333333333,0.05555555555555555,0.3333333333333333,0.9166666666666666,0.9444444444444444,0.3888888888888889,1.0,14,1411.1666666666667,198.5597222222222,0,1.7222222222222223,15965.027777777777
+Harbor multi-harness,0,opencode,0,223,0.0,13.179372197309418,11.448430493273543,14.0,18.0,0.5246636771300448,0.038140999359143606,0.3094170403587444,0.10762331838565023,0.4977578475336323,0.7488789237668162,0.33183856502242154,0.6322869955156951,0.3291139240506329,79,2014.8161434977578,275.3364125560538,3,1.1973094170403586,31591.430493273543
+Harbor multi-harness,0,opencode,1,27,1.0,9.296296296296296,6.962962962962963,7.0,12.400000000000002,0.18518518518518517,0.022289986228605157,0.14814814814814814,0.14814814814814814,0.037037037037037035,0.7407407407407407,0.8518518518518519,0.2962962962962963,1.0,7,818.4814814814815,255.86296296296297,0,1.3703703703703705,33369.59259259259
+Harbor multi-harness,100,claude-code,0,181,0.0,16.1878453038674,15.773480662983426,17.0,17.0,2.0939226519337018,0.129624268008588,0.5469613259668509,0.011049723756906077,0.8729281767955801,0.7348066298342542,0.3149171270718232,0.6795580110497238,0.2545454545454545,55,4083.745856353591,346.4222099447514,8,15.502762430939226,648460.8342541436
+Harbor multi-harness,100,claude-code,1,69,1.0,16.44927536231884,15.898550724637682,17.0,17.0,5.3768115942028984,0.3334485280726363,0.927536231884058,0.0,0.8840579710144928,0.8985507246376812,1.0,0.5072463768115942,1.0,31,3166.7971014492755,289.0965217391304,5,15.81159420289855,620988.5072463768
+Harbor multi-harness,100,codex,0,180,0.0,16.75,16.511111111111113,17.0,18.0,1.0444444444444445,0.06217952412224858,0.42777777777777776,0.0,0.9166666666666666,0.8722222222222222,0.15,0.75,0.16363636363636364,55,3696.4,353.48133333333334,0,1.0888888888888888,19165.077777777777
+Harbor multi-harness,100,codex,1,70,1.0,16.285714285714285,16.314285714285713,17.0,18.0,3.9285714285714284,0.23561877775012116,0.9,0.0,0.8714285714285714,0.9571428571428572,1.0,0.5285714285714286,1.0,31,2457.1285714285714,234.46214285714282,0,1.1285714285714286,14368.4
+Harbor multi-harness,100,mini-swe-agent,0,202,0.0,16.594059405940595,16.524752475247524,17.0,17.0,0.9554455445544554,0.05683738016658809,0.46534653465346537,0.0,0.905940594059406,0.9207920792079208,0.16336633663366337,0.6831683168316832,0.13636363636363635,66,2094.9752475247524,222.38960396039602,8,1.0792079207920793,15573.10891089109
+Harbor multi-harness,100,mini-swe-agent,1,48,1.0,12.770833333333334,12.770833333333334,12.0,17.0,1.1041666666666667,0.07575306297732769,0.5625,0.0,0.25,0.9791666666666666,1.0,0.2916666666666667,1.0,20,1214.5416666666667,166.41479166666667,1,1.0,8154.375
+Harbor multi-harness,100,opencode,0,189,0.0,16.523809523809526,15.587301587301587,16.0,16.0,3.798941798941799,0.2390842401610744,0.6349206349206349,0.010582010582010581,0.9153439153439153,0.8624338624338624,0.4126984126984127,0.6878306878306878,0.3709677419354839,62,2586.772486772487,266.1631216931217,19,2.3015873015873014,68499.28042328042
+Harbor multi-harness,100,opencode,1,61,1.0,16.934426229508198,15.98360655737705,16.0,16.0,9.704918032786885,0.605418591077501,0.9836065573770492,0.0,0.9836065573770492,0.8688524590163934,1.0,0.45901639344262296,1.0,24,1481.5245901639344,194.35557377049182,3,2.2950819672131146,59156.55737704918
+Harbor multi-harness,200,claude-code,0,175,0.0,16.46857142857143,15.862857142857143,17.0,17.0,2.2857142857142856,0.14150523287838443,0.5771428571428572,0.005714285714285714,0.8857142857142857,0.8742857142857143,0.26857142857142857,0.7371428571428571,0.2542372881355932,59,4198.674285714285,354.312,32,15.645714285714286,645421.8685714286
+Harbor multi-harness,200,claude-code,1,75,1.0,16.96,16.573333333333334,17.0,17.0,6.72,0.40244544997486176,0.92,0.0,0.96,0.9333333333333333,1.0,0.52,1.0,27,3256.3733333333334,281.4148,12,16.493333333333332,581790.8666666667
+Harbor multi-harness,200,codex,0,184,0.0,16.782608695652176,16.434782608695652,17.0,17.0,1.125,0.06736811858610721,0.4782608695652174,0.005434782608695652,0.9130434782608695,0.9184782608695652,0.23369565217391305,0.7880434782608695,0.21818181818181817,55,3704.461956521739,342.4726086956522,0,1.0108695652173914,18122.902173913044
+Harbor multi-harness,200,codex,1,66,1.0,16.954545454545453,16.863636363636363,17.0,17.5,4.848484848484849,0.2852143635231871,0.8787878787878788,0.0,0.9696969696969697,0.9848484848484849,1.0,0.5303030303030303,1.0,31,2319.8939393939395,230.32651515151514,0,1.0303030303030303,13138.469696969696
+Harbor multi-harness,200,mini-swe-agent,0,204,0.0,16.705882352941178,16.666666666666668,17.0,17.0,1.1715686274509804,0.07006045065084858,0.553921568627451,0.0,0.9313725490196079,0.8480392156862745,0.12745098039215685,0.6127450980392157,0.10606060606060606,66,1686.9117647058824,192.04999999999998,18,1.0735294117647058,14190.122549019608
+Harbor multi-harness,200,mini-swe-agent,1,46,1.0,13.826086956521738,13.826086956521738,16.0,17.0,1.673913043478261,0.10568712155988372,0.5434782608695652,0.0,0.4782608695652174,0.9347826086956522,1.0,0.45652173913043476,1.0,20,1125.9565217391305,155.02695652173912,3,1.0434782608695652,8373.065217391304
+Harbor multi-harness,200,opencode,0,174,0.0,16.67241379310345,15.672413793103448,16.0,16.0,3.6494252873563218,0.2275682268269276,0.632183908045977,0.005747126436781609,0.9597701149425287,0.9022988505747126,0.41379310344827586,0.6954022988505747,0.43137254901960786,51,2429.132183908046,261.697816091954,9,2.218390804597701,65169.862068965514
+Harbor multi-harness,200,opencode,1,76,1.0,17.0,16.039473684210527,16.0,16.0,8.881578947368421,0.5523832559339525,0.9473684210526315,0.0,1.0,0.9473684210526315,1.0,0.47368421052631576,1.0,35,1470.3552631578948,190.74039473684212,2,2.1184210526315788,48577.81578947369
+Harbor multi-harness,300,claude-code,0,167,0.0,15.988023952095809,15.676646706586826,17.0,17.400000000000006,2.3473053892215567,0.1516722823368047,0.6646706586826348,0.017964071856287425,0.844311377245509,0.8083832335329342,0.25748502994011974,0.7005988023952096,0.28846153846153844,52,3633.1437125748503,333.1297005988024,13,15.41317365269461,643764.1736526946
+Harbor multi-harness,300,claude-code,1,83,1.0,16.746987951807228,16.457831325301203,17.0,17.0,7.771084337349397,0.4685708039292656,0.9759036144578314,0.0,0.9397590361445783,0.9156626506024096,1.0,0.5542168674698795,1.0,34,2523.9156626506024,261.3520481927711,4,16.253012048192772,549294.4337349398
+Harbor multi-harness,300,codex,0,176,0.0,16.568181818181817,16.136363636363637,17.0,17.0,1.2443181818181819,0.07808195174923116,0.48295454545454547,0.0,0.9034090909090909,0.9318181818181818,0.25,0.8181818181818182,0.22033898305084745,59,3144.2954545454545,331.7754545454546,0,1.0284090909090908,18341.397727272728
+Harbor multi-harness,300,codex,1,74,1.0,15.594594594594595,15.297297297297296,17.0,17.0,5.472972972972973,0.3397225716111165,0.8918918918918919,0.0,0.7432432432432432,0.9324324324324325,1.0,0.47297297297297297,1.0,27,1847.7567567567567,205.8562162162162,0,1.0135135135135136,11819.445945945947
+Harbor multi-harness,300,mini-swe-agent,0,195,0.0,16.77948717948718,16.74871794871795,17.0,17.0,2.482051282051282,0.14659411101718794,0.7333333333333333,0.0,0.9487179487179487,0.764102564102564,0.17435897435897435,0.5384615384615384,0.12307692307692308,65,1517.7846153846153,191.68061538461538,4,1.041025641025641,11854.758974358974
+Harbor multi-harness,300,mini-swe-agent,1,55,1.0,14.763636363636364,14.763636363636364,17.0,17.0,3.2545454545454544,0.196820205937853,0.6545454545454545,0.0,0.6,0.8909090909090909,1.0,0.4,1.0,21,1023.5272727272727,151.77090909090907,0,1.0,6531.236363636363
+Harbor multi-harness,300,opencode,0,176,0.0,16.886363636363637,15.994318181818182,16.0,17.0,4.375,0.2719334893048128,0.6818181818181818,0.0,0.9715909090909091,0.8125,0.5056818181818182,0.6420454545454546,0.5454545454545454,55,1750.25,220.2347159090909,8,2.164772727272727,62165.60227272727
+Harbor multi-harness,300,opencode,1,74,1.0,16.364864864864863,15.378378378378379,16.0,16.700000000000003,7.45945945945946,0.4785542756130992,0.9594594594594594,0.0,0.8513513513513513,0.8108108108108109,1.0,0.36486486486486486,1.0,31,1148.945945945946,182.39418918918918,0,2.1486486486486487,50040.0
+Harbor multi-harness,400,claude-code,0,158,0.0,16.27848101265823,15.740506329113924,17.0,17.30000000000001,2.1835443037974684,0.13532350824009326,0.6582278481012658,0.012658227848101266,0.8670886075949367,0.759493670886076,0.2088607594936709,0.6772151898734177,0.24,50,4191.721518987341,352.5263291139241,11,15.550632911392405,657314.7468354431
+Harbor multi-harness,400,claude-code,1,92,1.0,16.815217391304348,16.315217391304348,17.0,17.0,6.510869565217392,0.3984565732647574,0.9891304347826086,0.0,0.9130434782608695,0.9565217391304348,1.0,0.5434782608695652,1.0,36,3078.1739130434785,296.3546739130435,1,16.25,624968.9891304348
+Harbor multi-harness,400,codex,0,163,0.0,16.7239263803681,16.380368098159508,17.0,17.0,1.5705521472392638,0.09735117472847533,0.49693251533742333,0.0,0.901840490797546,0.9447852760736196,0.19631901840490798,0.7914110429447853,0.12962962962962962,54,3678.5828220858893,347.2230674846626,0,1.0429447852760736,20032.79754601227
+Harbor multi-harness,400,codex,1,87,1.0,16.839080459770116,16.724137931034484,17.0,17.0,7.505747126436781,0.4469217188791225,0.9195402298850575,0.0,0.9655172413793104,1.0,1.0,0.6436781609195402,1.0,32,2115.1609195402298,217.14908045977012,0,1.0114942528735633,13202.206896551725
+Harbor multi-harness,400,mini-swe-agent,0,178,0.0,16.882022471910112,16.775280898876403,17.0,17.0,1.6910112359550562,0.09991325181758097,0.6853932584269663,0.0,0.9662921348314607,0.8089887640449438,0.15730337078651685,0.6460674157303371,0.2542372881355932,59,2064.4438202247193,223.49376404494382,7,1.1067415730337078,14779.460674157302
+Harbor multi-harness,400,mini-swe-agent,1,72,1.0,15.73611111111111,15.73611111111111,17.0,17.0,5.027777777777778,0.29767763101096434,0.75,0.0,0.8194444444444444,0.9305555555555556,1.0,0.3888888888888889,1.0,27,1342.1944444444443,165.27541666666667,0,1.0,8686.180555555555
+Harbor multi-harness,400,opencode,0,168,0.0,16.464285714285715,15.535714285714286,16.0,16.0,2.4047619047619047,0.15308887443866434,0.5178571428571429,0.0,0.9047619047619048,0.8154761904761905,0.2857142857142857,0.6964285714285714,0.3269230769230769,52,2696.1130952380954,277.2372619047619,18,2.3273809523809526,75044.54166666667
+Harbor multi-harness,400,opencode,1,82,1.0,16.951219512195124,15.987804878048781,16.0,16.0,8.134146341463415,0.5089614504338321,0.9634146341463414,0.0,0.9512195121951219,0.8780487804878049,1.0,0.3902439024390244,1.0,34,1410.4878048780488,185.70902439024388,7,2.1341463414634148,49940.865853658535
+Harbor multi-harness,500,claude-code,0,138,0.0,16.028985507246375,15.543478260869565,16.5,17.0,1.8840579710144927,0.12178888254150738,0.6594202898550725,0.0,0.8260869565217391,0.7681159420289855,0.18115942028985507,0.6811594202898551,0.2,40,4573.398550724638,370.31246376811595,9,15.318840579710145,653862.9492753623
+Harbor multi-harness,500,claude-code,1,112,1.0,16.955357142857142,16.705357142857142,17.0,17.0,7.473214285714286,0.44547463216055655,0.9732142857142857,0.0,0.9642857142857143,0.9553571428571429,1.0,0.5357142857142857,1.0,46,3165.9553571428573,274.3046428571428,7,16.589285714285715,614760.3928571428
+Harbor multi-harness,500,codex,0,152,0.0,16.92105263157895,16.394736842105264,17.0,17.0,1.9210526315789473,0.11469066118350173,0.5526315789473685,0.0,0.9276315789473685,0.9013157894736842,0.19078947368421054,0.8355263157894737,0.24,50,3921.5065789473683,359.5023684210526,0,1.0263157894736843,19961.88157894737
+Harbor multi-harness,500,codex,1,98,1.0,16.76530612244898,16.540816326530614,17.0,17.0,6.316326530612245,0.3783259507599243,0.9693877551020408,0.0,0.9489795918367347,0.9591836734693877,1.0,0.6938775510204082,1.0,36,2528.8571428571427,243.51612244897962,0,1.010204081632653,13861.84693877551
+Harbor multi-harness,500,mini-swe-agent,0,172,0.0,15.918604651162791,15.843023255813954,17.0,17.0,1.0290697674418605,0.06115766073871409,0.436046511627907,0.0,0.8313953488372093,0.8604651162790697,0.20348837209302326,0.6686046511627907,0.16071428571428573,56,2113.6337209302324,224.05284883720927,10,1.1686046511627908,17018.738372093023
+Harbor multi-harness,500,mini-swe-agent,1,78,1.0,11.0,10.987179487179487,11.0,17.0,0.6153846153846154,0.04216183410187935,0.28205128205128205,0.0,0.1282051282051282,0.8717948717948718,1.0,0.4358974358974359,1.0,30,1065.9615384615386,143.7998717948718,2,1.0256410256410255,7264.0641025641025
+Harbor multi-harness,500,opencode,0,168,0.0,16.36904761904762,15.267857142857142,16.0,16.0,3.1547619047619047,0.19931215541772265,0.5535714285714286,0.0,0.9107142857142857,0.7619047619047619,0.3333333333333333,0.6666666666666666,0.2830188679245283,53,2765.0476190476193,287.9326785714286,27,2.3988095238095237,77927.45238095238
+Harbor multi-harness,500,opencode,1,82,1.0,16.9390243902439,15.926829268292684,16.0,16.0,8.548780487804878,0.5360272110720461,1.0,0.0,0.9390243902439024,0.8780487804878049,1.0,0.43902439024390244,1.0,33,1578.7560975609756,205.88426829268292,10,2.231707317073171,57548.243902439026
+Harbor multi-harness,600,claude-code,0,175,0.0,16.18285714285714,15.857142857142858,17.0,18.0,2.8,0.1754122161194253,0.8,0.005714285714285714,0.8342857142857143,0.6742857142857143,0.2057142857142857,0.64,0.2,55,5347.417142857143,381.9341142857143,5,15.548571428571428,665576.6685714286
+Harbor multi-harness,600,claude-code,1,75,1.0,16.88,16.666666666666668,17.0,17.0,6.293333333333333,0.3761555396787595,0.96,0.0,0.96,0.8666666666666667,1.0,0.4533333333333333,1.0,31,3456.133333333333,277.3614666666667,0,16.506666666666668,590298.8266666667
+Harbor multi-harness,600,codex,0,169,0.0,16.70414201183432,15.828402366863905,16.0,17.0,2.106508875739645,0.1292100910347929,0.621301775147929,0.0,0.8224852071005917,0.8994082840236687,0.21893491124260356,0.7514792899408284,0.24074074074074073,54,4656.094674556213,394.3240828402367,0,1.0769230769230769,21019.08875739645
+Harbor multi-harness,600,codex,1,81,1.0,16.950617283950617,16.790123456790123,17.0,17.0,6.419753086419753,0.38044970441484605,0.9753086419753086,0.0,0.9753086419753086,0.9876543209876543,1.0,0.5061728395061729,1.0,32,3006.0864197530864,255.8167901234568,0,1.0246913580246915,14633.382716049382
+Harbor multi-harness,600,mini-swe-agent,0,173,0.0,16.190751445086704,15.947976878612717,17.0,17.0,2.398843930635838,0.1467314769932927,0.791907514450867,0.0,0.8323699421965318,0.7687861271676301,0.18497109826589594,0.49710982658959535,0.2413793103448276,58,2236.3294797687863,216.63260115606937,9,1.2658959537572254,13200.306358381504
+Harbor multi-harness,600,mini-swe-agent,1,77,1.0,11.454545454545455,11.415584415584416,11.0,17.0,0.6623376623376623,0.048452705939336954,0.3116883116883117,0.0,0.15584415584415584,0.8831168831168831,1.0,0.4155844155844156,1.0,28,1229.4935064935064,149.91506493506495,4,1.051948051948052,7048.636363636364
+Harbor multi-harness,600,opencode,0,165,0.0,16.01212121212121,14.587878787878788,16.0,16.0,2.9696969696969697,0.19405374037726977,0.6121212121212121,0.006060606060606061,0.8424242424242424,0.6787878787878788,0.30303030303030304,0.5818181818181818,0.28,50,2968.0363636363636,289.64424242424246,45,2.533333333333333,81446.47878787879
+Harbor multi-harness,600,opencode,1,85,1.0,16.88235294117647,15.83529411764706,16.0,16.0,8.4,0.5319574888779041,0.9764705882352941,0.0,0.8823529411764706,0.8352941176470589,1.0,0.4823529411764706,1.0,36,1910.1176470588234,208.67999999999998,20,2.541176470588235,66069.77647058823
+Harbor multi-harness,684,claude-code,0,161,0.0,15.782608695652174,15.46583850931677,17.0,18.0,3.2732919254658386,0.2125965942216928,0.8385093167701864,0.012422360248447204,0.8074534161490683,0.7080745341614907,0.2546583850931677,0.6708074534161491,0.30612244897959184,49,4663.981366459628,362.6972049689441,6,15.248447204968944,733663.4472049689
+Harbor multi-harness,684,claude-code,1,89,1.0,16.292134831460675,15.955056179775282,17.0,17.0,6.50561797752809,0.4024069059016184,0.9887640449438202,0.0,0.8651685393258427,0.8764044943820225,1.0,0.4943820224719101,1.0,37,2848.3595505617977,260.0788764044944,4,15.96629213483146,583406.1011235955
+Harbor multi-harness,684,codex,0,167,0.0,16.748502994011975,15.994011976047904,17.0,17.0,2.7784431137724552,0.17079379413526907,0.7305389221556886,0.0,0.8383233532934131,0.9281437125748503,0.19760479041916168,0.7904191616766467,0.16363636363636364,55,4250.748502994012,385.4654491017964,0,1.0479041916167664,20587.389221556885
+Harbor multi-harness,684,codex,1,83,1.0,16.867469879518072,16.602409638554217,17.0,17.0,6.9879518072289155,0.41958928149573077,0.9879518072289156,0.0,0.9036144578313253,0.9036144578313253,1.0,0.37349397590361444,1.0,31,2842.2289156626507,275.68036144578315,0,1.0240963855421688,14307.590361445784
+Harbor multi-harness,684,mini-swe-agent,0,174,0.0,15.454022988505747,15.172413793103448,17.0,17.0,2.839080459770115,0.17888920016201965,0.8045977011494253,0.0,0.7298850574712644,0.7528735632183908,0.28160919540229884,0.5517241379310345,0.2982456140350877,57,2137.6206896551726,222.47218390804596,8,1.264367816091954,15476.724137931034
+Harbor multi-harness,684,mini-swe-agent,1,76,1.0,10.460526315789474,10.421052631578947,10.0,16.0,0.7105263157894737,0.05720755744943052,0.42105263157894735,0.0,0.06578947368421052,0.8947368421052632,1.0,0.32894736842105265,1.0,29,908.6315789473684,128.98855263157895,0,1.0526315789473684,6987.4473684210525
+Harbor multi-harness,684,opencode,0,177,0.0,15.163841807909604,13.870056497175142,16.0,16.0,3.3389830508474576,0.22240027475570845,0.7062146892655368,0.005649717514124294,0.7457627118644068,0.655367231638418,0.2994350282485876,0.576271186440678,0.31666666666666665,60,2234.4180790960454,248.0429943502825,39,2.3220338983050848,70932.70056497175
+Harbor multi-harness,684,opencode,1,73,1.0,16.945205479452056,15.931506849315069,16.0,17.0,8.643835616438356,0.542459721539093,0.9726027397260274,0.0,0.958904109589041,0.7808219178082192,1.0,0.3972602739726027,1.0,26,1487.0958904109589,200.8408219178082,8,2.136986301369863,56677.82191780822
+Harbor multi-harness,700,claude-code,0,171,0.0,15.362573099415204,15.140350877192983,17.0,18.0,3.1228070175438596,0.2032035172535417,0.8421052631578947,0.011695906432748537,0.7602339181286549,0.6549707602339181,0.21637426900584794,0.6432748538011696,0.20754716981132076,53,4652.7192982456145,361.23532163742686,3,14.859649122807017,694100.2807017544
+Harbor multi-harness,700,claude-code,1,79,1.0,16.443037974683545,16.0,17.0,17.0,6.620253164556962,0.4081213219033104,1.0,0.0,0.8607594936708861,0.8607594936708861,1.0,0.45569620253164556,1.0,33,3123.9493670886077,280.37696202531646,2,15.91139240506329,602102.9620253164
+Harbor multi-harness,700,codex,0,170,0.0,16.74705882352941,16.194117647058825,17.0,17.0,3.1176470588235294,0.19078303568788,0.8352941176470589,0.0,0.8764705882352941,0.8529411764705882,0.2529411764705882,0.7294117647058823,0.21568627450980393,51,4079.929411764706,363.8954117647059,0,1.0058823529411764,19588.45294117647
+Harbor multi-harness,700,codex,1,80,1.0,16.8625,16.7,17.0,17.0,6.7625,0.405960617201426,0.975,0.0,0.9125,0.95,1.0,0.4625,1.0,35,2884.475,262.45475,0,1.0125,14580.1375
+Harbor multi-harness,700,mini-swe-agent,0,174,0.0,15.948275862068966,15.60919540229885,17.0,17.0,2.6839080459770117,0.16887949645102415,0.8390804597701149,0.0,0.7931034482758621,0.735632183908046,0.21839080459770116,0.5804597701149425,0.24528301886792453,53,2263.3563218390805,227.50166666666667,11,1.3793103448275863,16386.827586206895
+Harbor multi-harness,700,mini-swe-agent,1,76,1.0,10.81578947368421,10.697368421052632,11.0,15.5,0.8289473684210527,0.0667787449908193,0.5,0.0,0.09210526315789473,0.8552631578947368,1.0,0.27631578947368424,1.0,33,1056.0263157894738,140.4973684210526,0,1.0921052631578947,8168.815789473684
+Harbor multi-harness,700,opencode,0,197,0.0,14.345177664974619,13.126903553299492,16.0,16.0,3.0609137055837565,0.1993849283823903,0.6091370558375635,0.03553299492385787,0.6802030456852792,0.5431472081218274,0.26903553299492383,0.4467005076142132,0.1935483870967742,62,2101.964467005076,228.04253807106596,33,2.2944162436548226,71687.75126903554
+Harbor multi-harness,700,opencode,1,53,1.0,16.90566037735849,15.849056603773585,16.0,17.0,7.7924528301886795,0.4941648575250129,0.9622641509433962,0.0,0.9245283018867925,0.7924528301886793,1.0,0.3584905660377358,1.0,24,1739.0943396226414,211.79509433962264,13,2.169811320754717,54181.20754716981
+Harbor multi-harness,800,claude-code,0,170,0.0,15.158823529411764,14.770588235294118,16.0,17.0,3.4235294117647057,0.2306327284029433,0.8705882352941177,0.011764705882352941,0.7176470588235294,0.5764705882352941,0.1411764705882353,0.6294117647058823,0.11538461538461539,52,5225.105882352941,382.35835294117646,4,14.5,658475.4588235294
+Harbor multi-harness,800,claude-code,1,80,1.0,16.9625,16.7125,17.0,17.0,8.025,0.48083872364735364,1.0,0.0,0.9625,0.925,1.0,0.575,1.0,34,3774.8,313.24275,5,16.475,629729.4875
+Harbor multi-harness,800,codex,0,183,0.0,16.60655737704918,15.3551912568306,16.0,17.0,3.109289617486339,0.20031003631534008,0.8633879781420765,0.0,0.6830601092896175,0.8306010928961749,0.15846994535519127,0.5737704918032787,0.11666666666666667,60,5050.88524590164,443.0098360655738,0,1.0491803278688525,21167.169398907103
+Harbor multi-harness,800,codex,1,67,1.0,16.65671641791045,15.701492537313433,16.0,17.0,5.895522388059701,0.3727983970960266,1.0,0.0,0.7014925373134329,0.9850746268656716,1.0,0.3880597014925373,1.0,26,3771.955223880597,371.7289552238806,0,1.0298507462686568,14596.686567164179
+Harbor multi-harness,800,mini-swe-agent,0,185,0.0,16.572972972972973,16.427027027027027,17.0,17.0,2.708108108108108,0.16184273728391377,0.8108108108108109,0.0,0.8972972972972973,0.7189189189189189,0.08648648648648649,0.4810810810810811,0.04918032786885246,61,3029.054054054054,264.9401081081081,22,1.145945945945946,17322.75135135135
+Harbor multi-harness,800,mini-swe-agent,1,65,1.0,11.846153846153847,11.815384615384616,12.0,16.0,0.8307692307692308,0.060862606473466196,0.49230769230769234,0.0,0.07692307692307693,0.9538461538461539,1.0,0.35384615384615387,1.0,25,1533.3538461538462,167.18092307692308,3,1.0307692307692307,8874.661538461538
+Harbor multi-harness,800,opencode,0,192,0.0,15.739583333333334,14.453125,16.0,16.0,2.7135416666666665,0.17644656286659963,0.78125,0.036458333333333336,0.84375,0.5729166666666666,0.13541666666666666,0.46875,0.14754098360655737,61,3080.0416666666665,291.22421875000003,62,2.5260416666666665,81832.140625
+Harbor multi-harness,800,opencode,1,58,1.0,16.93103448275862,15.810344827586206,16.0,17.0,7.120689655172414,0.4502001618507704,0.9827586206896551,0.0,0.9482758620689655,0.7931034482758621,1.0,0.4482758620689655,1.0,25,1932.6724137931035,241.5848275862069,11,2.4482758620689653,62929.93103448276
+Harbor multi-harness,900,claude-code,0,159,0.0,14.81132075471698,14.075471698113208,16.0,17.0,3.981132075471698,0.26696447467266565,0.8238993710691824,0.031446540880503145,0.7484276729559748,0.6037735849056604,0.20125786163522014,0.6226415094339622,0.2222222222222222,45,10612.17610062893,454.2730188679246,29,14.779874213836479,634783.3018867924
+Harbor multi-harness,900,claude-code,1,91,1.0,16.736263736263737,16.439560439560438,17.0,18.0,9.263736263736265,0.55738028679758,1.0,0.0,0.945054945054945,0.8351648351648352,1.0,0.4175824175824176,1.0,41,7610.912087912088,357.4365934065934,5,16.736263736263737,717278.7692307692
+Harbor multi-harness,900,codex,0,202,0.0,7.965346534653466,7.138613861386139,6.0,17.0,0.995049504950495,0.0920445462451869,0.37623762376237624,0.14356435643564355,0.11386138613861387,0.5445544554455446,0.1188118811881188,0.3217821782178218,0.015151515151515152,66,3116.232673267327,324.8817326732673,2,1.108910891089109,13871.940594059406
+Harbor multi-harness,900,codex,1,48,1.0,14.458333333333334,13.9375,15.5,17.0,5.625,0.3789030363019334,0.9791666666666666,0.0,0.5208333333333334,0.9375,1.0,0.3958333333333333,1.0,20,6214.291666666667,388.205,0,1.0625,17239.0
+Harbor multi-harness,900,mini-swe-agent,0,168,0.0,15.529761904761905,14.476190476190476,16.0,17.0,1.4047619047619047,0.09060472431008144,0.625,0.0,0.6964285714285714,0.8214285714285714,0.17857142857142858,0.6071428571428571,0.1568627450980392,51,4686.678571428572,356.21607142857147,83,1.1488095238095237,17291.779761904763
+Harbor multi-harness,900,mini-swe-agent,1,82,1.0,10.329268292682928,9.939024390243903,9.0,14.900000000000006,0.45121951219512196,0.03692346430868669,0.2682926829268293,0.0,0.08536585365853659,0.9146341463414634,1.0,0.4146341463414634,1.0,35,2480.6585365853657,206.19500000000002,27,1.0365853658536586,8791.621951219513
+Harbor multi-harness,900,opencode,0,244,0.0,3.6639344262295084,1.8770491803278688,1.0,3.0,0.26229508196721313,0.020430672268907563,0.045081967213114756,0.8934426229508197,0.036885245901639344,0.040983606557377046,0.02459016393442623,0.036885245901639344,0.012048192771084338,83,4063.122950819672,226.0097131147541,3,2.057377049180328,51055.42622950819
+Harbor multi-harness,900,opencode,1,6,1.0,17.0,16.0,16.0,16.0,9.5,0.59375,1.0,0.0,1.0,0.6666666666666666,1.0,0.5,1.0,3,2838.0,459.5533333333333,0,2.1666666666666665,57072.5
+Harbor multi-harness,1000,claude-code,0,153,0.0,13.176470588235293,12.143790849673202,15.0,17.0,3.026143790849673,0.23638500454198869,0.8366013071895425,0.013071895424836602,0.5294117647058824,0.5816993464052288,0.23529411764705882,0.5620915032679739,0.11627906976744186,43,10960.522875816994,462.06437908496736,63,13.176470588235293,534125.8562091503
+Harbor multi-harness,1000,claude-code,1,97,1.0,13.164948453608247,12.329896907216495,12.0,17.0,4.979381443298969,0.36920607364576596,0.9072164948453608,0.0,0.3917525773195876,0.8556701030927835,1.0,0.6082474226804123,1.0,43,7160.865979381443,326.91278350515466,10,13.154639175257731,522237.4742268041
+Harbor multi-harness,1000,codex,0,188,0.0,8.595744680851064,7.718085106382978,6.0,17.0,1.5585106382978724,0.13339811045928066,0.44148936170212766,0.18085106382978725,0.15425531914893617,0.6063829787234043,0.18617021276595744,0.44680851063829785,0.06557377049180328,61,3815.446808510638,341.09920212765957,8,1.0638297872340425,14604.494680851063
+Harbor multi-harness,1000,codex,1,62,1.0,14.435483870967742,13.870967741935484,15.0,17.0,6.5,0.4503665044106221,1.0,0.0,0.45161290322580644,0.9516129032258065,1.0,0.46774193548387094,1.0,25,5575.887096774193,323.1374193548387,0,1.032258064516129,16035.064516129032
+Harbor multi-harness,1000,mini-swe-agent,0,160,0.0,13.71875,12.45,14.0,17.0,1.5875,0.1097099739517019,0.575,0.0,0.45625,0.7875,0.325,0.5,0.36,50,3934.6625,329.1018125,94,1.38125,15135.775
+Harbor multi-harness,1000,mini-swe-agent,1,90,1.0,8.8,8.433333333333334,8.0,13.100000000000009,0.3,0.02771149057913764,0.15555555555555556,0.0,0.03333333333333333,0.9111111111111111,0.9888888888888889,0.45555555555555555,1.0,36,1893.8555555555556,161.125,26,1.4555555555555555,10101.877777777778
+Harbor multi-harness,1000,opencode,0,236,0.0,4.758474576271187,3.0211864406779663,1.0,8.5,0.7161016949152542,0.04913525117288826,0.11016949152542373,0.7838983050847458,0.08050847457627118,0.13559322033898305,0.07203389830508475,0.11016949152542373,0.025974025974025976,77,3998.741525423729,205.5077118644068,4,2.152542372881356,56115.39406779661
+Harbor multi-harness,1000,opencode,1,14,1.0,14.714285714285714,13.5,16.0,16.0,8.0,0.5429946978791517,1.0,0.0,0.6428571428571429,0.7857142857142857,1.0,0.5,1.0,9,2043.7142857142858,147.84142857142857,0,2.5714285714285716,65820.35714285714
+Native OpenCode,0,claude-code,0,208,0.0,14.302884615384615,14.264423076923077,17.0,18.0,0.6586538461538461,0.04440941485464735,0.30288461538461536,0.014423076923076924,0.6298076923076923,0.75,0.3173076923076923,0.7163461538461539,0.29411764705882354,68,3756.2115384615386,315.4400961538461,175,14.052884615384615,570159.0384615385
+Native OpenCode,0,claude-code,1,42,1.0,9.904761904761905,9.380952380952381,8.0,16.9,0.4523809523809524,0.04256967744362702,0.30952380952380953,0.0,0.11904761904761904,0.9285714285714286,1.0,0.5238095238095238,1.0,18,2053.785714285714,224.98285714285717,32,9.761904761904763,339870.6904761905
+Native OpenCode,0,codex,0,212,0.0,15.415094339622641,16.169811320754718,17.0,19.0,0.9009433962264151,0.053540390426595565,0.45754716981132076,0.0,0.7547169811320755,0.7358490566037735,0.2641509433962264,0.6132075471698113,0.2753623188405797,69,3129.948113207547,308.76754716981134,184,1.1603773584905661,20631.580188679247
+Native OpenCode,0,codex,1,38,1.0,11.947368421052632,11.394736842105264,11.0,16.300000000000004,0.5263157894736842,0.041548275855551396,0.3684210526315789,0.0,0.2894736842105263,0.868421052631579,0.9736842105263158,0.47368421052631576,1.0,17,1782.8157894736842,239.67105263157896,30,1.1842105263157894,15461.157894736842
+Native OpenCode,0,mini-swe-agent,0,203,0.0,16.43349753694581,16.300492610837438,17.0,17.0,0.5369458128078818,0.03434724625321554,0.33004926108374383,0.0,0.8719211822660099,0.9064039408866995,0.21674876847290642,0.7339901477832512,0.2463768115942029,69,2708.073891625616,228.48142857142858,165,1.1379310344827587,18054.32512315271
+Native OpenCode,0,mini-swe-agent,1,47,1.0,13.48936170212766,12.680851063829786,13.0,17.0,1.0425531914893618,0.076475382842717,0.5319148936170213,0.0,0.3191489361702128,0.9574468085106383,1.0,0.425531914893617,1.0,17,1625.723404255319,172.65234042553192,39,1.702127659574468,14450.063829787234
+Native OpenCode,0,opencode,0,218,0.0,13.293577981651376,12.412844036697248,16.0,17.0,0.591743119266055,0.038509625964180456,0.3165137614678899,0.022935779816513763,0.5412844036697247,0.8623853211009175,0.40825688073394495,0.7201834862385321,0.4084507042253521,71,1849.3532110091744,228.68224770642203,179,1.1605504587155964,34423.706422018346
+Native OpenCode,0,opencode,1,32,1.0,8.65625,7.34375,7.0,12.600000000000009,0.125,0.007598039215686275,0.0625,0.0,0.09375,0.875,1.0,0.4375,1.0,15,853.15625,168.39625,26,1.3125,31859.375
+Native OpenCode,100,claude-code,0,199,0.0,13.763819095477388,13.763819095477388,17.0,18.0,0.6834170854271356,0.04006487274477853,0.32160804020100503,0.01507537688442211,0.6130653266331658,0.8592964824120602,0.41708542713567837,0.7336683417085427,0.375,64,4123.48743718593,150.66597989949747,0,13.683417085427136,571350.7336683417
+Native OpenCode,100,claude-code,1,51,1.0,8.294117647058824,7.470588235294118,6.0,14.0,0.17647058823529413,0.015685327104012225,0.13725490196078433,0.0196078431372549,0.058823529411764705,0.9215686274509803,1.0,0.5294117647058824,1.0,22,2041.4313725490197,113.89274509803921,0,8.294117647058824,246696.431372549
+Native OpenCode,100,codex,0,205,0.0,15.678048780487805,16.034146341463416,17.0,19.0,0.526829268292683,0.032170745981919505,0.35121951219512193,0.004878048780487805,0.7707317073170732,0.8634146341463415,0.2731707317073171,0.7219512195121951,0.23880597014925373,67,3219.7951219512197,162.1279512195122,0,1.3219512195121952,20868.331707317073
+Native OpenCode,100,codex,1,45,1.0,11.822222222222223,11.511111111111111,11.0,17.0,0.4666666666666667,0.03983310988212949,0.35555555555555557,0.0,0.15555555555555556,0.9777777777777777,1.0,0.5555555555555556,1.0,19,1963.8666666666666,142.86844444444444,0,1.2222222222222223,14224.6
+Native OpenCode,100,mini-swe-agent,0,198,0.0,16.18686868686869,16.08080808080808,17.0,17.0,0.4090909090909091,0.026111279375539625,0.3181818181818182,0.0,0.8636363636363636,0.9545454545454546,0.23737373737373738,0.7575757575757576,0.23076923076923078,65,2800.1161616161617,93.65085858585859,0,1.1666666666666667,16083.61616161616
+Native OpenCode,100,mini-swe-agent,1,52,1.0,12.288461538461538,11.711538461538462,12.0,17.0,0.4423076923076923,0.03660985350747794,0.28846153846153844,0.0,0.21153846153846154,0.9807692307692307,1.0,0.4423076923076923,1.0,21,1555.423076923077,77.1298076923077,1,1.7692307692307692,14584.115384615385
+Native OpenCode,100,opencode,0,201,0.0,13.298507462686567,12.00497512437811,16.0,16.0,0.5522388059701493,0.03520774841589499,0.2537313432835821,0.004975124378109453,0.527363184079602,0.945273631840796,0.5223880597014925,0.7412935323383084,0.5909090909090909,66,2472.7960199004974,149.8694527363184,0,2.2487562189054726,79615.70149253731
+Native OpenCode,100,opencode,1,49,1.0,7.6938775510204085,5.857142857142857,5.0,11.0,0.02040816326530612,0.002551020408163265,0.02040816326530612,0.0,0.02040816326530612,0.9795918367346939,1.0,0.3673469387755102,1.0,20,822.8163265306123,121.0073469387755,0,2.326530612244898,78707.26530612246
+Native OpenCode,200,claude-code,0,190,0.0,13.889473684210527,13.9,17.0,18.0,0.8894736842105263,0.05147545597541524,0.32105263157894737,0.010526315789473684,0.6421052631578947,0.9368421052631579,0.41578947368421054,0.8157894736842105,0.43333333333333335,60,4142.078947368421,147.35847368421054,0,13.878947368421052,545937.9157894737
+Native OpenCode,200,claude-code,1,60,1.0,7.516666666666667,6.633333333333334,6.0,11.100000000000001,0.05,0.005726495726495726,0.05,0.0,0.0,0.9666666666666667,1.0,0.48333333333333334,1.0,26,1709.3166666666666,108.92166666666667,0,7.5,238652.76666666666
+Native OpenCode,200,codex,0,186,0.0,14.634408602150538,14.935483870967742,17.0,18.0,0.5376344086021505,0.033831798553285615,0.3225806451612903,0.0,0.6827956989247311,0.9086021505376344,0.3817204301075269,0.7688172043010753,0.39344262295081966,61,2838.569892473118,153.04629032258066,0,1.3440860215053763,18924.704301075268
+Native OpenCode,200,codex,1,64,1.0,10.265625,9.625,9.0,15.700000000000003,0.25,0.01870659722222222,0.125,0.0,0.09375,0.921875,1.0,0.484375,1.0,25,1497.109375,130.18609375,0,1.359375,15162.0
+Native OpenCode,200,mini-swe-agent,0,196,0.0,15.948979591836734,15.755102040816327,17.0,17.0,0.3622448979591837,0.025260730332759145,0.25510204081632654,0.0,0.8214285714285714,0.9540816326530612,0.3163265306122449,0.7857142857142857,0.3484848484848485,66,2799.1428571428573,87.24484693877552,0,1.3112244897959184,16243.688775510203
+Native OpenCode,200,mini-swe-agent,1,54,1.0,12.092592592592593,10.907407407407407,11.0,15.700000000000003,0.5370370370370371,0.04723719151823727,0.3333333333333333,0.0,0.2222222222222222,0.9814814814814815,1.0,0.35185185185185186,1.0,20,1331.5740740740741,69.92592592592592,0,2.314814814814815,16194.37037037037
+Native OpenCode,200,opencode,0,207,0.0,12.429951690821255,11.096618357487923,12.0,16.0,0.5797101449275363,0.038225724240858454,0.26570048309178745,0.00966183574879227,0.4782608695652174,0.9758454106280193,0.5603864734299517,0.7971014492753623,0.5074626865671642,67,1963.6521739130435,146.0319806763285,0,1.826086956521739,81043.84057971014
+Native OpenCode,200,opencode,1,43,1.0,7.232558139534884,5.325581395348837,4.0,9.800000000000004,0.046511627906976744,0.004872646733111849,0.046511627906976744,0.0,0.023255813953488372,0.9767441860465116,1.0,0.3023255813953488,1.0,19,734.7441860465116,113.19883720930233,0,2.0930232558139537,79277.81395348837
+Native OpenCode,300,claude-code,0,177,0.0,14.124293785310735,14.01129943502825,17.0,18.0,0.5254237288135594,0.03287209548229121,0.2768361581920904,0.0,0.6214689265536724,0.9661016949152542,0.423728813559322,0.847457627118644,0.4482758620689655,58,4781.062146892656,150.71186440677965,0,14.07909604519774,577002.6101694915
+Native OpenCode,300,claude-code,1,73,1.0,8.479452054794521,7.47945205479452,6.0,13.799999999999997,0.0684931506849315,0.0065535491905354916,0.0684931506849315,0.0136986301369863,0.0273972602739726,0.9452054794520548,1.0,0.547945205479452,1.0,28,2040.013698630137,114.68150684931507,0,8.479452054794521,271751.4794520548
+Native OpenCode,300,codex,0,208,0.0,15.745192307692308,15.913461538461538,17.0,18.0,0.6153846153846154,0.03697912532084359,0.39903846153846156,0.0,0.8076923076923077,0.9134615384615384,0.24519230769230768,0.7548076923076923,0.1643835616438356,73,2957.014423076923,151.3935096153846,0,1.2788461538461537,17881.96153846154
+Native OpenCode,300,codex,1,42,1.0,12.095238095238095,11.452380952380953,12.5,17.0,0.6428571428571429,0.0437022244795354,0.35714285714285715,0.0,0.2857142857142857,0.9047619047619048,1.0,0.40476190476190477,1.0,13,1566.9761904761904,132.18380952380954,0,1.3571428571428572,14411.619047619048
+Native OpenCode,300,mini-swe-agent,0,201,0.0,16.36318407960199,16.149253731343283,17.0,17.0,0.5024875621890548,0.03314672312696895,0.3482587064676617,0.0,0.8706467661691543,0.9651741293532339,0.19900497512437812,0.8208955223880597,0.23943661971830985,71,2948.6517412935323,89.15626865671642,4,1.318407960199005,17349.243781094527
+Native OpenCode,300,mini-swe-agent,1,49,1.0,12.816326530612244,11.53061224489796,11.0,16.0,0.8775510204081632,0.0769577361414096,0.4489795918367347,0.0,0.1836734693877551,0.9795918367346939,1.0,0.3877551020408163,1.0,15,1338.7551020408164,68.10775510204083,0,2.2448979591836733,14512.469387755102
+Native OpenCode,300,opencode,0,198,0.0,13.691919191919192,12.474747474747474,16.0,17.0,0.6515151515151515,0.042636004155612,0.30808080808080807,0.0,0.6111111111111112,0.9949494949494949,0.4444444444444444,0.7878787878787878,0.38095238095238093,63,2405.9646464646466,145.3980303030303,2,1.606060606060606,52651.00505050505
+Native OpenCode,300,opencode,1,52,1.0,6.903846153846154,5.211538461538462,4.0,9.799999999999997,0.15384615384615385,0.013679029304029304,0.07692307692307693,0.0,0.019230769230769232,1.0,1.0,0.34615384615384615,1.0,23,707.9230769230769,107.16211538461539,0,1.9807692307692308,51971.0
+Native OpenCode,400,claude-code,0,169,0.0,14.29585798816568,14.248520710059172,17.0,18.0,0.7514792899408284,0.0437959892770562,0.28994082840236685,0.005917159763313609,0.6923076923076923,0.9230769230769231,0.3254437869822485,0.8224852071005917,0.3333333333333333,51,4858.84023668639,138.55562130177515,0,14.254437869822485,548807.5798816568
+Native OpenCode,400,claude-code,1,81,1.0,8.024691358024691,7.0,5.0,14.0,0.13580246913580246,0.009861885270382002,0.08641975308641975,0.0,0.04938271604938271,1.0,1.0,0.6666666666666666,1.0,35,1900.888888888889,111.3527160493827,0,8.024691358024691,231742.38271604938
+Native OpenCode,400,codex,0,186,0.0,16.032258064516128,16.182795698924732,17.0,18.0,0.6881720430107527,0.041173102205634915,0.3817204301075269,0.0,0.8440860215053764,0.946236559139785,0.22043010752688172,0.7634408602150538,0.18032786885245902,61,3020.1666666666665,146.95666666666665,0,1.2688172043010753,17990.56989247312
+Native OpenCode,400,codex,1,64,1.0,11.84375,11.4375,11.5,17.0,0.484375,0.03632220516411693,0.328125,0.0,0.21875,1.0,1.0,0.578125,1.0,25,1613.78125,129.23703125,0,1.328125,14538.671875
+Native OpenCode,400,mini-swe-agent,0,182,0.0,16.483516483516482,16.186813186813186,17.0,17.0,0.45054945054945056,0.03001271783592404,0.32967032967032966,0.0,0.8626373626373627,0.9615384615384616,0.24725274725274726,0.8571428571428571,0.3114754098360656,61,3207.5494505494507,85.6412087912088,0,1.4285714285714286,18521.351648351647
+Native OpenCode,400,mini-swe-agent,1,68,1.0,12.882352941176471,11.426470588235293,11.0,16.0,0.6470588235294118,0.05360530593013292,0.4411764705882353,0.0,0.25,0.9558823529411765,1.0,0.5588235294117647,1.0,25,1413.5735294117646,69.97279411764706,0,2.485294117647059,17904.45588235294
+Native OpenCode,400,opencode,0,199,0.0,13.07035175879397,11.72361809045226,15.0,16.0,0.5879396984924623,0.03827979587660344,0.2864321608040201,0.0,0.5025125628140703,0.9899497487437185,0.542713567839196,0.7839195979899497,0.5625,64,2161.718592964824,129.72386934673366,0,1.6984924623115578,37042.36180904523
+Native OpenCode,400,opencode,1,51,1.0,7.607843137254902,5.745098039215686,5.0,11.0,0.09803921568627451,0.007107843137254903,0.058823529411764705,0.0,0.0392156862745098,1.0,1.0,0.3137254901960784,1.0,22,712.8823529411765,106.30941176470587,0,2.1372549019607843,34225.470588235294
+Native OpenCode,500,claude-code,0,184,0.0,13.619565217391305,13.190217391304348,17.0,17.0,0.8804347826086957,0.05647794774044101,0.3641304347826087,0.010869565217391304,0.592391304347826,0.967391304347826,0.40217391304347827,0.8097826086956522,0.43333333333333335,60,4348.010869565217,147.87760869565219,0,13.576086956521738,536566.4076086957
+Native OpenCode,500,claude-code,1,66,1.0,8.181818181818182,6.833333333333333,5.5,13.0,0.36363636363636365,0.03604161361514302,0.24242424242424243,0.015151515151515152,0.07575757575757576,0.9545454545454546,1.0,0.45454545454545453,1.0,26,1441.7878787878788,107.24469696969696,0,8.181818181818182,247779.4696969697
+Native OpenCode,500,codex,0,203,0.0,15.551724137931034,15.52216748768473,17.0,17.0,0.645320197044335,0.039335488932669006,0.3891625615763547,0.0049261083743842365,0.7733990147783252,0.9556650246305419,0.2413793103448276,0.6798029556650246,0.2,70,2592.881773399015,152.5869458128079,0,1.354679802955665,18126.08866995074
+Native OpenCode,500,codex,1,47,1.0,11.27659574468085,10.51063829787234,10.0,16.4,0.2553191489361702,0.020055372245610045,0.14893617021276595,0.0,0.14893617021276595,0.9787234042553191,1.0,0.46808510638297873,1.0,16,1302.9148936170213,133.84765957446808,0,1.3404255319148937,13452.446808510638
+Native OpenCode,500,mini-swe-agent,0,177,0.0,16.372881355932204,15.824858757062147,17.0,17.0,0.3446327683615819,0.023830714446367487,0.2542372881355932,0.0,0.8757062146892656,0.96045197740113,0.23163841807909605,0.847457627118644,0.25,56,3247.4406779661017,96.2750847457627,3,1.96045197740113,21980.90395480226
+Native OpenCode,500,mini-swe-agent,1,73,1.0,12.821917808219178,10.452054794520548,10.0,15.799999999999997,0.7671232876712328,0.07374495747783419,0.5068493150684932,0.0,0.2602739726027397,1.0,1.0,0.4520547945205479,1.0,30,1418.2739726027398,72.02739726027397,0,3.3698630136986303,22001.712328767124
+Native OpenCode,500,opencode,0,205,0.0,12.302439024390244,10.917073170731708,13.0,16.0,1.824390243902439,0.12018429322948693,0.47804878048780486,0.004878048780487805,0.44390243902439025,0.9804878048780488,0.5463414634146342,0.7170731707317073,0.582089552238806,67,1360.0487804878048,132.32965853658538,0,2.3365853658536584,56299.10243902439
+Native OpenCode,500,opencode,1,45,1.0,7.155555555555556,5.2444444444444445,4.0,10.0,0.2222222222222222,0.022260702260702262,0.1111111111111111,0.0,0.0,1.0,1.0,0.37777777777777777,1.0,19,514.0222222222222,107.42822222222223,0,1.9555555555555555,36444.35555555556
+Native OpenCode,600,claude-code,0,169,0.0,13.094674556213018,12.591715976331361,16.0,17.0,1.2485207100591715,0.07677985064542303,0.42011834319526625,0.023668639053254437,0.5384615384615384,0.9349112426035503,0.44970414201183434,0.757396449704142,0.36363636363636365,55,3993.591715976331,139.73,1,13.053254437869823,512788.8402366864
+Native OpenCode,600,claude-code,1,81,1.0,8.11111111111111,6.518518518518518,5.0,13.0,0.32098765432098764,0.028901329445991757,0.18518518518518517,0.0,0.06172839506172839,0.9876543209876543,1.0,0.5061728395061729,1.0,31,1573.2345679012346,105.64456790123455,0,8.11111111111111,269689.3703703704
+Native OpenCode,600,codex,0,204,0.0,15.583333333333334,15.563725490196079,17.0,17.0,1.2990196078431373,0.07896935690256579,0.5980392156862745,0.0,0.7647058823529411,0.9362745098039216,0.3235294117647059,0.6519607843137255,0.24285714285714285,70,2547.2303921568628,147.71186274509805,0,1.4264705882352942,18717.892156862745
+Native OpenCode,600,codex,1,46,1.0,9.826086956521738,9.043478260869565,8.0,15.0,0.41304347826086957,0.034211114262265155,0.2826086956521739,0.0,0.10869565217391304,0.9565217391304348,1.0,0.41304347826086957,1.0,16,1209.4130434782608,124.62239130434783,0,1.5,14637.673913043478
+Native OpenCode,600,mini-swe-agent,0,168,0.0,15.577380952380953,14.363095238095237,16.0,17.0,0.5119047619047619,0.038403169154219575,0.36904761904761907,0.0,0.7321428571428571,0.9821428571428571,0.35714285714285715,0.8690476190476191,0.40384615384615385,52,3584.9583333333335,96.24154761904762,2,2.744047619047619,27519.95238095238
+Native OpenCode,600,mini-swe-agent,1,82,1.0,12.939024390243903,10.634146341463415,10.0,16.0,1.0731707317073171,0.10779894722685254,0.5365853658536586,0.0,0.2682926829268293,1.0,1.0,0.5487804878048781,1.0,34,1693.5,75.17646341463416,3,3.8902439024390243,26157.51219512195
+Native OpenCode,600,opencode,0,208,0.0,11.60576923076923,10.14423076923077,10.0,16.0,2.3028846153846154,0.15922536998258152,0.4230769230769231,0.014423076923076924,0.38461538461538464,0.9326923076923077,0.6153846153846154,0.6634615384615384,0.6716417910447762,67,1621.110576923077,127.25942307692307,8,2.706730769230769,60274.144230769234
+Native OpenCode,600,opencode,1,42,1.0,6.0476190476190474,4.238095238095238,3.5,7.0,0.23809523809523808,0.026785714285714284,0.07142857142857142,0.023809523809523808,0.0,0.9523809523809523,1.0,0.2857142857142857,1.0,19,410.95238095238096,106.43476190476191,0,2.119047619047619,44473.09523809524
+Native OpenCode,700,claude-code,0,176,0.0,12.829545454545455,12.170454545454545,16.0,17.0,1.1022727272727273,0.07071338780699708,0.375,0.03409090909090909,0.5170454545454546,0.8977272727272727,0.5795454545454546,0.8011363636363636,0.45454545454545453,55,3675.1704545454545,132.98482954545455,0,12.806818181818182,520574.35795454547
+Native OpenCode,700,claude-code,1,74,1.0,8.013513513513514,6.324324324324325,5.5,11.0,0.14864864864864866,0.021100296100296102,0.10810810810810811,0.0,0.04054054054054054,0.9594594594594594,1.0,0.5405405405405406,1.0,31,1597.918918918919,109.78432432432433,0,8.013513513513514,264528.13513513515
+Native OpenCode,700,codex,0,214,0.0,16.149532710280372,16.009345794392523,17.0,17.0,1.0046728971962617,0.06136985380388349,0.4532710280373832,0.0,0.8598130841121495,0.9906542056074766,0.2803738317757009,0.6308411214953271,0.17142857142857143,70,2724.4859813084113,146.12116822429905,0,1.341121495327103,17182.19158878505
+Native OpenCode,700,codex,1,36,1.0,11.527777777777779,10.694444444444445,10.0,17.0,0.6944444444444444,0.0536913839609918,0.3888888888888889,0.0,0.19444444444444445,0.9722222222222222,1.0,0.5555555555555556,1.0,16,1316.9166666666667,125.90972222222223,0,1.25,12513.777777777777
+Native OpenCode,700,mini-swe-agent,0,174,0.0,15.655172413793103,15.057471264367816,17.0,17.0,0.6206896551724138,0.04524330141723651,0.3505747126436782,0.0,0.7701149425287356,0.9885057471264368,0.3793103448275862,0.8390804597701149,0.38181818181818183,55,3373.057471264368,99.63563218390804,0,2.086206896551724,23998.488505747126
+Native OpenCode,700,mini-swe-agent,1,76,1.0,12.013157894736842,10.289473684210526,10.0,16.0,1.0657894736842106,0.1035692012665697,0.5263157894736842,0.0,0.3157894736842105,1.0,1.0,0.47368421052631576,1.0,31,1367.5394736842106,69.25381578947368,0,3.263157894736842,22327.776315789473
+Native OpenCode,700,opencode,0,204,0.0,10.892156862745098,9.245098039215685,8.0,16.0,0.5,0.03536008517626165,0.24019607843137256,0.014705882352941176,0.3235294117647059,0.9754901960784313,0.7058823529411765,0.7205882352941176,0.7313432835820896,67,1408.8970588235295,128.84166666666667,0,1.9068627450980393,39369.55392156863
+Native OpenCode,700,opencode,1,46,1.0,5.413043478260869,3.4130434782608696,3.0,5.0,0.0,0.0,0.0,0.0,0.0,1.0,1.0,0.17391304347826086,1.0,19,361.4782608695652,103.2104347826087,0,2.130434782608696,37931.36956521739
+Native OpenCode,800,claude-code,0,169,0.0,12.899408284023668,12.331360946745562,15.0,17.0,0.6035502958579881,0.03996253541383334,0.3076923076923077,0.029585798816568046,0.47928994082840237,0.9467455621301775,0.5739644970414202,0.8106508875739645,0.5740740740740741,54,4475.852071005917,139.79603550295857,0,12.8698224852071,534636.6804733728
+Native OpenCode,800,claude-code,1,81,1.0,8.345679012345679,6.765432098765432,5.0,12.0,0.08641975308641975,0.008167917372710401,0.06172839506172839,0.0,0.024691358024691357,0.9876543209876543,1.0,0.5802469135802469,1.0,32,1958.530864197531,111.65493827160493,0,8.333333333333334,276778.0987654321
+Native OpenCode,800,codex,0,205,0.0,15.692682926829269,15.526829268292683,17.0,17.0,0.624390243902439,0.03874638194580805,0.3463414634146341,0.00975609756097561,0.8146341463414634,0.975609756097561,0.28292682926829266,0.5951219512195122,0.2,70,2847.756097560976,147.91429268292683,0,1.3609756097560977,18079.11707317073
+Native OpenCode,800,codex,1,45,1.0,10.688888888888888,9.8,10.0,15.600000000000001,0.2222222222222222,0.018652868554829338,0.2,0.0,0.1111111111111111,0.9777777777777777,1.0,0.5555555555555556,1.0,16,1305.7555555555555,129.1468888888889,0,1.4444444444444444,14645.422222222222
+Native OpenCode,800,mini-swe-agent,0,170,0.0,16.0,15.635294117647058,17.0,17.0,0.4470588235294118,0.03224727886492593,0.3176470588235294,0.0,0.8352941176470589,1.0,0.3411764705882353,0.9294117647058824,0.3333333333333333,51,4005.1941176470586,103.559,0,1.8588235294117648,24782.74705882353
+Native OpenCode,800,mini-swe-agent,1,80,1.0,10.85,9.825,9.0,15.100000000000009,0.5,0.0563505642641672,0.4,0.0,0.15,0.9875,1.0,0.5375,1.0,35,1601.1,73.436625,0,2.625,20962.525
+Native OpenCode,800,opencode,0,200,0.0,10.49,8.865,6.0,16.0,0.25,0.016592728758169934,0.14,0.02,0.315,0.955,0.71,0.71,0.7692307692307693,65,1997.44,131.44165,0,1.88,54469.285
+Native OpenCode,800,opencode,1,50,1.0,6.9,4.96,3.0,12.0,0.02,0.0013333333333333333,0.02,0.0,0.04,1.0,1.0,0.36,1.0,21,712.5,110.8512,0,2.0,47741.16
+Native OpenCode,900,claude-code,0,165,0.0,11.969696969696969,11.345454545454546,12.0,17.0,0.4909090909090909,0.0314819058936706,0.22424242424242424,0.04242424242424243,0.44242424242424244,0.8545454545454545,0.5454545454545454,0.7333333333333333,0.6,50,4513.454545454545,141.17763636363637,0,11.951515151515151,497851.6424242424
+Native OpenCode,900,claude-code,1,85,1.0,9.070588235294117,7.623529411764705,6.0,14.600000000000009,0.2235294117647059,0.019638135797305348,0.12941176470588237,0.011764705882352941,0.09411764705882353,0.9529411764705882,1.0,0.6,1.0,36,2158.3058823529414,117.16929411764706,0,9.070588235294117,342238.4588235294
+Native OpenCode,900,codex,0,214,0.0,15.841121495327103,15.719626168224298,17.0,17.0,0.7336448598130841,0.04457102713826333,0.397196261682243,0.004672897196261682,0.8271028037383178,0.9813084112149533,0.19158878504672897,0.5887850467289719,0.15714285714285714,70,2882.448598130841,144.84471962616823,0,1.294392523364486,16528.140186915887
+Native OpenCode,900,codex,1,36,1.0,12.166666666666666,11.333333333333334,11.5,17.0,0.6388888888888888,0.05095076945567142,0.3888888888888889,0.0,0.2222222222222222,1.0,1.0,0.6944444444444444,1.0,16,1686.0,133.05027777777778,0,1.3055555555555556,14844.777777777777
+Native OpenCode,900,mini-swe-agent,0,165,0.0,16.024242424242424,15.50909090909091,17.0,17.0,0.5393939393939394,0.04098105202383277,0.3212121212121212,0.0,0.8,1.0,0.3878787878787879,0.8848484848484849,0.2830188679245283,53,4123.842424242424,101.6440606060606,1,1.896969696969697,26226.58787878788
+Native OpenCode,900,mini-swe-agent,1,85,1.0,11.941176470588236,10.6,10.0,16.0,0.9294117647058824,0.0903002764248439,0.5176470588235295,0.0,0.23529411764705882,1.0,1.0,0.5764705882352941,1.0,33,1540.9411764705883,71.20494117647058,1,2.5647058823529414,22007.776470588236
+Native OpenCode,900,opencode,0,203,0.0,9.443349753694582,7.783251231527093,6.0,16.0,0.2413793103448276,0.019007205808133074,0.15763546798029557,0.059113300492610835,0.2019704433497537,0.9408866995073891,0.7339901477832512,0.6945812807881774,0.7910447761194029,67,1543.2463054187192,125.21029556650245,1,1.9704433497536946,60316.12315270936
+Native OpenCode,900,opencode,1,47,1.0,6.148936170212766,4.212765957446808,3.0,7.0,0.02127659574468085,0.002127659574468085,0.02127659574468085,0.0425531914893617,0.0,0.9361702127659575,1.0,0.3404255319148936,1.0,19,544.0851063829788,107.44978723404256,0,2.106382978723404,55798.17021276596
+Native OpenCode,1000,claude-code,0,167,0.0,12.431137724550899,11.994011976047904,13.0,17.0,0.5808383233532934,0.036901164345501676,0.30538922155688625,0.017964071856287425,0.47904191616766467,0.9161676646706587,0.6287425149700598,0.8023952095808383,0.5961538461538461,52,4404.88622754491,139.98269461077845,0,12.431137724550899,514356.04191616765
+Native OpenCode,1000,claude-code,1,83,1.0,8.795180722891565,7.578313253012048,6.0,14.799999999999997,0.3253012048192771,0.028024013548464292,0.1686746987951807,0.0,0.04819277108433735,0.9879518072289156,1.0,0.6385542168674698,1.0,34,1971.1566265060242,121.15024096385541,0,8.795180722891565,317782.0843373494
+Native OpenCode,1000,codex,0,176,0.0,15.659090909090908,15.494318181818182,17.0,17.0,0.4602272727272727,0.028241647037602922,0.29545454545454547,0.005681818181818182,0.7670454545454546,1.0,0.4375,0.9261363636363636,0.3050847457627119,59,2594.130681818182,154.0847727272727,0,6.034090909090909,83996.02272727272
+Native OpenCode,1000,codex,1,74,1.0,12.716216216216216,12.0,11.0,17.0,0.6351351351351351,0.04693533921475098,0.3918918918918919,0.0,0.3108108108108108,1.0,1.0,0.7162162162162162,1.0,27,919.9459459459459,131.65594594594594,0,2.5,24360.202702702703
+Native OpenCode,1000,mini-swe-agent,0,160,0.0,15.7125,15.19375,17.0,17.0,0.55625,0.04185905444637063,0.30625,0.0,0.7625,0.99375,0.4625,0.85,0.6274509803921569,51,4085.3375,111.4579375,0,5.94375,83023.3875
+Native OpenCode,1000,mini-swe-agent,1,90,1.0,12.855555555555556,11.144444444444444,11.5,16.0,1.3333333333333333,0.11476076846174886,0.6333333333333333,0.0,0.3111111111111111,1.0,1.0,0.5,1.0,35,842.1333333333333,74.28633333333333,0,3.9,28343.78888888889
+Native OpenCode,1000,opencode,0,199,0.0,11.49748743718593,9.984924623115578,10.0,16.0,1.3417085427135678,0.10450114924324207,0.542713567839196,0.0,0.33668341708542715,0.9748743718592965,0.7537688442211056,0.7587939698492462,0.7096774193548387,62,1849.0452261306532,130.6827135678392,0,6.442211055276382,169374.60301507538
+Native OpenCode,1000,opencode,1,51,1.0,7.745098039215686,5.882352941176471,5.0,9.0,0.8627450980392157,0.11350055173584586,0.49019607843137253,0.0,0.0,1.0,1.0,0.37254901960784315,1.0,24,754.6470588235294,116.4713725490196,0,5.019607843137255,137442.9019607843
diff --git a/04-data-agent/reports/three-run-analysis-20260917/capture_warnings.csv b/04-data-agent/reports/three-run-analysis-20260917/capture_warnings.csv
new file mode 100644
index 0000000..6f3ac78
--- /dev/null
+++ b/04-data-agent/reports/three-run-analysis-20260917/capture_warnings.csv
@@ -0,0 +1,137 @@
+run,step,harness,truncated,context_limit,aux
+Harbor OpenCode-only,0,claude-code,4,8,54
+Harbor OpenCode-only,0,codex,1,0,28
+Harbor OpenCode-only,0,mini-swe-agent,0,0,44
+Harbor OpenCode-only,0,opencode,0,2,3
+Harbor OpenCode-only,100,claude-code,2,6,0
+Harbor OpenCode-only,100,codex,2,0,2
+Harbor OpenCode-only,100,mini-swe-agent,0,0,23
+Harbor OpenCode-only,100,opencode,0,2,0
+Harbor OpenCode-only,200,claude-code,1,7,0
+Harbor OpenCode-only,200,codex,4,0,4
+Harbor OpenCode-only,200,mini-swe-agent,0,0,18
+Harbor OpenCode-only,200,opencode,0,4,0
+Harbor OpenCode-only,300,claude-code,1,9,1
+Harbor OpenCode-only,300,codex,1,0,2
+Harbor OpenCode-only,300,mini-swe-agent,0,0,12
+Harbor OpenCode-only,300,opencode,0,12,0
+Harbor OpenCode-only,400,claude-code,0,7,0
+Harbor OpenCode-only,400,codex,2,0,3
+Harbor OpenCode-only,400,mini-swe-agent,1,0,9
+Harbor OpenCode-only,400,opencode,0,4,0
+Harbor OpenCode-only,500,claude-code,2,7,0
+Harbor OpenCode-only,500,codex,4,0,4
+Harbor OpenCode-only,500,mini-swe-agent,0,0,9
+Harbor OpenCode-only,500,opencode,2,5,0
+Harbor OpenCode-only,600,claude-code,1,5,0
+Harbor OpenCode-only,600,codex,3,0,3
+Harbor OpenCode-only,600,mini-swe-agent,0,0,10
+Harbor OpenCode-only,600,opencode,1,4,0
+Harbor OpenCode-only,700,claude-code,4,2,0
+Harbor OpenCode-only,700,codex,2,0,3
+Harbor OpenCode-only,700,mini-swe-agent,1,0,35
+Harbor OpenCode-only,700,opencode,0,5,0
+Harbor OpenCode-only,800,claude-code,5,4,0
+Harbor OpenCode-only,800,codex,3,0,4
+Harbor OpenCode-only,800,mini-swe-agent,1,0,69
+Harbor OpenCode-only,800,opencode,27,1,0
+Harbor OpenCode-only,900,claude-code,2,11,0
+Harbor OpenCode-only,900,codex,1,0,1
+Harbor OpenCode-only,900,mini-swe-agent,1,0,87
+Harbor OpenCode-only,900,opencode,7,33,0
+Harbor OpenCode-only,1000,claude-code,2,11,0
+Harbor OpenCode-only,1000,codex,5,1,4
+Harbor OpenCode-only,1000,mini-swe-agent,0,0,57
+Harbor OpenCode-only,1000,opencode,2,13,0
+Harbor multi-harness,0,claude-code,4,8,54
+Harbor multi-harness,0,codex,1,0,28
+Harbor multi-harness,0,mini-swe-agent,0,0,44
+Harbor multi-harness,0,opencode,0,2,3
+Harbor multi-harness,100,claude-code,8,12,102
+Harbor multi-harness,100,codex,0,0,64
+Harbor multi-harness,100,mini-swe-agent,0,0,14
+Harbor multi-harness,100,opencode,1,8,15
+Harbor multi-harness,200,claude-code,12,9,107
+Harbor multi-harness,200,codex,0,0,71
+Harbor multi-harness,200,mini-swe-agent,0,0,6
+Harbor multi-harness,200,opencode,0,5,22
+Harbor multi-harness,300,claude-code,7,12,85
+Harbor multi-harness,300,codex,0,0,58
+Harbor multi-harness,300,mini-swe-agent,0,0,5
+Harbor multi-harness,300,opencode,0,0,6
+Harbor multi-harness,400,claude-code,7,11,93
+Harbor multi-harness,400,codex,0,0,52
+Harbor multi-harness,400,mini-swe-agent,0,0,18
+Harbor multi-harness,400,opencode,1,8,12
+Harbor multi-harness,500,claude-code,5,12,85
+Harbor multi-harness,500,codex,1,0,62
+Harbor multi-harness,500,mini-swe-agent,0,0,13
+Harbor multi-harness,500,opencode,3,8,26
+Harbor multi-harness,600,claude-code,5,13,91
+Harbor multi-harness,600,codex,7,0,104
+Harbor multi-harness,600,mini-swe-agent,0,0,43
+Harbor multi-harness,600,opencode,15,7,46
+Harbor multi-harness,684,claude-code,6,21,73
+Harbor multi-harness,684,codex,7,0,89
+Harbor multi-harness,684,mini-swe-agent,0,0,43
+Harbor multi-harness,684,opencode,16,7,28
+Harbor multi-harness,700,claude-code,12,24,79
+Harbor multi-harness,700,codex,3,0,69
+Harbor multi-harness,700,mini-swe-agent,0,0,59
+Harbor multi-harness,700,opencode,11,10,17
+Harbor multi-harness,800,claude-code,16,30,90
+Harbor multi-harness,800,codex,35,0,149
+Harbor multi-harness,800,mini-swe-agent,1,0,24
+Harbor multi-harness,800,opencode,30,9,45
+Harbor multi-harness,900,claude-code,132,27,0
+Harbor multi-harness,900,codex,201,0,203
+Harbor multi-harness,900,mini-swe-agent,47,0,108
+Harbor multi-harness,900,opencode,226,0,0
+Harbor multi-harness,1000,claude-code,132,25,0
+Harbor multi-harness,1000,codex,162,0,162
+Harbor multi-harness,1000,mini-swe-agent,58,0,107
+Harbor multi-harness,1000,opencode,204,1,0
+Native OpenCode,0,claude-code,2,9,33
+Native OpenCode,0,codex,4,0,25
+Native OpenCode,0,mini-swe-agent,1,0,48
+Native OpenCode,0,opencode,0,3,7
+Native OpenCode,100,claude-code,0,2,0
+Native OpenCode,100,codex,2,0,3
+Native OpenCode,100,mini-swe-agent,0,0,36
+Native OpenCode,100,opencode,0,2,0
+Native OpenCode,200,claude-code,6,4,0
+Native OpenCode,200,codex,4,0,7
+Native OpenCode,200,mini-swe-agent,0,0,54
+Native OpenCode,200,opencode,0,3,0
+Native OpenCode,300,claude-code,1,2,0
+Native OpenCode,300,codex,1,0,3
+Native OpenCode,300,mini-swe-agent,1,0,58
+Native OpenCode,300,opencode,0,0,0
+Native OpenCode,400,claude-code,4,1,0
+Native OpenCode,400,codex,1,0,1
+Native OpenCode,400,mini-swe-agent,0,0,82
+Native OpenCode,400,opencode,0,1,0
+Native OpenCode,500,claude-code,3,3,0
+Native OpenCode,500,codex,1,0,4
+Native OpenCode,500,mini-swe-agent,2,0,123
+Native OpenCode,500,opencode,0,0,0
+Native OpenCode,600,claude-code,2,4,0
+Native OpenCode,600,codex,1,0,4
+Native OpenCode,600,mini-swe-agent,1,0,175
+Native OpenCode,600,opencode,1,1,0
+Native OpenCode,700,claude-code,2,5,0
+Native OpenCode,700,codex,0,0,2
+Native OpenCode,700,mini-swe-agent,0,0,117
+Native OpenCode,700,opencode,0,0,0
+Native OpenCode,800,claude-code,4,2,0
+Native OpenCode,800,codex,1,0,1
+Native OpenCode,800,mini-swe-agent,0,0,94
+Native OpenCode,800,opencode,0,1,0
+Native OpenCode,900,claude-code,6,4,0
+Native OpenCode,900,codex,1,0,1
+Native OpenCode,900,mini-swe-agent,0,0,109
+Native OpenCode,900,opencode,0,0,0
+Native OpenCode,1000,claude-code,1,7,0
+Native OpenCode,1000,codex,0,0,0
+Native OpenCode,1000,mini-swe-agent,0,0,117
+Native OpenCode,1000,opencode,0,0,0
diff --git a/04-data-agent/reports/three-run-analysis-20260917/completion.png b/04-data-agent/reports/three-run-analysis-20260917/completion.png
new file mode 100644
index 0000000..6240a27
Binary files /dev/null and b/04-data-agent/reports/three-run-analysis-20260917/completion.png differ
diff --git a/04-data-agent/reports/three-run-analysis-20260917/evaluation_vs_training_exposure.csv b/04-data-agent/reports/three-run-analysis-20260917/evaluation_vs_training_exposure.csv
new file mode 100644
index 0000000..8b00f3d
--- /dev/null
+++ b/04-data-agent/reports/three-run-analysis-20260917/evaluation_vs_training_exposure.csv
@@ -0,0 +1,35 @@
+run,step,score,supervised_tokens,forwarded_tokens
+Harbor multi-harness,0,0.146,0.0,0.0
+Harbor multi-harness,100,0.248,2341485.0,187688023.0
+Harbor multi-harness,200,0.263,3751080.0,288770381.0
+Harbor multi-harness,300,0.286,5083236.0,389874360.0
+Harbor multi-harness,400,0.333,6392092.0,474480006.0
+Harbor multi-harness,500,0.37,8194853.0,564646274.0
+Harbor multi-harness,600,0.318,9879725.0,650324146.0
+Harbor multi-harness,684,0.321,11257452.0,722450575.0
+Harbor multi-harness,700,0.288,11548209.0,736700946.0
+Harbor multi-harness,800,0.27,13319433.0,824426929.0
+Harbor multi-harness,900,0.227,17550674.0,922803659.0
+Harbor multi-harness,1000,0.263,22148436.0,1001322856.0
+Native OpenCode,0,0.159,0.0,0.0
+Native OpenCode,100,0.197,412431.0,28972901.0
+Native OpenCode,200,0.221,951366.0,59911629.0
+Native OpenCode,300,0.216,1695366.0,83719905.0
+Native OpenCode,400,0.264,2489783.0,98136103.0
+Native OpenCode,500,0.231,3086548.0,111429542.0
+Native OpenCode,600,0.251,3534140.0,131234844.0
+Native OpenCode,700,0.232,3982021.0,146777166.0
+Native OpenCode,800,0.256,4421519.0,162968313.0
+Native OpenCode,900,0.253,4895255.0,185471906.0
+Native OpenCode,1000,0.298,5327227.0,228413083.0
+Harbor OpenCode-only,0,0.146,0.0,0.0
+Harbor OpenCode-only,100,0.245,900139.0,22391797.0
+Harbor OpenCode-only,200,0.271,4248088.0,61601074.0
+Harbor OpenCode-only,300,0.285,6397309.0,94738314.0
+Harbor OpenCode-only,400,0.328,7451497.0,121807354.0
+Harbor OpenCode-only,500,0.33,8682354.0,154599334.0
+Harbor OpenCode-only,600,0.33,9761777.0,187407303.0
+Harbor OpenCode-only,700,0.395,10653106.0,231011537.0
+Harbor OpenCode-only,800,0.331,11935793.0,293584910.0
+Harbor OpenCode-only,900,0.296,13158260.0,356542913.0
+Harbor OpenCode-only,1000,0.264,14625038.0,419428466.0
diff --git a/04-data-agent/reports/three-run-analysis-20260917/harness_complementarity.csv b/04-data-agent/reports/three-run-analysis-20260917/harness_complementarity.csv
new file mode 100644
index 0000000..cf11f17
--- /dev/null
+++ b/04-data-agent/reports/three-run-analysis-20260917/harness_complementarity.csv
@@ -0,0 +1,35 @@
+run,step,solved_by_any,solved_by_all,solved_by_none,oracle_any_rate
+Harbor OpenCode-only,0,76,7,174,0.304
+Harbor OpenCode-only,100,114,20,136,0.456
+Harbor OpenCode-only,200,116,24,134,0.464
+Harbor OpenCode-only,300,113,31,137,0.452
+Harbor OpenCode-only,400,130,40,120,0.52
+Harbor OpenCode-only,500,121,40,129,0.484
+Harbor OpenCode-only,600,123,35,127,0.492
+Harbor OpenCode-only,700,149,50,101,0.596
+Harbor OpenCode-only,800,141,25,109,0.564
+Harbor OpenCode-only,900,128,24,122,0.512
+Harbor OpenCode-only,1000,119,26,131,0.476
+Harbor multi-harness,0,76,7,174,0.304
+Harbor multi-harness,100,117,17,133,0.468
+Harbor multi-harness,200,129,16,121,0.516
+Harbor multi-harness,300,127,21,123,0.508
+Harbor multi-harness,400,127,35,123,0.508
+Harbor multi-harness,500,141,45,109,0.564
+Harbor multi-harness,600,124,35,126,0.496
+Harbor multi-harness,684,124,32,126,0.496
+Harbor multi-harness,700,122,24,128,0.488
+Harbor multi-harness,800,107,25,143,0.428
+Harbor multi-harness,900,117,1,133,0.468
+Harbor multi-harness,1000,129,7,121,0.516
+Native OpenCode,0,95,3,155,0.38
+Native OpenCode,100,97,13,153,0.388
+Native OpenCode,200,99,19,151,0.396
+Native OpenCode,300,103,15,147,0.412
+Native OpenCode,400,120,21,130,0.48
+Native OpenCode,500,109,13,141,0.436
+Native OpenCode,600,116,17,134,0.464
+Native OpenCode,700,111,18,139,0.444
+Native OpenCode,800,122,11,128,0.488
+Native OpenCode,900,128,11,122,0.512
+Native OpenCode,1000,134,25,116,0.536
diff --git a/04-data-agent/reports/three-run-analysis-20260917/overall.csv b/04-data-agent/reports/three-run-analysis-20260917/overall.csv
new file mode 100644
index 0000000..ef9be67
--- /dev/null
+++ b/04-data-agent/reports/three-run-analysis-20260917/overall.csv
@@ -0,0 +1,35 @@
+run,step,cells,score,model_calls,tool_calls,tool_calls_median,tool_calls_p90,repeats,repeat_fraction,repeat_rollout_fraction,short_tool_fraction,budget_fraction,python_fraction,answer_path_fraction,tool_error_text_fraction,submission_fraction,submission_observations,trainable_tokens,wall_s,retry_cells,training_rows,packed_tokens
+Harbor OpenCode-only,0,1000,0.146,14.149,13.264,16.0,18.0,0.522,0.03575284067129578,0.285,0.066,0.602,0.779,0.38,0.618,0.4011627906976744,344,2721.986,273.92238000000003,11,4.122,143490.928
+Harbor OpenCode-only,100,1000,0.245,15.747,16.131,17.0,18.0,2.457,0.1476399107830088,0.62,0.0,0.82,0.967,0.537,0.714,0.5261627906976745,344,3049.281,266.45046,16,4.983,177707.37
+Harbor OpenCode-only,200,1000,0.271,16.046,16.63,17.0,19.0,1.699,0.10171712437576555,0.561,0.001,0.836,0.955,0.474,0.712,0.5174418604651163,344,3439.912,276.93613,8,5.109,182544.215
+Harbor OpenCode-only,300,1000,0.285,16.205,17.998,17.0,21.0,1.634,0.09349848499143593,0.56,0.003,0.864,0.972,0.421,0.699,0.44476744186046513,344,3382.215,293.72954,4,5.147,190025.617
+Harbor OpenCode-only,400,1000,0.328,16.162,18.168,18.0,21.0,2.026,0.10955959418379882,0.597,0.002,0.862,0.982,0.492,0.669,0.5261627906976745,344,2927.045,276.08891,5,5.24,184854.492
+Harbor OpenCode-only,500,1000,0.33,15.947,16.714,17.0,19.0,1.567,0.09457921511064785,0.521,0.002,0.822,0.991,0.494,0.685,0.5174418604651163,344,3410.234,291.9687,5,5.127,181689.72
+Harbor OpenCode-only,600,1000,0.33,15.815,16.986,17.0,20.0,2.063,0.12061561433857773,0.585,0.0,0.785,0.973,0.548,0.629,0.561046511627907,344,2640.898,275.99777,1,5.217,183274.1
+Harbor OpenCode-only,700,1000,0.395,14.451,16.62,17.0,22.0,2.092,0.12056283788646273,0.628,0.001,0.564,0.985,0.642,0.645,0.688953488372093,344,2909.286,276.39668,4,5.615,186714.849
+Harbor OpenCode-only,800,1000,0.331,13.89,13.97,16.0,18.0,1.075,0.0798111487259394,0.55,0.011,0.513,0.925,0.575,0.602,0.6162790697674418,344,3265.515,272.20868,20,6.168,194153.262
+Harbor OpenCode-only,900,1000,0.296,14.627,16.314,17.0,22.0,1.886,0.12036157527128212,0.68,0.001,0.612,0.927,0.549,0.649,0.4622093023255814,344,3135.778,293.36412,3,6.119,205588.621
+Harbor OpenCode-only,1000,1000,0.264,15.271,20.966,20.0,30.0,3.025,0.12659647516265488,0.766,0.0,0.702,0.945,0.562,0.712,0.4069767441860465,344,3403.287,293.54446,2,5.382,182515.884
+Harbor multi-harness,0,1000,0.146,14.149,13.264,16.0,18.0,0.522,0.03575284067129578,0.285,0.066,0.602,0.779,0.38,0.618,0.4011627906976744,344,2721.986,273.92238000000003,11,4.122,143490.928
+Harbor multi-harness,100,1000,0.248,16.341,15.938,17.0,17.0,2.769,0.1713903229938127,0.604,0.004,0.873,0.868,0.443,0.64,0.4680232558139535,344,2855.774,277.7602,44,5.013,184767.212
+Harbor multi-harness,200,1000,0.263,16.598,16.138,17.0,17.0,3.057,0.1868986447575995,0.636,0.003,0.914,0.902,0.451,0.651,0.49127906976744184,344,2744.07,267.66835,76,5.043,179096.311
+Harbor multi-harness,300,1000,0.286,16.397,15.987,17.0,17.0,3.646,0.2257868151714127,0.713,0.003,0.885,0.845,0.496,0.609,0.5203488372093024,344,2251.678,248.93432,29,4.977,174517.782
+Harbor multi-harness,400,1000,0.333,16.604,16.153,17.0,17.0,3.587,0.21946066816310625,0.698,0.002,0.913,0.869,0.474,0.634,0.5232558139534884,344,2761.828,271.93884,44,5.045,185725.598
+Harbor multi-harness,500,1000,0.37,16.061,15.582,16.0,17.0,3.464,0.2124553623921658,0.651,0.0,0.839,0.86,0.515,0.646,0.5494186046511628,344,2870.289,275.34286,65,5.094,184783.506
+Harbor multi-harness,600,1000,0.318,16.054,15.451,16.0,17.0,3.508,0.21791239233332885,0.741,0.002,0.806,0.799,0.473,0.569,0.5203488372093024,344,3359.023,289.55251,83,5.158,187366.857
+Harbor multi-harness,684,1000,0.321,15.593,15.009,16.0,17.0,3.92,0.2478296437781345,0.795,0.003,0.756,0.794,0.497,0.567,0.5319767441860465,344,2895.237,275.87375,65,5.003,194585.147
+Harbor multi-harness,700,1000,0.288,15.441,14.897,16.0,17.0,3.674,0.23182913666570787,0.798,0.009,0.748,0.743,0.459,0.536,0.5,344,2947.094,273.19223,62,4.94,191219.959
+Harbor multi-harness,800,1000,0.27,15.929,15.209,16.0,17.0,3.677,0.2331952162474608,0.842,0.009,0.759,0.741,0.365,0.512,0.39244186046511625,344,3730.791,325.84463,107,4.95,190314.026
+Harbor multi-harness,900,1000,0.227,10.633,9.646,10.0,17.0,2.341,0.15756903361464808,0.489,0.252,0.392,0.554,0.319,0.369,0.34593023255813954,344,5306.9,323.67207,149,4.941,186258.408
+Harbor multi-harness,1000,1000,0.263,10.12,9.018,9.0,17.0,2.204,0.16422693790987133,0.507,0.221,0.28,0.596,0.402,0.412,0.4127906976744186,344,5206.881,304.29470000000003,205,4.452,153613.721
+Native OpenCode,0,1000,0.159,14.258,14.068,16.0,18.0,0.658,0.04315955169345195,0.35,0.008,0.62,0.829,0.413,0.659,0.4418604651162791,344,2655.455,259.36484,830,4.23,150697.117
+Native OpenCode,100,1000,0.197,13.802,13.418,16.0,17.0,0.49,0.03143585735984547,0.289,0.006,0.579,0.917,0.488,0.686,0.5116279069767442,344,2845.774,134.06311,1,4.361,155000.771
+Native OpenCode,200,1000,0.221,13.132,12.636,16.0,17.0,0.51,0.0332330306530513,0.257,0.004,0.528,0.948,0.549,0.709,0.5726744186046512,344,2571.95,127.30409,0,4.274,146781.013
+Native OpenCode,300,1000,0.216,13.889,13.369,16.0,17.0,0.534,0.035410580252221116,0.309,0.001,0.598,0.959,0.47,0.722,0.45930232558139533,344,2847.512,127.70825,6,4.23,143617.756
+Native OpenCode,400,1000,0.264,13.671,13.066,16.0,17.0,0.545,0.035270807864306315,0.298,0.001,0.568,0.965,0.513,0.738,0.5523255813953488,344,2786.585,119.62226,0,4.256,129501.653
+Native OpenCode,500,1000,0.231,13.411,12.561,16.0,17.0,0.83,0.05693932569838142,0.354,0.005,0.543,0.97,0.507,0.686,0.5348837209302325,344,2463.056,125.81416,3,4.536,138071.582
+Native OpenCode,600,1000,0.251,12.847,11.82,14.0,17.0,1.184,0.0825353855552772,0.418,0.008,0.482,0.954,0.581,0.662,0.5901162790697675,344,2473.208,121.3107,14,4.655,134171.096
+Native OpenCode,700,1000,0.232,12.83,11.866,14.0,17.0,0.736,0.052030067820895996,0.335,0.009,0.509,0.969,0.604,0.673,0.5930232558139535,344,2390.413,120.96332000000001,0,4.277,130972.534
+Native OpenCode,800,1000,0.256,12.585,11.721,14.0,17.0,0.414,0.029572952142456786,0.252,0.011,0.474,0.974,0.611,0.692,0.627906976744186,344,2801.693,124.11447,0,4.196,136308.872
+Native OpenCode,900,1000,0.253,12.439,11.53,13.0,17.0,0.498,0.03663204338231428,0.277,0.023,0.459,0.953,0.597,0.675,0.6191860465116279,344,2755.981,122.3318,3,4.097,136371.959
+Native OpenCode,1000,1000,0.298,12.857,11.968,13.0,17.0,0.772,0.06054240726164101,0.385,0.004,0.459,0.979,0.704,0.754,0.7122093023255814,344,2559.758,126.75898,0,6.893,185409.144
diff --git a/04-data-agent/reports/three-run-analysis-20260917/paired_differences.csv b/04-data-agent/reports/three-run-analysis-20260917/paired_differences.csv
new file mode 100644
index 0000000..450ce60
--- /dev/null
+++ b/04-data-agent/reports/three-run-analysis-20260917/paired_differences.csv
@@ -0,0 +1,7 @@
+comparison,delta_pp,ci95_low_pp,ci95_high_pp,lost_correct_cells,gained_correct_cells
+Harbor multi-harness: baseline to selected peak,22.400000000000002,18.6,26.3,27,251
+Harbor multi-harness: selected peak to final,-10.7,-14.099999999999998,-7.3999999999999995,174,67
+Native OpenCode: baseline to selected peak,13.900000000000002,11.200000000000001,16.8,47,186
+Harbor OpenCode-only: baseline to selected peak,24.9,21.099999999999998,28.799999999999997,23,272
+Harbor OpenCode-only: selected peak to final,-13.100000000000001,-16.6,-9.700000000000001,190,59
+Multi-harness 500 to OpenCode-only 700,2.5,-0.6,5.7,94,119
diff --git a/04-data-agent/reports/three-run-analysis-20260917/summarize_training.py b/04-data-agent/reports/three-run-analysis-20260917/summarize_training.py
new file mode 100644
index 0000000..6bef835
--- /dev/null
+++ b/04-data-agent/reports/three-run-analysis-20260917/summarize_training.py
@@ -0,0 +1,134 @@
+"""Build training diagnostics from extracted captures and optimizer-step telemetry."""
+
+import argparse
+from pathlib import Path
+
+import matplotlib
+matplotlib.use("Agg")
+import matplotlib.pyplot as plt
+import numpy as np
+import pandas as pd
+
+
+RUNS = ["Harbor multi-harness", "Native OpenCode", "Harbor OpenCode-only"]
+COLORS = ["#4f46e5", "#e85c41", "#059669"]
+
+
+def summarize(frame, keys):
+ result = frame.groupby(keys).agg(
+ admitted_rollouts=("rollout_id", "size"),
+ unique_tasks=("task_index", "nunique"),
+ binary_reward_mean=("binary_reward", "mean"),
+ completion_tokens_mean=("completion_tokens", "mean"),
+ completion_tokens_median=("completion_tokens", "median"),
+ completion_tokens_p90=("completion_tokens", lambda x: x.quantile(.9)),
+ supervised_tokens=("supervised_tokens", "sum"),
+ completion_tokens=("completion_tokens", "sum"),
+ text_only_turn_tokens=("completion_tokens_in_text_only_turns", "sum"),
+ agent_turns_mean=("agent_turns", "mean"),
+ tool_calls_mean=("emitted_tool_calls", "mean"),
+ tool_calls_p90=("emitted_tool_calls", lambda x: x.quantile(.9)),
+ exact_repeated_calls_mean=("exact_repeated_calls", "mean"),
+ rollout_over4096_fraction=("has_response_over4096", "mean"),
+ rollout_finish_length_fraction=("has_finish_length", "mean"),
+ training_rows=("rows", "sum"),
+ receipt_token_mismatches=("receipt_token_delta", lambda x: (x != 0).sum()),
+ ).reset_index()
+ result["text_only_turn_token_share"] = result.text_only_turn_tokens / result.completion_tokens
+ result["rows_per_rollout"] = result.training_rows / result.admitted_rollouts
+ return result
+
+
+def main():
+ parser = argparse.ArgumentParser(description=__doc__)
+ parser.add_argument("--evidence", type=Path, required=True)
+ parser.add_argument("--output", type=Path, default=Path(__file__).resolve().parent)
+ args = parser.parse_args()
+ output = args.output
+ output.mkdir(parents=True, exist_ok=True)
+ captures = pd.read_csv(args.evidence / "admitted_training_behavior.csv")
+ steps = pd.read_csv(args.evidence / "training_metrics.csv")
+ for frame in (captures, steps):
+ frame["window_end"] = ((frame.step - 1) // 100 + 1) * 100
+ assert not captures.duplicated(["run", "rollout_id"]).any()
+ assert len(steps) == 3000 and not steps.duplicated(["run", "step"]).any()
+ assert np.isfinite(steps.grad_norm).all()
+ captures["has_response_over4096"] = captures.responses_over_4096 > 0
+ captures["has_finish_length"] = captures.responses_finish_length > 0
+ # Admission receipts cover steps 31 onward for multi-harness, every step otherwise.
+ for run, cohort in captures.groupby("run"):
+ first = 31 if run == "Harbor multi-harness" else 1
+ recorded = steps[(steps.run == run) & (steps.step >= first)]
+ assert cohort.supervised_tokens.sum() == recorded["batch/trained_tokens_per_step"].sum()
+ for name, keys in [
+ ("training_rollout_totals", ["run"]),
+ ("training_behavior_windows", ["run", "window_end"]),
+ ("training_behavior_by_harness", ["run", "harness", "window_end"]),
+ ("training_behavior_by_outcome", ["run", "window_end", "binary_reward"]),
+ ]:
+ summarize(captures, keys).to_csv(output / f"{name}.csv", index=False)
+ windows = summarize(captures, ["run", "window_end"])
+ steps["zero_gradient"] = steps.grad_norm == 0
+ assert (steps.zero_gradient == (steps.reward_std == 0)).all()
+ step_keys = ["reward", "reward_std", "tools/call_frequency", "tools/failure_frequency",
+ "rollout/turns_mean", "rollout/fork_frac", "rollout/samples_per_rollout",
+ "completions/mean_length", "completions/clipped_ratio", "zero_gradient",
+ "perf/step_s", "perf/fwd_bwd_s", "perf/rollout_wait_s"]
+ telemetry = steps.groupby(["run", "window_end"])[step_keys].mean().reset_index()
+ totals = steps.groupby(["run", "window_end"])[["batch/forwarded_tokens_per_step", "batch/trained_tokens_per_step"]].sum().reset_index()
+ telemetry = telemetry.merge(totals, on=["run", "window_end"], validate="one_to_one")
+ telemetry["forwarded_per_supervised_token"] = telemetry["batch/forwarded_tokens_per_step"] / telemetry["batch/trained_tokens_per_step"]
+ telemetry.to_csv(output / "training_step_diagnostics.csv", index=False)
+ accounting = []
+ for run, group in steps.groupby("run"):
+ forwarded = group["batch/forwarded_tokens_per_step"]
+ supervised = group["batch/trained_tokens_per_step"]
+ zero = group.zero_gradient
+ accounting.append({
+ "run": run, "steps": len(group), "zero_gradient_steps": int(zero.sum()),
+ "forwarded_tokens": forwarded.sum(), "supervised_tokens": supervised.sum(),
+ "forwarded_per_supervised_token": forwarded.sum() / supervised.sum(),
+ "supervised_tokens_in_zero_gradient_steps": supervised[zero].sum(),
+ "zero_gradient_supervised_token_fraction": supervised[zero].sum() / supervised.sum(),
+ "zero_gradient_forwarded_token_fraction": forwarded[zero].sum() / forwarded.sum(),
+ "mean_step_s": group["perf/step_s"].mean(),
+ "step_timing_observations": group["perf/step_s"].count(),
+ "receipt_token_mismatches": int((captures.loc[captures.run == run, "receipt_token_delta"] != 0).sum()),
+ })
+ pd.DataFrame(accounting).to_csv(output / "training_accounting.csv", index=False)
+
+ plt.rcParams.update({"font.family": "DejaVu Sans", "font.size": 10,
+ "axes.spines.top": False, "axes.spines.right": False})
+ fig, axes = plt.subplots(2, 3, figsize=(15, 9))
+ for run, color in zip(RUNS, COLORS):
+ w = windows[windows.run == run].sort_values("window_end")
+ t = telemetry[telemetry.run == run].sort_values("window_end")
+ options = dict(color=color, marker="o", markersize=4, label=run)
+ axes[0, 0].plot(w.window_end, w.completion_tokens_mean, **options)
+ axes[0, 1].plot(w.window_end, w.tool_calls_mean, **options)
+ axes[0, 2].plot(w.window_end, 100 * w.text_only_turn_token_share, **options)
+ axes[1, 0].plot(w.window_end, 100 * w.rollout_over4096_fraction, **options)
+ axes[1, 1].plot(t.window_end, t.forwarded_per_supervised_token, **options)
+ axes[1, 2].plot(t.window_end, 100 * t.zero_gradient, **options)
+ labels = ["Mean completion tokens / admitted rollout", "Mean emitted tool calls / admitted rollout",
+ "Completion tokens in text-only turns (%)", "Admitted rollouts with a response >4,096 tokens (%)",
+ "Forwarded / supervised tokens", "Steps with zero fresh gradient (%)"]
+ for ax, label in zip(axes.flat, labels):
+ ax.set_title(label, fontsize=11, pad=10)
+ ax.grid(alpha=.18)
+ ax.set_ylim(bottom=0)
+ ax.set_xlabel("End of 100-step window")
+ handles, labels = axes[0, 0].get_legend_handles_labels()
+ fig.legend(handles, labels, loc="upper center", bbox_to_anchor=(.5, .955), ncol=3, frameon=False)
+ fig.suptitle("Training behavior, token exposure and available learning signal", fontsize=19, y=.995)
+ fig.text(.05, .025, "Captured turns from 13,625 admitted rollouts; multi-harness receipts start at step 31. Bottom-right panels use all 3,000 step records.\nTool calls are emitted requests, not confirmed executions. Text-only includes reasoning and final answers. Zero fresh gradient does not rule out optimizer momentum.", fontsize=9, color="#475569")
+ fig.tight_layout(rect=(0, .075, 1, .90))
+ fig.savefig(output / "training_diagnostics.png", dpi=180)
+ fig.savefig(output / "training_diagnostics.pdf")
+ plt.close(fig)
+ print(windows[(windows.window_end.isin([500, 700, 1000]))].to_string(index=False))
+ print(pd.DataFrame(accounting).to_string(index=False))
+
+
+if __name__ == "__main__":
+ main()
diff --git a/04-data-agent/reports/three-run-analysis-20260917/tool_call_bins.csv b/04-data-agent/reports/three-run-analysis-20260917/tool_call_bins.csv
new file mode 100644
index 0000000..a3b7ac7
--- /dev/null
+++ b/04-data-agent/reports/three-run-analysis-20260917/tool_call_bins.csv
@@ -0,0 +1,510 @@
+run,step,harness,tool_call_bin,cells,score
+Harbor OpenCode-only,0,claude-code,0–2,8,0.25
+Harbor OpenCode-only,0,claude-code,3–8,63,0.42857142857142855
+Harbor OpenCode-only,0,claude-code,9–16,82,0.13414634146341464
+Harbor OpenCode-only,0,claude-code,17–24,97,0.020618556701030927
+Harbor OpenCode-only,0,codex,0–2,20,0.2
+Harbor OpenCode-only,0,codex,3–8,33,0.48484848484848486
+Harbor OpenCode-only,0,codex,9–16,54,0.24074074074074073
+Harbor OpenCode-only,0,codex,17–24,137,0.058394160583941604
+Harbor OpenCode-only,0,codex,25+,6,0.0
+Harbor OpenCode-only,0,mini-swe-agent,0–2,10,0.2
+Harbor OpenCode-only,0,mini-swe-agent,3–8,3,0.3333333333333333
+Harbor OpenCode-only,0,mini-swe-agent,9–16,68,0.4411764705882353
+Harbor OpenCode-only,0,mini-swe-agent,17–24,169,0.01775147928994083
+Harbor OpenCode-only,0,opencode,0–2,28,0.14285714285714285
+Harbor OpenCode-only,0,opencode,3–8,66,0.19696969696969696
+Harbor OpenCode-only,0,opencode,9–16,112,0.08035714285714286
+Harbor OpenCode-only,0,opencode,17–24,44,0.022727272727272728
+Harbor OpenCode-only,100,claude-code,3–8,22,0.5454545454545454
+Harbor OpenCode-only,100,claude-code,9–16,45,0.28888888888888886
+Harbor OpenCode-only,100,claude-code,17–24,178,0.21348314606741572
+Harbor OpenCode-only,100,claude-code,25+,5,0.0
+Harbor OpenCode-only,100,codex,3–8,25,0.56
+Harbor OpenCode-only,100,codex,9–16,26,0.5384615384615384
+Harbor OpenCode-only,100,codex,17–24,189,0.24867724867724866
+Harbor OpenCode-only,100,codex,25+,10,0.1
+Harbor OpenCode-only,100,mini-swe-agent,3–8,21,0.7619047619047619
+Harbor OpenCode-only,100,mini-swe-agent,9–16,67,0.3880597014925373
+Harbor OpenCode-only,100,mini-swe-agent,17–24,162,0.05555555555555555
+Harbor OpenCode-only,100,opencode,3–8,1,0.0
+Harbor OpenCode-only,100,opencode,9–16,36,0.2222222222222222
+Harbor OpenCode-only,100,opencode,17–24,213,0.22065727699530516
+Harbor OpenCode-only,200,claude-code,0–2,1,0.0
+Harbor OpenCode-only,200,claude-code,3–8,17,0.4117647058823529
+Harbor OpenCode-only,200,claude-code,9–16,44,0.5227272727272727
+Harbor OpenCode-only,200,claude-code,17–24,184,0.22282608695652173
+Harbor OpenCode-only,200,claude-code,25+,4,0.0
+Harbor OpenCode-only,200,codex,3–8,6,0.8333333333333334
+Harbor OpenCode-only,200,codex,9–16,40,0.725
+Harbor OpenCode-only,200,codex,17–24,197,0.2182741116751269
+Harbor OpenCode-only,200,codex,25+,7,0.0
+Harbor OpenCode-only,200,mini-swe-agent,3–8,9,0.8888888888888888
+Harbor OpenCode-only,200,mini-swe-agent,9–16,57,0.49122807017543857
+Harbor OpenCode-only,200,mini-swe-agent,17–24,184,0.07608695652173914
+Harbor OpenCode-only,200,opencode,3–8,2,0.0
+Harbor OpenCode-only,200,opencode,9–16,8,0.5
+Harbor OpenCode-only,200,opencode,17–24,240,0.2875
+Harbor OpenCode-only,300,claude-code,0–2,3,0.0
+Harbor OpenCode-only,300,claude-code,3–8,9,0.3333333333333333
+Harbor OpenCode-only,300,claude-code,9–16,34,0.5882352941176471
+Harbor OpenCode-only,300,claude-code,17–24,198,0.2676767676767677
+Harbor OpenCode-only,300,claude-code,25+,6,0.0
+Harbor OpenCode-only,300,codex,3–8,5,0.8
+Harbor OpenCode-only,300,codex,9–16,27,0.5925925925925926
+Harbor OpenCode-only,300,codex,17–24,209,0.2822966507177033
+Harbor OpenCode-only,300,codex,25+,9,0.1111111111111111
+Harbor OpenCode-only,300,mini-swe-agent,3–8,4,0.75
+Harbor OpenCode-only,300,mini-swe-agent,9–16,52,0.7692307692307693
+Harbor OpenCode-only,300,mini-swe-agent,17–24,194,0.08247422680412371
+Harbor OpenCode-only,300,opencode,3–8,3,0.0
+Harbor OpenCode-only,300,opencode,9–16,7,0.14285714285714285
+Harbor OpenCode-only,300,opencode,17–24,219,0.3059360730593607
+Harbor OpenCode-only,300,opencode,25+,21,0.09523809523809523
+Harbor OpenCode-only,400,claude-code,0–2,2,0.0
+Harbor OpenCode-only,400,claude-code,3–8,10,0.5
+Harbor OpenCode-only,400,claude-code,9–16,25,0.56
+Harbor OpenCode-only,400,claude-code,17–24,204,0.3137254901960784
+Harbor OpenCode-only,400,claude-code,25+,9,0.1111111111111111
+Harbor OpenCode-only,400,codex,3–8,5,0.8
+Harbor OpenCode-only,400,codex,9–16,33,0.5454545454545454
+Harbor OpenCode-only,400,codex,17–24,190,0.32105263157894737
+Harbor OpenCode-only,400,codex,25+,22,0.0
+Harbor OpenCode-only,400,mini-swe-agent,3–8,7,0.7142857142857143
+Harbor OpenCode-only,400,mini-swe-agent,9–16,54,0.6666666666666666
+Harbor OpenCode-only,400,mini-swe-agent,17–24,189,0.164021164021164
+Harbor OpenCode-only,400,opencode,3–8,1,0.0
+Harbor OpenCode-only,400,opencode,9–16,3,0.0
+Harbor OpenCode-only,400,opencode,17–24,223,0.39461883408071746
+Harbor OpenCode-only,400,opencode,25+,23,0.043478260869565216
+Harbor OpenCode-only,500,claude-code,0–2,2,0.0
+Harbor OpenCode-only,500,claude-code,3–8,18,0.6666666666666666
+Harbor OpenCode-only,500,claude-code,9–16,45,0.6666666666666666
+Harbor OpenCode-only,500,claude-code,17–24,182,0.21428571428571427
+Harbor OpenCode-only,500,claude-code,25+,3,0.0
+Harbor OpenCode-only,500,codex,3–8,15,0.6666666666666666
+Harbor OpenCode-only,500,codex,9–16,31,0.7096774193548387
+Harbor OpenCode-only,500,codex,17–24,197,0.30456852791878175
+Harbor OpenCode-only,500,codex,25+,7,0.14285714285714285
+Harbor OpenCode-only,500,mini-swe-agent,3–8,6,0.6666666666666666
+Harbor OpenCode-only,500,mini-swe-agent,9–16,64,0.78125
+Harbor OpenCode-only,500,mini-swe-agent,17–24,180,0.1111111111111111
+Harbor OpenCode-only,500,opencode,3–8,4,0.0
+Harbor OpenCode-only,500,opencode,9–16,3,0.0
+Harbor OpenCode-only,500,opencode,17–24,233,0.34334763948497854
+Harbor OpenCode-only,500,opencode,25+,10,0.2
+Harbor OpenCode-only,600,claude-code,3–8,16,0.5625
+Harbor OpenCode-only,600,claude-code,9–16,56,0.5178571428571429
+Harbor OpenCode-only,600,claude-code,17–24,172,0.22674418604651161
+Harbor OpenCode-only,600,claude-code,25+,6,0.16666666666666666
+Harbor OpenCode-only,600,codex,3–8,12,0.6666666666666666
+Harbor OpenCode-only,600,codex,9–16,37,0.6756756756756757
+Harbor OpenCode-only,600,codex,17–24,190,0.2789473684210526
+Harbor OpenCode-only,600,codex,25+,11,0.0
+Harbor OpenCode-only,600,mini-swe-agent,3–8,10,0.9
+Harbor OpenCode-only,600,mini-swe-agent,9–16,77,0.6883116883116883
+Harbor OpenCode-only,600,mini-swe-agent,17–24,163,0.09202453987730061
+Harbor OpenCode-only,600,opencode,3–8,3,0.0
+Harbor OpenCode-only,600,opencode,9–16,8,0.5
+Harbor OpenCode-only,600,opencode,17–24,227,0.3744493392070485
+Harbor OpenCode-only,600,opencode,25+,12,0.0
+Harbor OpenCode-only,700,claude-code,0–2,1,0.0
+Harbor OpenCode-only,700,claude-code,3–8,42,0.6428571428571429
+Harbor OpenCode-only,700,claude-code,9–16,89,0.7078651685393258
+Harbor OpenCode-only,700,claude-code,17–24,111,0.22522522522522523
+Harbor OpenCode-only,700,claude-code,25+,7,0.14285714285714285
+Harbor OpenCode-only,700,codex,3–8,32,0.6875
+Harbor OpenCode-only,700,codex,9–16,84,0.5833333333333334
+Harbor OpenCode-only,700,codex,17–24,115,0.1826086956521739
+Harbor OpenCode-only,700,codex,25+,19,0.05263157894736842
+Harbor OpenCode-only,700,mini-swe-agent,3–8,8,0.875
+Harbor OpenCode-only,700,mini-swe-agent,9–16,88,0.7045454545454546
+Harbor OpenCode-only,700,mini-swe-agent,17–24,152,0.10526315789473684
+Harbor OpenCode-only,700,mini-swe-agent,25+,2,0.5
+Harbor OpenCode-only,700,opencode,3–8,2,0.0
+Harbor OpenCode-only,700,opencode,9–16,62,0.6612903225806451
+Harbor OpenCode-only,700,opencode,17–24,149,0.33557046979865773
+Harbor OpenCode-only,700,opencode,25+,37,0.24324324324324326
+Harbor OpenCode-only,800,claude-code,3–8,70,0.6142857142857143
+Harbor OpenCode-only,800,claude-code,9–16,87,0.5057471264367817
+Harbor OpenCode-only,800,claude-code,17–24,93,0.10752688172043011
+Harbor OpenCode-only,800,codex,0–2,1,0.0
+Harbor OpenCode-only,800,codex,3–8,27,0.5185185185185185
+Harbor OpenCode-only,800,codex,9–16,110,0.5272727272727272
+Harbor OpenCode-only,800,codex,17–24,109,0.1559633027522936
+Harbor OpenCode-only,800,codex,25+,3,0.0
+Harbor OpenCode-only,800,mini-swe-agent,3–8,5,0.8
+Harbor OpenCode-only,800,mini-swe-agent,9–16,99,0.6565656565656566
+Harbor OpenCode-only,800,mini-swe-agent,17–24,146,0.0684931506849315
+Harbor OpenCode-only,800,opencode,0–2,10,0.0
+Harbor OpenCode-only,800,opencode,3–8,34,0.11764705882352941
+Harbor OpenCode-only,800,opencode,9–16,82,0.6219512195121951
+Harbor OpenCode-only,800,opencode,17–24,124,0.08870967741935484
+Harbor OpenCode-only,900,claude-code,0–2,1,0.0
+Harbor OpenCode-only,900,claude-code,3–8,36,0.5555555555555556
+Harbor OpenCode-only,900,claude-code,9–16,88,0.6590909090909091
+Harbor OpenCode-only,900,claude-code,17–24,121,0.08264462809917356
+Harbor OpenCode-only,900,claude-code,25+,4,0.0
+Harbor OpenCode-only,900,codex,3–8,16,0.3125
+Harbor OpenCode-only,900,codex,9–16,93,0.5376344086021505
+Harbor OpenCode-only,900,codex,17–24,126,0.09523809523809523
+Harbor OpenCode-only,900,codex,25+,15,0.06666666666666667
+Harbor OpenCode-only,900,mini-swe-agent,3–8,2,1.0
+Harbor OpenCode-only,900,mini-swe-agent,9–16,87,0.632183908045977
+Harbor OpenCode-only,900,mini-swe-agent,17–24,160,0.05
+Harbor OpenCode-only,900,mini-swe-agent,25+,1,0.0
+Harbor OpenCode-only,900,opencode,3–8,25,0.0
+Harbor OpenCode-only,900,opencode,9–16,47,0.6170212765957447
+Harbor OpenCode-only,900,opencode,17–24,152,0.3026315789473684
+Harbor OpenCode-only,900,opencode,25+,26,0.0
+Harbor OpenCode-only,1000,claude-code,3–8,19,0.631578947368421
+Harbor OpenCode-only,1000,claude-code,9–16,60,0.6666666666666666
+Harbor OpenCode-only,1000,claude-code,17–24,129,0.18604651162790697
+Harbor OpenCode-only,1000,claude-code,25+,42,0.023809523809523808
+Harbor OpenCode-only,1000,codex,3–8,12,0.75
+Harbor OpenCode-only,1000,codex,9–16,64,0.46875
+Harbor OpenCode-only,1000,codex,17–24,115,0.30434782608695654
+Harbor OpenCode-only,1000,codex,25+,59,0.11864406779661017
+Harbor OpenCode-only,1000,mini-swe-agent,9–16,36,0.8055555555555556
+Harbor OpenCode-only,1000,mini-swe-agent,17–24,189,0.10052910052910052
+Harbor OpenCode-only,1000,mini-swe-agent,25+,25,0.12
+Harbor OpenCode-only,1000,opencode,3–8,1,0.0
+Harbor OpenCode-only,1000,opencode,9–16,19,0.631578947368421
+Harbor OpenCode-only,1000,opencode,17–24,96,0.2916666666666667
+Harbor OpenCode-only,1000,opencode,25+,134,0.11194029850746269
+Harbor multi-harness,0,claude-code,0–2,8,0.25
+Harbor multi-harness,0,claude-code,3–8,63,0.42857142857142855
+Harbor multi-harness,0,claude-code,9–16,82,0.13414634146341464
+Harbor multi-harness,0,claude-code,17–24,97,0.020618556701030927
+Harbor multi-harness,0,codex,0–2,20,0.2
+Harbor multi-harness,0,codex,3–8,33,0.48484848484848486
+Harbor multi-harness,0,codex,9–16,54,0.24074074074074073
+Harbor multi-harness,0,codex,17–24,137,0.058394160583941604
+Harbor multi-harness,0,codex,25+,6,0.0
+Harbor multi-harness,0,mini-swe-agent,0–2,10,0.2
+Harbor multi-harness,0,mini-swe-agent,3–8,3,0.3333333333333333
+Harbor multi-harness,0,mini-swe-agent,9–16,68,0.4411764705882353
+Harbor multi-harness,0,mini-swe-agent,17–24,169,0.01775147928994083
+Harbor multi-harness,0,opencode,0–2,28,0.14285714285714285
+Harbor multi-harness,0,opencode,3–8,66,0.19696969696969696
+Harbor multi-harness,0,opencode,9–16,112,0.08035714285714286
+Harbor multi-harness,0,opencode,17–24,44,0.022727272727272728
+Harbor multi-harness,100,claude-code,0–2,2,0.0
+Harbor multi-harness,100,claude-code,3–8,9,0.1111111111111111
+Harbor multi-harness,100,claude-code,9–16,104,0.27884615384615385
+Harbor multi-harness,100,claude-code,17–24,135,0.28888888888888886
+Harbor multi-harness,100,codex,3–8,5,0.4
+Harbor multi-harness,100,codex,9–16,67,0.1791044776119403
+Harbor multi-harness,100,codex,17–24,177,0.3163841807909605
+Harbor multi-harness,100,codex,25+,1,0.0
+Harbor multi-harness,100,mini-swe-agent,3–8,4,0.75
+Harbor multi-harness,100,mini-swe-agent,9–16,63,0.5238095238095238
+Harbor multi-harness,100,mini-swe-agent,17–24,183,0.06557377049180328
+Harbor multi-harness,100,opencode,0–2,2,0.0
+Harbor multi-harness,100,opencode,3–8,2,0.0
+Harbor multi-harness,100,opencode,9–16,224,0.2544642857142857
+Harbor multi-harness,100,opencode,17–24,22,0.18181818181818182
+Harbor multi-harness,200,claude-code,0–2,1,0.0
+Harbor multi-harness,200,claude-code,3–8,4,0.0
+Harbor multi-harness,200,claude-code,9–16,103,0.22330097087378642
+Harbor multi-harness,200,claude-code,17–24,142,0.36619718309859156
+Harbor multi-harness,200,codex,0–2,1,0.0
+Harbor multi-harness,200,codex,3–8,1,0.0
+Harbor multi-harness,200,codex,9–16,71,0.15492957746478872
+Harbor multi-harness,200,codex,17–24,177,0.3107344632768362
+Harbor multi-harness,200,mini-swe-agent,3–8,6,0.3333333333333333
+Harbor multi-harness,200,mini-swe-agent,9–16,38,0.5789473684210527
+Harbor multi-harness,200,mini-swe-agent,17–24,206,0.10679611650485436
+Harbor multi-harness,200,opencode,0–2,1,0.0
+Harbor multi-harness,200,opencode,3–8,3,0.0
+Harbor multi-harness,200,opencode,9–16,225,0.30666666666666664
+Harbor multi-harness,200,opencode,17–24,21,0.3333333333333333
+Harbor multi-harness,300,claude-code,0–2,3,0.0
+Harbor multi-harness,300,claude-code,3–8,8,0.125
+Harbor multi-harness,300,claude-code,9–16,84,0.2619047619047619
+Harbor multi-harness,300,claude-code,17–24,155,0.3870967741935484
+Harbor multi-harness,300,codex,3–8,10,0.6
+Harbor multi-harness,300,codex,9–16,80,0.225
+Harbor multi-harness,300,codex,17–24,160,0.3125
+Harbor multi-harness,300,mini-swe-agent,3–8,9,0.7777777777777778
+Harbor multi-harness,300,mini-swe-agent,9–16,28,0.5357142857142857
+Harbor multi-harness,300,mini-swe-agent,17–24,213,0.15492957746478872
+Harbor multi-harness,300,opencode,3–8,4,0.75
+Harbor multi-harness,300,opencode,9–16,215,0.2930232558139535
+Harbor multi-harness,300,opencode,17–24,31,0.25806451612903225
+Harbor multi-harness,400,claude-code,0–2,2,0.0
+Harbor multi-harness,400,claude-code,3–8,5,0.0
+Harbor multi-harness,400,claude-code,9–16,90,0.32222222222222224
+Harbor multi-harness,400,claude-code,17–24,153,0.4117647058823529
+Harbor multi-harness,400,codex,3–8,3,0.3333333333333333
+Harbor multi-harness,400,codex,9–16,60,0.15
+Harbor multi-harness,400,codex,17–24,187,0.4117647058823529
+Harbor multi-harness,400,mini-swe-agent,3–8,5,0.8
+Harbor multi-harness,400,mini-swe-agent,9–16,31,0.2903225806451613
+Harbor multi-harness,400,mini-swe-agent,17–24,214,0.2757009345794392
+Harbor multi-harness,400,opencode,3–8,6,0.0
+Harbor multi-harness,400,opencode,9–16,223,0.34080717488789236
+Harbor multi-harness,400,opencode,17–24,21,0.2857142857142857
+Harbor multi-harness,500,claude-code,3–8,9,0.0
+Harbor multi-harness,500,claude-code,9–16,86,0.3023255813953488
+Harbor multi-harness,500,claude-code,17–24,155,0.5548387096774193
+Harbor multi-harness,500,codex,3–8,1,1.0
+Harbor multi-harness,500,codex,9–16,72,0.2222222222222222
+Harbor multi-harness,500,codex,17–24,177,0.4576271186440678
+Harbor multi-harness,500,mini-swe-agent,3–8,35,0.7142857142857143
+Harbor multi-harness,500,mini-swe-agent,9–16,73,0.589041095890411
+Harbor multi-harness,500,mini-swe-agent,17–24,142,0.07042253521126761
+Harbor multi-harness,500,opencode,3–8,8,0.0
+Harbor multi-harness,500,opencode,9–16,224,0.33035714285714285
+Harbor multi-harness,500,opencode,17–24,18,0.4444444444444444
+Harbor multi-harness,600,claude-code,0–2,1,0.0
+Harbor multi-harness,600,claude-code,3–8,7,0.0
+Harbor multi-harness,600,claude-code,9–16,91,0.2087912087912088
+Harbor multi-harness,600,claude-code,17–24,151,0.3708609271523179
+Harbor multi-harness,600,codex,3–8,1,0.0
+Harbor multi-harness,600,codex,9–16,109,0.11009174311926606
+Harbor multi-harness,600,codex,17–24,140,0.4928571428571429
+Harbor multi-harness,600,mini-swe-agent,3–8,23,0.7391304347826086
+Harbor multi-harness,600,mini-swe-agent,9–16,107,0.45794392523364486
+Harbor multi-harness,600,mini-swe-agent,17–24,120,0.09166666666666666
+Harbor multi-harness,600,opencode,0–2,1,0.0
+Harbor multi-harness,600,opencode,3–8,11,0.0
+Harbor multi-harness,600,opencode,9–16,224,0.3482142857142857
+Harbor multi-harness,600,opencode,17–24,14,0.5
+Harbor multi-harness,684,claude-code,0–2,2,0.0
+Harbor multi-harness,684,claude-code,3–8,13,0.38461538461538464
+Harbor multi-harness,684,claude-code,9–16,80,0.2875
+Harbor multi-harness,684,claude-code,17–24,155,0.3935483870967742
+Harbor multi-harness,684,codex,3–8,1,0.0
+Harbor multi-harness,684,codex,9–16,101,0.2079207920792079
+Harbor multi-harness,684,codex,17–24,148,0.4189189189189189
+Harbor multi-harness,684,mini-swe-agent,3–8,36,0.7222222222222222
+Harbor multi-harness,684,mini-swe-agent,9–16,119,0.3865546218487395
+Harbor multi-harness,684,mini-swe-agent,17–24,95,0.042105263157894736
+Harbor multi-harness,684,opencode,0–2,1,0.0
+Harbor multi-harness,684,opencode,3–8,26,0.0
+Harbor multi-harness,684,opencode,9–16,202,0.31683168316831684
+Harbor multi-harness,684,opencode,17–24,21,0.42857142857142855
+Harbor multi-harness,700,claude-code,0–2,2,0.0
+Harbor multi-harness,700,claude-code,3–8,21,0.14285714285714285
+Harbor multi-harness,700,claude-code,9–16,84,0.3333333333333333
+Harbor multi-harness,700,claude-code,17–24,143,0.3356643356643357
+Harbor multi-harness,700,codex,3–8,2,0.0
+Harbor multi-harness,700,codex,9–16,80,0.2
+Harbor multi-harness,700,codex,17–24,168,0.38095238095238093
+Harbor multi-harness,700,mini-swe-agent,3–8,31,0.8064516129032258
+Harbor multi-harness,700,mini-swe-agent,9–16,122,0.3770491803278688
+Harbor multi-harness,700,mini-swe-agent,17–24,97,0.05154639175257732
+Harbor multi-harness,700,opencode,0–2,7,0.0
+Harbor multi-harness,700,opencode,3–8,36,0.0
+Harbor multi-harness,700,opencode,9–16,183,0.25136612021857924
+Harbor multi-harness,700,opencode,17–24,24,0.2916666666666667
+Harbor multi-harness,800,claude-code,0–2,2,0.0
+Harbor multi-harness,800,claude-code,3–8,20,0.0
+Harbor multi-harness,800,claude-code,9–16,85,0.24705882352941178
+Harbor multi-harness,800,claude-code,17–24,143,0.4125874125874126
+Harbor multi-harness,800,codex,9–16,168,0.2619047619047619
+Harbor multi-harness,800,codex,17–24,82,0.2804878048780488
+Harbor multi-harness,800,mini-swe-agent,3–8,14,0.8571428571428571
+Harbor multi-harness,800,mini-swe-agent,9–16,85,0.5764705882352941
+Harbor multi-harness,800,mini-swe-agent,17–24,151,0.026490066225165563
+Harbor multi-harness,800,opencode,0–2,7,0.0
+Harbor multi-harness,800,opencode,3–8,10,0.0
+Harbor multi-harness,800,opencode,9–16,214,0.2336448598130841
+Harbor multi-harness,800,opencode,17–24,19,0.42105263157894735
+Harbor multi-harness,900,claude-code,0–2,5,0.0
+Harbor multi-harness,900,claude-code,3–8,24,0.08333333333333333
+Harbor multi-harness,900,claude-code,9–16,105,0.3047619047619048
+Harbor multi-harness,900,claude-code,17–24,115,0.4956521739130435
+Harbor multi-harness,900,claude-code,25+,1,0.0
+Harbor multi-harness,900,codex,0–2,29,0.0
+Harbor multi-harness,900,codex,3–8,115,0.034782608695652174
+Harbor multi-harness,900,codex,9–16,61,0.3442622950819672
+Harbor multi-harness,900,codex,17–24,45,0.5111111111111111
+Harbor multi-harness,900,mini-swe-agent,3–8,42,0.7142857142857143
+Harbor multi-harness,900,mini-swe-agent,9–16,161,0.3105590062111801
+Harbor multi-harness,900,mini-swe-agent,17–24,47,0.0425531914893617
+Harbor multi-harness,900,opencode,0–2,218,0.0
+Harbor multi-harness,900,opencode,3–8,16,0.0
+Harbor multi-harness,900,opencode,9–16,16,0.375
+Harbor multi-harness,1000,claude-code,0–2,2,0.0
+Harbor multi-harness,1000,claude-code,3–8,64,0.3125
+Harbor multi-harness,1000,claude-code,9–16,126,0.4603174603174603
+Harbor multi-harness,1000,claude-code,17–24,58,0.3275862068965517
+Harbor multi-harness,1000,codex,0–2,34,0.0
+Harbor multi-harness,1000,codex,3–8,89,0.056179775280898875
+Harbor multi-harness,1000,codex,9–16,73,0.410958904109589
+Harbor multi-harness,1000,codex,17–24,54,0.5
+Harbor multi-harness,1000,mini-swe-agent,3–8,87,0.5977011494252874
+Harbor multi-harness,1000,mini-swe-agent,9–16,137,0.27007299270072993
+Harbor multi-harness,1000,mini-swe-agent,17–24,26,0.038461538461538464
+Harbor multi-harness,1000,opencode,0–2,185,0.0
+Harbor multi-harness,1000,opencode,3–8,30,0.1
+Harbor multi-harness,1000,opencode,9–16,31,0.3225806451612903
+Harbor multi-harness,1000,opencode,17–24,4,0.25
+Native OpenCode,0,claude-code,0–2,3,0.0
+Native OpenCode,0,claude-code,3–8,55,0.4
+Native OpenCode,0,claude-code,9–16,74,0.20270270270270271
+Native OpenCode,0,claude-code,17–24,116,0.04310344827586207
+Native OpenCode,0,claude-code,25+,2,0.0
+Native OpenCode,0,codex,3–8,27,0.4074074074074074
+Native OpenCode,0,codex,9–16,71,0.323943661971831
+Native OpenCode,0,codex,17–24,144,0.027777777777777776
+Native OpenCode,0,codex,25+,8,0.0
+Native OpenCode,0,mini-swe-agent,3–8,5,0.6
+Native OpenCode,0,mini-swe-agent,9–16,70,0.5428571428571428
+Native OpenCode,0,mini-swe-agent,17–24,175,0.03428571428571429
+Native OpenCode,0,opencode,0–2,5,0.0
+Native OpenCode,0,opencode,3–8,85,0.2823529411764706
+Native OpenCode,0,opencode,9–16,119,0.05042016806722689
+Native OpenCode,0,opencode,17–24,41,0.04878048780487805
+Native OpenCode,100,claude-code,0–2,4,0.25
+Native OpenCode,100,claude-code,3–8,81,0.4444444444444444
+Native OpenCode,100,claude-code,9–16,46,0.2391304347826087
+Native OpenCode,100,claude-code,17–24,116,0.02586206896551724
+Native OpenCode,100,claude-code,25+,3,0.0
+Native OpenCode,100,codex,0–2,1,0.0
+Native OpenCode,100,codex,3–8,26,0.46153846153846156
+Native OpenCode,100,codex,9–16,56,0.44642857142857145
+Native OpenCode,100,codex,17–24,166,0.04819277108433735
+Native OpenCode,100,codex,25+,1,0.0
+Native OpenCode,100,mini-swe-agent,3–8,19,0.6842105263157895
+Native OpenCode,100,mini-swe-agent,9–16,58,0.5517241379310345
+Native OpenCode,100,mini-swe-agent,17–24,173,0.04046242774566474
+Native OpenCode,100,opencode,0–2,1,0.0
+Native OpenCode,100,opencode,3–8,101,0.39603960396039606
+Native OpenCode,100,opencode,9–16,130,0.06923076923076923
+Native OpenCode,100,opencode,17–24,18,0.0
+Native OpenCode,200,claude-code,0–2,2,0.0
+Native OpenCode,200,claude-code,3–8,88,0.5
+Native OpenCode,200,claude-code,9–16,40,0.4
+Native OpenCode,200,claude-code,17–24,119,0.0
+Native OpenCode,200,claude-code,25+,1,0.0
+Native OpenCode,200,codex,3–8,54,0.5370370370370371
+Native OpenCode,200,codex,9–16,67,0.4626865671641791
+Native OpenCode,200,codex,17–24,127,0.031496062992125984
+Native OpenCode,200,codex,25+,2,0.0
+Native OpenCode,200,mini-swe-agent,3–8,27,0.6296296296296297
+Native OpenCode,200,mini-swe-agent,9–16,65,0.5076923076923077
+Native OpenCode,200,mini-swe-agent,17–24,158,0.02531645569620253
+Native OpenCode,200,opencode,0–2,2,0.0
+Native OpenCode,200,opencode,3–8,109,0.3394495412844037
+Native OpenCode,200,opencode,9–16,128,0.046875
+Native OpenCode,200,opencode,17–24,11,0.0
+Native OpenCode,300,claude-code,0–2,1,1.0
+Native OpenCode,300,claude-code,3–8,88,0.5568181818181818
+Native OpenCode,300,claude-code,9–16,54,0.4074074074074074
+Native OpenCode,300,claude-code,17–24,106,0.009433962264150943
+Native OpenCode,300,claude-code,25+,1,0.0
+Native OpenCode,300,codex,3–8,29,0.5517241379310345
+Native OpenCode,300,codex,9–16,44,0.36363636363636365
+Native OpenCode,300,codex,17–24,176,0.056818181818181816
+Native OpenCode,300,codex,25+,1,0.0
+Native OpenCode,300,mini-swe-agent,3–8,13,0.5384615384615384
+Native OpenCode,300,mini-swe-agent,9–16,65,0.6
+Native OpenCode,300,mini-swe-agent,17–24,172,0.01744186046511628
+Native OpenCode,300,opencode,3–8,102,0.45098039215686275
+Native OpenCode,300,opencode,9–16,127,0.047244094488188976
+Native OpenCode,300,opencode,17–24,21,0.0
+Native OpenCode,400,claude-code,0–2,1,0.0
+Native OpenCode,400,claude-code,3–8,96,0.6354166666666666
+Native OpenCode,400,claude-code,9–16,39,0.46153846153846156
+Native OpenCode,400,claude-code,17–24,113,0.017699115044247787
+Native OpenCode,400,claude-code,25+,1,0.0
+Native OpenCode,400,codex,3–8,32,0.65625
+Native OpenCode,400,codex,9–16,52,0.6346153846153846
+Native OpenCode,400,codex,17–24,165,0.06060606060606061
+Native OpenCode,400,codex,25+,1,0.0
+Native OpenCode,400,mini-swe-agent,3–8,18,0.8333333333333334
+Native OpenCode,400,mini-swe-agent,9–16,86,0.5697674418604651
+Native OpenCode,400,mini-swe-agent,17–24,146,0.0273972602739726
+Native OpenCode,400,opencode,3–8,113,0.3805309734513274
+Native OpenCode,400,opencode,9–16,119,0.06722689075630252
+Native OpenCode,400,opencode,17–24,18,0.0
+Native OpenCode,500,claude-code,0–2,3,0.3333333333333333
+Native OpenCode,500,claude-code,3–8,92,0.532608695652174
+Native OpenCode,500,claude-code,9–16,52,0.25
+Native OpenCode,500,claude-code,17–24,103,0.02912621359223301
+Native OpenCode,500,codex,0–2,1,0.0
+Native OpenCode,500,codex,3–8,35,0.5142857142857142
+Native OpenCode,500,codex,9–16,54,0.4444444444444444
+Native OpenCode,500,codex,17–24,160,0.03125
+Native OpenCode,500,mini-swe-agent,3–8,29,0.7931034482758621
+Native OpenCode,500,mini-swe-agent,9–16,102,0.47058823529411764
+Native OpenCode,500,mini-swe-agent,17–24,119,0.01680672268907563
+Native OpenCode,500,opencode,0–2,1,0.0
+Native OpenCode,500,opencode,3–8,119,0.31092436974789917
+Native OpenCode,500,opencode,9–16,112,0.07142857142857142
+Native OpenCode,500,opencode,17–24,18,0.0
+Native OpenCode,600,claude-code,0–2,4,0.0
+Native OpenCode,600,claude-code,3–8,108,0.5740740740740741
+Native OpenCode,600,claude-code,9–16,52,0.3269230769230769
+Native OpenCode,600,claude-code,17–24,86,0.023255813953488372
+Native OpenCode,600,codex,3–8,42,0.6190476190476191
+Native OpenCode,600,codex,9–16,50,0.32
+Native OpenCode,600,codex,17–24,158,0.02531645569620253
+Native OpenCode,600,mini-swe-agent,3–8,44,0.5909090909090909
+Native OpenCode,600,mini-swe-agent,9–16,150,0.34
+Native OpenCode,600,mini-swe-agent,17–24,56,0.08928571428571429
+Native OpenCode,600,opencode,0–2,4,0.25
+Native OpenCode,600,opencode,3–8,126,0.30952380952380953
+Native OpenCode,600,opencode,9–16,109,0.01834862385321101
+Native OpenCode,600,opencode,17–24,11,0.0
+Native OpenCode,700,claude-code,0–2,6,0.0
+Native OpenCode,700,claude-code,3–8,115,0.5391304347826087
+Native OpenCode,700,claude-code,9–16,40,0.25
+Native OpenCode,700,claude-code,17–24,89,0.02247191011235955
+Native OpenCode,700,codex,3–8,20,0.6
+Native OpenCode,700,codex,9–16,45,0.4222222222222222
+Native OpenCode,700,codex,17–24,185,0.02702702702702703
+Native OpenCode,700,mini-swe-agent,3–8,44,0.7272727272727273
+Native OpenCode,700,mini-swe-agent,9–16,99,0.41414141414141414
+Native OpenCode,700,mini-swe-agent,17–24,107,0.028037383177570093
+Native OpenCode,700,opencode,0–2,3,0.0
+Native OpenCode,700,opencode,3–8,154,0.2987012987012987
+Native OpenCode,700,opencode,9–16,91,0.0
+Native OpenCode,700,opencode,17–24,2,0.0
+Native OpenCode,800,claude-code,0–2,5,0.0
+Native OpenCode,800,claude-code,3–8,106,0.5566037735849056
+Native OpenCode,800,claude-code,9–16,62,0.3387096774193548
+Native OpenCode,800,claude-code,17–24,77,0.012987012987012988
+Native OpenCode,800,codex,0–2,2,0.0
+Native OpenCode,800,codex,3–8,30,0.6
+Native OpenCode,800,codex,9–16,53,0.4528301886792453
+Native OpenCode,800,codex,17–24,165,0.01818181818181818
+Native OpenCode,800,mini-swe-agent,3–8,42,0.8095238095238095
+Native OpenCode,800,mini-swe-agent,9–16,81,0.49382716049382713
+Native OpenCode,800,mini-swe-agent,17–24,127,0.047244094488188976
+Native OpenCode,800,opencode,0–2,4,0.0
+Native OpenCode,800,opencode,3–8,158,0.27848101265822783
+Native OpenCode,800,opencode,9–16,84,0.07142857142857142
+Native OpenCode,800,opencode,17–24,4,0.0
+Native OpenCode,900,claude-code,0–2,8,0.125
+Native OpenCode,900,claude-code,3–8,117,0.48717948717948717
+Native OpenCode,900,claude-code,9–16,54,0.4444444444444444
+Native OpenCode,900,claude-code,17–24,71,0.04225352112676056
+Native OpenCode,900,codex,0–2,1,0.0
+Native OpenCode,900,codex,3–8,27,0.4074074074074074
+Native OpenCode,900,codex,9–16,40,0.475
+Native OpenCode,900,codex,17–24,182,0.03296703296703297
+Native OpenCode,900,mini-swe-agent,3–8,34,0.7941176470588235
+Native OpenCode,900,mini-swe-agent,9–16,98,0.5204081632653061
+Native OpenCode,900,mini-swe-agent,17–24,118,0.059322033898305086
+Native OpenCode,900,opencode,0–2,14,0.14285714285714285
+Native OpenCode,900,opencode,3–8,163,0.25766871165644173
+Native OpenCode,900,opencode,9–16,69,0.043478260869565216
+Native OpenCode,900,opencode,17–24,4,0.0
+Native OpenCode,1000,claude-code,0–2,3,0.0
+Native OpenCode,1000,claude-code,3–8,116,0.5172413793103449
+Native OpenCode,1000,claude-code,9–16,52,0.38461538461538464
+Native OpenCode,1000,claude-code,17–24,79,0.0379746835443038
+Native OpenCode,1000,codex,0–2,1,0.0
+Native OpenCode,1000,codex,3–8,27,0.5925925925925926
+Native OpenCode,1000,codex,9–16,71,0.5633802816901409
+Native OpenCode,1000,codex,17–24,151,0.11920529801324503
+Native OpenCode,1000,mini-swe-agent,3–8,34,0.7647058823529411
+Native OpenCode,1000,mini-swe-agent,9–16,107,0.5514018691588785
+Native OpenCode,1000,mini-swe-agent,17–24,109,0.045871559633027525
+Native OpenCode,1000,opencode,3–8,130,0.3076923076923077
+Native OpenCode,1000,opencode,9–16,110,0.1
+Native OpenCode,1000,opencode,17–24,10,0.0
diff --git a/04-data-agent/reports/three-run-analysis-20260917/training_accounting.csv b/04-data-agent/reports/three-run-analysis-20260917/training_accounting.csv
new file mode 100644
index 0000000..d018ebb
--- /dev/null
+++ b/04-data-agent/reports/three-run-analysis-20260917/training_accounting.csv
@@ -0,0 +1,4 @@
+run,steps,zero_gradient_steps,forwarded_tokens,supervised_tokens,forwarded_per_supervised_token,supervised_tokens_in_zero_gradient_steps,zero_gradient_supervised_token_fraction,zero_gradient_forwarded_token_fraction,mean_step_s,step_timing_observations,receipt_token_mismatches
+Harbor OpenCode-only,1000,380,419428466.0,14625038.0,28.67879495424217,5912749.0,0.40428947945297644,0.39100410509571853,79.62124525297799,999,0
+Harbor multi-harness,1000,349,1001322856.0,22148436.0,45.20964171014152,7357643.0,0.3321969551258608,0.34976226988271203,108.20385101597033,993,0
+Native OpenCode,1000,583,228413083.0,5327227.0,42.87654402562534,3418521.0,0.6417074023689998,0.5876681459616742,36.3195656516987,999,0
diff --git a/04-data-agent/reports/three-run-analysis-20260917/training_behavior.py b/04-data-agent/reports/three-run-analysis-20260917/training_behavior.py
new file mode 100644
index 0000000..dc00d26
--- /dev/null
+++ b/04-data-agent/reports/three-run-analysis-20260917/training_behavior.py
@@ -0,0 +1,114 @@
+"""Summarize saved optimizer-admitted rollouts without loading a model or launching jobs."""
+
+import argparse
+import collections
+import concurrent.futures
+import hashlib
+import json
+from pathlib import Path
+
+import orjson
+import pandas as pd
+
+
+def extract(row, audit_dir):
+ path = audit_dir / "rollouts" / (row["rollout_id"] + ".json")
+ raw = path.read_bytes()
+ saved = orjson.loads(raw)
+ result = saved["result"]
+ assert saved["task_index"] == row["task_index"], path
+ assert saved["harness"] == row["harness"], path
+ turns = [t for t in result["turns"] if t.get("trainable", True) and not t.get("discarded", False)]
+ tokens, supervised, action_tokens, prompt_tokens, calls = [], 0, 0, 0, []
+ for turn in turns:
+ count = len(turn["completion_token_ids"])
+ prompt_len = len(turn["prompt_token_ids"])
+ tokens.append(count)
+ prompt_tokens += prompt_len
+ mask = turn.get("loss_mask")
+ if mask is None:
+ supervised += count
+ elif len(mask) == prompt_len + count:
+ supervised += sum(mask[prompt_len:])
+ else:
+ assert len(mask) == count, (path, len(mask), prompt_len, count)
+ supervised += sum(mask)
+ if turn.get("tool_calls"):
+ action_tokens += count
+ for call in turn.get("tool_calls") or []:
+ fn = call.get("function") or call
+ args = fn.get("arguments", {})
+ if isinstance(args, str):
+ try:
+ args = orjson.loads(args)
+ except orjson.JSONDecodeError:
+ pass
+ signature = (fn.get("name", ""), json.dumps(args, sort_keys=True))
+ calls.append(signature)
+ counts = collections.Counter(calls)
+ reward = result.get("reward")
+ if row["run"] == "Native OpenCode":
+ correctness = result.get("correctness")
+ reward = float(correctness >= 1.0) if correctness is not None else None
+ return {
+ **row, "binary_reward": reward, "agent_turns": len(turns),
+ "emitted_tool_calls": len(calls), "exact_repeated_calls": sum(n - 1 for n in counts.values()),
+ "completion_tokens": sum(tokens), "masked_completion_tokens": supervised,
+ "receipt_token_delta": supervised - row["supervised_tokens"],
+ "completion_tokens_in_tool_turns": action_tokens,
+ "completion_tokens_in_text_only_turns": sum(tokens) - action_tokens,
+ "max_response_tokens": max(tokens, default=0),
+ "responses_over_4096": sum(n > 4096 for n in tokens),
+ "responses_finish_length": sum(t.get("finish_reason") == "length" for t in turns),
+ "sum_captured_prompt_tokens": prompt_tokens,
+ "last_turn_has_tool": bool(turns and turns[-1].get("tool_calls")),
+ "generation_s": saved["finished_at"] - saved["started_at"],
+ "tool_names": dict(collections.Counter(name for name, _ in calls)),
+ "source": str(path), "sha256": hashlib.sha256(raw).hexdigest(),
+ }
+
+
+def main():
+ parser = argparse.ArgumentParser(description=__doc__)
+ parser.add_argument("--evidence", type=Path, required=True, help="Directory containing optimizer_rollouts.csv and training_lineage.csv")
+ parser.add_argument("--workers", type=int, default=2)
+ args = parser.parse_args()
+ receipts = pd.read_csv(args.evidence / "optimizer_rollouts.csv")
+ lineage = pd.read_csv(args.evidence / "training_lineage.csv")
+ audit_dirs = {int(r.job): Path(r.coverage_source).parent for r in lineage.itertuples()}
+ cache_path = args.evidence / "admitted_training_behavior.jsonl"
+ cached = {}
+ if cache_path.exists():
+ with cache_path.open() as stream:
+ for line in stream:
+ item = json.loads(line)
+ cached[(item["run"], item["rollout_id"])] = item
+ rows = receipts.to_dict("records")
+ missing = [r for r in rows if (r["run"], r["rollout_id"]) not in cached]
+ print(f"Cached {len(cached)}; extracting {len(missing)} optimizer-admitted rollouts", flush=True)
+ with cache_path.open("a") as out, concurrent.futures.ThreadPoolExecutor(max_workers=args.workers) as pool:
+ futures = (pool.submit(extract, row, audit_dirs[row["job"]]) for row in missing)
+ # Keep only a small number of raw captures in flight.
+ pending = collections.deque()
+ for future in futures:
+ pending.append(future)
+ if len(pending) < args.workers * 2:
+ continue
+ item = pending.popleft().result()
+ out.write(json.dumps(item) + "\n")
+ cached[(item["run"], item["rollout_id"])] = item
+ if len(cached) % 500 == 0:
+ out.flush()
+ print(f"Extracted {len(cached)}/{len(rows)}", flush=True)
+ for future in pending:
+ item = future.result()
+ out.write(json.dumps(item) + "\n")
+ cached[(item["run"], item["rollout_id"])] = item
+ assert len(cached) == len(rows), (len(cached), len(rows))
+ frame = pd.DataFrame(cached.values())
+ frame.to_csv(args.evidence / "admitted_training_behavior.csv", index=False)
+ print(frame.groupby("run").agg(rollouts=("rollout_id", "size"), tool_calls=("emitted_tool_calls", "mean"), tokens=("completion_tokens", "mean"), receipt_mismatches=("receipt_token_delta", lambda x: (x != 0).sum())).to_string(), flush=True)
+
+
+if __name__ == "__main__":
+ main()
diff --git a/04-data-agent/reports/three-run-analysis-20260917/training_behavior_by_harness.csv b/04-data-agent/reports/three-run-analysis-20260917/training_behavior_by_harness.csv
new file mode 100644
index 0000000..d849645
--- /dev/null
+++ b/04-data-agent/reports/three-run-analysis-20260917/training_behavior_by_harness.csv
@@ -0,0 +1,61 @@
+run,harness,window_end,admitted_rollouts,unique_tasks,binary_reward_mean,completion_tokens_mean,completion_tokens_median,completion_tokens_p90,supervised_tokens,completion_tokens,text_only_turn_tokens,agent_turns_mean,tool_calls_mean,tool_calls_p90,exact_repeated_calls_mean,rollout_over4096_fraction,rollout_finish_length_fraction,training_rows,receipt_token_mismatches,text_only_turn_token_share,rows_per_rollout
+Harbor OpenCode-only,opencode,100,414,53,0.4613526570048309,2174.2487922705313,976.0,4195.299999999999,900139,900139,2772,14.152173913043478,14.577294685990339,17.0,6.584541062801932,0.033816425120772944,0.0,881,0,0.0030795243845672723,2.1280193236714977
+Harbor OpenCode-only,opencode,200,432,55,0.22916666666666666,7749.881944444444,2538.0,20144.0,3347949,3347949,16657,13.5,14.974537037037036,18.0,2.3518518518518516,0.2569444444444444,0.0023148148148148147,1101,0,0.004975284868437363,2.548611111111111
+Harbor OpenCode-only,opencode,300,410,52,0.36097560975609755,5242.002439024391,2272.0,11004.000000000018,2149221,2149221,17393,14.234146341463415,15.75609756097561,18.0,2.292682926829268,0.0951219512195122,0.0024390243902439024,1077,0,0.008092699633960398,2.626829268292683
+Harbor OpenCode-only,opencode,400,409,52,0.3154034229828851,2577.4767726161367,2011.0,4439.8,1054188,1054188,242,14.71393643031785,18.760391198044008,21.0,3.4376528117359415,0.007334963325183374,0.0,977,0,0.00022956057173862727,2.388753056234719
+Harbor OpenCode-only,opencode,500,415,54,0.4578313253012048,2965.9204819277106,1964.0,5172.6,1230857,1230857,352,14.573493975903615,16.896385542168673,19.0,2.746987951807229,0.043373493975903614,0.0,1144,0,0.00028597960607934145,2.756626506024096
+Harbor OpenCode-only,opencode,600,403,53,0.45161290322580644,2678.468982630273,2078.0,4732.8,1079423,1079423,653,14.468982630272953,16.523573200992555,18.0,2.967741935483871,0.004962779156327543,0.0,1061,0,0.0006049528312811567,2.632754342431762
+Harbor OpenCode-only,opencode,700,420,55,0.46190476190476193,2122.211904761905,1552.0,3914.3000000000006,891329,891329,27894,11.7,14.633333333333333,19.0,3.8214285714285716,0.011904761904761904,0.002380952380952381,1470,0,0.031294841747547764,3.5
+Harbor OpenCode-only,opencode,800,400,51,0.4625,3206.7175,1945.5,6440.900000000001,1282687,1282687,177689,10.455,15.2775,20.0,5.415,0.0725,0.015,1767,0,0.13852872914436648,4.4175
+Harbor OpenCode-only,opencode,900,412,54,0.44660194174757284,2967.152912621359,1510.0,6343.600000000004,1222467,1222467,144773,10.094660194174757,12.672330097087379,19.0,2.0679611650485437,0.05825242718446602,0.014563106796116505,1755,0,0.11842691868164948,4.259708737864078
+Harbor OpenCode-only,opencode,1000,404,51,0.3935643564356436,3630.638613861386,2102.0,5953.799999999999,1466778,1466778,75528,11.487623762376238,17.43069306930693,25.0,3.027227722772277,0.06435643564356436,0.0049504950495049506,1566,0,0.05149245489092419,3.876237623762376
+Harbor multi-harness,claude-code,100,82,10,0.2682926829268293,2714.890243902439,2291.5,5157.500000000001,222621,222621,1073,13.365853658536585,13.536585365853659,17.0,3.6707317073170733,0.0,0.0,1096,0,0.004819850777779275,13.365853658536585
+Harbor multi-harness,claude-code,200,130,17,0.3153846153846154,4026.346153846154,2943.5,8447.000000000004,523425,523425,263,15.107692307692307,15.215384615384615,17.0,4.076923076923077,0.015384615384615385,0.0,1964,0,0.0005024597602330802,15.107692307692307
+Harbor multi-harness,claude-code,300,140,18,0.37857142857142856,3982.5714285714284,3335.5,6859.6,557560,557560,16421,16.735714285714284,16.785714285714285,17.0,4.55,0.02142857142857143,0.007142857142857143,2343,0,0.029451538847837005,16.735714285714284
+Harbor multi-harness,claude-code,400,136,17,0.5220588235294118,3549.404411764706,2332.5,6253.5,482719,482719,16675,16.566176470588236,16.625,17.0,5.117647058823529,0.029411764705882353,0.007352941176470588,2253,0,0.03454390649632602,16.566176470588236
+Harbor multi-harness,claude-code,500,120,15,0.2916666666666667,5923.533333333334,5641.5,9395.4,710824,710824,16384,16.933333333333334,17.216666666666665,18.0,3.4166666666666665,0.008333333333333333,0.008333333333333333,2032,0,0.023049306157361036,16.933333333333334
+Harbor multi-harness,claude-code,600,128,16,0.40625,4824.53125,4213.5,8859.8,617540,617540,101,15.296875,15.390625,17.0,4.6328125,0.0234375,0.0,1958,0,0.00016355215856462738,15.296875
+Harbor multi-harness,claude-code,700,134,17,0.5,4083.1492537313434,2715.0,7031.500000000001,547142,547142,33063,15.171641791044776,15.514925373134329,17.0,5.485074626865671,0.04477611940298507,0.014925373134328358,2033,0,0.06042855419616845,15.171641791044776
+Harbor multi-harness,claude-code,800,128,16,0.3984375,4690.1484375,3595.5,8415.399999999998,600339,600339,855,16.703125,16.765625,17.0,5.9765625,0.0546875,0.0,2138,0,0.001424195329638754,16.703125
+Harbor multi-harness,claude-code,900,126,16,0.38095238095238093,10933.753968253968,8429.0,21573.0,1377653,1377653,66336,16.333333333333332,16.444444444444443,17.0,4.73015873015873,0.2857142857142857,0.031746031746031744,2058,0,0.04815145758765088,16.333333333333332
+Harbor multi-harness,claude-code,1000,119,17,0.3865546218487395,14147.81512605042,9810.0,25155.600000000002,1683590,1683590,345749,13.025210084033613,12.680672268907562,17.0,5.092436974789916,0.5126050420168067,0.058823529411764705,1550,0,0.20536413259760394,13.025210084033613
+Harbor multi-harness,codex,100,88,11,0.48863636363636365,3284.284090909091,2609.5,6985.2,289017,289017,1466,15.113636363636363,15.056818181818182,17.0,2.3295454545454546,0.0,0.0,100,0,0.005072365985391862,1.1363636363636365
+Harbor multi-harness,codex,200,136,16,0.3161764705882353,3008.904411764706,2459.0,5115.5,409211,409211,754,16.16176470588235,16.272058823529413,17.0,2.5661764705882355,0.0,0.0,151,0,0.0018425702143881762,1.1102941176470589
+Harbor multi-harness,codex,300,128,16,0.34375,2766.0859375,2386.5,4706.2,354059,354059,712,16.375,16.4375,17.0,3.2265625,0.0078125,0.0,131,0,0.0020109642743158626,1.0234375
+Harbor multi-harness,codex,400,128,16,0.6171875,2406.9609375,1978.0,4208.1,308091,308091,640,15.921875,15.890625,17.0,4.859375,0.0,0.0,132,0,0.0020773083277343383,1.03125
+Harbor multi-harness,codex,500,128,16,0.5078125,4199.0859375,2612.0,8614.8,537483,537483,13,16.4296875,16.4375,17.0,4.5546875,0.0390625,0.0,135,0,2.418681148985177e-05,1.0546875
+Harbor multi-harness,codex,600,128,16,0.625,3459.953125,2739.5,6188.999999999999,442874,442874,69,16.4296875,16.4375,17.0,4.609375,0.0078125,0.0,141,0,0.00015580052114145333,1.1015625
+Harbor multi-harness,codex,700,124,16,0.41935483870967744,4493.5161290322585,3596.5,8010.300000000001,557196,557196,117,16.080645161290324,16.072580645161292,17.0,4.395161290322581,0.024193548387096774,0.0,140,0,0.00020997997114121422,1.1290322580645162
+Harbor multi-harness,codex,800,118,16,0.635593220338983,4894.466101694915,3706.0,7741.899999999998,577547,577547,89,15.88135593220339,15.88135593220339,17.0,5.9491525423728815,0.07627118644067797,0.0,121,0,0.00015410001263966395,1.0254237288135593
+Harbor multi-harness,codex,900,136,17,0.4852941176470588,11910.426470588236,10029.0,24055.5,1619818,1619818,174,15.308823529411764,15.345588235294118,17.0,5.198529411764706,0.3602941176470588,0.0,154,0,0.000107419475521324,1.1323529411764706
+Harbor multi-harness,codex,1000,120,15,0.65,11192.558333333332,10052.5,20081.80000000001,1343107,1343107,7707,14.35,14.133333333333333,17.0,5.558333333333334,0.375,0.0,138,0,0.005738187649978743,1.15
+Harbor multi-harness,mini-swe-agent,100,107,11,0.375,1999.411214953271,1722.0,3551.2000000000007,213937,213937,0,14.261682242990654,14.261682242990654,17.0,1.0934579439252337,0.0,0.0,123,0,0.0,1.1495327102803738
+Harbor multi-harness,mini-swe-agent,200,134,16,0.1791044776119403,1702.1119402985075,1255.5,3141.4,228083,228083,0,14.962686567164178,14.962686567164178,17.0,1.507462686567164,0.0,0.0,142,0,0.0,1.0597014925373134
+Harbor multi-harness,mini-swe-agent,300,112,14,0.3063063063063063,1540.607142857143,1230.0,2424.0,172548,172548,0,15.464285714285714,15.464285714285714,17.0,2.3214285714285716,0.0,0.0,120,0,0.0,1.0714285714285714
+Harbor multi-harness,mini-swe-agent,400,144,18,0.3006993006993007,1901.6041666666667,1262.0,3841.000000000006,273831,273831,0,15.36111111111111,15.36111111111111,17.0,2.798611111111111,0.0,0.0,153,0,0.0,1.0625
+Harbor multi-harness,mini-swe-agent,500,130,17,0.36923076923076925,1602.7615384615385,1377.5,2364.7000000000003,208359,208359,0,15.107692307692307,15.107692307692307,17.0,2.6076923076923078,0.007692307692307693,0.0,143,0,0.0,1.1
+Harbor multi-harness,mini-swe-agent,600,134,17,0.43283582089552236,1897.955223880597,1659.5,3196.5,254326,254326,0,12.761194029850746,12.761194029850746,17.0,0.8507462686567164,0.0,0.0,151,0,0.0,1.126865671641791
+Harbor multi-harness,mini-swe-agent,700,136,17,0.45588235294117646,1323.7794117647059,1026.5,2662.5,180034,180034,0,11.794117647058824,11.794117647058824,17.0,1.8235294117647058,0.0,0.0,152,0,0.0,1.1176470588235294
+Harbor multi-harness,mini-swe-agent,800,134,17,0.417910447761194,1903.4626865671642,1504.5,4014.8,255064,255064,56,11.888059701492537,11.880597014925373,16.0,1.4701492537313432,0.007462686567164179,0.0,181,0,0.00021955273970454474,1.3507462686567164
+Harbor multi-harness,mini-swe-agent,900,112,14,0.41964285714285715,4311.580357142857,2825.0,8690.40000000001,482897,482897,0,12.508928571428571,12.508928571428571,17.0,1.6607142857142858,0.10714285714285714,0.0,120,0,0.0,1.0714285714285714
+Harbor multi-harness,mini-swe-agent,1000,128,16,0.359375,7110.0234375,4868.5,15039.199999999999,910083,910083,0,10.3671875,10.3671875,15.299999999999997,1.4921875,0.2421875,0.0,198,0,0.0,1.546875
+Harbor multi-harness,opencode,100,115,12,0.46956521739130436,1707.304347826087,1066.0,3837.8,196340,196340,972,12.982608695652173,12.747826086956522,16.0,3.8,0.0,0.0,257,0,0.004950595905062647,2.234782608695652
+Harbor multi-harness,opencode,200,120,15,0.44166666666666665,2073.9666666666667,1522.5,4272.4000000000015,248876,248876,68,15.475,15.458333333333334,16.0,6.583333333333333,0.008333333333333333,0.0,263,0,0.00027322843504395765,2.191666666666667
+Harbor multi-harness,opencode,300,136,17,0.2,1823.4485294117646,1494.5,3136.5,247989,247989,181,15.397058823529411,15.375,16.0,4.801470588235294,0.0,0.0,299,0,0.0007298710829915843,2.198529411764706
+Harbor multi-harness,opencode,400,130,17,0.23846153846153847,1878.576923076923,1350.0,3450.4000000000015,244215,244215,47,15.176923076923076,15.169230769230769,16.0,4.915384615384616,0.007692307692307693,0.0,278,0,0.000192453371005057,2.1384615384615384
+Harbor multi-harness,opencode,500,134,17,0.3283582089552239,2582.7985074626868,2016.5,4391.3,346095,346095,354,14.791044776119403,14.82089552238806,16.0,5.723880597014926,0.022388059701492536,0.0,320,0,0.0010228405495600918,2.388059701492537
+Harbor multi-harness,opencode,600,135,17,0.362962962962963,2741.7185185185185,1759.0,6205.000000000002,370132,370132,25316,13.985185185185186,13.955555555555556,16.0,7.066666666666666,0.05185185185185185,0.007407407407407408,333,0,0.06839722045108232,2.466666666666667
+Harbor multi-harness,opencode,700,121,16,0.35537190082644626,3174.4793388429753,2215.0,5755.0,384112,384112,56298,14.462809917355372,14.47107438016529,16.0,5.892561983471074,0.0743801652892562,0.024793388429752067,284,0,0.14656662640063314,2.347107438016529
+Harbor multi-harness,opencode,800,143,18,0.13986013986013987,2365.5524475524476,1940.0,4867.799999999999,338274,338274,19675,12.748251748251748,12.482517482517483,16.0,3.6153846153846154,0.03496503496503497,0.006993006993006993,319,0,0.05816290935750309,2.230769230769231
+Harbor multi-harness,opencode,900,113,15,0.25663716814159293,6644.8938053097345,5180.0,16415.0,750873,750873,335070,9.761061946902656,9.584070796460177,16.0,3.274336283185841,0.4247787610619469,0.1415929203539823,253,0,0.4462405759695714,2.2389380530973453
+Harbor multi-harness,opencode,1000,118,16,0.17796610169491525,5601.542372881356,1893.0,16330.0,660982,660982,217638,8.110169491525424,7.788135593220339,16.0,4.864406779661017,0.3813559322033898,0.06779661016949153,262,0,0.32926463958171326,2.2203389830508473
+Native OpenCode,opencode,100,413,52,0.4406779661016949,998.6222760290557,483.0,2787.8,412431,412431,13465,7.142857142857143,6.428571428571429,14.800000000000011,0.22760290556900725,0.002421307506053269,0.0,990,0,0.032647885343245295,2.3970944309927362
+Native OpenCode,opencode,200,428,55,0.18691588785046728,1259.193925233645,710.5,3299.6,538935,538935,8606,7.822429906542056,7.5046728971962615,15.0,0.2336448598130841,0.0,0.0,809,0,0.015968530527800198,1.8901869158878504
+Native OpenCode,opencode,300,453,58,0.2853982300884956,1642.3841059602648,1004.0,3866.0,744000,744000,9527,8.754966887417218,8.205298013245033,15.0,0.33774834437086093,0.0,0.0,866,0,0.01280510752688172,1.9116997792494481
+Native OpenCode,opencode,400,558,71,0.18100358422939067,1423.6863799283153,875.5,3166.5,794417,794417,10555,8.775985663082437,8.0663082437276,15.0,0.20430107526881722,0.0017921146953405018,0.0,1019,0,0.013286472973262154,1.8261648745519714
+Native OpenCode,opencode,500,521,66,0.2456813819577735,1145.42226487524,743.0,2489.0,596765,596765,7578,8.326295585412668,7.723608445297505,15.0,0.362763915547025,0.0019193857965451055,0.0,997,0,0.012698465895285414,1.9136276391554703
+Native OpenCode,opencode,600,451,57,0.12195121951219512,992.4434589800444,639.0,2268.0,447592,447592,1366,7.490022172949002,7.006651884700665,13.0,1.252771618625277,0.0,0.0,1200,0,0.0030518865395270695,2.6607538802660753
+Native OpenCode,opencode,700,425,54,0.30823529411764705,1053.8376470588234,504.0,2649.6000000000026,447881,447881,1134,6.842352941176471,6.061176470588236,13.0,0.2164705882352941,0.002352941176470588,0.0,856,0,0.002531922541925199,2.0141176470588236
+Native OpenCode,opencode,800,429,54,0.2727272727272727,1024.4708624708624,453.0,2760.1999999999994,439498,439498,1121,6.741258741258742,5.916083916083916,13.0,0.19347319347319347,0.002331002331002331,0.0,832,0,0.002550637318031026,1.9393939393939394
+Native OpenCode,opencode,900,413,53,0.36561743341404357,1147.0605326876514,508.0,2972.2000000000003,473736,473736,1381,6.7409200968523,5.9491525423728815,13.0,0.07263922518159806,0.002421307506053269,0.0,869,0,0.0029151257240319505,2.1041162227602905
+Native OpenCode,opencode,1000,402,53,0.19154228855721392,1074.5572139303483,636.5,2226.4000000000005,431972,431972,1588,6.800995024875622,6.052238805970149,13.0,0.24875621890547264,0.0,0.0,1835,0,0.0036761641958275074,4.564676616915423
diff --git a/04-data-agent/reports/three-run-analysis-20260917/training_behavior_by_outcome.csv b/04-data-agent/reports/three-run-analysis-20260917/training_behavior_by_outcome.csv
new file mode 100644
index 0000000..8273a53
--- /dev/null
+++ b/04-data-agent/reports/three-run-analysis-20260917/training_behavior_by_outcome.csv
@@ -0,0 +1,61 @@
+run,window_end,binary_reward,admitted_rollouts,unique_tasks,binary_reward_mean,completion_tokens_mean,completion_tokens_median,completion_tokens_p90,supervised_tokens,completion_tokens,text_only_turn_tokens,agent_turns_mean,tool_calls_mean,tool_calls_p90,exact_repeated_calls_mean,rollout_over4096_fraction,rollout_finish_length_fraction,training_rows,receipt_token_mismatches,text_only_turn_token_share,rows_per_rollout
+Harbor OpenCode-only,100,0.0,223,45,0.0,3094.9910313901346,1346.0,5601.000000000005,690183,690183,1850,14.094170403587444,14.582959641255606,17.0,4.7174887892376685,0.053811659192825115,0.0,481,0,0.002680448518726193,2.1569506726457397
+Harbor OpenCode-only,100,1.0,191,38,1.0,1099.2460732984293,888.0,1181.0,209956,209956,922,14.219895287958115,14.570680628272251,17.0,8.764397905759163,0.010471204188481676,0.0,400,0,0.004391396292556536,2.094240837696335
+Harbor OpenCode-only,200,0.0,333,53,0.0,8297.465465465466,2808.0,20519.6,2763056,2763056,16533,12.867867867867869,14.414414414414415,18.0,1.8198198198198199,0.2732732732732733,0.003003003003003003,832,0,0.005983592080652727,2.4984984984984986
+Harbor OpenCode-only,200,1.0,99,26,1.0,5908.010101010101,1394.0,18226.8,584893,584893,124,15.626262626262626,16.858585858585858,18.0,4.141414141414141,0.20202020202020202,0.0,269,0,0.00021200458887352046,2.717171717171717
+Harbor OpenCode-only,300,0.0,262,44,0.0,6887.614503816794,3353.5,17885.4,1804555,1804555,17094,13.606870229007633,15.229007633587786,18.0,1.4312977099236641,0.12595419847328243,0.003816793893129771,700,0,0.009472695484482324,2.6717557251908395
+Harbor OpenCode-only,300,1.0,148,32,1.0,2328.824324324324,1459.0,3045.500000000002,344666,344666,299,15.344594594594595,16.68918918918919,18.0,3.8175675675675675,0.04054054054054054,0.0,377,0,0.0008675065135522506,2.5472972972972974
+Harbor OpenCode-only,400,0.0,280,48,0.0,2983.432142857143,2451.0,4738.0999999999985,835361,835361,175,14.553571428571429,19.060714285714287,22.0,3.0892857142857144,0.010714285714285714,0.0,649,0,0.000209490268279223,2.317857142857143
+Harbor OpenCode-only,400,1.0,129,32,1.0,1696.3333333333333,1449.0,2663.600000000001,218827,218827,67,15.062015503875969,18.108527131782946,20.0,4.1937984496124034,0.0,0.0,328,0,0.0003061779396509572,2.5426356589147288
+Harbor OpenCode-only,500,0.0,225,43,0.0,3079.8933333333334,2464.0,5218.0,692976,692976,163,13.955555555555556,16.493333333333332,19.0,1.991111111111111,0.03111111111111111,0.0,601,0,0.00023521738126572927,2.671111111111111
+Harbor OpenCode-only,500,1.0,190,34,1.0,2830.952631578947,1759.0,4324.199999999999,537881,537881,189,15.305263157894737,17.373684210526317,19.0,3.642105263157895,0.05789473684210526,0.0,543,0,0.0003513788365828129,2.857894736842105
+Harbor OpenCode-only,600,0.0,221,43,0.0,3396.9366515837105,3017.0,5509.0,750723,750723,186,14.04524886877828,16.29864253393665,18.0,1.4434389140271493,0.00904977375565611,0.0,594,0,0.0002477611582434533,2.6877828054298645
+Harbor OpenCode-only,600,1.0,182,34,1.0,1806.043956043956,1615.5,2775.2000000000003,328700,328700,467,14.983516483516484,16.796703296703296,18.0,4.818681318681318,0.0,0.0,467,0,0.0014207484027989048,2.565934065934066
+Harbor OpenCode-only,700,0.0,226,46,0.0,2648.2433628318586,1924.5,5291.5,598503,598503,20850,10.641592920353983,13.929203539823009,20.0,2.256637168141593,0.017699115044247787,0.004424778761061947,939,0,0.03483691811068616,4.154867256637168
+Harbor OpenCode-only,700,1.0,194,36,1.0,1509.4123711340205,1405.0,2103.3,292826,292826,7044,12.93298969072165,15.45360824742268,18.0,5.644329896907217,0.005154639175257732,0.0,531,0,0.02405524099635961,2.7371134020618557
+Harbor OpenCode-only,800,0.0,215,43,0.0,4039.874418604651,2815.0,7427.799999999998,868573,868573,147535,10.604651162790697,15.190697674418605,21.0,4.511627906976744,0.09767441860465116,0.027906976744186046,1124,0,0.16985906768918674,5.227906976744186
+Harbor OpenCode-only,800,1.0,185,40,1.0,2238.4540540540543,1620.0,2883.9999999999995,414114,414114,30154,10.281081081081082,15.378378378378379,18.0,6.464864864864865,0.043243243243243246,0.0,643,0,0.0728156980927957,3.4756756756756757
+Harbor OpenCode-only,900,0.0,228,50,0.0,3985.8596491228072,2497.5,7192.600000000003,908776,908776,114572,10.609649122807017,13.62280701754386,20.0,1.6973684210526316,0.07894736842105263,0.02631578947368421,1220,0,0.12607287164273703,5.350877192982456
+Harbor OpenCode-only,900,1.0,184,36,1.0,1704.8423913043478,1322.0,2079.3000000000015,313691,313691,30201,9.456521739130435,11.494565217391305,14.700000000000017,2.527173913043478,0.03260869565217391,0.0,535,0,0.096276271872639,2.907608695652174
+Harbor OpenCode-only,1000,0.0,245,45,0.0,4519.910204081632,3128.0,7891.5999999999985,1107378,1107378,46071,12.346938775510203,18.93469387755102,27.0,2.8285714285714287,0.07346938775510205,0.00816326530612245,1162,0,0.04160368004421255,4.742857142857143
+Harbor OpenCode-only,1000,1.0,159,34,1.0,2260.377358490566,1583.0,2885.000000000001,359400,359400,29457,10.163522012578616,15.11320754716981,20.0,3.3333333333333335,0.050314465408805034,0.0,404,0,0.08196160267111853,2.540880503144654
+Harbor multi-harness,100,0.0,231,41,0.0,2939.3290043290044,2382.0,6648.0,678985,678985,1104,13.987012987012987,14.017316017316018,17.0,1.4415584415584415,0.0,0.0,994,0,0.0016259563907891927,4.303030303030303
+Harbor multi-harness,100,1.0,158,32,1.0,1500.2278481012659,1187.0,2712.3,237036,237036,2407,13.734177215189874,13.575949367088608,17.0,4.563291139240507,0.0,0.0,578,0,0.010154575676268584,3.6582278481012658
+Harbor multi-harness,200,0.0,359,59,0.0,3076.016713091922,2368.0,5619.0,1104290,1104290,476,15.309192200557103,15.384401114206128,17.0,2.211699164345404,0.005571030640668524,0.0,1645,0,0.00043104619257622545,4.582172701949861
+Harbor multi-harness,200,1.0,161,35,1.0,1896.304347826087,1555.0,3316.0,305305,305305,609,15.701863354037267,15.701863354037267,17.0,6.6894409937888195,0.006211180124223602,0.0,875,0,0.0019947265848905193,5.434782608695652
+Harbor multi-harness,300,0.0,356,62,0.0,2811.9438202247193,2210.0,5557.5,1001052,1001052,17102,16.06741573033708,16.098314606741575,17.0,2.938202247191011,0.008426966292134831,0.0028089887640449437,1862,0,0.017084027602961686,5.230337078651686
+Harbor multi-harness,300,1.0,158,38,1.0,2073.487341772152,1688.0,3585.6000000000013,327611,327611,212,15.936708860759493,15.943037974683545,17.0,5.765822784810126,0.006329113924050633,0.0,1027,0,0.0006471089188091975,6.5
+Harbor multi-harness,400,0.0,313,54,0.0,2777.4376996805113,2148.0,5305.800000000001,869338,869338,310,15.670926517571885,15.696485623003195,17.0,2.670926517571885,0.003194888178913738,0.0,1455,0,0.0003565931777973584,4.6485623003194885
+Harbor multi-harness,400,1.0,224,41,1.0,1958.861607142857,1565.5,3077.3000000000006,438785,438785,17052,15.888392857142858,15.866071428571429,17.0,6.799107142857143,0.017857142857142856,0.004464285714285714,1360,0,0.03886185717378671,6.071428571428571
+Harbor multi-harness,500,0.0,320,59,0.0,4161.546875,2809.5,8557.500000000002,1331695,1331695,16738,15.89375,15.984375,17.0,2.475,0.01875,0.003125,1823,0,0.01256894409005065,5.696875
+Harbor multi-harness,500,1.0,192,40,1.0,2453.46875,1742.0,4418.500000000001,471066,471066,13,15.598958333333334,15.651041666666666,17.0,6.807291666666667,0.020833333333333332,0.0,807,0,2.7596982163858142e-05,4.203125
+Harbor multi-harness,600,0.0,286,55,0.0,3893.9685314685316,2718.0,8150.5,1113675,1113675,25339,14.55944055944056,14.594405594405595,17.0,3.2412587412587412,0.024475524475524476,0.0034965034965034965,1442,0,0.022752598379239902,5.041958041958042
+Harbor multi-harness,600,1.0,239,51,1.0,2389.945606694561,1957.0,4490.200000000001,571197,571197,147,14.623430962343097,14.619246861924687,17.0,5.539748953974895,0.016736401673640166,0.0,1141,0,0.0002573542928271682,4.7740585774058575
+Harbor multi-harness,700,0.0,291,59,0.0,4066.979381443299,2837.0,7120.0,1183491,1183491,89227,14.353951890034365,14.426116838487973,17.0,3.5463917525773194,0.05154639175257732,0.01718213058419244,1271,0,0.07539305326360742,4.367697594501718
+Harbor multi-harness,700,1.0,224,41,1.0,2165.1473214285716,1920.0,3744.5000000000005,484993,484993,251,14.303571428571429,14.415178571428571,17.0,5.397321428571429,0.013392857142857142,0.0,1338,0,0.0005175332427478335,5.973214285714286
+Harbor multi-harness,800,0.0,321,61,0.0,3426.6417445482866,2493.0,6097.0,1099952,1099952,20586,14.077881619937695,13.962616822429906,17.0,3.6043613707165107,0.037383177570093455,0.003115264797507788,1720,0,0.018715362124892724,5.358255451713395
+Harbor multi-harness,800,1.0,202,42,1.0,3323.128712871287,2503.5,6228.000000000001,671272,671272,89,14.400990099009901,14.430693069306932,17.0,5.069306930693069,0.04950495049504951,0.0,1039,0,0.00013258410897519932,5.143564356435643
+Harbor multi-harness,900,0.0,297,56,0.0,9487.43771043771,6552.0,20822.400000000005,2817769,2817769,401242,12.973063973063972,12.929292929292929,17.0,2.6161616161616164,0.3164983164983165,0.06734006734006734,1609,0,0.14239705241983996,5.417508417508418
+Harbor multi-harness,900,1.0,190,40,1.0,7439.3263157894735,5507.5,16235.199999999999,1413472,1413472,338,14.689473684210526,14.75263157894737,17.0,5.6947368421052635,0.26842105263157895,0.0,976,0,0.00023912748183197122,5.136842105263158
+Harbor multi-harness,1000,0.0,294,56,0.0,9736.969387755102,6942.0,20111.899999999998,2862669,2862669,548463,10.214285714285714,9.982993197278912,17.0,3.183673469387755,0.3843537414965986,0.047619047619047616,1230,0,0.1915914833325124,4.183673469387755
+Harbor multi-harness,1000,1.0,191,44,1.0,9084.256544502618,6725.0,19028.0,1735093,1735093,22631,13.366492146596858,13.172774869109947,17.0,5.769633507853404,0.3612565445026178,0.005235602094240838,918,0,0.01304310489408925,4.806282722513089
+Native OpenCode,100,0.0,231,44,0.0,1311.090909090909,699.0,3337.0,302862,302862,7642,8.363636363636363,7.766233766233766,16.0,0.354978354978355,0.0,0.0,574,0,0.025232614193923305,2.484848484848485
+Native OpenCode,100,1.0,182,38,1.0,602.0274725274726,373.5,1096.9000000000005,109569,109569,5823,5.593406593406593,4.730769230769231,9.0,0.06593406593406594,0.005494505494505495,0.0,416,0,0.05314459381759439,2.2857142857142856
+Native OpenCode,200,0.0,348,51,0.0,1415.4166666666667,942.0,3478.7000000000003,492565,492565,6383,8.295977011494253,8.112068965517242,15.0,0.28160919540229884,0.0,0.0,643,0,0.012958695806644807,1.8477011494252873
+Native OpenCode,200,1.0,80,19,1.0,579.625,396.5,1067.8000000000006,46370,46370,2223,5.7625,4.8625,8.100000000000009,0.025,0.0,0.0,166,0,0.04794047875781755,2.075
+Native OpenCode,300,0.0,323,51,0.0,2044.108359133127,1563.0,4163.8,660247,660247,5858,10.052631578947368,9.613003095975232,16.0,0.43962848297213625,0.0,0.0,587,0,0.008872437133375842,1.8173374613003095
+Native OpenCode,300,1.0,129,26,1.0,631.3410852713179,408.0,1084.6000000000001,81443,81443,3669,5.4728682170542635,4.6434108527131785,8.0,0.07751937984496124,0.0,0.0,278,0,0.04504991220853849,2.1550387596899223
+Native OpenCode,400,0.0,457,67,0.0,1586.070021881838,1050.0,3408.4000000000005,724834,724834,8396,9.321663019693654,8.669584245076587,16.0,0.22319474835886213,0.002188183807439825,0.0,804,0,0.011583341841028428,1.7592997811816193
+Native OpenCode,400,1.0,101,21,1.0,688.940594059406,455.0,1311.0,69583,69583,2159,6.306930693069307,5.336633663366337,9.0,0.1188118811881188,0.0,0.0,215,0,0.031027693545837346,2.128712871287129
+Native OpenCode,500,0.0,393,57,0.0,1364.7328244274809,946.0,2695.6,536340,536340,5112,9.267175572519085,8.743002544529261,15.0,0.4681933842239186,0.002544529262086514,0.0,738,0,0.009531267479583846,1.8778625954198473
+Native OpenCode,500,1.0,128,27,1.0,472.0703125,356.0,863.0999999999999,60425,60425,2466,5.4375,4.59375,7.0,0.0390625,0.0,0.0,259,0,0.04081092263136119,2.0234375
+Native OpenCode,600,0.0,396,56,0.0,1072.871212121212,724.5,2357.5,424857,424857,1207,7.828282828282828,7.401515151515151,14.0,1.4065656565656566,0.0,0.0,1081,0,0.002840955898102185,2.7297979797979797
+Native OpenCode,600,1.0,55,18,1.0,413.3636363636364,326.0,680.4,22735,22735,159,5.054545454545455,4.163636363636364,6.0,0.14545454545454545,0.0,0.0,119,0,0.006993622168462723,2.1636363636363636
+Native OpenCode,700,0.0,294,48,0.0,1307.9761904761904,642.5,3257.8,384545,384545,741,7.476190476190476,6.76530612244898,14.0,0.272108843537415,0.003401360544217687,0.0,579,0,0.0019269526323317167,1.969387755102041
+Native OpenCode,700,1.0,131,28,1.0,483.48091603053433,351.0,889.0,63336,63336,393,5.419847328244275,4.480916030534351,7.0,0.0916030534351145,0.0,0.0,277,0,0.006205001894657067,2.114503816793893
+Native OpenCode,800,0.0,312,47,0.0,1242.4839743589744,603.5,3175.900000000002,387655,387655,770,7.4006410256410255,6.641025641025641,14.0,0.2564102564102564,0.003205128205128205,0.0,589,0,0.0019863022532922315,1.8878205128205128
+Native OpenCode,800,1.0,117,21,1.0,443.1025641025641,298.0,620.6,51843,51843,351,4.982905982905983,3.982905982905983,5.400000000000006,0.02564102564102564,0.0,0.0,243,0,0.006770441525374689,2.076923076923077
+Native OpenCode,900,0.0,262,42,0.0,1514.2519083969466,754.0,3885.500000000001,396734,396734,910,7.759541984732825,7.022900763358779,14.0,0.11068702290076336,0.003816793893129771,0.0,527,0,0.0022937282915000985,2.0114503816793894
+Native OpenCode,900,1.0,151,27,1.0,509.94701986754967,362.0,844.0,77002,77002,471,4.973509933774834,4.086092715231788,6.0,0.006622516556291391,0.0,0.0,342,0,0.006116724240928807,2.2649006622516556
+Native OpenCode,1000,0.0,325,49,0.0,1214.0553846153846,712.0,2890.2000000000007,394568,394568,1260,7.206153846153846,6.513846153846154,16.0,0.29846153846153844,0.0,0.0,1572,0,0.0031933659090448287,4.836923076923077
+Native OpenCode,1000,1.0,77,18,1.0,485.76623376623377,377.0,919.0000000000003,37404,37404,328,5.090909090909091,4.103896103896104,6.0,0.03896103896103896,0.0,0.0,263,0,0.008769115602609347,3.4155844155844157
diff --git a/04-data-agent/reports/three-run-analysis-20260917/training_behavior_windows.csv b/04-data-agent/reports/three-run-analysis-20260917/training_behavior_windows.csv
new file mode 100644
index 0000000..aeab0d0
--- /dev/null
+++ b/04-data-agent/reports/three-run-analysis-20260917/training_behavior_windows.csv
@@ -0,0 +1,31 @@
+run,window_end,admitted_rollouts,unique_tasks,binary_reward_mean,completion_tokens_mean,completion_tokens_median,completion_tokens_p90,supervised_tokens,completion_tokens,text_only_turn_tokens,agent_turns_mean,tool_calls_mean,tool_calls_p90,exact_repeated_calls_mean,rollout_over4096_fraction,rollout_finish_length_fraction,training_rows,receipt_token_mismatches,text_only_turn_token_share,rows_per_rollout
+Harbor OpenCode-only,100,414,53,0.4613526570048309,2174.2487922705313,976.0,4195.299999999999,900139,900139,2772,14.152173913043478,14.577294685990339,17.0,6.584541062801932,0.033816425120772944,0.0,881,0,0.0030795243845672723,2.1280193236714977
+Harbor OpenCode-only,200,432,55,0.22916666666666666,7749.881944444444,2538.0,20144.0,3347949,3347949,16657,13.5,14.974537037037036,18.0,2.3518518518518516,0.2569444444444444,0.0023148148148148147,1101,0,0.004975284868437363,2.548611111111111
+Harbor OpenCode-only,300,410,52,0.36097560975609755,5242.002439024391,2272.0,11004.000000000018,2149221,2149221,17393,14.234146341463415,15.75609756097561,18.0,2.292682926829268,0.0951219512195122,0.0024390243902439024,1077,0,0.008092699633960398,2.626829268292683
+Harbor OpenCode-only,400,409,52,0.3154034229828851,2577.4767726161367,2011.0,4439.8,1054188,1054188,242,14.71393643031785,18.760391198044008,21.0,3.4376528117359415,0.007334963325183374,0.0,977,0,0.00022956057173862727,2.388753056234719
+Harbor OpenCode-only,500,415,54,0.4578313253012048,2965.9204819277106,1964.0,5172.6,1230857,1230857,352,14.573493975903615,16.896385542168673,19.0,2.746987951807229,0.043373493975903614,0.0,1144,0,0.00028597960607934145,2.756626506024096
+Harbor OpenCode-only,600,403,53,0.45161290322580644,2678.468982630273,2078.0,4732.8,1079423,1079423,653,14.468982630272953,16.523573200992555,18.0,2.967741935483871,0.004962779156327543,0.0,1061,0,0.0006049528312811567,2.632754342431762
+Harbor OpenCode-only,700,420,55,0.46190476190476193,2122.211904761905,1552.0,3914.3000000000006,891329,891329,27894,11.7,14.633333333333333,19.0,3.8214285714285716,0.011904761904761904,0.002380952380952381,1470,0,0.031294841747547764,3.5
+Harbor OpenCode-only,800,400,51,0.4625,3206.7175,1945.5,6440.900000000001,1282687,1282687,177689,10.455,15.2775,20.0,5.415,0.0725,0.015,1767,0,0.13852872914436648,4.4175
+Harbor OpenCode-only,900,412,54,0.44660194174757284,2967.152912621359,1510.0,6343.600000000004,1222467,1222467,144773,10.094660194174757,12.672330097087379,19.0,2.0679611650485437,0.05825242718446602,0.014563106796116505,1755,0,0.11842691868164948,4.259708737864078
+Harbor OpenCode-only,1000,404,51,0.3935643564356436,3630.638613861386,2102.0,5953.799999999999,1466778,1466778,75528,11.487623762376238,17.43069306930693,25.0,3.027227722772277,0.06435643564356436,0.0049504950495049506,1566,0,0.05149245489092419,3.876237623762376
+Harbor multi-harness,100,392,44,0.40616966580976865,2351.8239795918366,1865.5,4838.400000000001,921915,921915,3511,13.89030612244898,13.84438775510204,17.0,2.704081632653061,0.0,0.0,1576,0,0.003808377128043258,4.020408163265306
+Harbor multi-harness,200,520,64,0.3096153846153846,2710.7596153846152,2055.0,5254.500000000002,1409595,1409595,1085,15.430769230769231,15.482692307692307,17.0,3.598076923076923,0.0057692307692307696,0.0,2520,0,0.0007697246372184918,4.846153846153846
+Harbor multi-harness,300,516,65,0.30739299610894943,2581.6976744186045,2040.0,4958.0,1332156,1332156,17314,16.017441860465116,16.040697674418606,17.0,3.804263565891473,0.007751937984496124,0.001937984496124031,2893,0,0.012996976330099478,5.6065891472868215
+Harbor multi-harness,400,538,68,0.4171322160148976,2432.817843866171,1798.5,4695.3,1308856,1308856,17362,15.754646840148698,15.760223048327138,17.0,4.386617100371748,0.00929368029739777,0.0018587360594795538,2816,0,0.013265019222893887,5.234200743494424
+Harbor multi-harness,500,512,65,0.375,3521.017578125,2359.0,7624.7000000000035,1802761,1802761,16751,15.783203125,15.859375,17.0,4.099609375,0.01953125,0.001953125,2630,0,0.009291858432704058,5.13671875
+Harbor multi-harness,600,525,66,0.4552380952380952,3209.28,2222.0,6944.000000000001,1684872,1684872,25486,14.588571428571429,14.605714285714285,17.0,4.287619047619048,0.02095238095238095,0.0019047619047619048,2583,0,0.015126371617547209,4.92
+Harbor multi-harness,700,515,66,0.4349514563106796,3239.7747572815533,2412.0,5951.200000000001,1668484,1668484,89478,14.332038834951456,14.421359223300971,17.0,4.351456310679612,0.03495145631067961,0.009708737864077669,2609,0,0.053628323675863836,5.0660194174757285
+Harbor multi-harness,800,523,67,0.3862332695984704,3386.661567877629,2495.0,6157.000000000001,1771224,1771224,20675,14.202676864244742,14.1434034416826,17.0,4.170172084130019,0.04206500956022945,0.0019120458891013384,2759,0,0.011672718978514292,5.275334608030593
+Harbor multi-harness,900,487,62,0.39014373716632444,8688.379876796715,6081.0,18372.4,4231241,4231241,401580,13.64271047227926,13.640657084188911,17.0,3.817248459958932,0.29774127310061604,0.04106776180698152,2585,0,0.09490832594976273,5.308008213552362
+Harbor multi-harness,1000,485,64,0.3938144329896907,9479.921649484537,6776.0,19807.000000000007,4597762,4597762,571094,11.455670103092784,11.239175257731958,17.0,4.202061855670103,0.3752577319587629,0.030927835051546393,2148,0,0.12421130106343042,4.4288659793814436
+Native OpenCode,100,413,52,0.4406779661016949,998.6222760290557,483.0,2787.8,412431,412431,13465,7.142857142857143,6.428571428571429,14.800000000000011,0.22760290556900725,0.002421307506053269,0.0,990,0,0.032647885343245295,2.3970944309927362
+Native OpenCode,200,428,55,0.18691588785046728,1259.193925233645,710.5,3299.6,538935,538935,8606,7.822429906542056,7.5046728971962615,15.0,0.2336448598130841,0.0,0.0,809,0,0.015968530527800198,1.8901869158878504
+Native OpenCode,300,453,58,0.2853982300884956,1642.3841059602648,1004.0,3866.0,744000,744000,9527,8.754966887417218,8.205298013245033,15.0,0.33774834437086093,0.0,0.0,866,0,0.01280510752688172,1.9116997792494481
+Native OpenCode,400,558,71,0.18100358422939067,1423.6863799283153,875.5,3166.5,794417,794417,10555,8.775985663082437,8.0663082437276,15.0,0.20430107526881722,0.0017921146953405018,0.0,1019,0,0.013286472973262154,1.8261648745519714
+Native OpenCode,500,521,66,0.2456813819577735,1145.42226487524,743.0,2489.0,596765,596765,7578,8.326295585412668,7.723608445297505,15.0,0.362763915547025,0.0019193857965451055,0.0,997,0,0.012698465895285414,1.9136276391554703
+Native OpenCode,600,451,57,0.12195121951219512,992.4434589800444,639.0,2268.0,447592,447592,1366,7.490022172949002,7.006651884700665,13.0,1.252771618625277,0.0,0.0,1200,0,0.0030518865395270695,2.6607538802660753
+Native OpenCode,700,425,54,0.30823529411764705,1053.8376470588234,504.0,2649.6000000000026,447881,447881,1134,6.842352941176471,6.061176470588236,13.0,0.2164705882352941,0.002352941176470588,0.0,856,0,0.002531922541925199,2.0141176470588236
+Native OpenCode,800,429,54,0.2727272727272727,1024.4708624708624,453.0,2760.1999999999994,439498,439498,1121,6.741258741258742,5.916083916083916,13.0,0.19347319347319347,0.002331002331002331,0.0,832,0,0.002550637318031026,1.9393939393939394
+Native OpenCode,900,413,53,0.36561743341404357,1147.0605326876514,508.0,2972.2000000000003,473736,473736,1381,6.7409200968523,5.9491525423728815,13.0,0.07263922518159806,0.002421307506053269,0.0,869,0,0.0029151257240319505,2.1041162227602905
+Native OpenCode,1000,402,53,0.19154228855721392,1074.5572139303483,636.5,2226.4000000000005,431972,431972,1588,6.800995024875622,6.052238805970149,13.0,0.24875621890547264,0.0,0.0,1835,0,0.0036761641958275074,4.564676616915423
diff --git a/04-data-agent/reports/three-run-analysis-20260917/training_diagnostics.png b/04-data-agent/reports/three-run-analysis-20260917/training_diagnostics.png
new file mode 100644
index 0000000..cd76770
Binary files /dev/null and b/04-data-agent/reports/three-run-analysis-20260917/training_diagnostics.png differ
diff --git a/04-data-agent/reports/three-run-analysis-20260917/training_exposure.png b/04-data-agent/reports/three-run-analysis-20260917/training_exposure.png
new file mode 100644
index 0000000..fd8b99e
Binary files /dev/null and b/04-data-agent/reports/three-run-analysis-20260917/training_exposure.png differ
diff --git a/04-data-agent/reports/three-run-analysis-20260917/training_rollout_totals.csv b/04-data-agent/reports/three-run-analysis-20260917/training_rollout_totals.csv
new file mode 100644
index 0000000..aaca2e7
--- /dev/null
+++ b/04-data-agent/reports/three-run-analysis-20260917/training_rollout_totals.csv
@@ -0,0 +1,4 @@
+run,admitted_rollouts,unique_tasks,binary_reward_mean,completion_tokens_mean,completion_tokens_median,completion_tokens_p90,supervised_tokens,completion_tokens,text_only_turn_tokens,agent_turns_mean,tool_calls_mean,tool_calls_p90,exact_repeated_calls_mean,rollout_over4096_fraction,rollout_finish_length_fraction,training_rows,receipt_token_mismatches,text_only_turn_token_share,rows_per_rollout
+Harbor OpenCode-only,4119,523,0.4032532168001942,3550.628307841709,1836.0,6105.800000000007,14625038,14625038,463953,12.944889536295218,15.738771546491867,19.0,3.4644331148336973,0.0657926681233309,0.004127215343529983,12799,0,0.03172319962519072,3.1073075989317798
+Harbor multi-harness,5013,410,0.3872578390253645,4135.022142429682,2441.0,9114.400000000001,20728866,20728866,1164336,14.553959704767605,14.55016955914622,17.0,3.974266906044285,0.07979253939756632,0.008976660682226212,25119,0,0.056169787580275736,5.010771992818672
+Native OpenCode,4493,566,0.256233303650935,1185.6726018250613,633.0,2940.8,5327227,5327227,56321,7.6107277987981306,6.9603828177164475,15.0,0.3383040284887603,0.0013354106387714222,0.0,10273,0,0.010572292113701932,2.28644558201647
diff --git a/04-data-agent/reports/three-run-analysis-20260917/training_step_diagnostics.csv b/04-data-agent/reports/three-run-analysis-20260917/training_step_diagnostics.csv
new file mode 100644
index 0000000..e17d3cc
--- /dev/null
+++ b/04-data-agent/reports/three-run-analysis-20260917/training_step_diagnostics.csv
@@ -0,0 +1,31 @@
+run,window_end,reward,reward_std,tools/call_frequency,tools/failure_frequency,rollout/turns_mean,rollout/fork_frac,rollout/samples_per_rollout,completions/mean_length,completions/clipped_ratio,zero_gradient,perf/step_s,perf/fwd_bwd_s,perf/rollout_wait_s,batch/forwarded_tokens_per_step,batch/trained_tokens_per_step,forwarded_per_supervised_token
+Harbor OpenCode-only,100,0.4613268398268398,0.2398699027380629,14.713121933621933,0.12279162456462402,14.271011904761904,0.08981528260297379,2.171547619047619,2565.4408452380953,0.0,0.35,72.34649321350504,9.118664061059535,60.09085775610933,22391797.0,900139.0,24.875932494870238
+Harbor OpenCode-only,200,0.24554958440252558,0.20753841008734042,15.661243906420378,0.22326203249355886,13.33732993197279,0.1541956229715745,2.571364795918367,8244.417920918368,0.008928571428571428,0.39,126.94011013682001,18.04403841590007,98.18580851653998,39209277.0,3347949.0,11.71143198417897
+Harbor OpenCode-only,300,0.3708239815739816,0.19694757694321144,16.069899295149295,0.19153211177615806,14.237745098039216,0.12875551244199257,2.6127450980392157,5064.541666666667,0.007352941176470588,0.39,96.98847900031993,14.141838750521856,78.69381559354166,33137240.0,2149221.0,15.418256196082208
+Harbor OpenCode-only,400,0.3105725108225108,0.2271403104089478,19.034032231657232,0.1724393838204193,14.713235294117647,0.1021339224280583,2.394607843137255,2574.9313725490197,0.0,0.33,76.02990748340002,10.865588490648951,62.00304648158941,27069040.0,1054188.0,25.67762106948666
+Harbor OpenCode-only,500,0.46779365079365076,0.18029720685931688,17.237106809856808,0.14127172032796412,14.5453197945845,0.12750780485480157,2.7301159352629942,3332.42470821662,0.0024509803921568627,0.48,75.64743303306008,13.32731223725903,57.816416095330354,32791980.0,1230857.0,26.641583872050123
+Harbor OpenCode-only,600,0.4499239094239094,0.19571289410877754,16.578901348651346,0.1763313325828645,14.329111458716723,0.12504802676395887,2.660502874318664,3108.048309439099,0.0,0.52,60.88787030558,13.256466008439602,43.034404825770906,32807969.0,1079423.0,30.393987343238006
+Harbor OpenCode-only,700,0.4436214711715873,0.2003848720445334,15.192719801408172,0.22907039219006176,11.674995168521955,0.2492364847873423,3.63704745155638,2175.0650726010103,0.002232142857142857,0.43,64.5336026951499,17.211821312580142,43.838990044047826,43604234.0,891329.0,48.920470443573585
+Harbor OpenCode-only,800,0.438452406417842,0.2612226987006718,15.826681698851091,0.2192432145564864,10.194886958279815,0.35625899786629783,4.21397018160411,3152.636801442605,0.014668367346938774,0.26,72.59849311632003,25.298051341829705,42.60313191080044,62573373.0,1282687.0,48.783041381100766
+Harbor OpenCode-only,900,0.4337144914822021,0.2566402646216497,13.61259492586263,0.22333017742691244,10.406444991789819,0.35978850404867146,4.564449917898194,3654.4251231527096,0.029248768472906403,0.31,70.1935425982601,26.00453474922746,38.00356280478198,62958003.0,1222467.0,51.500779162136894
+Harbor OpenCode-only,1000,0.33886414499712514,0.22169366935889592,17.96717597866881,0.14836250005459165,11.04720133667502,0.2651926560421726,3.785609857978279,3874.793149540518,0.004699248120300751,0.34,79.97377342697,26.870135988060063,47.05893725607006,62885553.0,1466778.0,42.87325893898054
+Harbor multi-harness,100,0.37215481064105627,0.2817602841205276,14.091755350686729,0.16644353977207949,13.608209208247729,0.27170567711999943,4.371736225174313,2434.2689411490683,0.0,0.17,219.83294780647236,78.68988377495785,160.96184653661913,187688023.0,2341485.0,80.15768753590136
+Harbor multi-harness,200,0.33429811481932725,0.2209406219192409,15.786707289453354,0.15555696292824975,15.463118242466068,0.26966753878799526,4.875117897400506,2749.4112031285945,0.0,0.41,94.76055106370606,44.1395378899423,47.15803065143642,101082358.0,1409595.0,71.71021321727163
+Harbor multi-harness,300,0.3078947098658278,0.237504531226522,16.205064360533626,0.19417233135332557,16.002894709959925,0.29052063159214797,5.542150281280716,2981.8417820705863,0.0,0.34,89.43567845421029,43.12261901319493,44.12112560480717,101103979.0,1332156.0,75.89499953458905
+Harbor multi-harness,400,0.4208965591133613,0.1347151666009812,16.046035668255467,0.16542645323759192,15.65814536340852,0.25465502643949534,4.856035923141186,2405.253268588137,0.0,0.49,91.19121140880044,33.57061344736023,54.94743524379679,84605646.0,1308856.0,64.64091236927516
+Harbor multi-harness,500,0.32965815112938374,0.21353757183881694,15.979141145963936,0.20272489386589615,15.858448827292111,0.25258061664179454,4.957356076759062,3658.3221304193316,0.0,0.36,87.1897372502496,36.77998640247155,47.74810984188574,90166268.0,1802761.0,50.01565265722966
+Harbor multi-harness,600,0.42514402674199275,0.21481521660147151,14.885919789409268,0.14239050757746663,14.5796066252588,0.28829488482569093,4.903295376121463,3503.119858523119,0.0018115942028985507,0.3,83.39630996603984,34.40547226127121,46.63930073371972,85677872.0,1684872.0,50.8512646658025
+Harbor multi-harness,700,0.43366307997193104,0.21850890470961898,14.809511820629087,0.13326109980800552,13.886473014165322,0.2655647788294409,4.441551397320628,3437.0248391352234,0.008974358974358974,0.36,87.71859020842334,35.783885467979125,49.40421284527634,86376800.0,1668484.0,51.76963039501727
+Harbor multi-harness,800,0.35281411250347294,0.19344944471574962,14.598113697768602,0.12470995081799482,13.979096924317513,0.25206306693000136,4.7138252270605205,3910.5884460514976,0.001225490196078431,0.39,92.73759917171206,36.07318357932381,52.21698128161486,87725983.0,1771224.0,49.528452076078466
+Harbor multi-harness,900,0.35901134931123724,0.2291403835501686,14.18688828568166,0.1429223534360349,13.163304988662132,0.317435083122598,4.824875798804371,9480.489273345702,0.04736394557823129,0.35,115.72125592437573,42.08636902739294,67.09815469327849,98376730.0,4231241.0,23.250089040071224
+Harbor multi-harness,1000,0.41036747714576377,0.24216930597349595,12.230913871451857,0.12421673095427341,11.471771610742199,0.31062348236496173,4.37483839689722,10657.320653243816,0.027328431372549016,0.32,125.29679813764058,33.5303732728539,84.98472009332384,78519197.0,4597762.0,17.077699324149446
+Native OpenCode,100,0.4384733044733045,0.23678973565545625,6.595128038628038,0.1250655540736819,7.402300692602417,0.2642018642181204,2.435050366300366,1097.8185752705149,0.0,0.34,31.587826412050504,11.618848592969988,18.02173868855992,28972901.0,412431.0,70.24908651386535
+Native OpenCode,200,0.20371825396825397,0.11595934084676054,7.46128434065934,0.21892108315969483,7.8879166666666665,0.1851386637967458,1.9196230158730159,1262.6713690476192,0.0,0.58,50.66147141980001,12.501171760049928,34.67551194384007,30938728.0,538935.0,57.40716041823225
+Native OpenCode,300,0.2764642857142857,0.13116962071445037,8.043817460317461,0.24209013075995742,8.756902958152958,0.16471670689892573,1.9301722582972582,1646.2756998557,0.0,0.57,51.83079009951,9.259974337550211,40.43463070367999,23808276.0,744000.0,32.00037096774194
+Native OpenCode,400,0.17345833333333333,0.10571283425688895,8.150039682539681,0.23606811371498868,8.903753306878308,0.13761309659417711,1.8108134920634922,1482.5227237654321,0.0,0.66,41.88394934417998,5.251358338560167,34.75259857053005,14416198.0,794417.0,18.14689010935063
+Native OpenCode,500,0.262,0.10691747367582768,7.858011904761904,0.23863880565256199,8.499986881805064,0.1576233237569586,1.8871507280598188,1167.200844483799,0.0,0.63,36.184499155539996,4.691997400669898,29.273478976599144,13293439.0,596765.0,22.27583554665572
+Native OpenCode,600,0.12442532467532467,0.1306467763651302,7.764659241575418,0.2731742470870204,7.495316159250586,0.266928864144792,2.6837919594067134,1031.9630171740828,0.0,0.59,33.34266252039001,7.050879189619446,22.431043816850252,19805302.0,447592.0,44.248561189654865
+Native OpenCode,700,0.31109498834498833,0.17358571064202213,6.122058441558441,0.26768672319235504,6.757709176788124,0.216580856021291,1.9953385065227172,972.7612516869096,0.0,0.51,27.225210022760002,5.668830248999875,19.533168102390228,15542322.0,447881.0,34.70190072809518
+Native OpenCode,800,0.27081349206349203,0.10170429946793036,5.850321428571428,0.23879520652958153,6.734375,0.21248057847110624,1.9387019230769231,1022.0012019230769,0.0,0.64,26.7776295466,5.835523945260138,18.29469989886984,16191147.0,439498.0,36.84009256014817
+Native OpenCode,900,0.3780271534021534,0.12676802095135348,5.842643134643135,0.24723269360709835,6.7568012290739565,0.23655465123164057,2.0432350982350984,1192.0391753700844,0.0,0.59,30.724514038079977,8.463444579549396,20.111304883080336,22503593.0,473736.0,47.50239162740429
+Native OpenCode,1000,0.18717783882783884,0.10568101863877383,6.7947657674487445,0.28285419085527297,6.825864955357143,0.5819421111620064,4.616029456654457,1564.7171581673535,0.015625,0.72,32.92978656568004,15.960202438620545,14.157918948470396,42941177.0,431972.0,99.40731575194688
diff --git a/04-data-agent/reports/three-run-analysis-20260917/training_totals.csv b/04-data-agent/reports/three-run-analysis-20260917/training_totals.csv
new file mode 100644
index 0000000..6aeb1f4
--- /dev/null
+++ b/04-data-agent/reports/three-run-analysis-20260917/training_totals.csv
@@ -0,0 +1,4 @@
+run,forwarded_tokens,supervised_tokens,mean_step_s,step_timing_observations,max_staleness_observed,fraction_steps_staleness_gt4,nonfinite_grad_steps
+Harbor OpenCode-only,419428466.0,14625038.0,79.62124525297799,999,4.0,0.0,0
+Harbor multi-harness,1001322856.0,22148436.0,108.20385101597033,993,4.0,0.0,0
+Native OpenCode,228413083.0,5327227.0,36.3195656516987,999,4.0,0.0,0
diff --git a/04-data-agent/reports/three-run-analysis-20260917/training_windows.csv b/04-data-agent/reports/three-run-analysis-20260917/training_windows.csv
new file mode 100644
index 0000000..b20b3df
--- /dev/null
+++ b/04-data-agent/reports/three-run-analysis-20260917/training_windows.csv
@@ -0,0 +1,31 @@
+run,window_end,reward,entropy,grad_norm,sample/staleness_mean,sample/staleness_max,tools/call_frequency,tools/failure_frequency,batch/forwarded_tokens_per_step,batch/trained_tokens_per_step,perf/step_s,rollout/samples_per_rollout,completions/mean_length,clip_ratio/region_mean,kl
+Harbor OpenCode-only,100,0.4613268398268398,0.09685979937518965,2.667939453125,2.0316666666666667,2.24,14.713121933621933,0.12279162456462402,223917.97,9001.39,72.34649321350504,2.171547619047619,2565.4408452380953,0.004187458868193879,0.005040083591849535
+Harbor OpenCode-only,200,0.24554958440252558,0.09733831891737652,1.1925244140625,1.7242142857142857,2.32,15.661243906420378,0.22326203249355886,392092.77,33479.49,126.94011013682001,2.571364795918367,8244.417920918368,0.002383259688850152,0.0030197780759285025
+Harbor OpenCode-only,300,0.3708239815739816,0.14049491090903823,1.64328125,1.7858333333333334,2.42,16.069899295149295,0.19153211177615806,331372.4,21492.21,96.98847900031993,2.6127450980392157,5064.541666666667,0.003933157082318967,0.002670543493925051
+Harbor OpenCode-only,400,0.3105725108225108,0.13660548355254132,1.80888671875,2.042333333333333,2.46,19.034032231657232,0.1724393838204193,270690.4,10541.88,76.02990748340002,2.394607843137255,2574.9313725490197,0.0030803969387157336,0.0018282461108336781
+Harbor OpenCode-only,500,0.46779365079365076,0.13306280535237497,1.560234375,2.022333333333333,2.33,17.237106809856808,0.14127172032796412,327919.8,12308.57,75.64743303306008,2.7301159352629942,3332.42470821662,0.0025820452011899382,0.002231793405654206
+Harbor OpenCode-only,600,0.4499239094239094,0.1470729348302163,1.78265625,2.184,2.29,16.578901348651346,0.1763313325828645,328079.69,10794.23,60.88787030558,2.660502874318664,3108.048309439099,0.001838536804866822,0.001952234455951008
+Harbor OpenCode-only,700,0.4436214711715873,0.14341516735157175,2.1547265625,1.9445,2.26,15.192719801408172,0.22907039219006176,436042.34,8913.29,64.5336026951499,3.63704745155638,2175.0650726010103,0.002594599951658072,0.0018724857904997049
+Harbor OpenCode-only,800,0.438452406417842,0.17447699038519943,2.55837890625,2.11,2.46,15.826681698851091,0.2192432145564864,625733.73,12826.87,72.59849311632003,4.21397018160411,3152.636801442605,0.003508212072951627,0.001373625607360752
+Harbor OpenCode-only,900,0.4337144914822021,0.17869817124870213,2.6642578125,2.1835,2.35,13.61259492586263,0.22333017742691244,629580.03,12224.67,70.1935425982601,4.564449917898194,3654.4251231527096,0.0030231491391100518,0.001325954932362795
+Harbor OpenCode-only,1000,0.33886414499712514,0.1568996988469791,2.265947265625,2.019,2.47,17.96717597866881,0.14836250005459165,628855.53,14667.78,79.97377342697,3.785609857978279,3874.793149540518,0.0020346568106116418,0.000934227482170192
+Harbor multi-harness,100,0.37215481064105627,0.2637117265651165,4.17544921875,1.7269317640313784,2.1,14.091755350686729,0.16644353977207949,1876880.23,23414.85,219.83294780647236,4.371736225174313,2434.2689411490683,0.009547546282052555,0.008544461813041776
+Harbor multi-harness,200,0.33429811481932725,0.2817592404597404,1.7822265625,1.4976190476190476,1.89,15.786707289453354,0.15555696292824975,1010823.58,14095.95,94.76055106370606,4.875117897400506,2749.4112031285945,0.003997303393562508,0.0039874680662799035
+Harbor multi-harness,300,0.3078947098658278,0.2556433835762293,1.98423828125,1.5767619047619048,1.82,16.205064360533626,0.19417233135332557,1011039.79,13321.56,89.43567845421029,5.542150281280716,2981.8417820705863,0.004483454293340759,0.0031596616756331096
+Harbor multi-harness,400,0.4208965591133613,0.2602957298295186,1.326943359375,1.2613214285714285,1.79,16.046035668255467,0.16542645323759192,846056.46,13088.56,91.19121140880044,4.856035923141186,2405.253268588137,0.0027571686920980134,0.00267292184568319
+Harbor multi-harness,500,0.32965815112938374,0.3421668481281644,1.8252734375,1.516702380952381,1.93,15.979141145963936,0.20272489386589615,901662.68,18027.61,87.1897372502496,4.957356076759062,3658.3221304193316,0.003900711896447942,0.002308183441914882
+Harbor multi-harness,600,0.42514402674199275,0.31104822814197736,2.1425537109375,1.474047619047619,1.85,14.885919789409268,0.14239050757746663,856778.72,16848.72,83.39630996603984,4.903295376121463,3503.119858523119,0.005506542724431796,0.0024274147227585107
+Harbor multi-harness,700,0.43366307997193104,0.34037941961742574,2.196533203125,1.6215357142857143,1.85,14.809511820629087,0.13326109980800552,863768.0,16684.84,87.71859020842334,4.441551397320628,3437.0248391352234,0.004985732428707765,0.0028271833525996036
+Harbor multi-harness,800,0.35281411250347294,0.2978964158105625,1.893984375,1.555404761904762,1.86,14.598113697768602,0.12470995081799482,877259.83,17712.24,92.73759917171206,4.7138252270605205,3910.5884460514976,0.004122921543132058,0.0021502267781378782
+Harbor multi-harness,900,0.35901134931123724,0.3623014800418172,1.49798828125,1.5702738095238096,2.04,14.18688828568166,0.1429223534360349,983767.3,42312.41,115.72125592437573,4.824875798804371,9480.489273345702,0.0035995473452451092,0.0015889896107803862
+Harbor multi-harness,1000,0.41036747714576377,0.32723013623204006,1.3534666442871093,1.5440833333333333,1.96,12.230913871451857,0.12421673095427341,785191.97,45977.62,125.29679813764058,4.37483839689722,10657.320653243816,0.003878371968903037,0.001586671160716524
+Native OpenCode,100,0.4384733044733045,0.17068306005160314,4.484921875,2.1381666666666668,2.38,6.595128038628038,0.1250655540736819,289729.01,4124.31,31.587826412050504,2.435050366300366,1097.8185752705149,0.004409961396235686,0.008285643573207067
+Native OpenCode,200,0.20371825396825397,0.18744950996944607,2.21181640625,1.7630238095238095,2.36,7.46128434065934,0.21892108315969483,309387.28,5389.35,50.66147141980001,1.9196230158730159,1262.6713690476192,0.0007772616376360563,0.003431557870288053
+Native OpenCode,300,0.2764642857142857,0.17872697230910006,2.28291015625,1.7443214285714286,2.11,8.043817460317461,0.24209013075995742,238082.76,7440.0,51.83079009951,1.9301722582972582,1646.2756998557,0.002291983241470562,0.006575444186290422
+Native OpenCode,400,0.17345833333333333,0.19328260360224725,1.5931640625,1.2819761904761904,1.71,8.150039682539681,0.23606811371498868,144161.98,7944.17,41.88394934417998,1.8108134920634922,1482.5227237654321,0.0031121111611040002,0.004288135047909705
+Native OpenCode,500,0.262,0.15426974010064506,1.8798828125,1.4659166666666668,1.88,7.858011904761904,0.23863880565256199,132934.39,5967.65,36.184499155539996,1.8871507280598188,1167.200844483799,0.002459962850667994,0.003597480714192069
+Native OpenCode,600,0.12442532467532467,0.14728269294145113,2.8753125,1.8172857142857142,2.21,7.764659241575418,0.2731742470870204,198053.02,4475.92,33.34266252039001,2.6837919594067134,1031.9630171740828,0.002019672447124215,0.0032964696410401496
+Native OpenCode,700,0.31109498834498833,0.10232620991616216,4.194765625,1.9360238095238094,2.33,6.122058441558441,0.26768672319235504,155423.22,4478.81,27.225210022760002,1.9953385065227172,972.7612516869096,0.001808775189040755,0.001616135894910377
+Native OpenCode,800,0.27081349206349203,0.10113436551513161,2.444013671875,1.7724761904761905,2.31,5.850321428571428,0.23879520652958153,161911.47,4394.98,26.7776295466,1.9387019230769231,1022.0012019230769,0.000773144740999812,0.0009989796439796794
+Native OpenCode,900,0.3780271534021534,0.12660436471971986,2.2771875,1.7313571428571428,2.38,5.842643134643135,0.24723269360709835,225035.93,4737.36,30.724514038079977,2.0432350982350984,1192.0391753700844,0.000942620709915634,0.0011704037453183133
+Native OpenCode,1000,0.18717783882783884,0.14192835536074921,1.9496484375,2.1566666666666667,2.25,6.7947657674487445,0.28285419085527297,429411.77,4319.72,32.92978656568004,4.616029456654457,1564.7171581673535,0.0009637467219425069,0.0021438640694901962
diff --git a/04-data-agent/reproduce.md b/04-data-agent/reproduce.md
new file mode 100644
index 0000000..515ab62
--- /dev/null
+++ b/04-data-agent/reproduce.md
@@ -0,0 +1,149 @@
+# Reproduce training and evaluation
+
+Use one recipe at a time initially. A training smoke performs **two optimizer steps → save/upload → verified remote restore → two more steps**. It must show exact-token capture, finite updates, changed weights and native optimizer state before a long run is admitted.
+
+## 1. Requirements and credentials
+
+Use Linux, Python 3.12, Git, [`uv`](https://docs.astral.sh/uv/), and an HF account. Local runs need Slurm and two suitable CUDA GPUs on one node; HF Jobs need organization Jobs permissions and GPU quota. The default namespace is `HuggingEnvs`; use `--namespace YOUR_ORG` during preparation to create resources in your own organization.
+
+The frozen task/runtime bundle is public at [HuggingEnvs/data-agent-daytona-repro](https://huggingface.co/datasets/HuggingEnvs/data-agent-daytona-repro). Use the exact archive named in `hf/configs/sources.json`; `hf/build.py --seed-archive PATH` verifies its SHA-256. It never substitutes today's dataset for the measured train/test split. [The public artifact index](https://huggingface.co/datasets/HuggingEnvs/data-agent-experiment-results) links environments, dashboards, results and published checkpoints.
+
+```bash
+cd 04-data-agent
+uv venv --python 3.12 .venv-launcher
+uv pip install --python .venv-launcher/bin/python 'huggingface-hub==1.26.0' python-dotenv
+source .venv-launcher/bin/activate
+```
+
+Create `.env` (ignored by Git):
+
+```dotenv
+HF_API_KEY=your_hf_token
+DAYTONA_API_KEY=your_daytona_key
+# Optional for a non-default Daytona deployment:
+# DAYTONA_API_URL=https://app.daytona.io/api
+# DAYTONA_TARGET=eu
+```
+
+The HF token needs permission to launch Jobs and manage the selected Spaces and run storage. Public bundles and published results can be read without organization membership. The launcher sends credentials as secrets; they are excluded from the bundle and launch metadata. Live raw artifacts remain private: captured tool output can contain credentials. Public releases use audited copies, with redacted files identified in a manifest. The native adapter also supports HF/E2B sandboxes, but these training recipes select Daytona.
+
+## 2. Select and freeze a recipe
+
+Choose `harbor-multi`, `harbor-opencode`, `native-opencode`, or `seta`. The first two use Harbor; the third uses `envs/blackbox-opencode` directly.
+
+```bash
+python reproduce.py prepare --recipe harbor-opencode --env-file .env \
+ --run-id my-harbor-opencode-01
+```
+
+The default output is `temp/reproduction/harbor-opencode/`. Use the same `--recipe` and, if supplied, `--out` in subsequent commands. Preparing an existing run directory is rejected to preserve its identity. A new run should use a new output directory and run ID.
+
+The preparation pins OpenEnv and TRL commits, Qwen3.5-2B revision, the 1,000 training tasks, the 250 test tasks and schedule hashes. Python and shell paths are relocated inside the bundle. Package versions are frozen in separate environment/training lockfiles.
+
+| Setting | Async Harbor / native OpenCode | Sync SETA |
+| --- | --- | --- |
+| Model | Qwen3.5-2B, pinned revision | Same |
+| Training tasks | 150 easy / 600 medium / 250 hard | Same fixed order |
+| Initial curriculum | First 32 tasks easy; then shuffled | Same |
+| Learning rate / generations | `3e-6` / 8 | Same |
+| Sampling | Temperature 0.8, top-p 1, top-k disabled; thinking off | Same |
+| Max context / output per call | 131,072 / 16,384 | Same model limits; tool-loop budget differs |
+| Staleness | 4 | Synchronous |
+| Backpressure | 32 workers, 16 outstanding rollouts; whole-group admission | Native sync batches |
+| Save / eval | Every 50 / 100 optimizer steps | Same |
+| Step / wall-clock ceiling | 1,000 / about 23 hours plus checkpoint grace | Same |
+| Checkpoint suite | Four harnesses × 250 tests | Native SETA × 250 tests |
+
+The 32-worker ceiling is not a claim that eight generations always run simultaneously. Backpressure, group readiness and provider capacity determine active work. Multi-harness training assigns one harness per task per pass and rotates assignments on later passes. Four thousand task/harness pairs are not four thousand optimizer steps.
+
+## 3A. HF Jobs and Spaces
+
+First upload the prepared bundle. Deploy only into an idle environment or a new owned Space; deployment restarts that Space. Existing ongoing runs must finish before changing their environment.
+
+```bash
+python reproduce.py upload --recipe harbor-opencode --env-file .env
+python reproduce.py spaces --recipe harbor-opencode --env-file .env
+python reproduce.py eval --recipe harbor-opencode --env-file .env --flavor a100-large
+python reproduce.py smoke --recipe harbor-opencode --env-file .env --flavor a100x4
+python reproduce.py status --recipe harbor-opencode --env-file .env
+```
+
+Hub eval uses **TP1/DP1 on one A100 80GB**, concurrency **35**, and the fixed pass@1 cohort. To change concurrency, set `--concurrency` during preparation so the chosen value is frozen. Increasing it does not create more sandbox quota. Training uses separate inference and optimizer GPUs: `h200x2` or `a100x4` (the recipe uses two of the four A100s). SETA's validated HF training allocation is `h200x2`.
+
+For a smoke against an already-deployed Space, skip the deployment command and pass its exact `/deployment` `bundle_sha256`:
+
+```bash
+python reproduce.py smoke --recipe seta --env-file .env --flavor h200x2 \
+ --space-bundle-sha EXACT_DEPLOYED_SHA256
+```
+
+This explicitly records two source identities: the trainer bundle and the existing environment bundle. It does not upgrade the live Space or certify untested server changes. Async training checks the advertised rollout API before allocating a Job; an older native OpenCode server without explicit sampling support must be upgraded while idle. The current qualification preserves the active Harbor/SETA deployments and updates the idle native OpenCode Space.
+
+Once both jobs complete and their evidence passes:
+
+```bash
+python reproduce.py train --recipe harbor-opencode --env-file .env \
+ --flavor a100x4 --baseline-job BASELINE_JOB_ID --smoke-job SMOKE_JOB_ID
+```
+
+The launcher submits a separate CPU coordinator. At steps 100, 200, … it waits for the completed checkpoint manifest, submits a separate A100 eval Job, and verifies the model hash before serving it. Step-50 checkpoints remain available for later evaluations. The final checkpoint is also eligible. A failed or ambiguous eval submission is recorded for reconciliation; it is not blindly duplicated.
+
+Native OpenCode's baseline command measures its **native** 250-task protocol on the configured sandboxes. Its **checkpoint comparisons** use the four-harness Harbor Space. Deploy the Harbor environment as well when reproducing native OpenCode in a new namespace. Keep that Space's task/harness pins fixed. Native and Harbor baseline percentages are different cohorts and must be labelled accordingly.
+
+For native OpenCode, run the shared four-harness baseline with the `harbor-opencode` recipe too (or reuse a completed matching baseline). Long-run admission requires both: the native diagnostic verifies that environment's grading, and the Harbor baseline supplies step 0 of the checkpoint curve.
+
+```bash
+python reproduce.py train --recipe native-opencode --env-file .env \
+ --baseline-job NATIVE_DIAGNOSTIC_JOB --comparison-baseline-job HARBOR_BASELINE_JOB \
+ --smoke-job NATIVE_SMOKE_JOB --flavor a100x4
+```
+
+The launcher checks fixed task/model/sampling identity and 250 results for each of the four harnesses. It never places the native diagnostic percentage on the four-harness curve.
+
+## 3B. Local / Slurm
+
+Use the same preparation command. The local launcher runs both the environment service and vLLM inside the allocation. Training uses one inference GPU and one optimizer GPU; eval uses TP1/DP2 on two GPUs. Local eval defaults to concurrency 50.
+
+```bash
+python reproduce.py eval --platform local --recipe harbor-opencode --env-file .env \
+ --partition YOUR_GPU_PARTITION --submit
+python reproduce.py smoke --platform local --recipe harbor-opencode --env-file .env \
+ --partition YOUR_GPU_PARTITION --submit
+```
+
+Omit `--submit` to inspect generated Slurm scripts first. The allocation creates its own hash-locked venvs. To reuse validated local environments without modifying them, supply both `--train-venv /path/to/train-venv` and `--env-venv /path/to/env-venv`. Use a shared filesystem with room for the model, optimizer states and captures.
+
+Before the long run, qualify the checkpoint controller against the smoke's checkpoint 4. The advanced helper prints the resulting plan:
+
+```bash
+python hf/local_long.py --arm blackbox \
+ --smoke-run /absolute/path/to/repro/outputs/local-train-blackbox-JOB_ID \
+ --baseline-score /absolute/path/to/baseline/canonical_scores.json \
+ --env-file .env --out temp/checkpoint-qualification \
+ --coordination-dir temp/eval-admission --qualify-checkpoint-eval
+python hf/local_followup.py watch --plan temp/checkpoint-qualification/plan.json --submit
+```
+
+After the independent checkpoint eval produces `checkpoint_eval_verified.json`:
+
+```bash
+python reproduce.py train --platform local --recipe harbor-opencode --env-file .env \
+ --partition YOUR_GPU_PARTITION --cpu-partition YOUR_CPU_PARTITION \
+ --smoke-run /absolute/path/to/repro/outputs/local-train-blackbox-JOB_ID \
+ --baseline-score /absolute/path/to/baseline/canonical_scores.json \
+ --checkpoint-eval-proof /absolute/path/to/checkpoint_eval_verified.json --submit
+```
+
+Use internal arm `opencode` for native OpenCode and `whitebox` for SETA when invoking advanced helpers. Native long-run admission also checks the frozen task grading/tolerance audit; preserve the `verification.json` next to its canonical baseline score. No recipe uses `hopper-extra` or `hopper-atl` by default. Partition names are explicit deployment settings, not source edits.
+
+For local native training, add `--comparison-baseline-score /absolute/path/to/harbor-baseline/repro/outputs/local-eval-blackbox-JOB_ID/canonical_scores.json` to the qualification and long-run commands. Keep that baseline inside its prepared runtime: admission reads the adjacent frozen configuration to verify the comparison protocol. `--baseline-score` remains the separate native diagnostic. The controller records the matching baseline at step 0 and rejects a modified score file.
+
+## Evaluation protocol and logs
+
+Pass@1 keeps the **first graded** attempt for each task/harness cell, including a zero. Only an ungraded infrastructure failure can be retried. Publication requires complete fixed coverage, exact-token audit, task hashes, harness versions and checkpoint provenance. The test set is 33 easy, 118 medium and 99 hard; overall scores are computed from counts, not an unweighted average of difficulty percentages.
+
+Job outputs contain `status.json`, `training_recipe.json`, `space_identity.json`, `training_smoke_verified.json`, `canonical_scores.json`, capture audits and native checkpoints. Remote artifacts are stored under the run ID and unique job owner. Checkpoints publish their ready marker only after all files and hashes are verified. Bucket transfers retry transient transport, rate-limit and server errors up to three attempts; permission or validation errors fail immediately. Full optimizer checkpoints are larger than inference-only exports, so the smoke includes their upload and restore time. [HF Bucket sync](https://huggingface.co/docs/huggingface_hub/guides/buckets) compares existing content when retrying. Local source transformations have their own manifest; the original portable bundle remains intact.
+
+Training logs locally and uploads Trackio events/databases asynchronously. Environment Spaces do not contain a dashboard. The [shared comparison dashboard](https://huggingface.co/spaces/HuggingEnvs/data-agent-training-comparison-trackio) replays audited training and eval artifacts; see `hf/consolidate_async_runs.py` for the historical collector and `hf/runtime/logging_sync.py` for per-run logging. A new reproduction keeps its own run identity rather than overwriting these measured runs.
+
+`--dry-run` on the beginner CLI prints the exact command without allocating resources. Never reuse a run directory to silently overwrite a baseline. Superseded files go into the ignored `temp/` archive; credentials, raw traces, checkpoints and SQLite databases are not committed.
diff --git a/04-data-agent/reproduce.py b/04-data-agent/reproduce.py
new file mode 100644
index 0000000..fb64cd8
--- /dev/null
+++ b/04-data-agent/reproduce.py
@@ -0,0 +1,137 @@
+"""One entry point for the pinned data-agent training and evaluation recipes.
+
+Start with `python reproduce.py prepare --recipe harbor-opencode --env-file .env`.
+See reproduce.md for credentials, hardware and checkpoint evaluation.
+"""
+from __future__ import annotations
+
+import argparse
+import json
+from pathlib import Path
+import shlex
+import subprocess
+import sys
+
+PROJECT = Path(__file__).resolve().parent
+RECIPES = {"harbor-multi": "blackbox", "harbor-opencode": "blackbox",
+ "native-opencode": "opencode", "seta": "whitebox"}
+
+
+def run(command, dry_run=False):
+ command = list(map(str, command))
+ print(shlex.join(command), flush=True)
+ if not dry_run:
+ subprocess.run(command, check=True)
+
+
+def main():
+ p = argparse.ArgumentParser(description=__doc__)
+ p.add_argument("action", choices=["prepare", "upload", "spaces", "smoke", "eval", "train", "status"])
+ p.add_argument("--recipe", choices=RECIPES, default="harbor-opencode")
+ p.add_argument("--platform", choices=["hub", "local"], default="hub")
+ p.add_argument("--out", type=Path, help="Isolated run directory (default: temp/reproduction/)")
+ p.add_argument("--env-file", type=Path, default=PROJECT / ".env")
+ p.add_argument("--namespace", default="HuggingEnvs", help="HF namespace where you can create resources")
+ p.add_argument("--run-id", help="Unique artifact identity; set when preparing a new experiment")
+ p.add_argument("--flavor", help="HF hardware: h200x2/a100x4 for training, a100-large for eval")
+ p.add_argument("--partition", default="hopper-prod", help="Local Slurm GPU partition")
+ p.add_argument("--cpu-partition", default="hopper-cpu")
+ p.add_argument("--timeout", help="HF duration, e.g. 2h or 24h")
+ p.add_argument("--concurrency", type=int, help="Eval concurrency (Hub default 35; local default 50)")
+ p.add_argument("--space-bundle-sha", help="Pin an existing Space for a training smoke without redeploying it")
+ p.add_argument("--baseline-job")
+ p.add_argument("--comparison-baseline-job")
+ p.add_argument("--smoke-job")
+ p.add_argument("--smoke-run", type=Path, help="Local smoke output directory")
+ p.add_argument("--baseline-score", type=Path)
+ p.add_argument("--comparison-baseline-score", type=Path)
+ p.add_argument("--checkpoint-eval-proof", type=Path)
+ p.add_argument("--qualify-checkpoint-eval", action="store_true")
+ p.add_argument("--train-venv", type=Path, help="Optional existing local training venv; otherwise create locked venvs")
+ p.add_argument("--env-venv", type=Path)
+ p.add_argument("--submit", action="store_true", help="Submit a prepared local Slurm script")
+ p.add_argument("--dry-run", action="store_true", help="Print commands without allocating resources")
+ a = p.parse_args()
+ out = (a.out or PROJECT / "temp/reproduction" / a.recipe).resolve()
+ arm = RECIPES[a.recipe]
+ if a.action == "prepare":
+ if a.concurrency is not None and a.concurrency < 1:
+ p.error("Concurrency must be positive")
+ if (out / "config.json").exists():
+ p.error("This run directory already exists. Use a new --out to preserve its configuration and artifacts.")
+ config = json.loads((PROJECT / "hf/configs/deployment.json").read_text())
+ config["namespace"] = a.namespace
+ config["run_id"] = a.run_id or f"reproduction-{a.recipe}"
+ config["recipe"] = a.recipe
+ if a.recipe == "harbor-multi":
+ config["arms"][arm]["training_harnesses"] = ["opencode", "claude-code", "codex", "mini-swe-agent"]
+ if a.namespace != "HuggingEnvs":
+ # All resources become owned copies. The immutable task seed still
+ # requires access to the original dataset repository.
+ def relocate(value):
+ if isinstance(value, str) and value.startswith("HuggingEnvs/"):
+ return a.namespace + value[len("HuggingEnvs"):]
+ if isinstance(value, dict): return {k: relocate(v) for k, v in value.items()}
+ if isinstance(value, list): return [relocate(v) for v in value]
+ return value
+ config["resources"] = relocate(config["resources"])
+ if a.concurrency:
+ config["evaluation"]["concurrency_per_job"] = a.concurrency
+ config["evaluation"]["concurrency_per_arm"] = {k: a.concurrency for k in RECIPES.values()}
+ config["evaluation"]["opencode_backend_concurrency"] = {k: a.concurrency for k in ("daytona", "hf")}
+ if not a.dry_run:
+ out.mkdir(parents=True, exist_ok=True)
+ (out / "config.json").write_text(json.dumps(config, indent=2) + "\n")
+ run([sys.executable, PROJECT / "hf/build.py", "--out", out / "bundle",
+ "--config", out / "config.json", "--env-file", a.env_file], a.dry_run)
+ return
+ if not (out / "config.json").is_file() and not a.dry_run:
+ p.error("Run prepare first, using the same --recipe and --out")
+ if a.platform == "local" and a.action == "status":
+ for path in (out / "baseline/launch.json", out / "smoke/launch.json", out / "long/plan.json"):
+ if not path.exists():
+ continue
+ record = json.loads(path.read_text())
+ jobs = [str(record[k]) for k in ("slurm_job", "training_job", "controller_job") if record.get(k)]
+ print(json.dumps({"record": str(path), "jobs": jobs}))
+ if jobs:
+ if not all(job.isdigit() for job in jobs):
+ p.error("Stored Slurm job IDs must be numeric")
+ run(["sacct", "-X", "-j", ",".join(jobs), "--format=JobID,State,Elapsed,ExitCode"], a.dry_run)
+ return
+ if a.platform == "local" and a.action in {"eval", "smoke", "train"}:
+ if a.action == "train":
+ required = (a.smoke_run, a.baseline_score)
+ if not all(required): p.error("Local training needs --smoke-run and --baseline-score")
+ cmd = [sys.executable, PROJECT / "hf/local_long.py", "--arm", arm,
+ "--smoke-run", a.smoke_run, "--baseline-score", a.baseline_score,
+ "--env-file", a.env_file.resolve(), "--out", out / "long",
+ "--coordination-dir", out.parent / "eval-admission", "--partition", a.partition,
+ "--cpu-partition", a.cpu_partition]
+ if a.qualify_checkpoint_eval: cmd += ["--qualify-checkpoint-eval"]
+ if a.checkpoint_eval_proof: cmd += ["--checkpoint-eval-proof", a.checkpoint_eval_proof]
+ if a.comparison_baseline_score: cmd += ["--comparison-baseline-score", a.comparison_baseline_score]
+ else:
+ cmd = [sys.executable, PROJECT / "hf/cluster.py", "--bundle", out / "bundle",
+ "--out", out / ("baseline" if a.action == "eval" else "smoke"),
+ "--env-file", a.env_file.resolve(), "--arm", arm,
+ "--phase", "baseline" if a.action == "eval" else "smoke", "--partition", a.partition]
+ for key in ("train_venv", "env_venv"):
+ if getattr(a, key): cmd += ["--" + key.replace("_", "-"), getattr(a, key).resolve()]
+ if a.submit: cmd += ["--submit"]
+ else:
+ action = "job" if a.action in {"smoke", "eval", "train"} else a.action
+ cmd = [sys.executable, PROJECT / "hf/deploy.py", action, "--config", out / "config.json",
+ "--out", out, "--env-file", a.env_file.resolve(), "--arm", arm]
+ if action == "spaces": cmd += ["--only", arm]
+ if action == "job":
+ cmd += ["--role", "eval" if a.action == "eval" else "train", "--phase",
+ {"smoke": "smoke", "eval": "baseline", "train": "long"}[a.action]]
+ cmd += ["--timeout", a.timeout or ("24h" if a.action == "train" else "4h" if a.action == "eval" else "2h")]
+ for key in ("flavor", "space_bundle_sha", "baseline_job", "comparison_baseline_job", "smoke_job"):
+ if getattr(a, key): cmd += ["--" + key.replace("_", "-"), getattr(a, key)]
+ run(cmd, a.dry_run)
+
+
+if __name__ == "__main__":
+ main()
diff --git a/04-data-agent/results.md b/04-data-agent/results.md
new file mode 100644
index 0000000..de8dc66
--- /dev/null
+++ b/04-data-agent/results.md
@@ -0,0 +1,137 @@
+# Data Agent: completed training and evaluation
+
+Updated September 17, 2026. Three async Qwen3.5-2B runs reached 1,000 optimizer steps.
+Every scheduled 100-step checkpoint evaluation is complete: 250 fixed test tasks ×
+four harnesses, pass@1. The test set contains 33 easy, 118 medium and 99 hard tasks.
+
+| Run | Baseline | Best measured checkpoint | Final step 1,000 |
+| --- | ---: | ---: | ---: |
+| Harbor multi-harness | 14.6% | **37.0% at 500** | 26.3% |
+| Native OpenCode | 15.9% | **29.8% at 1,000** | 29.8% |
+| Harbor OpenCode-only | 14.6% | **39.5% at 700** | 26.4% |
+
+[Every checkpoint, harness and difficulty](reports/async-comparison-20260916/REPORT.md) ·
+[Public Trackio comparison](https://huggingface.co/spaces/HuggingEnvs/data-agent-training-comparison-trackio)
+
+## What training and evaluation show
+
+**Multi-harness produces longer responses but takes fewer actions.** Between training
+steps 401–500 and 901–1,000, completion tokens per admitted rollout grow **3,521 → 9,480**,
+while emitted tool calls fall **15.86 → 11.24**. In the final window, 37.5% of admitted
+rollouts contain a response longer than the evaluation's 4,096-token output cap.
+At evaluation, output-truncated rollouts rise **9/1,000 → 556/1,000** from peak to final.
+OpenCode accounts for about 64% of the net lost successful evaluations. Budget mismatch
+is a plausible contributor, not a proven explanation for the whole decline.
+
+**Harbor OpenCode-only continues working but finishes less reliably.** Peak-to-final
+eval tool calls rise **16.62 → 20.97**, while submission falls **68.9% → 40.7%** on the
+86-task subset with explicit submission instrumentation. Output truncation remains rare.
+Training also shows longer outputs (**2,122 → 3,631 tokens**) and more tool use
+(**14.63 → 17.43 calls**) per admitted rollout, comparing steps 601–700 with 901–1,000.
+
+**Native OpenCode finishes at its best aggregate score with shorter training outputs.**
+Its final 100 steps average 1,075 completion tokens and 6.05 emitted tool calls per
+admitted rollout. However, late prompt forking increases context overhead: forwarded
+tokens per supervised token rise **22.3× → 99.4×** between steps 401–500 and 901–1,000.
+
+**One-harness training transfers to other harnesses.** Harbor OpenCode-only's best
+checkpoint scores 46.4% under Claude Code and 40.0% under OpenCode. Native OpenCode
+improves all four evaluation harnesses; its largest gains are outside OpenCode.
+
+## Training accounting changes the interpretation
+
+| Run | Distinct tasks covered | Supervised tokens | Forwarded tokens | Zero-fresh-gradient steps |
+| --- | ---: | ---: | ---: | ---: |
+| Harbor multi-harness | 482 | 22.15M | 1,001.32M | 349/1,000 |
+| Native OpenCode | 566 | 5.33M | 228.41M | 583/1,000 |
+| Harbor OpenCode-only | 523 | 14.63M | 419.43M | 380/1,000 |
+
+The 1,000-step cap did not cover the whole 1,000-task training pool. Equal optimizer
+steps also did not provide equal token exposure. Zero-gradient steps coincide with
+zero within-group reward variance and supply no fresh GRPO contrast; optimizer
+momentum may still update weights.
+
+The multi-harness resume after step 684 revisits previously seen tasks in **1,575 of
+1,579 admitted rollouts**. The saved schedule cursor does not preserve later completed
+groups. This changes late data exposure but cannot explain the initial decline after
+step 500, which happened before that restart.
+
+## Next experiments
+
+1. Fix and test resume accounting for out-of-order completed groups and unfinished work.
+2. Diagnose the train/eval output-budget mismatch on a small, separately labeled cohort;
+ preserve the canonical scores. Check submission and executed actions, not just reward.
+3. Track per-harness tokens, calls, submission, truncation, context duplication, task
+ coverage and zero-advantage groups. Compare future runs at matched exposure and budgets.
+
+This is an observational comparison. Backend, rollout filtering, training histories and
+historical eval retries differ. The 39.5% versus 37.0% peak difference is not clearly
+separated by paired task-level uncertainty. All three trainers use binary correctness;
+native raw efficiency bonuses are removed before training.
+
+Evidence: [evaluation analysis and limitations](reports/three-run-analysis-20260917/REPORT.md),
+[training token/tool analysis and reproduction](reports/three-run-analysis-20260917/TRAINING.md).
+Raw captures and accepted scores are unchanged.
+
+[Short message for sharing](reports/three-run-analysis-20260917/TLDR.md)
+
+## Earlier SETA and infrastructure snapshot
+
+The following September 16 snapshot is retained for SETA results and qualification provenance. Its pending async evaluations were subsequently completed; use the September 17 tables above for the three async runs.
+
+# Historical data-agent results
+
+Snapshot: **2026-09-16 UTC**. Metric: **pass@1** on the fixed 250-task test set (33 easy, 118 medium, 99 hard). Each accepted async checkpoint has 250 tasks × four harnesses = **1,000 first-graded cells**. SETA uses its native bash/SETA evaluator, 250 cells.
+
+[Public artifact index](https://huggingface.co/datasets/HuggingEnvs/data-agent-experiment-results): code, environments, dashboards, report downloads, qualification evidence and published checkpoints. Credential-bearing raw evidence stays private; redacted public copies are explicitly marked.
+
+
+
+The [complete report](results/2026-09-16/REPORT.md) includes training history and **harness × difficulty at every accepted checkpoint**. [CSV](results/2026-09-16/checkpoint_scores.csv) provides the underlying correct/graded counts; [snapshot](results/2026-09-16/snapshot.json.gz) retains audited provenance and training metrics. The [live Trackio dashboard](https://huggingface.co/spaces/HuggingEnvs/data-agent-training-comparison-trackio) may contain newer observations.
+
+| Checkpoint | Harbor multi-harness | Native OpenCode training, four-harness eval | SETA native eval |
+| --- | ---: | ---: | ---: |
+| Base | 14.6% | 15.9% | 18.8% |
+| 100 | 24.8% | 19.7% | 34.8% |
+| 150 (final SETA) | — | — | **38.0%** |
+| 200 | 26.3% | 22.1% | — |
+| 300 | 28.6% | 21.6% | — |
+| 400 | 33.3% | 26.4% | — |
+| 500 | **37.0%** | 23.1% | — |
+| 600 | 31.8% | 25.1% | — |
+| 684 (recovery) | 32.1% | — | — |
+| 700 | 28.8% | 23.2% | — |
+| 800 | 27.0% | 25.6% | — |
+| 900 | Incomplete | 25.3% | — |
+| 1000 | Incomplete | **29.8%** | — |
+
+Harbor multi-harness and native OpenCode reached 1,000 training steps. Native OpenCode's final four-harness evaluation is complete at 29.8%; Harbor multi-harness still lacks accepted step-900/1000 scores. SETA was intentionally stopped after a verified checkpoint 150; its final evaluation completed at **38.0%**. Harbor OpenCode-only is a new run in progress; no post-training checkpoint score is claimed here.
+
+The async report/CSV/figure retain their 10:50 UTC snapshot. SETA's later result has a separate [checkpoint-150 receipt](results/2026-09-16/seta-checkpoint-150.json), including the verified model manifest, complete scoring and job identity.
+
+## Baselines and difficulty
+
+| Measured cohort | Easy | Medium | Hard | Overall |
+| --- | ---: | ---: | ---: | ---: |
+| Harbor multi-harness base (E2B) | 53/132 = 40.2% | 68/472 = 14.4% | 25/396 = 6.3% | 146/1000 = 14.6% |
+| SETA base (HF Job / Daytona) | 14/33 = 42.4% | 27/118 = 22.9% | 6/99 = 6.1% | 47/250 = 18.8% |
+| SETA checkpoint 100 | 23/33 = 69.7% | 45/118 = 38.1% | 19/99 = 19.2% | 87/250 = 34.8% |
+| SETA checkpoint 150 | 28/33 = 84.8% | 49/118 = 41.5% | 18/99 = 18.2% | 95/250 = 38.0% |
+
+Native OpenCode's **standalone** base evaluation scored **21/250 = 8.4%**. That is a different protocol from the **15.9%** four-harness Harbor/Daytona baseline used for its checkpoint comparison. Do not mix these denominators or relabel one cohort as the other. The shared Harbor OpenCode-only run reuses the recorded E2B base cohort and has no new measured gain yet.
+
+## What the runs established
+
+- Exact captured prompt/completion IDs, real aligned log probabilities and authoritative loss masks are usable across the selected harnesses. Lossless forks preserve supervision when prompts change; more rows still affect token cost and weighting.
+- The async recipe admits complete rollout groups and checks retained supervision against capture records. The optimizer, checkpoint, upload and remote-resume paths have real GPU evidence.
+- The native OpenCode baseline completed 250 tasks at local concurrency 50. HF SETA exercised 8, 32 and 53 concurrent slots. Earlier scaling failures are preserved; the reproduction defaults to **35** on Hub infrastructure.
+- Checkpoint evaluations run on separate GPUs, with fixed test identities and first-graded results. A graded zero is never replaced by a retry.
+- Task parsing now preserves explicit zero numerical tolerances. Native baseline qualification deterministically rechecks the unchanged submitted answers and frozen grading parameters.
+
+Historical qualification receipts: native optimizer smoke **80593**, local SETA **80555**, HF SETA **6aa9a487f76d6a098a70e3d2**; independent checkpoint smokes **80603**, **80576**, and **6aa9af55f76d6a098a70e52d** respectively. These are evidence for their recorded source snapshots, not substitutes for qualifying a changed bundle. Fresh PR qualification is recorded separately in [validation.md](results/validation.md).
+
+## Limits of the comparison
+
+Infrastructure, harness protocols, batching and recipe versions changed during bring-up. Async atomic batching and synchronous GRPO/DAPO have different scheduling and token accounting. These curves are observational; they do not isolate a causal effect of sync versus async or multi-harness versus one harness. Training reward is a sampled training signal, not held-out pass@1.
+
+Scores only enter the accepted table after complete coverage and their recorded TiTO/version/provenance checks. The Harbor decline after step 500 is observed; this report does not assign a cause without a controlled ablation. Failed or incomplete cohorts remain visible in the snapshot's pending section and are not estimated.
diff --git a/04-data-agent/results/2026-09-16/REPORT.md b/04-data-agent/results/2026-09-16/REPORT.md
new file mode 100644
index 0000000..8664fad
--- /dev/null
+++ b/04-data-agent/results/2026-09-16/REPORT.md
@@ -0,0 +1,245 @@
+# Harbor and OpenCode — consolidated training and pass@1
+
+Updated: 2026-09-16T10:50:30.515660+00:00
+
+[Live Trackio dashboard](https://huggingface.co/spaces/HuggingEnvs/data-agent-training-comparison-trackio) · [Overview image](comparison.png) · [Snapshot](snapshot.json.gz)
+
+Qwen3.5-2B; 1,000 optimizer-step target per run. Recorded training: Harbor multi-harness: 1000 steps; Native OpenCode: 1000 steps; Harbor OpenCode-only: 74 steps. Every accepted checkpoint has 250 fixed tasks × four harnesses = 1,000 grades. Task difficulty: 33 easy, 118 medium, 99 hard (13.2% / 47.2% / 39.6%). Scores retain first graded attempts; incomplete and failed-audit evaluations are excluded. Missing scores are not estimated.
+
+Baselines are separate measured cohorts: Harbor/E2B 14.6%; Harbor/Daytona 15.9% for the native OpenCode checkpoint evaluator. The standalone native OpenCode 8.4% baseline uses a different harness protocol and is excluded here. Infrastructure and training recipe histories differ; this is an observational comparison, not a controlled causal experiment.
+
+## Overall checkpoint curve
+
+| Checkpoint | Harbor multi-harness | Native OpenCode | Harbor OpenCode-only |
+| --- | ---: | ---: | ---: |
+| 0 (baseline) | 14.6% | 15.9% | 14.6% |
+| 100 | 24.8% | 19.7% | Pending |
+| 200 | 26.3% | 22.1% | Pending |
+| 300 | 28.6% | 21.6% | Pending |
+| 400 | 33.3% | 26.4% | Pending |
+| 500 | 37.0% | 23.1% | Pending |
+| 600 | 31.8% | 25.1% | Pending |
+| 684 (recovery) | 32.1% | Not scheduled | Not scheduled |
+| 700 | 28.8% | 23.2% | Pending |
+| 800 | 27.0% | 25.6% | Pending |
+| 900 | Pending | 25.3% | Pending |
+| 1000 | Pending | 29.8% | Pending |
+
+## Harbor multi-harness
+
+### Overall and difficulty
+
+| Checkpoint | Overall | Easy (132 cells) | Medium (472) | Hard (396) |
+| --- | ---: | ---: | ---: | ---: |
+| 0 | 14.6% | 40.2% | 14.4% | 6.3% |
+| 100 | 24.8% | 53.0% | 30.5% | 8.6% |
+| 200 | 26.3% | 58.3% | 30.1% | 11.1% |
+| 300 | 28.6% | 64.4% | 32.8% | 11.6% |
+| 400 | 33.3% | 73.5% | 39.6% | 12.4% |
+| 500 | 37.0% | 72.7% | 44.3% | 16.4% |
+| 600 | 31.8% | 72.7% | 37.3% | 11.6% |
+| 684 | 32.1% | 75.8% | 35.8% | 13.1% |
+| 700 | 28.8% | 60.6% | 33.9% | 12.1% |
+| 800 | 27.0% | 59.1% | 32.6% | 9.6% |
+
+### Harness × difficulty at every checkpoint
+
+| Checkpoint | Harness | Overall (250) | Easy (33) | Medium (118) | Hard (99) |
+| --- | --- | ---: | ---: | ---: | ---: |
+| 0 | opencode | 10.8% | 33.3% (11/33) | 8.5% (10/118) | 6.1% (6/99) |
+| 0 | claude-code | 16.8% | 42.4% (14/33) | 18.6% (22/118) | 6.1% (6/99) |
+| 0 | codex | 16.4% | 42.4% (14/33) | 16.9% (20/118) | 7.1% (7/99) |
+| 0 | mini-swe-agent | 14.4% | 42.4% (14/33) | 13.6% (16/118) | 6.1% (6/99) |
+| 100 | opencode | 24.4% | 51.5% (17/33) | 31.4% (37/118) | 7.1% (7/99) |
+| 100 | claude-code | 27.6% | 60.6% (20/33) | 32.2% (38/118) | 11.1% (11/99) |
+| 100 | codex | 28.0% | 57.6% (19/33) | 35.6% (42/118) | 9.1% (9/99) |
+| 100 | mini-swe-agent | 19.2% | 42.4% (14/33) | 22.9% (27/118) | 7.1% (7/99) |
+| 200 | opencode | 30.4% | 51.5% (17/33) | 34.7% (41/118) | 18.2% (18/99) |
+| 200 | claude-code | 30.0% | 63.6% (21/33) | 33.1% (39/118) | 15.2% (15/99) |
+| 200 | codex | 26.4% | 63.6% (21/33) | 29.7% (35/118) | 10.1% (10/99) |
+| 200 | mini-swe-agent | 18.4% | 54.5% (18/33) | 22.9% (27/118) | 1.0% (1/99) |
+| 300 | opencode | 29.6% | 60.6% (20/33) | 33.1% (39/118) | 15.2% (15/99) |
+| 300 | claude-code | 33.2% | 66.7% (22/33) | 39.0% (46/118) | 15.2% (15/99) |
+| 300 | codex | 29.6% | 69.7% (23/33) | 33.1% (39/118) | 12.1% (12/99) |
+| 300 | mini-swe-agent | 22.0% | 60.6% (20/33) | 26.3% (31/118) | 4.0% (4/99) |
+| 400 | opencode | 32.8% | 72.7% (24/33) | 38.1% (45/118) | 13.1% (13/99) |
+| 400 | claude-code | 36.8% | 75.8% (25/33) | 44.9% (53/118) | 14.1% (14/99) |
+| 400 | codex | 34.8% | 72.7% (24/33) | 42.4% (50/118) | 13.1% (13/99) |
+| 400 | mini-swe-agent | 28.8% | 72.7% (24/33) | 33.1% (39/118) | 9.1% (9/99) |
+| 500 | opencode | 32.8% | 69.7% (23/33) | 40.7% (48/118) | 11.1% (11/99) |
+| 500 | claude-code | 44.8% | 75.8% (25/33) | 52.5% (62/118) | 25.3% (25/99) |
+| 500 | codex | 39.2% | 75.8% (25/33) | 46.6% (55/118) | 18.2% (18/99) |
+| 500 | mini-swe-agent | 31.2% | 69.7% (23/33) | 37.3% (44/118) | 11.1% (11/99) |
+| 600 | opencode | 34.0% | 66.7% (22/33) | 41.5% (49/118) | 14.1% (14/99) |
+| 600 | claude-code | 30.0% | 72.7% (24/33) | 33.9% (40/118) | 11.1% (11/99) |
+| 600 | codex | 32.4% | 66.7% (22/33) | 39.8% (47/118) | 12.1% (12/99) |
+| 600 | mini-swe-agent | 30.8% | 84.8% (28/33) | 33.9% (40/118) | 9.1% (9/99) |
+| 684 | opencode | 29.2% | 72.7% (24/33) | 30.5% (36/118) | 13.1% (13/99) |
+| 684 | claude-code | 35.6% | 72.7% (24/33) | 41.5% (49/118) | 16.2% (16/99) |
+| 684 | codex | 33.2% | 84.8% (28/33) | 36.4% (43/118) | 12.1% (12/99) |
+| 684 | mini-swe-agent | 30.4% | 72.7% (24/33) | 34.7% (41/118) | 11.1% (11/99) |
+| 700 | opencode | 21.2% | 30.3% (10/33) | 28.0% (33/118) | 10.1% (10/99) |
+| 700 | claude-code | 31.6% | 78.8% (26/33) | 34.7% (41/118) | 12.1% (12/99) |
+| 700 | codex | 32.0% | 72.7% (24/33) | 34.7% (41/118) | 15.2% (15/99) |
+| 700 | mini-swe-agent | 30.4% | 60.6% (20/33) | 38.1% (45/118) | 11.1% (11/99) |
+| 800 | opencode | 23.2% | 36.4% (12/33) | 33.1% (39/118) | 7.1% (7/99) |
+| 800 | claude-code | 32.0% | 66.7% (22/33) | 37.3% (44/118) | 14.1% (14/99) |
+| 800 | codex | 26.8% | 66.7% (22/33) | 31.4% (37/118) | 8.1% (8/99) |
+| 800 | mini-swe-agent | 26.0% | 66.7% (22/33) | 28.8% (34/118) | 9.1% (9/99) |
+
+### Training history
+
+| Allocation | First optimizer step | Last optimizer step |
+| --- | ---: | ---: |
+| 78647 | 1 | 17 |
+| 78681 | 18 | 25 |
+| 78767 | 26 | 30 |
+| 78831 | 31 | 53 |
+| 78956 | 54 | 196 |
+| 79083 | 197 | 684 |
+| 80608 | 685 | 1000 |
+
+### Score provenance
+
+- Step 0: `/fsx/adithyaskolavi/projects/trl_prod/experiments/async_grpo_harbor_data_agent/logs/multi4-baseline-20260914/job-78215/canonical_results.json`; SHA256 `8c4f5bced4eff04b0c2e5f41806da9ae1b8a4c0fe356ddf926e1f781c6bb9ac6`.
+- Step 100: `/fsx/adithyaskolavi/projects/trl_prod/experiments/async_grpo_harbor_data_agent/logs/multi4-long-bounded-20260915/checkpoint-evals/step-000100/scores.json`; SHA256 `1e4f42f54526c9b8b71156f5b1f3673529519201401bc88f88ccff805f291181`.
+- Step 200: `/fsx/adithyaskolavi/projects/trl_prod/experiments/async_grpo_harbor_data_agent/logs/multi4-long-prod-20260915/checkpoint-evals/step-000200/scores.json`; SHA256 `699dce549d1002e879c675555ac06142448b0cbeef534a0816616faf669862af`.
+- Step 300: `/fsx/adithyaskolavi/projects/trl_prod/experiments/async_grpo_harbor_data_agent/logs/multi4-long-prod-20260915/checkpoint-evals/step-000300/scores.json`; SHA256 `299e5ed3528922d9912a591f3cff0c4d85070fef4de732d37723967934bfd691`.
+- Step 400: `/fsx/adithyaskolavi/projects/trl_prod/experiments/async_grpo_harbor_data_agent/logs/multi4-long-prod-20260915/checkpoint-evals/step-000400/scores.json`; SHA256 `f8909af81669f4ec092317c5f9889ef8735754c0dcbc826da82a270d2d92dc7a`.
+- Step 500: `/fsx/adithyaskolavi/projects/trl_prod/experiments/async_grpo_harbor_data_agent/logs/multi4-long-prod-20260915/checkpoint-evals/step-000500/scores.json`; SHA256 `86d56b65edbdc2f5a3f5d54151d0888dae83ca2b81753a7a9dbc2e80ee4f6130`.
+- Step 600: `/fsx/adithyaskolavi/projects/trl_prod/experiments/async_grpo_harbor_data_agent/logs/multi4-long-prod-20260915/checkpoint-evals/step-000600/scores.json`; SHA256 `02fc5a5e84978c198dc880c143fe223507c30485563b3545cb64b2794e3e60b0`.
+- Step 684: `/fsx/adithyaskolavi/projects/trl_prod/experiments/async_grpo_harbor_data_agent/logs/multi4-long-prod-20260915/checkpoint-evals/step-000684/scores.json`; SHA256 `fb54bd52f352463e405bee8067ef24b27147bfd02d41429561f440a916f5d776`.
+- Step 700: `/fsx/adithyaskolavi/projects/trl_prod/experiments/async_grpo_harbor_data_agent/logs/multi4-long-prod-cont-20260915/checkpoint-evals/step-000700/scores.json`; SHA256 `39c2ead8681847c6c398eb6b22ae8919aabaff3ad6b668d81b5961564139f8be`.
+- Step 800: `/fsx/adithyaskolavi/projects/trl_prod/experiments/async_grpo_harbor_data_agent/logs/multi4-long-prod-cont-20260915/checkpoint-evals/step-000800/scores.json`; SHA256 `65130786707d07f68cfa145fcd5ad7308890cb89a305b65382dfbec36ef7a150`.
+
+## Native OpenCode
+
+### Overall and difficulty
+
+| Checkpoint | Overall | Easy (132 cells) | Medium (472) | Hard (396) |
+| --- | ---: | ---: | ---: | ---: |
+| 0 | 15.9% | 37.9% | 18.0% | 6.1% |
+| 100 | 19.7% | 44.7% | 22.5% | 8.1% |
+| 200 | 22.1% | 59.8% | 24.6% | 6.6% |
+| 300 | 21.6% | 51.5% | 25.6% | 6.8% |
+| 400 | 26.4% | 59.1% | 29.4% | 11.9% |
+| 500 | 23.1% | 53.0% | 27.3% | 8.1% |
+| 600 | 25.1% | 56.1% | 28.8% | 10.4% |
+| 700 | 23.2% | 49.2% | 28.8% | 7.8% |
+| 800 | 25.6% | 52.3% | 29.7% | 11.9% |
+| 900 | 25.3% | 50.0% | 30.3% | 11.1% |
+| 1000 | 29.8% | 59.8% | 35.8% | 12.6% |
+
+### Harness × difficulty at every checkpoint
+
+| Checkpoint | Harness | Overall (250) | Easy (33) | Medium (118) | Hard (99) |
+| --- | --- | ---: | ---: | ---: | ---: |
+| 0 | opencode | 12.8% | 24.2% (8/33) | 16.9% (20/118) | 4.0% (4/99) |
+| 0 | claude-code | 16.8% | 42.4% (14/33) | 18.6% (22/118) | 6.1% (6/99) |
+| 0 | codex | 15.2% | 33.3% (11/33) | 16.9% (20/118) | 7.1% (7/99) |
+| 0 | mini-swe-agent | 18.8% | 51.5% (17/33) | 19.5% (23/118) | 7.1% (7/99) |
+| 100 | opencode | 19.6% | 42.4% (14/33) | 24.6% (29/118) | 6.1% (6/99) |
+| 100 | claude-code | 20.4% | 42.4% (14/33) | 22.9% (27/118) | 10.1% (10/99) |
+| 100 | codex | 18.0% | 27.3% (9/33) | 21.2% (25/118) | 11.1% (11/99) |
+| 100 | mini-swe-agent | 20.8% | 66.7% (22/33) | 21.2% (25/118) | 5.1% (5/99) |
+| 200 | opencode | 17.2% | 48.5% (16/33) | 21.2% (25/118) | 2.0% (2/99) |
+| 200 | claude-code | 24.0% | 60.6% (20/33) | 26.3% (31/118) | 9.1% (9/99) |
+| 200 | codex | 25.6% | 63.6% (21/33) | 28.8% (34/118) | 9.1% (9/99) |
+| 200 | mini-swe-agent | 21.6% | 66.7% (22/33) | 22.0% (26/118) | 6.1% (6/99) |
+| 300 | opencode | 20.8% | 48.5% (16/33) | 28.0% (33/118) | 3.0% (3/99) |
+| 300 | claude-code | 29.2% | 66.7% (22/33) | 30.5% (36/118) | 15.2% (15/99) |
+| 300 | codex | 16.8% | 39.4% (13/33) | 20.3% (24/118) | 5.1% (5/99) |
+| 300 | mini-swe-agent | 19.6% | 51.5% (17/33) | 23.7% (28/118) | 4.0% (4/99) |
+| 400 | opencode | 20.4% | 48.5% (16/33) | 24.6% (29/118) | 6.1% (6/99) |
+| 400 | claude-code | 32.4% | 60.6% (20/33) | 35.6% (42/118) | 19.2% (19/99) |
+| 400 | codex | 25.6% | 57.6% (19/33) | 28.0% (33/118) | 12.1% (12/99) |
+| 400 | mini-swe-agent | 27.2% | 69.7% (23/33) | 29.7% (35/118) | 10.1% (10/99) |
+| 500 | opencode | 18.0% | 48.5% (16/33) | 19.5% (23/118) | 6.1% (6/99) |
+| 500 | claude-code | 26.4% | 51.5% (17/33) | 33.1% (39/118) | 10.1% (10/99) |
+| 500 | codex | 18.8% | 48.5% (16/33) | 22.0% (26/118) | 5.1% (5/99) |
+| 500 | mini-swe-agent | 29.2% | 63.6% (21/33) | 34.7% (41/118) | 11.1% (11/99) |
+| 600 | opencode | 16.8% | 42.4% (14/33) | 18.6% (22/118) | 6.1% (6/99) |
+| 600 | claude-code | 32.4% | 63.6% (21/33) | 37.3% (44/118) | 16.2% (16/99) |
+| 600 | codex | 18.4% | 45.5% (15/33) | 23.7% (28/118) | 3.0% (3/99) |
+| 600 | mini-swe-agent | 32.8% | 72.7% (24/33) | 35.6% (42/118) | 16.2% (16/99) |
+| 700 | opencode | 18.4% | 42.4% (14/33) | 23.7% (28/118) | 4.0% (4/99) |
+| 700 | claude-code | 29.6% | 63.6% (21/33) | 33.1% (39/118) | 14.1% (14/99) |
+| 700 | codex | 14.4% | 24.2% (8/33) | 22.9% (27/118) | 1.0% (1/99) |
+| 700 | mini-swe-agent | 30.4% | 66.7% (22/33) | 35.6% (42/118) | 12.1% (12/99) |
+| 800 | opencode | 20.0% | 36.4% (12/33) | 26.3% (31/118) | 7.1% (7/99) |
+| 800 | claude-code | 32.4% | 72.7% (24/33) | 35.6% (42/118) | 15.2% (15/99) |
+| 800 | codex | 18.0% | 30.3% (10/33) | 21.2% (25/118) | 10.1% (10/99) |
+| 800 | mini-swe-agent | 32.0% | 69.7% (23/33) | 35.6% (42/118) | 15.2% (15/99) |
+| 900 | opencode | 18.8% | 33.3% (11/33) | 23.7% (28/118) | 8.1% (8/99) |
+| 900 | claude-code | 34.0% | 60.6% (20/33) | 42.4% (50/118) | 15.2% (15/99) |
+| 900 | codex | 14.4% | 27.3% (9/33) | 17.8% (21/118) | 6.1% (6/99) |
+| 900 | mini-swe-agent | 34.0% | 78.8% (26/33) | 37.3% (44/118) | 15.2% (15/99) |
+| 1000 | opencode | 20.4% | 42.4% (14/33) | 25.4% (30/118) | 7.1% (7/99) |
+| 1000 | claude-code | 33.2% | 66.7% (22/33) | 37.3% (44/118) | 17.2% (17/99) |
+| 1000 | codex | 29.6% | 57.6% (19/33) | 35.6% (42/118) | 13.1% (13/99) |
+| 1000 | mini-swe-agent | 36.0% | 72.7% (24/33) | 44.9% (53/118) | 13.1% (13/99) |
+
+### Training history
+
+| Allocation | First optimizer step | Last optimizer step |
+| --- | ---: | ---: |
+| 80626 | 1 | 1000 |
+
+### Score provenance
+
+- Step 0: `/fsx/adithyaskolavi/projects/trl_prod/experiments/daytona_harness_comparison/logs/20260915/blackbox/canonical_scores.json`; SHA256 `ece2b0e03c7c7e0e54988eaaa473ba6b53bd6028315b1e99ec39df5137c7632e`.
+- Step 100: `/fsx/adithyaskolavi/projects/trl_prod/experiments/daytona_harness_comparison/logs/hf-20260915/local-opencode-smoke-v4/repro/outputs/local-eval-opencode-80657/canonical_scores.json`; SHA256 `37eab8fe9e97fda23e9803846f965b08c3de2a6e4142363219a4467af41c6f5c`.
+- Step 200: `/fsx/adithyaskolavi/projects/trl_prod/experiments/daytona_harness_comparison/logs/hf-20260915/local-opencode-smoke-v4/repro/outputs/local-eval-opencode-80675/canonical_scores.json`; SHA256 `fa67547d19c7f4e63166fc7c1518ca15eb5e2c28ec8f1f3739445029006617ad`.
+- Step 300: `/fsx/adithyaskolavi/projects/trl_prod/experiments/daytona_harness_comparison/logs/hf-20260915/local-opencode-smoke-v4/repro/outputs/local-eval-opencode-80748/canonical_scores.json`; SHA256 `a0e0cc489e3195827d5ac035945277bfd9ca67e985a015148e5b3c94ca938b0f`.
+- Step 400: `/fsx/adithyaskolavi/projects/trl_prod/experiments/daytona_harness_comparison/logs/hf-20260915/local-opencode-smoke-v4/repro/outputs/local-eval-opencode-80807/canonical_scores.json`; SHA256 `f30bb541e207a2e8b83b2aabd05bf2e3d96eeac40c2fd8226ff1985606397a7b`.
+- Step 500: `/fsx/adithyaskolavi/projects/trl_prod/experiments/daytona_harness_comparison/logs/hf-20260915/local-opencode-smoke-v4/repro/outputs/local-eval-opencode-80861/canonical_scores.json`; SHA256 `b6df2570695c2a15ba43f185719b647dc22319eb82ca1494d56e705572e3f1a2`.
+- Step 600: `/fsx/adithyaskolavi/projects/trl_prod/experiments/daytona_harness_comparison/logs/hf-20260915/local-opencode-smoke-v4/repro/outputs/local-eval-opencode-80902/canonical_scores.json`; SHA256 `d86128c2cae4813a7ae8b99f06d1cd8ca109cefade9944c1cdbebf2b1550e1d9`.
+- Step 700: `/fsx/adithyaskolavi/projects/trl_prod/experiments/daytona_harness_comparison/logs/hf-20260915/local-opencode-smoke-v4/repro/outputs/local-eval-opencode-80956/canonical_scores.json`; SHA256 `7c60cfab333948e63bf44bd01ed4ce3f0a788f76de84c4ceda2177144e9bc9ba`.
+- Step 800: `/fsx/adithyaskolavi/projects/trl_prod/experiments/daytona_harness_comparison/logs/hf-20260915/local-opencode-smoke-v4/repro/outputs/local-eval-opencode-80993/canonical_scores.json`; SHA256 `b689c9dd76c3e2230bfea49eea393f2d5842fe8f3630f04f59380a131497ae98`.
+- Step 900: `/fsx/adithyaskolavi/projects/trl_prod/experiments/daytona_harness_comparison/logs/hf-20260915/local-opencode-smoke-v4/repro/outputs/local-eval-opencode-81034/canonical_scores.json`; SHA256 `996f5ea262419b9639fa8f33c1b33fef9b49959c1cbe61e62ba922c0d642985f`.
+- Step 1000: `/fsx/adithyaskolavi/projects/trl_prod/experiments/daytona_harness_comparison/logs/hf-20260915/local-opencode-smoke-v4/repro/outputs/local-eval-opencode-81098/canonical_scores.json`; SHA256 `1355a9a2ecc1ec165cf413120dacfc672e5d8d59ef2807b28bcf02322dca142b`.
+
+## Harbor OpenCode-only
+
+### Overall and difficulty
+
+| Checkpoint | Overall | Easy (132 cells) | Medium (472) | Hard (396) |
+| --- | ---: | ---: | ---: | ---: |
+| 0 | 14.6% | 40.2% | 14.4% | 6.3% |
+
+### Harness × difficulty at every checkpoint
+
+| Checkpoint | Harness | Overall (250) | Easy (33) | Medium (118) | Hard (99) |
+| --- | --- | ---: | ---: | ---: | ---: |
+| 0 | opencode | 10.8% | 33.3% (11/33) | 8.5% (10/118) | 6.1% (6/99) |
+| 0 | claude-code | 16.8% | 42.4% (14/33) | 18.6% (22/118) | 6.1% (6/99) |
+| 0 | codex | 16.4% | 42.4% (14/33) | 16.9% (20/118) | 7.1% (7/99) |
+| 0 | mini-swe-agent | 14.4% | 42.4% (14/33) | 13.6% (16/118) | 6.1% (6/99) |
+
+### Training history
+
+| Allocation | First optimizer step | Last optimizer step |
+| --- | ---: | ---: |
+| 81075 | 1 | 74 |
+
+### Score provenance
+
+- Step 0: `/fsx/adithyaskolavi/projects/trl_prod/experiments/async_grpo_harbor_data_agent/logs/multi4-baseline-20260914/job-78215/canonical_results.json`; SHA256 `8c4f5bced4eff04b0c2e5f41806da9ae1b8a4c0fe356ddf926e1f781c6bb9ac6`.
+
+## Dashboard metric guide
+
+Both runs use identical metric names and optimizer-step axes. `eval/pass_at_1` is the overall score; `eval/difficulty/*` aggregates each difficulty; `eval/harness/*` compares each harness; `eval/harness_difficulty/*` contains all twelve intersections. `train/*` preserves recorded loss, reward, learning rate, gradient norm, entropy, KL, staleness, throughput, token, batching and rollout metrics where observed. Missing metrics are not filled with zeros. `train/reward_rolling20` and `train/nonzero_gradient_rolling20` are explicitly derived trailing windows. Raw metrics remain available. Use zero dashboard smoothing for exact checkpoint values.
+
+The independent CPU publisher refreshes every 60 seconds and admits new evaluations only after their full comparison gates pass. It never changes trainer state. Local SQLite backup, event ledger and remote exact-content verification receipts are kept alongside this report.
+
+Storage and deployment follow the [Trackio guide](https://huggingface.co/docs/trackio/quickstart) and [environment configuration](https://huggingface.co/docs/trackio/environment_variables).
+
+- [Overview](https://huggingenvs-data-agent-training-comparison-trackio.hf.space/?project=qwen35-2b-harbor-vs-opencode-20260916&run_ids=3ae29a23763093285702b71a1f76805a%2Cbeb2604c8f737da4262b1ab19b8b0cbd%2Cc5b445fa337b56b139c1e6b34ae35409&smoothing=0&metric_filter=%5E%28eval%2Fpass_at_1%7Ctrain%2Freward_rolling20%29%24)
+- [Difficulty](https://huggingenvs-data-agent-training-comparison-trackio.hf.space/?project=qwen35-2b-harbor-vs-opencode-20260916&run_ids=3ae29a23763093285702b71a1f76805a%2Cbeb2604c8f737da4262b1ab19b8b0cbd%2Cc5b445fa337b56b139c1e6b34ae35409&smoothing=0&metric_filter=%5Eeval%2Fdifficulty%2F)
+- [Harness](https://huggingenvs-data-agent-training-comparison-trackio.hf.space/?project=qwen35-2b-harbor-vs-opencode-20260916&run_ids=3ae29a23763093285702b71a1f76805a%2Cbeb2604c8f737da4262b1ab19b8b0cbd%2Cc5b445fa337b56b139c1e6b34ae35409&smoothing=0&metric_filter=%5Eeval%2Fharness%2F)
+- [Harness × difficulty](https://huggingenvs-data-agent-training-comparison-trackio.hf.space/?project=qwen35-2b-harbor-vs-opencode-20260916&run_ids=3ae29a23763093285702b71a1f76805a%2Cbeb2604c8f737da4262b1ab19b8b0cbd%2Cc5b445fa337b56b139c1e6b34ae35409&smoothing=0&metric_filter=%5Eeval%2Fharness_difficulty%2F)
+- [Optimizer diagnostics](https://huggingenvs-data-agent-training-comparison-trackio.hf.space/?project=qwen35-2b-harbor-vs-opencode-20260916&run_ids=3ae29a23763093285702b71a1f76805a%2Cbeb2604c8f737da4262b1ab19b8b0cbd%2Cc5b445fa337b56b139c1e6b34ae35409&smoothing=0&metric_filter=%5Etrain%2F%28loss%7Cgrad_norm%7Centropy%7Ckl%7Clearning_rate%7Cnonzero_gradient_rolling20%29%24)
+- [Throughput and rollout diagnostics](https://huggingenvs-data-agent-training-comparison-trackio.hf.space/?project=qwen35-2b-harbor-vs-opencode-20260916&run_ids=3ae29a23763093285702b71a1f76805a%2Cbeb2604c8f737da4262b1ab19b8b0cbd%2Cc5b445fa337b56b139c1e6b34ae35409&smoothing=0&metric_filter=%5Etrain%2F%28perf%7Crollout%7Csample%7Cbatch%29%2F)
+- [All metrics](https://huggingenvs-data-agent-training-comparison-trackio.hf.space/?project=qwen35-2b-harbor-vs-opencode-20260916&run_ids=3ae29a23763093285702b71a1f76805a%2Cbeb2604c8f737da4262b1ab19b8b0cbd%2Cc5b445fa337b56b139c1e6b34ae35409&smoothing=0&metric_filter=)
+
+[Download checkpoint scores as CSV](checkpoint_scores.csv)
diff --git a/04-data-agent/results/2026-09-16/checkpoint_scores.csv b/04-data-agent/results/2026-09-16/checkpoint_scores.csv
new file mode 100644
index 0000000..147f505
--- /dev/null
+++ b/04-data-agent/results/2026-09-16/checkpoint_scores.csv
@@ -0,0 +1,265 @@
+run,checkpoint,harness,difficulty,correct,graded,pass_at_1
+Harbor multi-harness,0,opencode,easy,11,33,0.3333333333333333
+Harbor multi-harness,0,opencode,medium,10,118,0.0847457627118644
+Harbor multi-harness,0,opencode,hard,6,99,0.06060606060606061
+Harbor multi-harness,0,claude-code,easy,14,33,0.42424242424242425
+Harbor multi-harness,0,claude-code,medium,22,118,0.1864406779661017
+Harbor multi-harness,0,claude-code,hard,6,99,0.06060606060606061
+Harbor multi-harness,0,codex,easy,14,33,0.42424242424242425
+Harbor multi-harness,0,codex,medium,20,118,0.1694915254237288
+Harbor multi-harness,0,codex,hard,7,99,0.0707070707070707
+Harbor multi-harness,0,mini-swe-agent,easy,14,33,0.42424242424242425
+Harbor multi-harness,0,mini-swe-agent,medium,16,118,0.13559322033898305
+Harbor multi-harness,0,mini-swe-agent,hard,6,99,0.06060606060606061
+Harbor multi-harness,100,opencode,easy,17,33,0.5151515151515151
+Harbor multi-harness,100,opencode,medium,37,118,0.3135593220338983
+Harbor multi-harness,100,opencode,hard,7,99,0.0707070707070707
+Harbor multi-harness,100,claude-code,easy,20,33,0.6060606060606061
+Harbor multi-harness,100,claude-code,medium,38,118,0.3220338983050847
+Harbor multi-harness,100,claude-code,hard,11,99,0.1111111111111111
+Harbor multi-harness,100,codex,easy,19,33,0.5757575757575758
+Harbor multi-harness,100,codex,medium,42,118,0.3559322033898305
+Harbor multi-harness,100,codex,hard,9,99,0.09090909090909091
+Harbor multi-harness,100,mini-swe-agent,easy,14,33,0.42424242424242425
+Harbor multi-harness,100,mini-swe-agent,medium,27,118,0.2288135593220339
+Harbor multi-harness,100,mini-swe-agent,hard,7,99,0.0707070707070707
+Harbor multi-harness,200,opencode,easy,17,33,0.5151515151515151
+Harbor multi-harness,200,opencode,medium,41,118,0.3474576271186441
+Harbor multi-harness,200,opencode,hard,18,99,0.18181818181818182
+Harbor multi-harness,200,claude-code,easy,21,33,0.6363636363636364
+Harbor multi-harness,200,claude-code,medium,39,118,0.3305084745762712
+Harbor multi-harness,200,claude-code,hard,15,99,0.15151515151515152
+Harbor multi-harness,200,codex,easy,21,33,0.6363636363636364
+Harbor multi-harness,200,codex,medium,35,118,0.2966101694915254
+Harbor multi-harness,200,codex,hard,10,99,0.10101010101010101
+Harbor multi-harness,200,mini-swe-agent,easy,18,33,0.5454545454545454
+Harbor multi-harness,200,mini-swe-agent,medium,27,118,0.2288135593220339
+Harbor multi-harness,200,mini-swe-agent,hard,1,99,0.010101010101010102
+Harbor multi-harness,300,opencode,easy,20,33,0.6060606060606061
+Harbor multi-harness,300,opencode,medium,39,118,0.3305084745762712
+Harbor multi-harness,300,opencode,hard,15,99,0.15151515151515152
+Harbor multi-harness,300,claude-code,easy,22,33,0.6666666666666666
+Harbor multi-harness,300,claude-code,medium,46,118,0.3898305084745763
+Harbor multi-harness,300,claude-code,hard,15,99,0.15151515151515152
+Harbor multi-harness,300,codex,easy,23,33,0.696969696969697
+Harbor multi-harness,300,codex,medium,39,118,0.3305084745762712
+Harbor multi-harness,300,codex,hard,12,99,0.12121212121212122
+Harbor multi-harness,300,mini-swe-agent,easy,20,33,0.6060606060606061
+Harbor multi-harness,300,mini-swe-agent,medium,31,118,0.2627118644067797
+Harbor multi-harness,300,mini-swe-agent,hard,4,99,0.04040404040404041
+Harbor multi-harness,400,opencode,easy,24,33,0.7272727272727273
+Harbor multi-harness,400,opencode,medium,45,118,0.3813559322033898
+Harbor multi-harness,400,opencode,hard,13,99,0.13131313131313133
+Harbor multi-harness,400,claude-code,easy,25,33,0.7575757575757576
+Harbor multi-harness,400,claude-code,medium,53,118,0.4491525423728814
+Harbor multi-harness,400,claude-code,hard,14,99,0.1414141414141414
+Harbor multi-harness,400,codex,easy,24,33,0.7272727272727273
+Harbor multi-harness,400,codex,medium,50,118,0.423728813559322
+Harbor multi-harness,400,codex,hard,13,99,0.13131313131313133
+Harbor multi-harness,400,mini-swe-agent,easy,24,33,0.7272727272727273
+Harbor multi-harness,400,mini-swe-agent,medium,39,118,0.3305084745762712
+Harbor multi-harness,400,mini-swe-agent,hard,9,99,0.09090909090909091
+Harbor multi-harness,500,opencode,easy,23,33,0.696969696969697
+Harbor multi-harness,500,opencode,medium,48,118,0.4067796610169492
+Harbor multi-harness,500,opencode,hard,11,99,0.1111111111111111
+Harbor multi-harness,500,claude-code,easy,25,33,0.7575757575757576
+Harbor multi-harness,500,claude-code,medium,62,118,0.5254237288135594
+Harbor multi-harness,500,claude-code,hard,25,99,0.25252525252525254
+Harbor multi-harness,500,codex,easy,25,33,0.7575757575757576
+Harbor multi-harness,500,codex,medium,55,118,0.4661016949152542
+Harbor multi-harness,500,codex,hard,18,99,0.18181818181818182
+Harbor multi-harness,500,mini-swe-agent,easy,23,33,0.696969696969697
+Harbor multi-harness,500,mini-swe-agent,medium,44,118,0.3728813559322034
+Harbor multi-harness,500,mini-swe-agent,hard,11,99,0.1111111111111111
+Harbor multi-harness,600,opencode,easy,22,33,0.6666666666666666
+Harbor multi-harness,600,opencode,medium,49,118,0.4152542372881356
+Harbor multi-harness,600,opencode,hard,14,99,0.1414141414141414
+Harbor multi-harness,600,claude-code,easy,24,33,0.7272727272727273
+Harbor multi-harness,600,claude-code,medium,40,118,0.3389830508474576
+Harbor multi-harness,600,claude-code,hard,11,99,0.1111111111111111
+Harbor multi-harness,600,codex,easy,22,33,0.6666666666666666
+Harbor multi-harness,600,codex,medium,47,118,0.3983050847457627
+Harbor multi-harness,600,codex,hard,12,99,0.12121212121212122
+Harbor multi-harness,600,mini-swe-agent,easy,28,33,0.8484848484848485
+Harbor multi-harness,600,mini-swe-agent,medium,40,118,0.3389830508474576
+Harbor multi-harness,600,mini-swe-agent,hard,9,99,0.09090909090909091
+Harbor multi-harness,684,opencode,easy,24,33,0.7272727272727273
+Harbor multi-harness,684,opencode,medium,36,118,0.3050847457627119
+Harbor multi-harness,684,opencode,hard,13,99,0.13131313131313133
+Harbor multi-harness,684,claude-code,easy,24,33,0.7272727272727273
+Harbor multi-harness,684,claude-code,medium,49,118,0.4152542372881356
+Harbor multi-harness,684,claude-code,hard,16,99,0.16161616161616163
+Harbor multi-harness,684,codex,easy,28,33,0.8484848484848485
+Harbor multi-harness,684,codex,medium,43,118,0.3644067796610169
+Harbor multi-harness,684,codex,hard,12,99,0.12121212121212122
+Harbor multi-harness,684,mini-swe-agent,easy,24,33,0.7272727272727273
+Harbor multi-harness,684,mini-swe-agent,medium,41,118,0.3474576271186441
+Harbor multi-harness,684,mini-swe-agent,hard,11,99,0.1111111111111111
+Harbor multi-harness,700,opencode,easy,10,33,0.30303030303030304
+Harbor multi-harness,700,opencode,medium,33,118,0.2796610169491525
+Harbor multi-harness,700,opencode,hard,10,99,0.10101010101010101
+Harbor multi-harness,700,claude-code,easy,26,33,0.7878787878787878
+Harbor multi-harness,700,claude-code,medium,41,118,0.3474576271186441
+Harbor multi-harness,700,claude-code,hard,12,99,0.12121212121212122
+Harbor multi-harness,700,codex,easy,24,33,0.7272727272727273
+Harbor multi-harness,700,codex,medium,41,118,0.3474576271186441
+Harbor multi-harness,700,codex,hard,15,99,0.15151515151515152
+Harbor multi-harness,700,mini-swe-agent,easy,20,33,0.6060606060606061
+Harbor multi-harness,700,mini-swe-agent,medium,45,118,0.3813559322033898
+Harbor multi-harness,700,mini-swe-agent,hard,11,99,0.1111111111111111
+Harbor multi-harness,800,opencode,easy,12,33,0.36363636363636365
+Harbor multi-harness,800,opencode,medium,39,118,0.3305084745762712
+Harbor multi-harness,800,opencode,hard,7,99,0.0707070707070707
+Harbor multi-harness,800,claude-code,easy,22,33,0.6666666666666666
+Harbor multi-harness,800,claude-code,medium,44,118,0.3728813559322034
+Harbor multi-harness,800,claude-code,hard,14,99,0.1414141414141414
+Harbor multi-harness,800,codex,easy,22,33,0.6666666666666666
+Harbor multi-harness,800,codex,medium,37,118,0.3135593220338983
+Harbor multi-harness,800,codex,hard,8,99,0.08080808080808081
+Harbor multi-harness,800,mini-swe-agent,easy,22,33,0.6666666666666666
+Harbor multi-harness,800,mini-swe-agent,medium,34,118,0.288135593220339
+Harbor multi-harness,800,mini-swe-agent,hard,9,99,0.09090909090909091
+Native OpenCode,0,opencode,easy,8,33,0.24242424242424243
+Native OpenCode,0,opencode,medium,20,118,0.1694915254237288
+Native OpenCode,0,opencode,hard,4,99,0.04040404040404041
+Native OpenCode,0,claude-code,easy,14,33,0.42424242424242425
+Native OpenCode,0,claude-code,medium,22,118,0.1864406779661017
+Native OpenCode,0,claude-code,hard,6,99,0.06060606060606061
+Native OpenCode,0,codex,easy,11,33,0.3333333333333333
+Native OpenCode,0,codex,medium,20,118,0.1694915254237288
+Native OpenCode,0,codex,hard,7,99,0.0707070707070707
+Native OpenCode,0,mini-swe-agent,easy,17,33,0.5151515151515151
+Native OpenCode,0,mini-swe-agent,medium,23,118,0.19491525423728814
+Native OpenCode,0,mini-swe-agent,hard,7,99,0.0707070707070707
+Native OpenCode,100,opencode,easy,14,33,0.42424242424242425
+Native OpenCode,100,opencode,medium,29,118,0.2457627118644068
+Native OpenCode,100,opencode,hard,6,99,0.06060606060606061
+Native OpenCode,100,claude-code,easy,14,33,0.42424242424242425
+Native OpenCode,100,claude-code,medium,27,118,0.2288135593220339
+Native OpenCode,100,claude-code,hard,10,99,0.10101010101010101
+Native OpenCode,100,codex,easy,9,33,0.2727272727272727
+Native OpenCode,100,codex,medium,25,118,0.211864406779661
+Native OpenCode,100,codex,hard,11,99,0.1111111111111111
+Native OpenCode,100,mini-swe-agent,easy,22,33,0.6666666666666666
+Native OpenCode,100,mini-swe-agent,medium,25,118,0.211864406779661
+Native OpenCode,100,mini-swe-agent,hard,5,99,0.050505050505050504
+Native OpenCode,200,opencode,easy,16,33,0.48484848484848486
+Native OpenCode,200,opencode,medium,25,118,0.211864406779661
+Native OpenCode,200,opencode,hard,2,99,0.020202020202020204
+Native OpenCode,200,claude-code,easy,20,33,0.6060606060606061
+Native OpenCode,200,claude-code,medium,31,118,0.2627118644067797
+Native OpenCode,200,claude-code,hard,9,99,0.09090909090909091
+Native OpenCode,200,codex,easy,21,33,0.6363636363636364
+Native OpenCode,200,codex,medium,34,118,0.288135593220339
+Native OpenCode,200,codex,hard,9,99,0.09090909090909091
+Native OpenCode,200,mini-swe-agent,easy,22,33,0.6666666666666666
+Native OpenCode,200,mini-swe-agent,medium,26,118,0.22033898305084745
+Native OpenCode,200,mini-swe-agent,hard,6,99,0.06060606060606061
+Native OpenCode,300,opencode,easy,16,33,0.48484848484848486
+Native OpenCode,300,opencode,medium,33,118,0.2796610169491525
+Native OpenCode,300,opencode,hard,3,99,0.030303030303030304
+Native OpenCode,300,claude-code,easy,22,33,0.6666666666666666
+Native OpenCode,300,claude-code,medium,36,118,0.3050847457627119
+Native OpenCode,300,claude-code,hard,15,99,0.15151515151515152
+Native OpenCode,300,codex,easy,13,33,0.3939393939393939
+Native OpenCode,300,codex,medium,24,118,0.2033898305084746
+Native OpenCode,300,codex,hard,5,99,0.050505050505050504
+Native OpenCode,300,mini-swe-agent,easy,17,33,0.5151515151515151
+Native OpenCode,300,mini-swe-agent,medium,28,118,0.23728813559322035
+Native OpenCode,300,mini-swe-agent,hard,4,99,0.04040404040404041
+Native OpenCode,400,opencode,easy,16,33,0.48484848484848486
+Native OpenCode,400,opencode,medium,29,118,0.2457627118644068
+Native OpenCode,400,opencode,hard,6,99,0.06060606060606061
+Native OpenCode,400,claude-code,easy,20,33,0.6060606060606061
+Native OpenCode,400,claude-code,medium,42,118,0.3559322033898305
+Native OpenCode,400,claude-code,hard,19,99,0.1919191919191919
+Native OpenCode,400,codex,easy,19,33,0.5757575757575758
+Native OpenCode,400,codex,medium,33,118,0.2796610169491525
+Native OpenCode,400,codex,hard,12,99,0.12121212121212122
+Native OpenCode,400,mini-swe-agent,easy,23,33,0.696969696969697
+Native OpenCode,400,mini-swe-agent,medium,35,118,0.2966101694915254
+Native OpenCode,400,mini-swe-agent,hard,10,99,0.10101010101010101
+Native OpenCode,500,opencode,easy,16,33,0.48484848484848486
+Native OpenCode,500,opencode,medium,23,118,0.19491525423728814
+Native OpenCode,500,opencode,hard,6,99,0.06060606060606061
+Native OpenCode,500,claude-code,easy,17,33,0.5151515151515151
+Native OpenCode,500,claude-code,medium,39,118,0.3305084745762712
+Native OpenCode,500,claude-code,hard,10,99,0.10101010101010101
+Native OpenCode,500,codex,easy,16,33,0.48484848484848486
+Native OpenCode,500,codex,medium,26,118,0.22033898305084745
+Native OpenCode,500,codex,hard,5,99,0.050505050505050504
+Native OpenCode,500,mini-swe-agent,easy,21,33,0.6363636363636364
+Native OpenCode,500,mini-swe-agent,medium,41,118,0.3474576271186441
+Native OpenCode,500,mini-swe-agent,hard,11,99,0.1111111111111111
+Native OpenCode,600,opencode,easy,14,33,0.42424242424242425
+Native OpenCode,600,opencode,medium,22,118,0.1864406779661017
+Native OpenCode,600,opencode,hard,6,99,0.06060606060606061
+Native OpenCode,600,claude-code,easy,21,33,0.6363636363636364
+Native OpenCode,600,claude-code,medium,44,118,0.3728813559322034
+Native OpenCode,600,claude-code,hard,16,99,0.16161616161616163
+Native OpenCode,600,codex,easy,15,33,0.45454545454545453
+Native OpenCode,600,codex,medium,28,118,0.23728813559322035
+Native OpenCode,600,codex,hard,3,99,0.030303030303030304
+Native OpenCode,600,mini-swe-agent,easy,24,33,0.7272727272727273
+Native OpenCode,600,mini-swe-agent,medium,42,118,0.3559322033898305
+Native OpenCode,600,mini-swe-agent,hard,16,99,0.16161616161616163
+Native OpenCode,700,opencode,easy,14,33,0.42424242424242425
+Native OpenCode,700,opencode,medium,28,118,0.23728813559322035
+Native OpenCode,700,opencode,hard,4,99,0.04040404040404041
+Native OpenCode,700,claude-code,easy,21,33,0.6363636363636364
+Native OpenCode,700,claude-code,medium,39,118,0.3305084745762712
+Native OpenCode,700,claude-code,hard,14,99,0.1414141414141414
+Native OpenCode,700,codex,easy,8,33,0.24242424242424243
+Native OpenCode,700,codex,medium,27,118,0.2288135593220339
+Native OpenCode,700,codex,hard,1,99,0.010101010101010102
+Native OpenCode,700,mini-swe-agent,easy,22,33,0.6666666666666666
+Native OpenCode,700,mini-swe-agent,medium,42,118,0.3559322033898305
+Native OpenCode,700,mini-swe-agent,hard,12,99,0.12121212121212122
+Native OpenCode,800,opencode,easy,12,33,0.36363636363636365
+Native OpenCode,800,opencode,medium,31,118,0.2627118644067797
+Native OpenCode,800,opencode,hard,7,99,0.0707070707070707
+Native OpenCode,800,claude-code,easy,24,33,0.7272727272727273
+Native OpenCode,800,claude-code,medium,42,118,0.3559322033898305
+Native OpenCode,800,claude-code,hard,15,99,0.15151515151515152
+Native OpenCode,800,codex,easy,10,33,0.30303030303030304
+Native OpenCode,800,codex,medium,25,118,0.211864406779661
+Native OpenCode,800,codex,hard,10,99,0.10101010101010101
+Native OpenCode,800,mini-swe-agent,easy,23,33,0.696969696969697
+Native OpenCode,800,mini-swe-agent,medium,42,118,0.3559322033898305
+Native OpenCode,800,mini-swe-agent,hard,15,99,0.15151515151515152
+Native OpenCode,900,opencode,easy,11,33,0.3333333333333333
+Native OpenCode,900,opencode,medium,28,118,0.23728813559322035
+Native OpenCode,900,opencode,hard,8,99,0.08080808080808081
+Native OpenCode,900,claude-code,easy,20,33,0.6060606060606061
+Native OpenCode,900,claude-code,medium,50,118,0.423728813559322
+Native OpenCode,900,claude-code,hard,15,99,0.15151515151515152
+Native OpenCode,900,codex,easy,9,33,0.2727272727272727
+Native OpenCode,900,codex,medium,21,118,0.17796610169491525
+Native OpenCode,900,codex,hard,6,99,0.06060606060606061
+Native OpenCode,900,mini-swe-agent,easy,26,33,0.7878787878787878
+Native OpenCode,900,mini-swe-agent,medium,44,118,0.3728813559322034
+Native OpenCode,900,mini-swe-agent,hard,15,99,0.15151515151515152
+Native OpenCode,1000,opencode,easy,14,33,0.42424242424242425
+Native OpenCode,1000,opencode,medium,30,118,0.2542372881355932
+Native OpenCode,1000,opencode,hard,7,99,0.0707070707070707
+Native OpenCode,1000,claude-code,easy,22,33,0.6666666666666666
+Native OpenCode,1000,claude-code,medium,44,118,0.3728813559322034
+Native OpenCode,1000,claude-code,hard,17,99,0.1717171717171717
+Native OpenCode,1000,codex,easy,19,33,0.5757575757575758
+Native OpenCode,1000,codex,medium,42,118,0.3559322033898305
+Native OpenCode,1000,codex,hard,13,99,0.13131313131313133
+Native OpenCode,1000,mini-swe-agent,easy,24,33,0.7272727272727273
+Native OpenCode,1000,mini-swe-agent,medium,53,118,0.4491525423728814
+Native OpenCode,1000,mini-swe-agent,hard,13,99,0.13131313131313133
+Harbor OpenCode-only,0,opencode,easy,11,33,0.3333333333333333
+Harbor OpenCode-only,0,opencode,medium,10,118,0.0847457627118644
+Harbor OpenCode-only,0,opencode,hard,6,99,0.06060606060606061
+Harbor OpenCode-only,0,claude-code,easy,14,33,0.42424242424242425
+Harbor OpenCode-only,0,claude-code,medium,22,118,0.1864406779661017
+Harbor OpenCode-only,0,claude-code,hard,6,99,0.06060606060606061
+Harbor OpenCode-only,0,codex,easy,14,33,0.42424242424242425
+Harbor OpenCode-only,0,codex,medium,20,118,0.1694915254237288
+Harbor OpenCode-only,0,codex,hard,7,99,0.0707070707070707
+Harbor OpenCode-only,0,mini-swe-agent,easy,14,33,0.42424242424242425
+Harbor OpenCode-only,0,mini-swe-agent,medium,16,118,0.13559322033898305
+Harbor OpenCode-only,0,mini-swe-agent,hard,6,99,0.06060606060606061
diff --git a/04-data-agent/results/2026-09-16/comparison.png b/04-data-agent/results/2026-09-16/comparison.png
new file mode 100644
index 0000000..ca38580
Binary files /dev/null and b/04-data-agent/results/2026-09-16/comparison.png differ
diff --git a/04-data-agent/results/2026-09-16/seta-checkpoint-150.json b/04-data-agent/results/2026-09-16/seta-checkpoint-150.json
new file mode 100644
index 0000000..8107eb0
--- /dev/null
+++ b/04-data-agent/results/2026-09-16/seta-checkpoint-150.json
@@ -0,0 +1,65 @@
+{
+ "status": {
+ "arm": "whitebox",
+ "phase": "checkpoint",
+ "started_at": 1789555915.8065052,
+ "passed": true,
+ "finished_at": 1789557493.4326816
+ },
+ "canonical_scores": {
+ "metric": "pass@1",
+ "arm": "whitebox",
+ "complete": true,
+ "graded_cells": 250,
+ "expected_cells": 250,
+ "harnesses": {
+ "whitebox_seta": {
+ "graded": 250,
+ "correct": 95.0,
+ "pass_at_1": 0.38,
+ "difficulty": {
+ "easy": {
+ "graded": 33,
+ "correct": 28.0,
+ "pass_at_1": 0.8484848484848485
+ },
+ "medium": {
+ "graded": 118,
+ "correct": 49.0,
+ "pass_at_1": 0.4152542372881356
+ },
+ "hard": {
+ "graded": 99,
+ "correct": 18.0,
+ "pass_at_1": 0.18181818181818182
+ }
+ }
+ }
+ },
+ "tito_pass": true,
+ "ungraded_attempts": 0,
+ "harness_versions": {
+ "whitebox_seta": {}
+ },
+ "harness_versions_match_baseline": true,
+ "comparison_ready": true,
+ "average_pass_at_1": 0.38,
+ "selection": "first graded attempt per fixed task/harness; infrastructure failures excluded and retried"
+ },
+ "checkpoint_evaluation": {
+ "source": "hf://buckets/HuggingEnvs/data-agent-daytona-artifacts/20260915-hf/jobs/train-whitebox-1789507273/run/checkpoint-150",
+ "manifest_sha256": "78252d1f78dc0109c0252d53e22d1bc2fd58d720f41164e3a8bae4c5d62bd8ea",
+ "step": 150,
+ "bundle_sha256": "f4a288eaafefddf3cb686eb89de184437a0496f75074a00f1903a9335dcd1a0d"
+ },
+ "services": {
+ "job_id": "6aaa7488f76d6a098a710836",
+ "public_vllm": "https://6aaa7488f76d6a098a710836--8000.hf.jobs",
+ "server": "http://127.0.0.1:8100",
+ "space": "https://huggingenvs-data-agent-seta-whitebox-env.hf.space",
+ "tp": 1,
+ "dp": 1,
+ "flavor": "a100-large"
+ },
+ "job_id": "6aaa7488f76d6a098a710836"
+}
diff --git a/04-data-agent/results/2026-09-16/snapshot.json.gz b/04-data-agent/results/2026-09-16/snapshot.json.gz
new file mode 100644
index 0000000..61b6763
Binary files /dev/null and b/04-data-agent/results/2026-09-16/snapshot.json.gz differ
diff --git a/04-data-agent/results/qualification/harbor-v3-upload-failure.json b/04-data-agent/results/qualification/harbor-v3-upload-failure.json
new file mode 100644
index 0000000..d262789
--- /dev/null
+++ b/04-data-agent/results/qualification/harbor-v3-upload-failure.json
@@ -0,0 +1,71 @@
+{
+ "id": "6aaa7b18f76d6a098a71093d",
+ "owner": "train-blackbox-1789557528",
+ "stage": "ERROR",
+ "status.json": {
+ "arm": "blackbox",
+ "phase": "smoke",
+ "started_at": 1789557599.0150673,
+ "passed": false,
+ "error_type": "TimeoutError",
+ "finished_at": 1789560537.914167
+ },
+ "services.json": {
+ "job_id": "6aaa7b18f76d6a098a71093d",
+ "public_vllm": "https://6aaa7b18f76d6a098a71093d--8000.hf.jobs",
+ "server": "http://127.0.0.1:8100",
+ "space": "https://huggingenvs-data-agent-blackbox-harbor-env.hf.space",
+ "tp": 1,
+ "dp": 1,
+ "flavor": "a100x4"
+ },
+ "trackio_verified.json": {
+ "passed": true,
+ "project": "daytona-blackbox-qwen35-2b-smoke",
+ "run": "train-blackbox-1789557528",
+ "local_database": "/workspace/repro/outputs/train-blackbox-1789557528/trackio/daytona-blackbox-qwen35-2b-smoke.db",
+ "mode": "offline",
+ "remote_storage": "run artifact bucket",
+ "native_remote_readback": false,
+ "updated_at": 1789559547.7700548,
+ "unique_events": 6
+ },
+ "upload_status.json": {
+ "last_success": 1789560580.5810785,
+ "destination": "hf://buckets/HuggingEnvs/data-agent-daytona-artifacts/data-agent-reproduction-20260916/jobs/train-blackbox-1789557528",
+ "published_checkpoints": [
+ "checkpoint-2",
+ "checkpoint-4"
+ ]
+ },
+ "failure": "Transient HF Xet upload TimeoutError after all four optimizer updates; final cleanup published both checkpoints. Final integrated smoke validator was not reached.",
+ "posthoc_capture_audit": {
+ "blackbox": {
+ "completed_results": 37,
+ "tito_pass": 37,
+ "eligible_tokens": 19880,
+ "retained_tokens": 19880,
+ "rows_over_token_budget": 0,
+ "optimizer_rollouts_verified": 25
+ }
+ },
+ "optimizer_updates": [
+ {
+ "step": 1,
+ "grad_norm": 11.0
+ },
+ {
+ "step": 2,
+ "grad_norm": 10.4375
+ },
+ {
+ "step": 3,
+ "grad_norm": 6.9375
+ },
+ {
+ "step": 4,
+ "grad_norm": 5.59375
+ }
+ ],
+ "qualifies_long_run": false
+}
diff --git a/04-data-agent/results/qualification/harbor-v4.json b/04-data-agent/results/qualification/harbor-v4.json
new file mode 100644
index 0000000..bc5551f
--- /dev/null
+++ b/04-data-agent/results/qualification/harbor-v4.json
@@ -0,0 +1,56 @@
+{
+ "id": "6aaa8a06f76d6a098a710a5e",
+ "owner": "train-blackbox-1789561350",
+ "stage": "COMPLETED",
+ "status.json": {
+ "arm": "blackbox",
+ "phase": "smoke",
+ "started_at": 1789561426.4942746,
+ "passed": true,
+ "finished_at": 1789565459.8160994
+ },
+ "services.json": {
+ "job_id": "6aaa8a06f76d6a098a710a5e",
+ "public_vllm": "https://6aaa8a06f76d6a098a710a5e--8000.hf.jobs",
+ "server": "http://127.0.0.1:8100",
+ "space": "https://huggingenvs-data-agent-blackbox-harbor-env.hf.space",
+ "tp": 1,
+ "dp": 1,
+ "flavor": "a100x4"
+ },
+ "training_smoke_verified.json": {
+ "arm": "blackbox",
+ "passed": true,
+ "bundle_sha256": "d24c3641bdda424259741e27d451430830df1c54e0a31245ac2c8ae6210a44da",
+ "optimizer_steps": [
+ 1,
+ 2,
+ 3,
+ 4
+ ],
+ "native_optimizer_state_verified": true,
+ "remote_restore_verified": true,
+ "tito_pass": true,
+ "weights_updated": true,
+ "nonzero_gradient_updates": 4
+ },
+ "trackio_verified.json": {
+ "passed": true,
+ "project": "daytona-blackbox-qwen35-2b-smoke",
+ "run": "train-blackbox-1789561350",
+ "local_database": "/workspace/repro/outputs/train-blackbox-1789561350/trackio/daytona-blackbox-qwen35-2b-smoke.db",
+ "mode": "offline",
+ "remote_storage": "run artifact bucket",
+ "native_remote_readback": false,
+ "updated_at": 1789564559.6583176,
+ "unique_events": 6
+ },
+ "upload_status.json": {
+ "last_success": 1789565418.31508,
+ "destination": "hf://buckets/HuggingEnvs/data-agent-daytona-artifacts/data-agent-reproduction-20260916/jobs/train-blackbox-1789561350",
+ "published_checkpoints": [
+ "checkpoint-2",
+ "checkpoint-4"
+ ]
+ }
+}
diff --git a/04-data-agent/results/qualification/opencode-v3.json b/04-data-agent/results/qualification/opencode-v3.json
new file mode 100644
index 0000000..8eef827
--- /dev/null
+++ b/04-data-agent/results/qualification/opencode-v3.json
@@ -0,0 +1,56 @@
+{
+ "id": "6aaa7b875527934177ee9d15",
+ "owner": "train-opencode-1789557638",
+ "stage": "COMPLETED",
+ "status.json": {
+ "arm": "opencode",
+ "phase": "smoke",
+ "started_at": 1789557710.0655792,
+ "passed": true,
+ "finished_at": 1789561700.7866313
+ },
+ "services.json": {
+ "job_id": "6aaa7b875527934177ee9d15",
+ "public_vllm": "https://6aaa7b875527934177ee9d15--8000.hf.jobs",
+ "server": "http://127.0.0.1:8100",
+ "space": "https://huggingenvs-data-agent-blackbox-opencode-env.hf.space",
+ "tp": 1,
+ "dp": 1,
+ "flavor": "a100x4"
+ },
+ "training_smoke_verified.json": {
+ "arm": "opencode",
+ "passed": true,
+ "bundle_sha256": "6527c25ae379ab10f055577c5b87374c9018df3c8d28bb983f84fbd1e7f6302e",
+ "optimizer_steps": [
+ 1,
+ 2,
+ 3,
+ 4
+ ],
+ "native_optimizer_state_verified": true,
+ "remote_restore_verified": true,
+ "tito_pass": true,
+ "weights_updated": true,
+ "nonzero_gradient_updates": 4
+ },
+ "trackio_verified.json": {
+ "passed": true,
+ "project": "daytona-opencode-qwen35-2b-smoke",
+ "run": "train-opencode-1789557638",
+ "local_database": "/workspace/repro/outputs/train-opencode-1789557638/trackio/daytona-opencode-qwen35-2b-smoke.db",
+ "mode": "offline",
+ "remote_storage": "run artifact bucket",
+ "native_remote_readback": false,
+ "updated_at": 1789561370.6661468,
+ "unique_events": 6
+ },
+ "upload_status.json": {
+ "last_success": 1789561666.6907096,
+ "destination": "hf://buckets/HuggingEnvs/data-agent-daytona-artifacts/data-agent-reproduction-20260916/jobs/train-opencode-1789557638",
+ "published_checkpoints": [
+ "checkpoint-2",
+ "checkpoint-4"
+ ]
+ }
+}
diff --git a/04-data-agent/results/qualification/seta-v2.json b/04-data-agent/results/qualification/seta-v2.json
new file mode 100644
index 0000000..5fecf43
--- /dev/null
+++ b/04-data-agent/results/qualification/seta-v2.json
@@ -0,0 +1,47 @@
+{
+ "id": "6aaa77a65527934177ee9c34",
+ "owner": "train-whitebox-1789556646",
+ "stage": "COMPLETED",
+ "status.json": {
+ "arm": "whitebox",
+ "phase": "smoke",
+ "started_at": 1789556698.284571,
+ "passed": true,
+ "finished_at": 1789558558.68511
+ },
+ "training_smoke_verified.json": {
+ "arm": "whitebox",
+ "passed": true,
+ "bundle_sha256": "8b02b40687414905830799a458bf253d3552f9a40860f9980c983fb4ededa45a",
+ "optimizer_steps": [
+ 1,
+ 2,
+ 3,
+ 4
+ ],
+ "native_optimizer_state_verified": true,
+ "remote_restore_verified": true,
+ "tito_pass": true,
+ "weights_updated": true,
+ "nonzero_gradient_updates": 2
+ },
+ "trackio_verified.json": {
+ "passed": true,
+ "project": "daytona-whitebox-qwen35-2b-smoke",
+ "run": "train-whitebox-1789556646",
+ "local_database": "/workspace/repro/outputs/train-whitebox-1789556646/trackio/daytona-whitebox-qwen35-2b-smoke.db",
+ "mode": "offline",
+ "remote_storage": "run artifact bucket",
+ "native_remote_readback": false,
+ "updated_at": 1789558378.5921993,
+ "unique_events": 6
+ },
+ "upload_status.json": {
+ "last_success": 1789558531.14131,
+ "destination": "hf://buckets/HuggingEnvs/data-agent-daytona-artifacts/data-agent-reproduction-20260916/jobs/train-whitebox-1789556646",
+ "published_checkpoints": [
+ "checkpoint-2",
+ "checkpoint-4"
+ ]
+ }
+}
diff --git a/04-data-agent/results/qualification/seta-v3.json b/04-data-agent/results/qualification/seta-v3.json
new file mode 100644
index 0000000..afc0c09
--- /dev/null
+++ b/04-data-agent/results/qualification/seta-v3.json
@@ -0,0 +1,56 @@
+{
+ "id": "6aaa7f915527934177ee9da4",
+ "owner": "train-whitebox-1789558672",
+ "stage": "COMPLETED",
+ "status.json": {
+ "arm": "whitebox",
+ "phase": "smoke",
+ "started_at": 1789558725.4122534,
+ "passed": true,
+ "finished_at": 1789560315.8249245
+ },
+ "services.json": {
+ "job_id": "6aaa7f915527934177ee9da4",
+ "public_vllm": "https://6aaa7f915527934177ee9da4--8000.hf.jobs",
+ "server": "http://127.0.0.1:8100",
+ "space": "https://huggingenvs-data-agent-seta-whitebox-env.hf.space",
+ "tp": 1,
+ "dp": 1,
+ "flavor": "h200x2"
+ },
+ "training_smoke_verified.json": {
+ "arm": "whitebox",
+ "passed": true,
+ "bundle_sha256": "6527c25ae379ab10f055577c5b87374c9018df3c8d28bb983f84fbd1e7f6302e",
+ "optimizer_steps": [
+ 1,
+ 2,
+ 3,
+ 4
+ ],
+ "native_optimizer_state_verified": true,
+ "remote_restore_verified": true,
+ "tito_pass": true,
+ "weights_updated": true,
+ "nonzero_gradient_updates": 2
+ },
+ "trackio_verified.json": {
+ "passed": true,
+ "project": "daytona-whitebox-qwen35-2b-smoke",
+ "run": "train-whitebox-1789558672",
+ "local_database": "/workspace/repro/outputs/train-whitebox-1789558672/trackio/daytona-whitebox-qwen35-2b-smoke.db",
+ "mode": "offline",
+ "remote_storage": "run artifact bucket",
+ "native_remote_readback": false,
+ "updated_at": 1789560105.7279177,
+ "unique_events": 6
+ },
+ "upload_status.json": {
+ "last_success": 1789560265.77855,
+ "destination": "hf://buckets/HuggingEnvs/data-agent-daytona-artifacts/data-agent-reproduction-20260916/jobs/train-whitebox-1789558672",
+ "published_checkpoints": [
+ "checkpoint-2",
+ "checkpoint-4"
+ ]
+ }
+}
diff --git a/04-data-agent/results/validation.md b/04-data-agent/results/validation.md
new file mode 100644
index 0000000..d577898
--- /dev/null
+++ b/04-data-agent/results/validation.md
@@ -0,0 +1,53 @@
+# PR preparation validation — 2026-09-16
+
+This records validation of the prepared sources separately from the historical learning curves.
+
+| Check | Evidence |
+| --- | --- |
+| HuggingEnvs CPU regression suite | 149 passed, 1 skipped; 24 subtests. Includes real CPU optimizer grouping, save/resume boundaries, capture budgets, task dispatch, HTTP controls, eval recovery and artifact provenance. |
+| Portable archive | 10,606 packaged files hash-verified; source runtime matches the reviewed files; no configured credential values included. No external local experiments checkout required to build. |
+| Frozen native grading | 1,250 task configurations verified; all 250 original first-graded baseline answers replayed with identical scores. |
+| Local/Hub commands | CLI help, dry-run commands, Python compilation and fatal-error lint passed; generated project index checked. |
+| OpenEnv | 2,393 CPU tests passed with unrelated QED service tests excluded; 107 additional upstream MCP integration tests and 65 Gradio/MCP/TBench tests passed after the current-main merge; 39 client/TiTO and 59 rollout/session regressions passed for the final fixes; 334 passed and 7 skipped for client cancellation/discovery/Harbor regressions after the last upstream merge. GitHub CI is green on Python 3.11/3.12. Harbor capture/UI checks include concurrent trace isolation, session budgets and browser layout. |
+| TRL | 245 CPU tests passed, plus HTTP controls and pre-commit checks. Main is merged; all PR CI passed, including the distributed GPU smoke. |
+
+## Current training qualification
+
+| Implementation | HF Job | Trainer bundle | State |
+| --- | --- | --- | --- |
+| Harbor / OpenCode | [6aaa8a06f76d6a098a710a5e](https://huggingface.co/jobs/HuggingEnvs/6aaa8a06f76d6a098a710a5e) | v4 | **Passed**: four nonzero-gradient updates, exact-token retention, native optimizer state, remote restore, changed weights; [receipt](qualification/harbor-v4.json) |
+| Native OpenCode | [6aaa7b875527934177ee9d15](https://huggingface.co/jobs/HuggingEnvs/6aaa7b875527934177ee9d15) | v3 | **Passed**: four nonzero-gradient updates, exact-token retention, native optimizer state, remote restore, changed weights; [receipt](qualification/opencode-v3.json) |
+| SETA whitebox | [6aaa7f915527934177ee9da4](https://huggingface.co/jobs/HuggingEnvs/6aaa7f915527934177ee9da4) | v3 | **Passed**: four steps, exact-token audit, native optimizer state, remote restore, changed weights; [receipt](qualification/seta-v3.json) |
+
+- **v2**: SHA256 `8b02b40687414905830799a458bf253d3552f9a40860f9980c983fb4ededa45a`, Hub revision `0e59f18b0ddf0df0f46aa8925b4d8bb66aa95bb5`.
+- **v3**: SHA256 `6527c25ae379ab10f055577c5b87374c9018df3c8d28bb983f84fbd1e7f6302e`. Hub revision `599efbda7c93056e9d0a6a2a3324d24ac1ba2f3f`. Uses OpenEnv `b13aeb9f8ecd4817e02d3a37c2a9ae15e41710e3` and TRL `8e87edb45eac7c52d749256379714fa40f0eb746`; 10,604 packaged files. Subsequent OpenEnv PR commits preserve verifier warning diagnostics, clarify timeout scope and merge upstream client cancellation/discovery fixes. The later TRL merge changes only a tiny Gemma2 test-model generator; runtime qualification remains tied to the explicit pins above.
+
+- **v4**: SHA256 `d24c3641bdda424259741e27d451430830df1c54e0a31245ac2c8ae6210a44da`, Hub revision `d7622b44f55c65387778327229543d36446e6597`; 10,606 packaged files. Model, OpenEnv/TRL pins, training code and settings match v3. It adds the shared transfer retry path and host baseline-cohort checks.
+
+Harbor and SETA use their existing separately pinned environments. Native OpenCode uses v3, deployed only after confirming no active Jobs used that Space; CPU Basic and sandbox capacity 100 are retained. The active SETA evaluation and Harbor-only Slurm trainer were not restarted. Every future long run still requires proofs matching its own exact bundle, environment and baseline; these receipts do not waive that gate for another bundle.
+
+The final v3 SETA smoke completed successfully. The earlier successful v2 [Job 6aaa77a65527934177ee9c34](https://huggingface.co/jobs/HuggingEnvs/6aaa77a65527934177ee9c34) and its [receipt](qualification/seta-v2.json) are retained independently. Both have two nonzero-gradient updates and four completed optimizer steps.
+
+The final host-launcher regression suite additionally verifies that native diagnostic and four-harness comparison baselines remain separate, that checkpoint curves receive a matching baseline at step 0, and that changed score files fail validation. These host-only admission/reporting changes do not alter the GPU trainer runtime used by the qualification bundle.
+
+## Earlier qualification attempts
+
+The first trainer bundle was `ccfe97822cf7c88931acda8a4894bd7515e40a939ebfcc4c20c14db45e607de3`, uploaded to `HuggingEnvs/data-agent-daytona-repro` at revision `3808a6d5c48320b5e7745c877dc9b7ed2819310b`. Source pins are in [sources.json](../hf/configs/sources.json). The OpenEnv runtime pin includes the TiTO/UI changes; later OpenEnv PR commits update documentation, optional tests and merge newer upstream MCP behavior.
+
+These jobs qualify the new trainer against existing separately pinned Spaces. The first attempts did not restart or upgrade those Spaces; the later idle native OpenCode update is recorded above. A smoke proves four optimizer updates with a checkpoint-2 remote restore; it is not a new baseline or evidence of a reward gain.
+
+| Implementation | Job | GPUs | State |
+| --- | --- | --- | --- |
+| Harbor / OpenCode | [6aaa75bb5527934177ee9b8b](https://huggingface.co/jobs/HuggingEnvs/6aaa75bb5527934177ee9b8b) | A100 ×4 allocation; two used | Failed before optimizer startup: missing endpoint directory |
+| Native OpenCode | [6aaa75bb5527934177ee9b8d](https://huggingface.co/jobs/HuggingEnvs/6aaa75bb5527934177ee9b8d) | A100 ×4 allocation; two used | Failed before optimizer startup: missing endpoint directory |
+| SETA whitebox | [6aaa75bbf76d6a098a710867](https://huggingface.co/jobs/HuggingEnvs/6aaa75bbf76d6a098a710867) | H200 ×2 | Failed before optimizer startup: missing endpoint directory |
+
+The clean-Job failure is fixed by creating the endpoint/log parent directories in `serve/vllm.sh`. The failed cohort is preserved. A second cohort exposed a deployed-server API mismatch in both async arms; those two jobs were stopped before optimizer updates (`6aaa77a65527934177ee9c30`, `6aaa77a65527934177ee9c32`). The Harbor client now omits only default provider/eval arguments. Explicit settings are still sent. The idle native OpenCode Space was upgraded to accept and enforce sampling. Training submission now checks the remote tool schema before allocating a GPU Job.
+
+Completion requires `training_smoke_verified.json`: exact capture, retained supervision, native optimizer state, remote restoration and changed weights. Pending jobs are not counted as passed.
+
+The v3 Harbor Job later ended with an HF Xet upload `TimeoutError` after completing all four updates. Its final cleanup published both checkpoints. A separate audit reconciled 37/37 completed captures, all 19,880 eligible supervised tokens and 25 optimizer rollout receipts; this does **not** waive the failed integrated qualification. The [failure receipt](qualification/harbor-v3-upload-failure.json) is preserved. The shared publisher now retries transient transport/429/5xx errors up to three attempts, keeps ready markers last, and allows an hour for an already-active full-checkpoint upload during shutdown. Permission and validation errors remain fatal. Fault-injection and full CPU regression tests passed. The v4 Harbor rerun completed successfully, including both checkpoint publications.
+
+## Preserved material
+
+Preparation uses separate Git worktrees. Original dirty worktrees, active services, source snapshots, raw captures and checkpoint files remain intact. Superseded local guides/build inputs moved into ignored `04-data-agent/temp/historical-notes/` and `temp/legacy-hf/`; replaced bundle outputs are archived in `temp/build-archive/`. Committed results contain compact scores, a static figure and a compressed full metrics/provenance snapshot. No raw task answers, credentials, model checkpoints or Trackio databases are committed.
diff --git a/04-data-agent/serve/vllm.sh b/04-data-agent/serve/vllm.sh
new file mode 100755
index 0000000..20d761a
--- /dev/null
+++ b/04-data-agent/serve/vllm.sh
@@ -0,0 +1,238 @@
+#!/bin/bash
+# Core launcher: vLLM OpenAI-compatible server + cloudflared tunnel.
+# Invoked from per-model .slurm files that pre-set environment variables.
+#
+# Required env vars:
+# MODEL HF model id (e.g. Qwen/Qwen3-4B)
+# TP_SIZE tensor-parallel size (per replica)
+# DP_SIZE data-parallel size (replicas, vLLM-managed)
+# MAX_MODEL_LEN max prompt+output tokens
+# SHORT_NAME short slug for log/url filenames (e.g. qwen3-4b)
+#
+# Optional env vars:
+# PORT default 8000
+# GPU_MEMORY_UTILIZATION default 0.92
+# TOOL_CALL_PARSER default hermes
+# REASONING_PARSER default qwen3 (empty disables)
+# EXTRA_VLLM_ARGS extra args appended verbatim
+# READY_TIMEOUT_SEC default 1800 (30 min, larger models need longer)
+# TRL_PROD default current working directory
+#
+# Outputs:
+# VLLM_URL_FILE (default: $TRL_PROD/temp/vllm-url-.txt)
+# VLLM_LOG (default: $TRL_PROD/temp/vllm-server-.log)
+
+set -e
+
+: "${MODEL:?MODEL is required}"
+: "${TP_SIZE:?TP_SIZE is required}"
+: "${DP_SIZE:?DP_SIZE is required}"
+: "${MAX_MODEL_LEN:?MAX_MODEL_LEN is required}"
+: "${SHORT_NAME:?SHORT_NAME is required}"
+
+# Derive a per-job port so co-located jobs (this cluster does not allocate
+# nodes exclusively) don't all collide on 8000. Range 8000-8999.
+PORT="${PORT:-$((8000 + ${SLURM_JOB_ID:-0} % 1000))}"
+GPU_MEMORY_UTILIZATION="${GPU_MEMORY_UTILIZATION:-0.92}"
+TOOL_CALL_PARSER="${TOOL_CALL_PARSER:-hermes}"
+# Use `:-` only when var is UNSET (not when explicitly empty), so per-model
+# slurm scripts can opt out of a reasoning parser with REASONING_PARSER="".
+REASONING_PARSER="${REASONING_PARSER-qwen3}"
+EXTRA_VLLM_ARGS="${EXTRA_VLLM_ARGS:-}"
+READY_TIMEOUT_SEC="${READY_TIMEOUT_SEC:-1800}"
+TRL_PROD="${TRL_PROD:-$PWD}"
+# gradio by default: it needs no binary download and no ingress, and these endpoints are reached
+# from off-cluster (evals, sandboxed agents). TUNNEL=cloudflared keeps the old path; TUNNEL=none
+# serves locally only.
+TUNNEL="${TUNNEL:-gradio}"
+CLOUDFLARED="${CLOUDFLARED:-$HOME/.local/bin/cloudflared}"
+
+cd "$TRL_PROD"
+# shellcheck disable=SC1091
+source "${VENV:-.venv312}/bin/activate" # .venv312 = the current (cuda-13/vllm-0.25) env
+export UV_LINK_MODE="${UV_LINK_MODE:-copy}"
+export VLLM_USE_AOT_COMPILE="${VLLM_USE_AOT_COMPILE:-0}" # avoids vLLM distributed-startup errors (trl-internal #206)
+
+# The API-server process waits VLLM_ENGINE_READY_TIMEOUT_S (default 600s) for
+# the engine cores to come up. With full CUDA-graph capture at large context
+# on a CPU-contended node, engine init can exceed 600s and the API server
+# bails out *after* the engines were nearly ready. Give it the same generous
+# budget we use for /health polling.
+export VLLM_ENGINE_READY_TIMEOUT_S="${VLLM_ENGINE_READY_TIMEOUT_S:-$READY_TIMEOUT_SEC}"
+
+NODE_HOSTNAME=$(hostname)
+NODE_IP=$(hostname -I | awk '{print $1}')
+
+TOTAL_GPUS=$((TP_SIZE * DP_SIZE))
+
+echo "============================================================"
+echo "vLLM + Cloudflare tunnel"
+echo " Model: $MODEL"
+echo " Short name: $SHORT_NAME"
+echo " Node: $NODE_HOSTNAME ($NODE_IP)"
+echo " Port: $PORT"
+echo " TP size: $TP_SIZE"
+echo " DP size: $DP_SIZE"
+echo " Total GPUs: $TOTAL_GPUS"
+echo " Max model len: $MAX_MODEL_LEN"
+echo " GPU mem util: $GPU_MEMORY_UTILIZATION"
+echo " Tool parser: $TOOL_CALL_PARSER"
+echo " Reasoning parser:$REASONING_PARSER"
+echo " Extra args: $EXTRA_VLLM_ARGS"
+echo " vllm bin: $(command -v vllm)"
+echo " python: $(which python)"
+echo " START TIME: $(date)"
+echo "============================================================"
+nvidia-smi --query-gpu=name,memory.total --format=csv,noheader 2>/dev/null || true
+
+# Install cloudflared if missing (skipped unless TUNNEL=cloudflared)
+if [ "$TUNNEL" = cloudflared ] && [ ! -f "$CLOUDFLARED" ]; then
+ echo ">>> Installing cloudflared to $CLOUDFLARED ..."
+ mkdir -p "$(dirname "$CLOUDFLARED")"
+ curl -sSL https://github.com/cloudflare/cloudflared/releases/latest/download/cloudflared-linux-amd64 \
+ -o "$CLOUDFLARED"
+ chmod +x "$CLOUDFLARED"
+fi
+
+VLLM_LOG="${VLLM_LOG:-$TRL_PROD/temp/vllm-server-${SLURM_JOB_ID:-$$}.log}"
+mkdir -p "$(dirname "$VLLM_LOG")"
+: > "$VLLM_LOG"
+echo ">>> vLLM log: $VLLM_LOG"
+
+# Build vllm args
+VLLM_ARGS=(
+ "$MODEL"
+ --host 0.0.0.0
+ --port "$PORT"
+ --tensor-parallel-size "$TP_SIZE"
+ --max-model-len "$MAX_MODEL_LEN"
+ --gpu-memory-utilization "$GPU_MEMORY_UTILIZATION"
+ --trust-remote-code
+ --enable-auto-tool-choice
+ --tool-call-parser "$TOOL_CALL_PARSER"
+)
+if [ "$DP_SIZE" -gt 1 ]; then
+ VLLM_ARGS+=(--data-parallel-size "$DP_SIZE")
+fi
+if [ -n "$REASONING_PARSER" ]; then
+ VLLM_ARGS+=(--reasoning-parser "$REASONING_PARSER")
+fi
+# ENFORCE_EAGER=1 skips torch.compile + CUDA-graph capture. Slower decode but
+# much faster, crash-free startup (the Inductor autotune path can fault under
+# memory pressure on shared nodes). Recommended for the large TP>=2 models.
+if [ "${ENFORCE_EAGER:-0}" = "1" ]; then
+ VLLM_ARGS+=(--enforce-eager)
+fi
+# ENABLE_THINKING=0 sets a SERVER-SIDE default of enable_thinking=false for the
+# chat template, so every client (incl. opencode, which makes its own API calls
+# and can't send per-request kwargs) gets thinking off. vLLM merges this with
+# request-level chat_template_kwargs (request wins). NOTE the flag is
+# --default-chat-template-kwargs (vLLM 0.18); plain --chat-template-kwargs does
+# NOT exist and errors with "unrecognized arguments".
+if [ "${ENABLE_THINKING:-1}" = "0" ]; then
+ VLLM_ARGS+=(--default-chat-template-kwargs '{"enable_thinking": false}')
+fi
+# shellcheck disable=SC2206
+EXTRA_ARR=($EXTRA_VLLM_ARGS)
+VLLM_ARGS+=("${EXTRA_ARR[@]}")
+
+echo ">>> Starting vLLM:"
+echo " vllm serve ${VLLM_ARGS[*]}"
+vllm serve "${VLLM_ARGS[@]}" >> "$VLLM_LOG" 2>&1 &
+VLLM_PID=$!
+
+# Stream the vllm log into job stdout so failures are immediately visible.
+tail -f -n +1 --pid=$$ "$VLLM_LOG" &
+TAIL_PID=$!
+
+# Wait for /health
+echo ">>> Waiting up to ${READY_TIMEOUT_SEC}s for vLLM /health ..."
+READY=0
+for i in $(seq 1 "$READY_TIMEOUT_SEC"); do
+ if curl -s "http://localhost:$PORT/health" > /dev/null 2>&1; then
+ echo ">>> vLLM ready after ${i}s"
+ READY=1
+ break
+ fi
+ if ! kill -0 "$VLLM_PID" 2>/dev/null; then
+ echo "ERROR: vLLM exited prematurely. Tail of log:"
+ tail -80 "$VLLM_LOG"
+ exit 1
+ fi
+ sleep 1
+done
+if [ "$READY" -ne 1 ]; then
+ echo "ERROR: vLLM did not become ready within ${READY_TIMEOUT_SEC}s. Tail:"
+ tail -80 "$VLLM_LOG"
+ kill "$VLLM_PID" 2>/dev/null || true
+ exit 1
+fi
+
+echo ">>> /v1/models:"
+curl -s "http://localhost:$PORT/v1/models" | python -m json.tool || true
+
+# Start the tunnel
+TUNNEL_LOG="/tmp/tunnel_${SHORT_NAME}_${SLURM_JOB_ID:-$$}.log"
+TUNNEL_URL=""
+TUNNEL_PID=""
+echo ""
+echo "============================================================"
+echo ">>> Starting tunnel: $TUNNEL (log: $TUNNEL_LOG)"
+echo "============================================================"
+case "$TUNNEL" in
+ gradio)
+ # The helper only prints TUNNEL_URL= after proving a request THROUGH the tunnel reaches this
+ # server, so a URL here is a working endpoint rather than merely a resolving hostname.
+ python "$(dirname "${BASH_SOURCE[0]}")/../gradio_tunnel.py" \
+ --port "$PORT" --verify-path /health > "$TUNNEL_LOG" 2>>"$TUNNEL_LOG" &
+ TUNNEL_PID=$!
+ for i in $(seq 1 60); do
+ TUNNEL_URL=$(grep -o 'TUNNEL_URL=https://[^ ]*' "$TUNNEL_LOG" 2>/dev/null | head -1 | cut -d= -f2-)
+ [ -n "$TUNNEL_URL" ] && break
+ if grep -q "TUNNEL_ERROR=" "$TUNNEL_LOG" 2>/dev/null; then
+ echo "ERROR: gradio tunnel did not reach the server:"; cat "$TUNNEL_LOG"; break
+ fi
+ sleep 5
+ done
+ ;;
+ cloudflared)
+ "$CLOUDFLARED" tunnel --url "http://localhost:$PORT" 2>&1 | tee "$TUNNEL_LOG" &
+ TUNNEL_PID=$!
+ for i in $(seq 1 60); do
+ TUNNEL_URL=$(grep -o 'https://[a-z0-9-]*\.trycloudflare\.com' "$TUNNEL_LOG" 2>/dev/null | head -1)
+ [ -n "$TUNNEL_URL" ] && break
+ sleep 1
+ done
+ ;;
+ none)
+ echo ">>> TUNNEL=none, serving locally only"
+ ;;
+esac
+
+URL_FILE="${VLLM_URL_FILE:-$TRL_PROD/temp/vllm-url-${SLURM_JOB_ID:-$$}.txt}"
+mkdir -p "$(dirname "$URL_FILE")"
+echo "$TUNNEL_URL" > "$URL_FILE"
+
+echo ""
+echo "============================================================"
+echo ">>> TUNNEL URL: ${TUNNEL_URL:-}"
+echo ""
+echo " OpenAI endpoint: ${TUNNEL_URL}/v1"
+echo " Models: ${TUNNEL_URL}/v1/models"
+echo " Health: ${TUNNEL_URL}/health"
+echo " Local: http://${NODE_HOSTNAME}:${PORT}/v1"
+echo " URL file: $URL_FILE"
+echo "============================================================"
+echo ""
+
+# Quick self-test against the tunnel (optional, ignore failures)
+if [ -n "$TUNNEL_URL" ]; then
+ echo ">>> Self-test (chat completion via tunnel) ..."
+ curl -sS --max-time 60 -X POST "${TUNNEL_URL}/v1/chat/completions" \
+ -H "Content-Type: application/json" \
+ -d "{\"model\":\"$MODEL\",\"messages\":[{\"role\":\"user\",\"content\":\"Reply with the single word PONG.\"}],\"max_tokens\":16,\"temperature\":0}" \
+ | python -m json.tool 2>/dev/null | head -40 || echo " (self-test failed or no JSON returned — server may still be warming up)"
+fi
+
+# Keep running until vllm exits (or job time elapses).
+wait "$VLLM_PID"
diff --git a/04-data-agent/tools/audit_blackbox.py b/04-data-agent/tools/audit_blackbox.py
new file mode 100644
index 0000000..a5963d2
--- /dev/null
+++ b/04-data-agent/tools/audit_blackbox.py
@@ -0,0 +1,10 @@
+"""Use the original baseline's exact-token auditor on the Daytona captures."""
+import argparse
+from pathlib import Path
+from baseline_checks import audit_captures
+
+p=argparse.ArgumentParser();p.add_argument('--phase',choices=['smoke','resume','final'],required=True)
+p.add_argument('--output-root',type=Path,default=Path(__file__).resolve().parents[1]/'logs/20260915/blackbox')
+args=p.parse_args()
+root=args.output_root
+audit_captures(root,root,args.phase)
diff --git a/04-data-agent/tools/audit_multiharness_training.py b/04-data-agent/tools/audit_multiharness_training.py
new file mode 100644
index 0000000..a34d4aa
--- /dev/null
+++ b/04-data-agent/tools/audit_multiharness_training.py
@@ -0,0 +1,73 @@
+"""Replay saved training captures through the same lossless TITO checks as the smoke test."""
+
+import argparse
+import collections
+import json
+import os
+from pathlib import Path
+
+from openenv.harbor.models import HarborRolloutResult
+from smoke_multiharness_tito import audit
+
+
+def save_json(path, value):
+ temporary = path.with_suffix(f'.{os.getpid()}.tmp')
+ temporary.write_text(json.dumps(value, indent=2) + '\n')
+ temporary.replace(path)
+
+
+def summarize_checks(checks, token_budget):
+ by_harness = collections.defaultdict(list)
+ for check in checks:
+ by_harness[check['harness']].append(check)
+ return {h: {
+ 'rollouts': len(rows), 'completed_results': sum(r['completed_result'] for r in rows),
+ 'incomplete_results': sum(not r['completed_result'] for r in rows),
+ 'tito_pass': sum(r['tito_pass'] for r in rows),
+ 'graded': sum(r.get('reward') is not None for r in rows),
+ 'reward_sum': sum(r.get('reward') or 0 for r in rows),
+ 'task_indices': sorted({r['task_index'] for r in rows}), 'token_budget': token_budget,
+ 'largest_row_tokens': max(r.get('largest_row_tokens', 0) for r in rows),
+ **{key: sum(r.get(key, 0) for r in rows)
+ for key in ('rows', 'eligible_tokens', 'retained_tokens', 'rows_over_40960', 'rows_over_token_budget')},
+ } for h, rows in sorted(by_harness.items())}
+
+
+def main():
+ parser = argparse.ArgumentParser(description=__doc__)
+ parser.add_argument("directory", type=Path, help="Job's audit directory")
+ args = parser.parse_args()
+ config_path = args.directory.parent / "run_config.json"
+ config = json.loads(config_path.read_text()) if config_path.exists() else {}
+ training = config.get("training", config)
+ token_budget = (training['max_model_len'] if training.get('atomic_rollouts')
+ else training.get("token_budget", 40960))
+ output = args.directory / "tito_checks"
+ output.mkdir(parents=True, exist_ok=True)
+ checks = []
+ for path in sorted((args.directory / "rollouts").glob("*.json")):
+ cached = output / path.name
+ if cached.exists():
+ previous = json.loads(cached.read_text())
+ if previous.get("token_budget") == token_budget and previous.get('audit_schema_version') == 2:
+ checks.append(previous)
+ continue
+ record = json.loads(path.read_text())
+ check = {k: record[k] for k in ("group_id", "episode_id", "harness", "task_index")}
+ check.update(audit_schema_version=2, token_budget=token_budget,
+ completed_result=record.get('result') is not None)
+ try:
+ result = HarborRolloutResult.model_validate(record["result"])
+ validation, _ = audit(result, token_budget=token_budget)
+ check.update(validation, reward=result.reward, ok=result.ok, error=result.error)
+ except Exception as exc: # noqa: BLE001 - record a failed capture and audit the remaining ones
+ check.update(tito_pass=False, error=f"{type(exc).__name__}: {exc}")
+ save_json(cached, check)
+ checks.append(check)
+ summary = summarize_checks(checks, token_budget)
+ save_json(args.directory / "tito_summary.json", summary)
+ print(json.dumps(summary, indent=2))
+
+
+if __name__ == "__main__":
+ main()
diff --git a/04-data-agent/tools/cleanup_daytona.py b/04-data-agent/tools/cleanup_daytona.py
new file mode 100644
index 0000000..8809d34
--- /dev/null
+++ b/04-data-agent/tools/cleanup_daytona.py
@@ -0,0 +1,50 @@
+"""Delete only this comparison's owned Daytona sandboxes after the owning job exits."""
+import argparse
+import concurrent.futures
+import json
+import time
+from pathlib import Path
+
+from dotenv import load_dotenv
+from daytona import Daytona, ListSandboxesQuery
+
+
+def main():
+ p=argparse.ArgumentParser(description=__doc__)
+ group=p.add_mutually_exclusive_group(required=True)
+ group.add_argument('--legacy-baseline',action='store_true')
+ group.add_argument('--owner')
+ p.add_argument('--out',type=Path,required=True)
+ args=p.parse_args()
+ repo=Path(__file__).resolve().parents[3]
+ load_dotenv(repo/'experiments/.env')
+ client=Daytona()
+ labels={'experiment':'daytona-harness-comparison','run':'20260915'}
+ if args.owner: labels['owner']=args.owner
+ sandboxes=list(client.list(ListSandboxesQuery(labels=labels),request_timeout=30))
+ if args.legacy_baseline:
+ sandboxes=[s for s in sandboxes if 'owner' not in s.labels]
+ def remove(sandbox):
+ try:
+ client.delete(sandbox,timeout=90,wait=True)
+ return {'sandbox_id':sandbox.id,'deleted':True}
+ except Exception as exc:
+ return {'sandbox_id':sandbox.id,'deleted':False,'error_type':type(exc).__name__}
+ with concurrent.futures.ThreadPoolExecutor(max_workers=8) as pool:
+ results=list(pool.map(remove,sandboxes))
+ # A successful wait=True delete can precede the list index update. Verify
+ # disappearance with a bounded poll before reporting a leaked sandbox.
+ deadline=time.monotonic()+60
+ while True:
+ remaining=list(client.list(ListSandboxesQuery(labels=labels),request_timeout=30))
+ if args.legacy_baseline: remaining=[s for s in remaining if 'owner' not in s.labels]
+ if not remaining or time.monotonic()>=deadline: break
+ time.sleep(5)
+ report={'labels':labels,'legacy_baseline':args.legacy_baseline,'results':results,'remaining':len(remaining)}
+ args.out.parent.mkdir(parents=True,exist_ok=True)
+ args.out.write_text(json.dumps(report,indent=2)+'\n')
+ print(json.dumps({'selected':len(sandboxes),'remaining':len(remaining)}))
+ return 2 if remaining else 0
+
+
+if __name__=='__main__': raise SystemExit(main())
diff --git a/04-data-agent/tools/eval_whitebox_native.py b/04-data-agent/tools/eval_whitebox_native.py
new file mode 100644
index 0000000..ee6b3e9
--- /dev/null
+++ b/04-data-agent/tools/eval_whitebox_native.py
@@ -0,0 +1,234 @@
+"""Resumable pass@1 using frozen TRL's actual token-preserving bash/SETA loop."""
+from __future__ import annotations
+
+import argparse
+import concurrent.futures
+import copy
+import inspect
+import json
+import math
+import time
+import uuid
+from pathlib import Path
+from types import SimpleNamespace
+
+import httpx
+from transformers import AutoProcessor
+from transformers.utils import get_json_schema
+from trl.chat_template_utils import (
+ add_response_schema, get_training_chat_template,
+ is_chat_template_prefix_preserving, parse_response,
+)
+from trl.trainer.grpo_trainer import GRPOTrainer
+from whitebox_bash import white_box_bash_env
+
+MODEL = 'Qwen/Qwen3.5-2B'
+REVISION = '15852e8c16360a2fea060d615a32b45270f8a8fc'
+SYSTEM = (
+ 'You are a terminal agent working in a sandbox. Use the available tools to inspect the '
+ 'filesystem and solve the task. Work step by step: look before you act. When you are confident, '
+ 'call submit_solution with the final answer and nothing else -- not the command that would '
+ 'produce it.'
+)
+
+
+class NativeLoop:
+ _get_tool_suffix_ids = GRPOTrainer._get_tool_suffix_ids
+ _tool_call_loop = GRPOTrainer._tool_call_loop
+
+ def __init__(self, env, processor, url, model, deadline):
+ self.processing_class = processor
+ self._tokenizer = processor.tokenizer
+ self._is_vlm = True
+ self.chat_template = (
+ None if is_chat_template_prefix_preserving(processor)
+ else get_training_chat_template(processor)
+ )
+ self.chat_template_kwargs = {'enable_thinking': False}
+ methods = {
+ n:m for n,m in inspect.getmembers(env,predicate=inspect.ismethod)
+ if not n.startswith('_') and n not in {'reset','get_reward'}
+ }
+ self.tools = [get_json_schema(m) for m in methods.values()]
+ self._sync_tool_dicts = [methods]
+ self._async_tool_dicts = [{}]
+ self.max_tool_calling_iterations = 16 # first generation + 16 continuations = 17 calls
+ self.max_completion_length = 16384 # includes masked tool-result tokens, as in sync training
+ self.use_vllm = True
+ self.vllm_mode = 'server'
+ self.model = SimpleNamespace(config=SimpleNamespace(text_config=SimpleNamespace(max_position_embeddings=131072)))
+ self.url, self.model_name, self.deadline = url.rstrip('/'), model, deadline
+ self.client = httpx.Client(timeout=120)
+ self.calls = []
+ self.stop_reason = None
+
+ def _generate_single_turn(self, prompts, images, multimodal_fields, has_tool_images=False):
+ assert len(prompts)==1 and not images and not has_tool_images
+ remaining = self.deadline-time.monotonic()
+ if remaining <= 0:
+ self.stop_reason = 'episode_deadline'
+ return [[]], [[]]
+ prompt=prompts[0]
+ if self.calls:
+ parent=self.calls[-1]['prompt_ids']+self.calls[-1]['completion_ids']
+ assert prompt[:len(parent)]==parent, 'native loop lost the sampled prefix'
+ try:
+ response=self.client.post(self.url+'/completions',json={
+ 'model':self.model_name,'prompt':prompt,'max_tokens':min(4096,131072-len(prompt)),
+ 'temperature':0.8,'top_p':1.0,'top_k':-1,'n':1,'logprobs':0,
+ 'return_token_ids':True,'return_tokens_as_token_ids':True,
+ },timeout=remaining).raise_for_status().json()
+ except httpx.ReadTimeout:
+ if time.monotonic() < self.deadline:
+ raise # An earlier transport failure remains an ungraded attempt.
+ self.stop_reason = 'episode_deadline'
+ # The native loop already accepts an empty continuation on truncation.
+ # Stop without inventing tokens; grade work done within the episode budget.
+ return [[]], [[]]
+ choice=response['choices'][0]
+ ids=choice['token_ids']
+ lp=choice['logprobs']
+ assert choice['prompt_token_ids']==prompt, 'engine prompt differs from supplied token IDs'
+ assert ids and len(ids)==len(lp['tokens'])==len(lp['token_logprobs'])
+ assert lp['tokens']==[f'token_id:{i}' for i in ids], 'sampled token/logprob pairing mismatch'
+ assert all(math.isfinite(p) for p in lp['token_logprobs'])
+ self.calls.append({'prompt_ids':prompt.copy(),'completion_ids':ids.copy(),
+ 'logprobs':lp['token_logprobs'].copy(),'finish_reason':choice['finish_reason']})
+ return [ids], [lp['token_logprobs']]
+
+ def run(self, prompt):
+ prompts=[[{'role':'system','content':SYSTEM},{'role':'user','content':'Solve the task.'+prompt}]]
+ tokenized=self.processing_class.apply_chat_template(
+ prompts[0], tools=self.tools, add_generation_prompt=True,tokenize=True,
+ chat_template=self.chat_template,return_dict=False,**self.chat_template_kwargs,
+ )
+ ids=tokenized[0] if isinstance(tokenized[0],list) else tokenized
+ generated,lps=self._generate_single_turn([ids],None,{})
+ completions=[[parse_response(self._tokenizer,generated[0],prefix=ids)]]
+ masks,completions,generated,lps,tools,failures,_=self._tool_call_loop(
+ copy.deepcopy(prompts),[ids],generated,completions,lps,None,{},
+ )
+ mask,completion,logprobs=masks[0],generated[0],lps[0]
+ all_sampled=[t for call in self.calls for t in call['completion_ids']]
+ all_lp=[p for call in self.calls for p in call['logprobs']]
+ supervised=[t for t,m in zip(completion,mask,strict=True) if m]
+ supervised_lp=[p for p,m in zip(logprobs,mask,strict=True) if m]
+ checks={
+ 'sampled_ids_preserved':supervised==all_sampled[:len(supervised)],
+ 'sampled_logprobs_preserved':supervised_lp==all_lp[:len(supervised_lp)],
+ 'tool_context_masked':all(p==0.0 for p,m in zip(logprobs,mask,strict=True) if not m),
+ 'loss_mask_binary':set(mask)<={0,1},
+ 'real_logprobs':bool(supervised_lp) and any(p<0 for p in supervised_lp),
+ 'finite_logprobs':all(math.isfinite(p) for p in logprobs),
+ }
+ assert all(checks.values()), checks
+ return {'prompt_ids':ids,'completion_ids':completion,'logprobs':logprobs,'loss_mask':mask,
+ 'calls':self.calls,'messages':completions,'tito_checks':checks,'tito_pass':True,
+ 'turns':len(self.calls),'tool_calls':tools,'tool_failures':failures,
+ 'stop_reason':self.stop_reason,'budget_policy':'600s episode; empty native continuation at deadline; no fabricated token IDs'}
+
+
+def episode(args,index,processor):
+ start=time.monotonic()
+ env=white_box_bash_env(args.server,toolsets='bash,seta',step_limit=17,timeout_s=600)()
+ loop=None
+ rec={'index':index,'reward':None,'tito_pass':False,'harness':'whitebox_seta','pass_k':1}
+ try:
+ prompt=env.reset(split='test',index=index)
+ loop=NativeLoop(env,processor,args.vllm_url,args.model,time.monotonic()+600)
+ capture=loop.run(prompt)
+ reward=env.get_reward()
+ if not math.isfinite(reward):
+ raise RuntimeError('episode ungraded')
+ assert reward in (0,1)
+ path=args.out/'captures'/f'{index:03d}-{uuid.uuid4().hex}.json'
+ path.write_text(json.dumps(capture)+'\n')
+ rec.update({k:capture[k] for k in ('tito_pass','turns','tool_calls','tool_failures','stop_reason')})
+ rec.update(reward=reward,capture_file=str(path),supervised_tokens=sum(capture['loss_mask']),
+ forwarded_tokens=len(capture['prompt_ids'])+len(capture['completion_ids']))
+ except Exception as exc:
+ rec['error_type']=type(exc).__name__
+ # Local exception diagnostic; do not put prompts, provider bodies or signed URLs in reports.
+ import traceback
+ (args.out/f'error-{index:03d}-{uuid.uuid4().hex}.txt').write_text(traceback.format_exc())
+ finally:
+ if env._session is not None:
+ try: env.get_reward()
+ except Exception: pass
+ env._mcp.close()
+ if loop is not None: loop.client.close()
+ rec['elapsed_s']=time.monotonic()-start
+ return rec
+
+
+def main():
+ p=argparse.ArgumentParser(description=__doc__)
+ p.add_argument('--run',type=Path,required=True)
+ p.add_argument('--server',required=True)
+ p.add_argument('--vllm-url',required=True)
+ p.add_argument('--model',default=MODEL)
+ p.add_argument('--out',type=Path,required=True)
+ p.add_argument('--concurrency',type=int,default=4)
+ p.add_argument('--max-new-rollouts',type=int,default=0)
+ p.add_argument('--require-complete',action='store_true')
+ p.add_argument('--ramp',action='store_true',help='Measure 8 slots, then 32, then 100; stop if a ramp grades under 90%.')
+ args=p.parse_args()
+ args.out=args.out.resolve()
+ args.out.mkdir(parents=True,exist_ok=True)
+ (args.out/'captures').mkdir(exist_ok=True)
+ config={'model':args.model,'revision':REVISION,'dataset':str(args.run/'datasets/test'),
+ 'vllm_url':args.vllm_url,'server':args.server,'pass_k':1,'temperature':0.8,'top_p':1.0,
+ 'max_output_tokens_per_call':4096,'max_episode_completion_tokens':16384,'max_model_calls':17,
+ 'toolsets':['bash','seta'],'native_loop_source':inspect.getfile(GRPOTrainer),
+ 'source_manifest':str(args.run/'source_input_hashes.json')}
+ manifest=args.out/'eval_config.json'
+ if manifest.exists(): assert json.loads(manifest.read_text())==config, 'resume config mismatch'
+ else: manifest.write_text(json.dumps(config,indent=2)+'\n')
+ records=args.out/'attempts.jsonl'
+ selected={}
+ if records.exists():
+ for line in records.read_text().splitlines():
+ r=json.loads(line)
+ if r.get('reward') in (0,1) and r.get('tito_pass'):
+ selected.setdefault(r['index'],r)
+ indices=list(map(int,(args.run/'test_indices.txt').read_text().replace(',',' ').split()))
+ processor=AutoProcessor.from_pretrained(MODEL,revision=REVISION)
+ tokenizer=processor.tokenizer
+ if not (getattr(tokenizer,'response_template',None) or getattr(tokenizer,'response_schema',None)):
+ processor=add_response_schema(processor)
+ start=time.monotonic()
+ phases=[(8,8),(32,32),(100,0)] if args.ramp else [(args.concurrency,args.max_new_rollouts)]
+ ramp=[]
+ for concurrency,limit in phases:
+ pending=[i for i in indices if i not in selected]
+ if limit: pending=pending[:limit]
+ before=len(selected);phase_start=time.monotonic()
+ with records.open('a') as out, concurrent.futures.ThreadPoolExecutor(max_workers=concurrency) as pool:
+ futures=[pool.submit(episode,args,i,processor) for i in pending]
+ for f in concurrent.futures.as_completed(futures):
+ r=f.result()
+ out.write(json.dumps(r)+'\n');out.flush()
+ if r['reward'] in (0,1) and r['tito_pass']: selected.setdefault(r['index'],r)
+ print(json.dumps({'graded':len(selected),'latest':r}),flush=True)
+ ramp.append({'concurrency':concurrency,'attempted':len(pending),'graded':len(selected)-before,
+ 'elapsed_s':time.monotonic()-phase_start})
+ (args.out/'ramp.json').write_text(json.dumps(ramp,indent=2)+'\n')
+ if args.ramp and len(selected)-before < 0.9*len(pending): break
+ manifest=json.loads((args.run/'test_manifest.json').read_text())
+ difficulty={}
+ for level in ('easy','medium','hard'):
+ subset=[r for i,r in selected.items() if manifest['tasks'][i]['difficulty']==level]
+ difficulty[level]={'graded':len(subset),'correct':sum(r['reward'] for r in subset)}
+ report={'metric':'pass@1','complete':len(selected)==250,'graded':len(selected),'expected':250,
+ 'correct':sum(r['reward'] for r in selected.values()),
+ 'pass_at_1':sum(r['reward'] for r in selected.values())/len(selected) if selected else None,
+ 'graded_indices':sorted(selected),'difficulty':difficulty,'concurrency':args.concurrency,
+ 'phase_elapsed_s':time.monotonic()-start,'ramp':ramp,
+ 'tito_pass':bool(selected) and all(r['tito_pass'] for r in selected.values())}
+ (args.out/'scores.json').write_text(json.dumps(report,indent=2)+'\n')
+ print(json.dumps(report),flush=True)
+ return 2 if args.require_complete and not report['complete'] else 0
+
+
+if __name__=='__main__':
+ raise SystemExit(main())
diff --git a/04-data-agent/tools/generation_routing.py b/04-data-agent/tools/generation_routing.py
new file mode 100644
index 0000000..48ff9a1
--- /dev/null
+++ b/04-data-agent/tools/generation_routing.py
@@ -0,0 +1,12 @@
+"""Protect native server generation's duplicate-prompt optimization after tools."""
+
+
+def generation_group_size(prompts, requested):
+ if requested<1:
+ raise ValueError('generation group size must be positive')
+ if len(prompts)%requested:
+ return 1
+ for start in range(0,len(prompts),requested):
+ if any(prompt!=prompts[start] for prompt in prompts[start:start+requested]):
+ return 1
+ return requested
diff --git a/04-data-agent/tools/monitor_multi4.py b/04-data-agent/tools/monitor_multi4.py
new file mode 100644
index 0000000..524d18e
--- /dev/null
+++ b/04-data-agent/tools/monitor_multi4.py
@@ -0,0 +1,228 @@
+"""CPU-only Slurm watchdog. Writes local status/alerts; never changes or stops training."""
+import argparse
+from datetime import datetime, timezone
+import json
+import math
+import os
+from pathlib import Path
+import subprocess
+import sys
+import time
+
+TERMINAL = {'COMPLETED', 'FAILED', 'CANCELLED', 'TIMEOUT', 'OUT_OF_MEMORY',
+ 'NODE_FAIL', 'PREEMPTED', 'BOOT_FAIL', 'DEADLINE', 'REVOKED'}
+
+
+def read_json(path, default=None):
+ return json.loads(path.read_text()) if path.exists() else default
+
+
+def read_metrics(path):
+ if not path.exists():
+ return []
+ lines = path.read_text().splitlines(keepends=True)
+ # A trainer can be in the middle of appending its latest line.
+ return [json.loads(line) for line in lines if line.endswith('\n') and line.strip()]
+
+
+def slurm_states(jobs):
+ if not jobs:
+ return {}
+ result = subprocess.check_output(['sacct', '-X', '-n', '-P', '-j', ','.join(map(str, jobs)),
+ '--format=JobID,State'], text=True, timeout=30)
+ found = {line.split('|')[0]: line.split('|')[1].split()[0].rstrip('+')
+ for line in result.splitlines() if '|' in line}
+ return {str(job): found.get(str(job), 'UNKNOWN') for job in jobs}
+
+
+def resolved_empty_captures(root, logs, harness):
+ """Resolve individually investigated incidents without waiving token validation.
+
+ Historical failed attempts remain failed in the audit and dashboard. Only named
+ incidents with verified recovery can stop paging; every new failure still alerts.
+ An acknowledgement can never resolve a capture containing model tokens.
+ """
+ incidents = read_json(root / 'monitor/resolved_incidents.json', {}).get('episodes', {})
+ failed, resolved = set(), set()
+ for path in (logs / 'audit/tito_checks').glob('*.json'):
+ check = read_json(path)
+ if (check.get('harness') != harness or not check.get('completed_result')
+ or check.get('tito_pass')):
+ continue
+ failed.add(path.stem)
+ incident = incidents.get(path.stem, {})
+ if not incident.get('verified_at') or not incident.get('recovery_evidence'):
+ continue
+ result = (read_json(logs / 'audit/rollouts' / path.name, {}) or {}).get('result') or {}
+ fatal = [f for f in result.get('findings', []) if '[FATAL]' in f]
+ if (result.get('n_turns') == 0 and result.get('n_trainable_tokens') == 0
+ and result.get('turns') == [] and fatal
+ and all(f.startswith('[FATAL] no_turns:') for f in fatal)):
+ resolved.add(path.stem)
+ return failed, resolved
+
+
+def inspect_run(root, job, states, *, now=None):
+ now = time.time() if now is None else now
+ logs = root / f'job-{job}'
+ config = read_json(root / 'run_config.json', {})
+ metrics_path = logs / 'audit/metrics.jsonl'
+ metrics = read_metrics(metrics_path)
+ updates = [m for m in metrics if 'grad_norm' in m]
+ latest = updates[-1] if updates else {}
+ resume_step = config.get('resume_state', {}).get('step', 0)
+ step = latest.get('step', resume_step)
+ alerts = []
+ state = states.get(str(job), 'UNKNOWN')
+ if state in TERMINAL - {'COMPLETED'}:
+ alerts.append(f'training_{state.lower()}')
+ if state == 'UNKNOWN':
+ alerts.append('training_state_unknown')
+ for m in updates[-5:]:
+ if any(isinstance(m.get(k), (float, int)) and not math.isfinite(m[k])
+ for k in ('loss', 'grad_norm', 'ratio')):
+ alerts.append('nonfinite_training_metric')
+ if len(updates) >= 5 and all(m.get('grad_norm', 0) == 0 for m in updates[-5:]):
+ alerts.append('five_updates_without_gradient')
+ for metric, label in [('sample/dropped_stale_total', 'stale_rows_dropped'),
+ ('batch/dropped_oversize_total', 'oversize_rows_dropped')]:
+ if any(m.get(metric, 0) > 0 for m in updates[-5:]):
+ alerts.append(label)
+ age = now - metrics_path.stat().st_mtime if metrics_path.exists() else None
+ if state == 'RUNNING' and age is not None and age > 1800:
+ alerts.append('no_optimizer_update_for_30_minutes')
+ if state == 'RUNNING' and age is None and logs.exists() and now - logs.stat().st_mtime > 1800:
+ alerts.append('no_optimizer_metrics_after_startup')
+ tito_path = logs / 'audit/tito_summary.json'
+ tito = read_json(tito_path, {})
+ resolved_incidents = {}
+ for harness, summary in tito.items():
+ if summary['tito_pass'] != summary.get('completed_results', summary['rollouts']):
+ failed, resolved = resolved_empty_captures(root, logs, harness)
+ expected_failures = summary.get('completed_results', summary['rollouts']) - summary['tito_pass']
+ if failed != resolved or len(failed) != expected_failures:
+ alerts.append(f'tito_failure:{harness}')
+ if resolved:
+ resolved_incidents[harness] = sorted(resolved)
+ if summary.get('incomplete_results', 0) and state == 'RUNNING':
+ alerts.append(f'incomplete_rollout:{harness}')
+ if summary['retained_tokens'] != summary['eligible_tokens'] or summary['rows_over_token_budget']:
+ alerts.append(f'token_retention_failure:{harness}')
+ evals = read_json(root / 'checkpoint-evals/state.json', {})
+ evaluation = []
+ for key, record in evals.items():
+ eval_state = states.get(str(record.get('job_id')), record.get('slurm_state', 'UNKNOWN'))
+ result = record.get('scores', {})
+ if eval_state in TERMINAL - {'COMPLETED'}:
+ alerts.append(f'eval_{eval_state.lower()}:{Path(key).name}')
+ if eval_state == 'COMPLETED' and result and not result.get('comparison_ready', False):
+ alerts.append(f'eval_not_comparable:{Path(key).name}')
+ evaluation.append({'checkpoint': key, 'job_id': record.get('job_id'), 'state': eval_state,
+ 'scores': result})
+ submission = read_json(root / 'submission.json', {})
+ watcher = submission.get('eval_watcher')
+ if watcher and (states.get(str(watcher)) in TERMINAL - {'COMPLETED'}
+ or states.get(str(watcher)) == 'COMPLETED' and state not in TERMINAL):
+ alerts.append('eval_watcher_stopped_early')
+ saved = sorted(int(p.name.split('-')[1]) for p in (logs / 'run').glob('checkpoint-*')
+ if (p / 'checkpoint.saved.json').exists() or (p / 'checkpoint.ready.json').exists())
+ interval = config.get('evaluation', {}).get('interval_optimizer_steps', 100)
+ submitted = {int(Path(k).name.split('-')[1]) for k in evals}
+ pending = [s for s in saved if s % interval == 0 and s not in submitted]
+ return {'checked_at': datetime.fromtimestamp(now, timezone.utc).isoformat(),
+ 'job_id': job, 'training_state': state, 'optimizer_step': step,
+ 'resumed_from_step': resume_step,
+ 'new_optimizer_updates': len(updates),
+ 'awaiting_first_optimizer_update': not updates and state == 'RUNNING',
+ 'metrics_age_seconds': age, 'last_metrics': latest,
+ 'counters': {'stale_rows_dropped': sum(m.get('sample/dropped_stale_total', 0) for m in updates),
+ 'oversize_rows_dropped': sum(m.get('batch/dropped_oversize_total', 0) for m in updates),
+ 'nonzero_gradient_updates': sum(m.get('grad_norm', 0) > 0 for m in updates),
+ 'resolved_empty_capture_incidents': sum(map(len, resolved_incidents.values()))},
+ 'resolved_empty_capture_incidents': resolved_incidents,
+ 'coverage': read_json(logs / 'audit/coverage.json', {}), 'tito': tito,
+ 'tito_audit_age_seconds': now - tito_path.stat().st_mtime if tito_path.exists() else None,
+ 'saved_checkpoints': saved, 'queued_eval_steps': pending, 'evaluations': evaluation,
+ 'job_states': states, 'alerts': sorted(set(alerts)),
+ 'monitoring_scope': 'local artifacts only; no chat wakeups or external notifications'}
+
+
+def next_interval(previous, current, stable_checks, config):
+ threshold = config.get('stable_after_optimizer_step', 10)
+ required = config.get('required_progressing_checks', 2)
+ if (current['alerts'] or not previous or current['optimizer_step'] < threshold
+ or current['optimizer_step'] < previous['optimizer_step']):
+ stable_checks = 0
+ elif current['optimizer_step'] > previous['optimizer_step']:
+ stable_checks += 1
+ # An unchanged step during an ordinary rollout wait is not a failure. Keep the
+ # evidence of healthy updates; inspect_run's stall/error alerts reset it above.
+ interval = config['stable_interval_seconds'] if stable_checks >= required else config['startup_interval_seconds']
+ return stable_checks, interval
+
+
+def main():
+ p = argparse.ArgumentParser(description=__doc__)
+ p.add_argument('--run', type=Path, required=True)
+ p.add_argument('--train-job', required=True)
+ p.add_argument('--watch', action='store_true')
+ p.add_argument('--audit', action='store_true', help='Replay newly captured rollouts on this CPU job')
+ args = p.parse_args()
+ config = read_json(args.run / 'run_config.json')['monitoring']
+ destination = args.run / 'monitor'
+ destination.mkdir(exist_ok=True)
+ previous, stable_checks = None, 0
+ while True:
+ cycle_started = time.monotonic()
+ try:
+ submission = read_json(args.run / 'submission.json', {})
+ evals = read_json(args.run / 'checkpoint-evals/state.json', {})
+ jobs = {args.train_job} | {str(j) for j in submission.values() if str(j).isdigit()}
+ jobs.update(str(r[k]) for r in evals.values() for k in ('job_id', 'cleanup_job_id') if r.get(k))
+ audit_error = None
+ if args.audit:
+ audit_dir = args.run / f'job-{args.train_job}/audit'
+ if audit_dir.exists():
+ with (destination / 'audit.log').open('a') as stream:
+ try:
+ subprocess.run(['nice', '-n', '10', sys.executable,
+ str(Path(__file__).with_name('audit_multiharness_training.py')), str(audit_dir)],
+ check=True, timeout=600, stdout=stream, stderr=subprocess.STDOUT)
+ except (subprocess.SubprocessError, OSError) as exc:
+ audit_error = type(exc).__name__
+ status = inspect_run(args.run, args.train_job, slurm_states(jobs))
+ if audit_error:
+ status['alerts'].append('capture_audit_failed:' + audit_error)
+ stable_checks, interval = next_interval(previous, status, stable_checks, config)
+ status['next_check_seconds'] = interval
+ temporary = destination / 'status.json.tmp'
+ temporary.write_text(json.dumps(status, indent=2) + '\n')
+ temporary.replace(destination / 'status.json')
+ with (destination / 'history.jsonl').open('a') as stream:
+ stream.write(json.dumps(status) + '\n')
+ old_alerts = set(previous['alerts']) if previous else set()
+ changes = {'new': sorted(set(status['alerts']) - old_alerts),
+ 'resolved': sorted(old_alerts - set(status['alerts']))}
+ if changes['new'] or changes['resolved']:
+ with (destination / 'alerts.jsonl').open('a') as stream:
+ stream.write(json.dumps({'checked_at': status['checked_at'], **changes}) + '\n')
+ print(json.dumps({'step': status['optimizer_step'], 'state': status['training_state'],
+ 'alerts': status['alerts'], 'next_check_seconds': interval}), flush=True)
+ previous = status
+ # Continue through final audit/cleanup and outstanding evals; exclude our own monitor job.
+ other_jobs = {j: s for j, s in status['job_states'].items() if j != str(submission.get('monitor'))}
+ if status['training_state'] in TERMINAL and all(s in TERMINAL for s in other_jobs.values()):
+ break
+ except Exception as exc:
+ with (destination / 'alerts.jsonl').open('a') as stream:
+ stream.write(json.dumps({'checked_at': time.time(), 'monitor_error': str(exc)}) + '\n')
+ if not args.watch:
+ raise
+ interval = config['startup_interval_seconds']
+ if not args.watch:
+ break
+ time.sleep(max(1, interval - (time.monotonic() - cycle_started)))
+
+
+if __name__ == '__main__':
+ main()
diff --git a/04-data-agent/tools/score_comparison.py b/04-data-agent/tools/score_comparison.py
new file mode 100644
index 0000000..e16a876
--- /dev/null
+++ b/04-data-agent/tools/score_comparison.py
@@ -0,0 +1,65 @@
+"""Canonical pass@1 summaries: fixed identities, first graded attempt, pinned harnesses."""
+import argparse
+import json
+from pathlib import Path
+
+ROOT=Path(__file__).resolve().parents[1]/'logs/20260915'
+HARNESS_VERSIONS={'opencode':'1.18.31','claude-code':'2.1.270','codex':'0.154.0','mini-swe-agent':'2.4.6'}
+
+
+def summarize(arm,output):
+ manifest=json.loads((ROOT/'test_manifest.json').read_text())
+ selected={};ungraded=0
+ files=sorted((output/'traces').glob('*.jsonl')) if arm=='blackbox' else [output/'attempts.jsonl']
+ for p in files:
+ for line in p.read_text().splitlines():
+ r=json.loads(line)
+ valid=r.get('reward') in (0,1) and (r.get('n_turns',0)>0 if arm=='blackbox' else r.get('tito_pass',False))
+ if not valid:
+ ungraded+=1;continue
+ assert r.get('rep',0)==0 and 0<=r['index']<250
+ selected.setdefault((r['harness'],r['index']),r)
+ harnesses=list(HARNESS_VERSIONS) if arm=='blackbox' else ['whitebox_seta']
+ expected={(h,i) for h in harnesses for i in range(250)}
+ complete=set(selected)==expected
+ if arm=='blackbox':
+ audit_path=output/'final_tito.json'
+ audit=json.loads(audit_path.read_text()) if audit_path.exists() else {}
+ tito=complete and audit.get('tito_pass',False) and audit.get('counts')=={h:250 for h in harnesses}
+ else:
+ tito=complete and all(r['tito_pass'] for r in selected.values())
+ scores={};versions={h:{} for h in harnesses}
+ for h in harnesses:
+ rows=[r for (name,_),r in selected.items() if name==h]
+ difficulty={}
+ for level in ('easy','medium','hard'):
+ subset=[r for r in rows if manifest['tasks'][r['index']]['difficulty']==level]
+ correct=sum(r['reward'] for r in subset)
+ difficulty[level]={'graded':len(subset),'correct':correct,'pass_at_1':correct/len(subset) if subset else None}
+ correct=sum(r['reward'] for r in rows)
+ scores[h]={'graded':len(rows),'correct':correct,'pass_at_1':correct/len(rows) if rows else None,'difficulty':difficulty}
+ if arm=='blackbox':
+ for row in rows:
+ p=output/'trials'/row.get('trial_name','missing')/'result.json'
+ data=json.loads(p.read_text()) if p.is_file() else {}
+ version=(data.get('agent_info') or {}).get('version') or 'unverified'
+ versions[h][version]=versions[h].get(version,0)+1
+ pins=all(versions[h]=={HARNESS_VERSIONS[h]:250} for h in harnesses) if arm=='blackbox' else True
+ result={'metric':'pass@1','arm':arm,'complete':complete,'graded_cells':len(selected),
+ 'expected_cells':len(expected),'harnesses':scores,'tito_pass':tito,
+ 'ungraded_attempts':ungraded,'harness_versions':versions,'harness_versions_match_baseline':pins,
+ 'comparison_ready':complete and tito and pins,
+ 'average_pass_at_1':sum(r['reward'] for r in selected.values())/len(selected) if selected else None,
+ 'selection':'first graded attempt per fixed task/harness; infrastructure failures excluded and retried'}
+ (output/'canonical_scores.json').write_text(json.dumps(result,indent=2)+'\n')
+ return result
+
+
+if __name__=='__main__':
+ p=argparse.ArgumentParser(description=__doc__)
+ p.add_argument('--arm',choices=['blackbox','whitebox'],required=True)
+ p.add_argument('--output',type=Path,required=True)
+ p.add_argument('--require-complete',action='store_true')
+ a=p.parse_args();result=summarize(a.arm,a.output)
+ print(json.dumps({k:v for k,v in result.items() if k not in ('harnesses','harness_versions')}))
+ raise SystemExit(2 if a.require_complete and not result['comparison_ready'] else 0)
diff --git a/04-data-agent/tools/smoke_multiharness_tito.py b/04-data-agent/tools/smoke_multiharness_tito.py
new file mode 100644
index 0000000..021d4d2
--- /dev/null
+++ b/04-data-agent/tools/smoke_multiharness_tito.py
@@ -0,0 +1,222 @@
+#!/usr/bin/env python
+"""Real Harbor harness smoke with exact token/context retention through TRL.
+
+Reuses OpenEnv capture, Harbor rollout, and TRL's actual trace reader and builder.
+Artifacts contain task data, so keep the output under the experiment's ignored logs/.
+Prefix drift is a packing metric, never the TITO admission test.
+"""
+from __future__ import annotations
+
+import argparse
+import asyncio
+import collections
+import hashlib
+import importlib.metadata
+import json
+from pathlib import Path
+import secrets
+import shutil
+import socket
+import struct
+import subprocess
+import time
+
+from harbor_env.harness import to_trace_entries
+from openenv.core.harness.capture.export import export_session
+from openenv.core.harness.capture.forwarding import GradioForwarder
+from openenv.core.harness.capture.runner import CaptureServer
+from openenv.core.harness.capture.upstream import training_sampling
+from openenv.core.harness.capture.validate import validate_training_turn
+from openenv.harbor.rollout import run_rollout
+from openenv.harbor.seams import SEAMS
+from trl.experimental.async_grpo.async_rollout_worker import _chain_to_sequences
+from trl.experimental.async_grpo.openenv_harness import _turns_from_trace
+
+ROOT = Path(__file__).resolve().parents[3]
+
+
+def write(path, value):
+ path.write_text(json.dumps(value, indent=2, allow_nan=False, default=str) + "\n")
+
+
+def supervised_positions(ids, masks, logprobs):
+ """Multiset of exact causal context, sampled id, and behavior logprob.
+
+ Incremental hashing bounds memory even when a harness repeats a long prompt.
+ Include multiplicity: equal generated tokens are separate training positions.
+ """
+ out = collections.Counter()
+ context = hashlib.sha256()
+ for token, mask, lp in zip(ids, masks, logprobs, strict=True):
+ if mask:
+ out[(context.hexdigest(), token, float(lp).hex())] += 1
+ context.update(struct.pack(">q", token))
+ return out
+
+
+def audit(result, token_budget=40960):
+ entries = to_trace_entries(result)
+ expected = collections.Counter()
+ policy = training_sampling({"temperature": 0.8})
+ for entry in entries:
+ p, c, lp, mask = (entry[k] for k in (
+ "prompt_token_ids", "completion_token_ids", "per_token_logps", "loss_mask"))
+ validate_training_turn(p, c, lp, mask)
+ assert entry["metadata"]["sampling_params"] == policy, "sampling policy mismatch"
+ expected.update(supervised_positions(p + c, mask, [0.0] * len(p) + lp))
+ turns = _turns_from_trace(entries)
+ rows, tally = _chain_to_sequences(turns, result.session_id, fork_threshold=0)
+ retained = collections.Counter()
+ for row in rows:
+ retained.update(supervised_positions(row.input_ids, row.completion_mask, row.old_log_probs))
+ fatal = [x for x in result.findings if "[FATAL]" in x]
+ checks = {
+ "train_tier": result.rollout_type == "train" and result.capture_level == "tokens",
+ "nonempty_supervision": bool(expected),
+ "exact_context_ids_logprobs_masks_retained": retained == expected,
+ "no_capture_fatal": not fatal,
+ "token_count_matches_export": sum(expected.values()) == result.n_trainable_tokens,
+ "nonzero_logprobs_present": any(lp < 0 for e in entries for lp in e["per_token_logps"]),
+ }
+ action_entries = [e for e in entries if e["response"]["choices"][0]["message"].get("tool_calls")]
+ return {
+ "checks": checks, "tito_pass": all(checks.values()), "fatal": fatal,
+ "entries": len(entries), "rows": len(rows), "eligible_tokens": sum(expected.values()),
+ "retained_tokens": sum(retained.values()),
+ "packed_tokens": sum(len(row.input_ids) for row in rows),
+ "largest_row_tokens": max((len(row.input_ids) for row in rows), default=0),
+ "rows_over_40960": sum(len(row.input_ids) > 40960 for row in rows),
+ "token_budget": token_budget,
+ "rows_over_token_budget": sum(len(row.input_ids) > token_budget for row in rows),
+ "action_entries": len(action_entries),
+ "action_tokens": sum(sum(e["loss_mask"]) for e in action_entries), "drift": tally,
+ }, entries
+
+
+async def main(args):
+ args.output.mkdir(parents=True, exist_ok=True)
+ tasks = []
+ manifests = []
+ source_tasks = sorted(p for p in args.tasks.iterdir() if (p / "task.toml").exists())
+ for index in args.indices:
+ source = source_tasks[index]
+ target = args.output / "tasks" / source.name
+ if not target.exists():
+ shutil.copytree(source, target)
+ # Isolated resource override, recorded below. Never modify the cached dataset.
+ config = target / "task.toml"
+ config.write_text(config.read_text().replace("memory_mb = 1024", "memory_mb = 4096"))
+ files = {str(p.relative_to(source)): hashlib.sha256(p.read_bytes()).hexdigest()
+ for p in sorted(source.rglob("*")) if p.is_file()}
+ manifests.append({"index": index, "source": str(source), "files_sha256": files,
+ "resource_override": {"memory_mb": 4096},
+ "effective_task_sha256": hashlib.sha256((target / "task.toml").read_bytes()).hexdigest()})
+ tasks.append((index, target))
+ heads = {repo: subprocess.check_output(["git", "rev-parse", "HEAD"], cwd=ROOT / repo, text=True).strip()
+ for repo in ("OpenEnv", "trl", "HuggingEnvs")}
+ write(args.output / "manifest.json", {
+ "started_utc": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()),
+ "model": args.model, "inference": args.inference, "git_heads": heads,
+ "working_tree_diff_sha256": {repo: hashlib.sha256(subprocess.check_output(
+ ["git", "diff", "HEAD"], cwd=ROOT / repo)).hexdigest() for repo in heads},
+ "runner_sha256": hashlib.sha256(Path(__file__).read_bytes()).hexdigest(),
+ "local_changes": True, "harbor_version": importlib.metadata.version("harbor"),
+ "harnesses": args.harnesses, "tasks": manifests, "concurrency": args.concurrency,
+ "sampling": training_sampling({"temperature": 0.8}), "model_call_limit": 17,
+ "max_output_tokens": 4096, "agent_timeout_sec": 600,
+ "scope": "Frozen-policy smoke; not optimizer, weight sync, or scale certification.",
+ })
+ with socket.socket() as sock:
+ sock.bind(("", 0))
+ port = sock.getsockname()[1]
+ capture = CaptureServer(llm_url=args.inference, model=args.model, port=port,
+ max_output_tokens=4096, admin_key=secrets.token_hex(32))
+ capture.start()
+ forwarder = GradioForwarder()
+ url = forwarder.start(port)
+ print(json.dumps({"event": "proxy_ready", "port": port, "url": url}), flush=True)
+ # Record the upstream response before ingest and dialect replay. No credentials or headers.
+ completion = capture.inference.completion
+ proxy_loop = None
+ raw_dir = args.output / "engine_responses"
+ raw_dir.mkdir(exist_ok=True)
+
+ async def observed(request):
+ nonlocal proxy_loop
+ proxy_loop = asyncio.get_running_loop()
+ response = await completion(request)
+ key = hashlib.sha256(json.dumps(response, sort_keys=True).encode()).hexdigest()
+ write(raw_dir / f"{key}.json", {"request": request, "response": response})
+ return response
+
+ capture.inference.completion = observed
+ # Save graph evidence before run_rollout's finally releases each session, including cancellation.
+ delete = capture.registry.delete
+
+ def save_and_delete(sid):
+ session = capture.registry.get(sid)
+ if session is not None:
+ doc = export_session(session, include_messages=True, include_discarded=True)
+ write(args.output / f"capture-{sid}.json", doc)
+ return delete(sid)
+
+ capture.registry.delete = save_and_delete
+ results = []
+ semaphore = asyncio.Semaphore(args.concurrency)
+
+ async def run(harness, index, task):
+ async with semaphore:
+ case = f"{harness}-{index}"
+ print(json.dumps({"event": "start", "case": case}), flush=True)
+ record = {"harness": harness, "task_index": index, "task": task.name}
+ started = time.monotonic()
+ try:
+ result = await asyncio.wait_for(run_rollout(
+ task_dir=task, harness=harness, sandbox="e2b", registry=capture.registry,
+ intercept_url=url, model=args.model, trials_dir=args.output / "trials",
+ dataset="AdithyaSK/data_agent_rl_environment_train (local pinned tasks)",
+ agent_timeout_sec=600, agent_step_limit=17, session_prefix="tito-smoke",
+ inference=capture.inference, sampling={"temperature": 0.8}), timeout=1200)
+ write(args.output / f"result-{case}.json", result.model_dump())
+ record.update(ok=result.ok, reward=result.reward, error=result.error,
+ exception_type=result.exception_type, turns=result.n_turns,
+ roots=result.n_roots, findings=result.findings,
+ session_id=result.session_id, trial_name=result.trial_name)
+ audited, entries = audit(result)
+ record.update(audited)
+ write(args.output / f"trace-{case}.json", entries)
+ except Exception as exc:
+ record.update(tito_pass=False, error=f"{type(exc).__name__}: {exc}")
+ record["wall_s"] = round(time.monotonic() - started, 2)
+ results.append(record)
+ write(args.output / "matrix.json", results)
+ print(json.dumps({"event": "done", **record}), flush=True)
+
+ try:
+ cases = [(h, i, p) for h in args.harnesses for i, p in tasks]
+ # Warm the shared template once before concurrent trials can race its first build.
+ await run(*cases[0])
+ await asyncio.gather(*(run(*case) for case in cases[1:]))
+ finally:
+ write(args.output / "cleanup.json", {"remaining_sessions": capture.registry.list_ids(),
+ "completed_cases": len(results)})
+ # Harbor shields sandbox deletion. Let its already-scheduled cleanup finish before closing
+ # this loop; only the capture client belongs to the server's separate loop.
+ await asyncio.sleep(2)
+ if proxy_loop is not None:
+ future = asyncio.run_coroutine_threadsafe(capture.inference.aclose(), proxy_loop)
+ await asyncio.wrap_future(future)
+ forwarder.stop()
+ capture.stop()
+
+
+if __name__ == "__main__":
+ p = argparse.ArgumentParser(description=__doc__)
+ p.add_argument("--inference", required=True)
+ p.add_argument("--model", default="Qwen/Qwen3.5-4B")
+ p.add_argument("--tasks", type=Path, default=Path("/admin/home/adithyaskolavi/.cache/openenv/harbor-datasets/AdithyaSK__data_agent_rl_environment_train/tasks"))
+ p.add_argument("--indices", nargs="+", type=int, default=[8, 9])
+ p.add_argument("--harnesses", nargs="+", default=[k for k, v in SEAMS.items() if v.status == "validated"])
+ p.add_argument("--concurrency", type=int, default=4)
+ p.add_argument("--output", type=Path, required=True)
+ asyncio.run(main(p.parse_args()))
diff --git a/04-data-agent/tools/trackio_multi4.py b/04-data-agent/tools/trackio_multi4.py
new file mode 100644
index 0000000..8401f11
--- /dev/null
+++ b/04-data-agent/tools/trackio_multi4.py
@@ -0,0 +1,421 @@
+"""Replay durable scalar metrics into Trackio; sync on a CPU job, never on the trainer.
+
+The small fragment adapter is pinned to Trackio 0.33.0. Its existing log_id deduplication
+makes replay/restarts idempotent. SQLite lives on node-local disk; a consistent backup is
+atomically published to FSx before any network call. Raw captures/completions stay local.
+"""
+import argparse
+from datetime import datetime, timezone
+import fcntl
+import hashlib
+from importlib.metadata import version
+import json
+import math
+import os
+from pathlib import Path
+import re
+import sqlite3
+import subprocess
+import sys
+import tempfile
+import time
+
+from monitor_multi4 import TERMINAL, read_json, read_metrics, slurm_states
+
+TRACKIO_VERSION = '0.33.0'
+REMOTE_ENV = ('TRACKIO_SPACE_ID', 'TRACKIO_SERVER_URL', 'TRACKIO_BUCKET_ID',
+ 'TRACKIO_DATASET_ID', 'TRACKIO_WEBHOOK_URL')
+
+
+def digest(value):
+ return hashlib.sha256(json.dumps(value, sort_keys=True, allow_nan=False).encode()).hexdigest()
+
+
+def scalars(data, prefix=''):
+ result = {}
+ for key, value in data.items():
+ name = f'{prefix}{key}'
+ if isinstance(value, dict):
+ result.update(scalars(value, name + '/'))
+ elif isinstance(value, (int, float)):
+ if math.isfinite(value):
+ result[name] = value
+ else:
+ result[name + '/nonfinite'] = 1
+ return result
+
+
+def event(project, run, step, metrics, config, *, identity=None):
+ """Stable IDs survive source replay, worker restart, and out-of-order eval completion."""
+ content = [project, run, step, metrics, identity]
+ return {'v': 1, 'kind': 'metric', 'project': project, 'run': run,
+ 'run_id': digest([project, run])[:32], 'step': int(step), 'metrics': metrics,
+ 'config': config, 'log_id': digest(content),
+ 'timestamp': datetime.now(timezone.utc).isoformat()}
+
+
+def score_metrics(scores, protocol):
+ metrics = {'eval/pass_at_1': scores['average_pass_at_1'],
+ 'eval/delta_from_baseline': scores['average_pass_at_1'] - protocol['average_pass_at_1'],
+ 'eval/graded_cells': scores['graded_cells']}
+ totals = {}
+ for harness, values in scores['harnesses'].items():
+ metrics[f'eval/{harness}/pass_at_1'] = values['pass_at_1']
+ metrics[f'eval/{harness}/delta_from_baseline'] = values['pass_at_1'] - protocol['scores'][harness]['pass_at_1']
+ for difficulty, counts in values.get('difficulty', {}).items():
+ n = counts['graded']
+ metrics[f'eval/{harness}/{difficulty}/pass_at_1'] = counts['correct'] / n
+ total = totals.setdefault(difficulty, [0, 0])
+ total[0] += counts['correct']; total[1] += n
+ for difficulty, (correct, n) in totals.items():
+ metrics[f'eval/{difficulty}/pass_at_1'] = correct / n
+ return metrics
+
+
+def evaluation_roots(root, config=None):
+ """Include explicitly configured earlier allocations whose evals can finish late."""
+ config = config if config is not None else read_json(root / 'run_config.json', {})
+ roots = [root.resolve()]
+ for value in config.get('logging', {}).get('evaluation_sources', []):
+ source = Path(value)
+ if not source.is_absolute() or not (source / 'run_config.json').is_file():
+ raise ValueError('Evaluation source must be an existing absolute run directory')
+ if source.resolve() not in roots:
+ roots.append(source.resolve())
+ return roots
+
+
+def training_lineage(root, job):
+ """Follow actual resume checkpoints, excluding abandoned post-checkpoint updates."""
+ segments, seen = [], set()
+ current, current_job, upper = root.resolve(), str(job), None
+ while True:
+ if current in seen:
+ raise ValueError('Cycle in training checkpoint lineage')
+ seen.add(current)
+ config = read_json(current / 'run_config.json')
+ resume = config.get('training', {}).get('resume_from_checkpoint')
+ lower = 0
+ parent = None
+ if resume:
+ checkpoint = Path(resume)
+ checkpoint_match = re.fullmatch(r'checkpoint-(\d+)', checkpoint.name)
+ job_match = re.fullmatch(r'job-(\d+)', checkpoint.parent.parent.name)
+ if not checkpoint.is_absolute() or checkpoint.parent.name != 'run' or not checkpoint_match or not job_match:
+ raise ValueError('Invalid training checkpoint lineage path')
+ lower = int(checkpoint_match[1])
+ parent = (checkpoint.parents[2], job_match[1], lower)
+ selected = {}
+ for row in read_metrics(current / f'job-{current_job}/audit/metrics.jsonl'):
+ # Trainer's final aggregate summary can reuse its last optimizer step.
+ if 'grad_norm' not in row:
+ continue
+ step = row['step']
+ if step <= lower or (upper is not None and step > upper):
+ continue
+ if step in selected and selected[step] != row:
+ raise ValueError(f'Conflicting optimizer records at step {step}')
+ selected[step] = row
+ if upper is None:
+ upper = max(selected, default=lower)
+ if sorted(selected) != list(range(lower + 1, upper + 1)):
+ raise ValueError(f'Incomplete optimizer history in job {current_job}: expected {lower + 1}..{upper}')
+ segments.append({'job': current_job, 'root': str(current), 'start_step': lower + 1,
+ 'end_step': upper, 'resume_from_checkpoint': resume,
+ 'training': config.get('training', {}), 'rows': list(selected.values())})
+ if parent is None:
+ break
+ current, current_job, upper = parent
+ current = current.resolve()
+ return list(reversed(segments))
+
+
+def collect(root, job):
+ config = read_json(root / 'run_config.json')
+ project = config['logging']['project']
+ protocol = read_json(Path(config['evaluation']['protocol_file']))
+ # Explicit allowlist: never serialize process environment, credentials, task text or captures.
+ metadata = {k: config[k] for k in ('model', 'model_revision', 'harnesses', 'harness_versions',
+ 'training', 'sampling')}
+ metadata.update(train_job=job, run_directory=str(root),
+ protocol_sha256=digest(protocol), trackio_version=TRACKIO_VERSION,
+ schedule_sha256=config['dataset']['schedule_sha256'],
+ metric_axis='optimizer step', tito_scope='capture and sequence builder; not optimizer consumption')
+ records, withheld = [], []
+ logs = root / f'job-{job}' if job else None
+ if logs:
+ for row in read_metrics(logs / 'audit/metrics.jsonl'):
+ metrics = scalars({k: v for k, v in row.items() if k != 'step'}, 'train/')
+ metrics['train/global_step'] = row['step']
+ records.append(event(project, f'training-{job}', row['step'], metrics, metadata))
+ for row in read_metrics(root / 'monitor/history.jsonl'):
+ metrics = scalars({k: row.get(k, {}) for k in ('counters', 'coverage', 'tito')}, 'audit/')
+ metrics.update({'audit/alert_count': len(row.get('alerts', [])),
+ 'audit/eval_backlog': len(row.get('queued_eval_steps', []))})
+ for harness, values in row.get('tito', {}).items():
+ completed = values.get('completed_results')
+ if completed:
+ metrics[f'audit/tito/{harness}/pass_fraction_completed'] = values['tito_pass'] / completed
+ eligible = values.get('eligible_tokens', 0)
+ if eligible:
+ metrics[f'audit/tito/{harness}/retention_fraction'] = values['retained_tokens'] / eligible
+ graded = values.get('graded', 0)
+ if graded:
+ metrics[f'audit/{harness}/reward_mean'] = values['reward_sum'] / graded
+ records.append(event(project, f'audit-{job}', row['optimizer_step'], metrics, metadata,
+ identity=row.get('checked_at')))
+ if job and config['logging'].get('stitch_training_history'):
+ segments = training_lineage(root, job)
+ lineage_metadata = {k: v for k, v in metadata.items() if k != 'training'}
+ lineage_metadata.update(
+ description='Continuous optimizer history along the checkpoint resume chain; configuration changed between some allocations.',
+ training_segments=[{k: v for k, v in segment.items() if k != 'rows'} for segment in segments])
+ for segment in segments:
+ for row in segment['rows']:
+ metrics = scalars({k: v for k, v in row.items() if k != 'step'}, 'train/')
+ metrics.update({'train/global_step': row['step'], 'train/source_job_id': int(segment['job'])})
+ records.append(event(project, 'training-full-history', row['step'], metrics, lineage_metadata))
+ baseline_path = Path(protocol['baseline_run']) / f"job-{protocol['baseline_job']}" / 'canonical_results.json'
+ baseline = read_json(baseline_path)
+ if not baseline['coverage_complete']:
+ raise ValueError('Baseline coverage is incomplete')
+ baseline_scores = {'average_pass_at_1': protocol['average_pass_at_1'], 'graded_cells': 1000,
+ 'harnesses': {h: {'pass_at_1': protocol['scores'][h]['pass_at_1'],
+ 'difficulty': baseline['harnesses'][h]['difficulty']} for h in protocol['harnesses']}}
+ records.append(event(project, 'evaluation-curve', 0, score_metrics(baseline_scores, protocol), metadata))
+ evaluations = {}
+ for source in evaluation_roots(root, config):
+ for directory in sorted((source / 'checkpoint-evals').glob('step-*')):
+ if not directory.is_dir():
+ continue
+ scores = read_json(directory / 'scores.json', {})
+ step = int(directory.name.split('-')[1])
+ eval_config = read_json(directory / 'eval_plan.json', {})
+ comparable = (scores.get('complete') and scores.get('comparison_ready')
+ and scores.get('graded_cells') == 1000 and eval_config.get('protocol') == protocol
+ and set(scores.get('harnesses', {})) == set(protocol['harnesses'])
+ and all(v.get('graded') == 250 for v in scores.get('harnesses', {}).values()))
+ if not comparable:
+ withheld.append({'step': step, 'source': str(directory),
+ 'reason': 'incomplete, audit/version failure, or protocol mismatch'})
+ continue
+ metrics = score_metrics(scores, protocol)
+ if step in evaluations and evaluations[step] != metrics:
+ raise ValueError(f'Conflicting evaluation scores at optimizer step {step}')
+ evaluations[step] = metrics
+ for step, metrics in sorted(evaluations.items()):
+ records.append(event(project, 'evaluation-curve', step, metrics, metadata))
+ return records, withheld
+
+
+def import_events(records):
+ if version('trackio') != TRACKIO_VERSION:
+ raise RuntimeError(f'Trackio adapter requires {TRACKIO_VERSION}; installed {version("trackio")}')
+ from trackio.fragments import import_records
+ import_records(records)
+
+
+def backup_project(project, destination):
+ from trackio.sqlite_storage import SQLiteStorage
+ destination.mkdir(parents=True, exist_ok=True)
+ target = destination / SQLiteStorage.get_project_db_filename(project)
+ temporary = target.with_suffix('.db.tmp')
+ with sqlite3.connect(SQLiteStorage.get_project_db_path(project)) as source:
+ with sqlite3.connect(temporary) as output:
+ source.backup(output)
+ output.execute('PRAGMA journal_mode=DELETE')
+ temporary.replace(target)
+
+
+def write_json(path, value):
+ temp = path.with_suffix('.tmp')
+ temp.write_text(json.dumps(value, indent=2) + '\n')
+ temp.replace(path)
+
+
+def configuration_records(project):
+ """Replay one existing log per run with its allowlisted configuration.
+
+ Trackio 0.33.0's get_all_logs_for_sync emits config=None. Its native bulk_log
+ endpoint can still store config on an existing log_id without adding a metric.
+ """
+ from trackio.sqlite_storage import SQLiteStorage
+ records, seen = [], set()
+ for entry in SQLiteStorage.get_all_logs_for_sync(project):
+ identity = entry.get('run_id') or entry['run']
+ if identity in seen:
+ continue
+ seen.add(identity)
+ config = SQLiteStorage.get_run_config(project, entry['run'], run_id=entry.get('run_id'))
+ if config:
+ records.append({**entry, 'config': config})
+ return records
+
+
+def sync_project(config):
+ # Token is inherited or loaded only in the sync subprocess; never logged or put in config.
+ from dotenv import dotenv_values
+ values = dotenv_values(os.environ.get('DATA_AGENT_ENV_FILE', '.env'))
+ token = os.environ.get('HF_TOKEN') or values.get('HF_TOKEN') or values.get('HF_API_KEY')
+ if token:
+ os.environ['HF_TOKEN'] = token
+ from trackio.deploy import create_space_if_not_exists
+ logging = config['logging']
+ # Provision once, then send stable log IDs through Trackio's bulk API. Replacing
+ # the live Space's mounted SQLite file can leave open readers on an old inode.
+ from huggingface_hub import HfApi
+ from huggingface_hub.errors import RepositoryNotFoundError
+ try:
+ info = HfApi().space_info(logging['space_id'])
+ if not info.private or info.sdk != 'gradio':
+ raise ValueError('Online metrics require the configured private Gradio Space')
+ except RepositoryNotFoundError:
+ create_space_if_not_exists(logging['space_id'], bucket_id=logging['bucket_id'], private=True)
+ from trackio.remote_client import RemoteClient
+ from trackio.sqlite_storage import SQLiteStorage
+ client = RemoteClient(logging['space_id'], hf_token=token, httpx_kwargs={'timeout': 30})
+ records = SQLiteStorage.get_all_logs_for_sync(logging['project'])
+ # Multiple allocation collectors share evaluation-curve. Trackio 0.33.0's
+ # sync_incremental waits for exact remote/local row-count equality, which
+ # cannot hold when a peer has already published additional checkpoint scores.
+ # Keep native bulk_log deduplication, then verify our exact event contents.
+ for start in range(0, len(records), 500):
+ client.predict(api_name='/bulk_log', logs=records[start:start + 500], hf_token=token)
+ metadata = configuration_records(logging['project'])
+ if metadata:
+ client.predict(api_name='/bulk_log', logs=metadata, hf_token=token)
+ proof = verify_remote_records(client, logging['project'], records)
+ for entry in metadata:
+ summary = client.predict(api_name='/get_run_summary', project=logging['project'],
+ run_id=entry['run_id'])
+ if summary.get('config') != entry['config']:
+ raise RuntimeError('Remote Trackio configuration differs for ' + entry['run'])
+ return {**proof, 'configuration_runs_verified': len(metadata),
+ 'checked_at': datetime.now(timezone.utc).isoformat()}
+
+
+def verify_remote_records(client, project, records, timeout=90):
+ """Allow other publishers' events; require every local ID and scalar payload."""
+ expected = {}
+ for entry in records:
+ identity = entry.get('log_id', '')
+ if entry.get('project') != project or not re.fullmatch(r'[0-9a-f]{64}', identity):
+ raise ValueError('Invalid scalar event identity')
+ value = {k: entry[k] for k in ('run_id', 'step', 'metrics')}
+ value['run_name'] = entry['run']
+ if identity in expected and expected[identity] != value:
+ raise ValueError('Conflicting local scalar event')
+ expected[identity] = value
+ pending = dict(expected)
+ deadline = time.monotonic() + timeout
+ while pending:
+ keys = sorted(pending)
+ for start in range(0, len(keys), 100):
+ # IDs are validated SHA256 hex strings, never arbitrary SQL input.
+ ids = ','.join("'" + k + "'" for k in keys[start:start + 100])
+ result = client.predict(api_name='/query_project', project=project,
+ query='SELECT log_id, run_id, run_name, step, CAST(metrics AS TEXT) AS metrics FROM metrics WHERE log_id IN (' + ids + ')')
+ for row in result['rows']:
+ identity = row['log_id']
+ actual = {k: row[k] for k in ('run_id', 'run_name', 'step')}
+ actual['metrics'] = json.loads(row['metrics']) if isinstance(row['metrics'], str) else row['metrics']
+ if identity not in expected or actual != expected[identity]:
+ raise RuntimeError('Remote Trackio event content mismatch')
+ pending.pop(identity, None)
+ if not pending:
+ break
+ if time.monotonic() >= deadline:
+ raise TimeoutError(f'{len(pending)} Trackio events are not remotely visible')
+ time.sleep(min(2, max(0, deadline - time.monotonic())))
+ return {'ok': True, 'events_verified': len(expected), 'verification': 'exact log_id/run_id/step/metrics'}
+
+
+def bounded_sync(root, seconds, log):
+ try:
+ with log.open('a') as stream:
+ result = subprocess.run([sys.executable, '-u', __file__, '--run', str(root), '--sync-only'],
+ stdout=stream, stderr=subprocess.STDOUT, timeout=seconds)
+ return {'ok': result.returncode == 0, 'returncode': result.returncode}
+ except subprocess.TimeoutExpired:
+ return {'ok': False, 'error': 'sync_timeout', 'timeout_seconds': seconds}
+
+
+def work_finished(root, job):
+ submission = read_json(root / 'submission.json', {})
+ # Do not wait for the monitor: it waits for us. Include watcher to prevent early exit
+ # between training finishing and a just-saved checkpoint being submitted for evaluation.
+ jobs = [str(job)] + [str(submission[k]) for k in ('cleanup', 'eval_watcher') if k in submission]
+ for source in evaluation_roots(root):
+ for record in read_json(source / 'checkpoint-evals/state.json', {}).values():
+ jobs += [str(record[k]) for k in ('job_id', 'cleanup_job_id') if record.get(k)]
+ # Recovery jobs can be submitted after an allocation's watcher exits.
+ for path in (source / 'checkpoint-evals').glob('step-*/submission.json'):
+ record = read_json(path, {})
+ jobs += [str(record[k]) for k in ('job_id', 'cleanup_job_id') if record.get(k)]
+ states = slurm_states(sorted(set(jobs)))
+ return set(states) == set(jobs) and all(value in TERMINAL for value in states.values())
+
+
+def main():
+ p = argparse.ArgumentParser(description=__doc__)
+ p.add_argument('--run', type=Path, required=True)
+ p.add_argument('--train-job', default='')
+ p.add_argument('--watch', action='store_true')
+ p.add_argument('--online', action='store_true')
+ p.add_argument('--sync-only', action='store_true', help=argparse.SUPPRESS)
+ args = p.parse_args()
+ root = args.run.resolve()
+ config = read_json(root / 'run_config.json')
+ if args.sync_only:
+ proof = sync_project(config)
+ write_json(root / 'trackio/sync-receipt.json', proof)
+ return
+ if args.watch and not args.train_job:
+ p.error('--watch requires --train-job')
+ destination = root / 'trackio'
+ destination.mkdir(exist_ok=True)
+ for key in REMOTE_ENV:
+ os.environ.pop(key, None)
+ with (destination / '.collector.lock').open('w') as lock, tempfile.TemporaryDirectory(prefix='multi4-trackio-') as scratch:
+ fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
+ os.environ['TRACKIO_DIR'] = scratch
+ os.environ['TRACKIO_STORAGE_MODE'] = 'sqlite'
+ last_synced = None
+ while True:
+ start = time.monotonic()
+ state = {'checked_at': datetime.now(timezone.utc).isoformat(), 'online': args.online}
+ try:
+ records, withheld = collect(root, args.train_job)
+ import_events(records)
+ backup_project(config['logging']['project'], destination / 'dashboard')
+ state.update(events=len(records), withheld_evaluations=withheld, local_ok=True,
+ evaluation_sources=[str(p) for p in evaluation_roots(root, config)],
+ evaluation_steps=sorted({r['step'] for r in records if r['run'] == 'evaluation-curve'}),
+ full_history_steps=len([r for r in records if r['run'] == 'training-full-history']),
+ training_step=max((r['step'] for r in records if r['run'] == f'training-{args.train_job}'), default=None))
+ current = digest([r['log_id'] for r in records])
+ if args.online and current != last_synced:
+ state['sync'] = bounded_sync(root, config['logging']['sync_timeout_seconds'], destination / 'sync.log')
+ if state['sync']['ok']:
+ last_synced = current
+ else:
+ state['sync'] = {'ok': bool(last_synced), 'skipped_unchanged': True}
+ finished = args.watch and work_finished(root, args.train_job)
+ except Exception as exc:
+ state.update(error=type(exc).__name__ + ': ' + str(exc), local_ok=False)
+ finished = False
+ write_json(destination / 'status.json', state)
+ with (destination / 'history.jsonl').open('a') as stream:
+ stream.write(json.dumps(state) + '\n')
+ print(json.dumps(state), flush=True)
+ if not args.watch:
+ if not state.get('local_ok') or (args.online and not state.get('sync', {}).get('ok')):
+ raise SystemExit(1)
+ break
+ if finished and (not args.online or state.get('sync', {}).get('ok')):
+ break
+ time.sleep(max(1, config['logging']['poll_seconds'] - (time.monotonic() - start)))
+
+
+if __name__ == '__main__':
+ main()
diff --git a/04-data-agent/tools/train_whitebox_daytona.py b/04-data-agent/tools/train_whitebox_daytona.py
new file mode 100644
index 0000000..f47aa69
--- /dev/null
+++ b/04-data-agent/tools/train_whitebox_daytona.py
@@ -0,0 +1,151 @@
+"""Isolated synchronous GRPO comparison with actual token and loss-mask audits."""
+from __future__ import annotations
+
+import argparse
+import hashlib
+import json
+import math
+import os
+from pathlib import Path
+
+import torch
+from datasets import Dataset
+from transformers import TrainerCallback
+from trl import GRPOConfig, GRPOTrainer
+from whitebox_bash import white_box_bash_env
+from eval_whitebox_native import MODEL, REVISION, SYSTEM
+from generation_routing import generation_group_size
+
+
+class AuditedTrainer(GRPOTrainer):
+ """Observe the native loop without replacing its generation, rewards or loss."""
+
+ def _generate_single_turn(self, prompt_ids, *args, **kwargs):
+ completion_ids, logprobs = super()._generate_single_turn(prompt_ids,*args,**kwargs)
+ records=getattr(self,'_captured_calls',[])
+ for prompt,completion,lps in zip(prompt_ids,completion_ids,logprobs,strict=True):
+ assert len(completion)==len(lps) and all(math.isfinite(p) for p in lps)
+ records.append({'prompt_ids':prompt.copy(),'completion_ids':completion.copy(),'logprobs':lps.copy()})
+ self._captured_calls=records
+ return completion_ids,logprobs
+
+ def _generate(self,*args,**kwargs):
+ self._captured_calls=[]
+ return super()._generate(*args,**kwargs)
+
+ def _tool_call_loop(self,prompts,prompt_ids,*args,**kwargs):
+ result=super()._tool_call_loop(prompts,prompt_ids,*args,**kwargs)
+ masks,_,completions,lps,_,_,_=result
+ evidence=Path(self.args.output_dir)/'capture_audit'
+ evidence.mkdir(exist_ok=True)
+ (evidence/f'step-{self.state.global_step}.json').write_text(json.dumps({
+ 'prompt_ids':prompt_ids,'completions':completions,'masks':masks,'logprobs':lps,
+ 'calls':self._captured_calls})+'\n')
+ from whitebox_tito import audit_rows
+ checks=audit_rows(prompt_ids,completions,masks,lps,self._captured_calls)
+ path=Path(self.args.output_dir)/'token_audit.jsonl'
+ with path.open('a') as stream:
+ stream.write(json.dumps({'step':self.state.global_step,'rows':checks,'calls':len(self._captured_calls)})+'\n')
+ return result
+
+ def compute_loss(self,model,inputs,*args,**kwargs):
+ if 'tool_mask' in inputs:
+ mask=inputs['completion_mask']*inputs['tool_mask']
+ assert torch.isfinite(inputs['sampling_per_token_logps'][mask.bool()]).all()
+ assert set(inputs['tool_mask'].unique().tolist())<={0,1}
+ return super().compute_loss(model,inputs,*args,**kwargs)
+
+
+def digest(model):
+ h=hashlib.sha256()
+ for name,param in model.named_parameters():
+ if param.requires_grad:
+ h.update(name.encode());h.update(param.detach().flatten()[:256].float().cpu().numpy().tobytes())
+ return h.hexdigest()
+
+
+class SmokeEvidence(TrainerCallback):
+ def __init__(self,path): self.path=path
+ def on_train_begin(self,args,state,control,model=None,**kwargs):
+ self.initial=digest(model)
+ self.initial_step=state.global_step
+ def on_log(self,args,state,control,logs=None,**kwargs):
+ for key,value in (logs or {}).items():
+ if isinstance(value,float): assert math.isfinite(value), f'nonfinite metric: {key}'
+ with (self.path/'metrics.jsonl').open('a') as stream:
+ stream.write(json.dumps({'step':state.global_step,**(logs or {})})+'\n')
+ def on_train_end(self,args,state,control,model=None,**kwargs):
+ result={'initial_step':self.initial_step,'final_step':state.global_step,
+ 'initial_weight_digest':self.initial,'final_weight_digest':digest(model)}
+ result['weights_changed']=result['initial_weight_digest']!=result['final_weight_digest']
+ (self.path/f'optimizer_evidence_from_{self.initial_step}.json').write_text(json.dumps(result,indent=2)+'\n')
+
+
+def main():
+ p=argparse.ArgumentParser(description=__doc__)
+ p.add_argument('--run',type=Path,required=True)
+ p.add_argument('--server',required=True)
+ p.add_argument('--vllm-url',required=True)
+ p.add_argument('--output-dir',type=Path,required=True)
+ p.add_argument('--max-steps',type=int,default=1000)
+ p.add_argument('--save-steps',type=int,default=50)
+ p.add_argument('--resume-from-checkpoint')
+ p.add_argument('--max-train-seconds',type=float,default=0)
+ p.add_argument('--checkpoint-max-seconds',type=float,default=0)
+ args=p.parse_args()
+ args.output_dir.mkdir(parents=True,exist_ok=True)
+ schedule=json.loads((args.run/'reference_schedule.json').read_text())
+ rows=[{'prompt':[{'role':'system','content':SYSTEM},{'role':'user','content':'Solve the task.'}],
+ 'split':'train','index':g['task_index']} for g in schedule['groups'][:schedule['task_count']]]
+ config=GRPOConfig(
+ output_dir=str(args.output_dir),model_init_kwargs={'revision':REVISION,'dtype':'bfloat16'},
+ learning_rate=3e-6,lr_scheduler_type='constant',warmup_steps=0,beta=0.0,loss_type='dapo',
+ num_generations=8,per_device_train_batch_size=1,gradient_accumulation_steps=8,
+ max_steps=args.max_steps,max_completion_length=16384,max_tool_calling_iterations=16,
+ temperature=0.8,top_p=1.0,top_k=0,chat_template_kwargs={'enable_thinking':False},
+ gradient_checkpointing=True,gradient_checkpointing_kwargs={'use_reentrant':False},
+ bf16=True,optim='paged_adamw_8bit',max_grad_norm=1.0,seed=0,shuffle_dataset=False,
+ use_vllm=True,vllm_mode='server',vllm_server_base_url=args.vllm_url,
+ vllm_max_model_length=131072,vllm_server_timeout=900,
+ vllm_group_port=49000+int(os.environ.get('SLURM_JOB_ID','0'))%1000,
+ generation_kwargs={'max_tokens':16384},
+ save_strategy='steps',save_steps=args.save_steps,save_total_limit=None,
+ logging_steps=1,log_completions=True,num_completions_to_print=1,
+ report_to='trackio',project='daytona-whitebox-qwen35-2b',
+ run_name=f'whitebox-{os.environ.get("RUN_OWNER", os.environ.get("SLURM_JOB_ID", "local"))}',
+ trackio_space_id=None,trackio_static_space_id=False,
+ )
+ (args.output_dir/'comparison_config.json').write_text(json.dumps(config.to_dict(),indent=2,default=str)+'\n')
+ trainer=AuditedTrainer(model=MODEL,args=config,train_dataset=Dataset.from_list(rows),reward_funcs=[],
+ environment_factory=white_box_bash_env(args.server,split='train',toolsets='bash,seta',step_limit=17),
+ callbacks=[SmokeEvidence(args.output_dir)])
+ from training_audit import CheckpointReadyCallback
+ trainer.add_callback(CheckpointReadyCallback(MODEL,REVISION))
+ if args.max_train_seconds:
+ from training_audit import WallTimeCallback
+ trainer.add_callback(WallTimeCallback(args.max_train_seconds))
+ if args.checkpoint_max_seconds:
+ from training_audit import PeriodicCheckpointCallback
+ trainer.add_callback(PeriodicCheckpointCallback(args.checkpoint_max_seconds))
+ # Check the response's engine IDs and sampled logprob IDs before GRPO consumes them.
+ generate=trainer.vllm_generation.generate
+ def checked_generate(*a,**kw):
+ # Native server generation groups duplicated initial prompts by G. After tools,
+ # histories differ and need one continuation each; grouping those would replace
+ # seven trajectories' contexts with the first one's context.
+ prompts=kw.get('prompts',a[0] if a else None)
+ kw={**kw,'num_generations':generation_group_size(prompts,kw.get('num_generations',1))}
+ output=generate(*a,**kw)
+ returned_prompts,completions,logprobs,token_ids=output
+ assert returned_prompts==prompts
+ for ids,lps,lp_ids in zip(completions,logprobs,token_ids,strict=True):
+ assert len(ids)==len(lps)==len(lp_ids)
+ assert all(len(ids_at_pos)==1 and ids_at_pos[0]==tok for tok,ids_at_pos in zip(ids,lp_ids,strict=True))
+ return output
+ trainer.vllm_generation.generate=checked_generate
+ trainer.train(resume_from_checkpoint=args.resume_from_checkpoint)
+ trainer.save_state()
+ trainer.save_model(str(args.output_dir/'final'))
+
+
+if __name__=='__main__': main()
diff --git a/04-data-agent/tools/validate_training_smoke.py b/04-data-agent/tools/validate_training_smoke.py
new file mode 100644
index 0000000..1e50f87
--- /dev/null
+++ b/04-data-agent/tools/validate_training_smoke.py
@@ -0,0 +1,55 @@
+"""Require real optimizer, save/resume and token-provenance evidence before long runs."""
+import argparse
+import json
+import math
+from pathlib import Path
+import sys
+
+ROOT=Path(__file__).resolve().parents[1]/'logs/20260915'
+sys.path.insert(0,str(ROOT/'source/HuggingEnvs/04-data-agent/train'))
+from checkpoint_artifacts import finalize_saved, verify_ready
+
+
+def validate(arm,job):
+ logs=ROOT/arm/f'training-smoke/job-{job}';run=logs/'run'
+ assert (logs/'exit_code.txt').read_text().strip()=='0'
+ markers={}
+ for step in (2,4):
+ checkpoint=run/f'checkpoint-{step}'
+ finalize_saved(checkpoint);markers[step]=verify_ready(checkpoint)
+ assert markers[step]['step']==step
+ rows=[json.loads(l) for l in (logs/('audit/metrics.jsonl' if arm=='blackbox' else 'run/metrics.jsonl')).read_text().splitlines()]
+ updates=[r for r in rows if 'grad_norm' in r]
+ assert {r['step'] for r in updates}>={1,2,3,4}
+ assert all(math.isfinite(v) for r in updates for v in r.values() if isinstance(v,float))
+ assert any(r['grad_norm']>0 for r in updates), 'No optimizer learning signal observed'
+ if arm=='blackbox':
+ from checkpoint_artifacts import resume_info
+ resume=resume_info(run/'checkpoint-2',markers[2]['base_model'],markers[2]['base_revision'])
+ assert f"resume checkpoint step=2, next schedule group={resume['group_offset']}" in (logs/'train-resumed.log').read_text()
+ summary=json.loads((logs/'audit/tito_summary.json').read_text())
+ assert summary and all(v['tito_pass']==v['completed_results'] and v['retained_tokens']==v['eligible_tokens'] and v['rows_over_token_budget']==0 for v in summary.values())
+ else:
+ initial=json.loads((run/'optimizer_evidence_from_0.json').read_text())
+ resumed=json.loads((run/'optimizer_evidence_from_2.json').read_text())
+ assert initial['final_step']==2 and resumed['final_step']==4
+ assert initial['weights_changed'] or resumed['weights_changed']
+ assert initial['final_weight_digest']==resumed['initial_weight_digest'], 'Resume did not load saved parameters'
+ token_rows=[json.loads(l) for l in (run/'token_audit.jsonl').read_text().splitlines()]
+ assert {r['step'] for r in token_rows}>={0,1,2,3}
+ assert all(row['tito_pass'] and row['supervised']>0 for r in token_rows for row in r['rows'])
+ # A native optimizer has persisted nonempty state at the resumed checkpoint.
+ import torch
+ optimizer=torch.load(run/'checkpoint-4/optimizer.pt',map_location='cpu',weights_only=False)
+ assert optimizer['state'] and optimizer['param_groups']
+ report={'arm':arm,'job_id':str(job),'passed':True,'optimizer_steps':[1,2,3,4],
+ 'nonzero_gradient_updates':sum(r['grad_norm']>0 for r in updates),
+ 'checkpoint_steps':[2,4],'native_optimizer_state_verified':True,
+ 'resume_verified':True,'tito_pass':True,'weights_updated':True}
+ target=ROOT/arm/'training-smoke/validation.json';target.write_text(json.dumps(report,indent=2)+'\n')
+ return report
+
+
+if __name__=='__main__':
+ p=argparse.ArgumentParser(description=__doc__);p.add_argument('--arm',choices=['blackbox','whitebox'],required=True)
+ p.add_argument('--job',required=True);a=p.parse_args();print(json.dumps(validate(a.arm,a.job)))
diff --git a/04-data-agent/train/LOGGING.md b/04-data-agent/train/LOGGING.md
new file mode 100644
index 0000000..2bbc413
--- /dev/null
+++ b/04-data-agent/train/LOGGING.md
@@ -0,0 +1,13 @@
+# Training and evaluation logging
+
+The durable scalar source is `audit/metrics.jsonl` for async training and `run/metrics.jsonl` for SETA. Optimizer steps are the horizontal axis. Raw captures and token/mask audits are stored separately.
+
+`hf/runtime/logging_sync.py` replays events into Trackio in a separate process. Deterministic log IDs prevent duplicates after a restart. It writes `trackio-events.jsonl`, a consistent `trackio-backup/` database snapshot and `trackio_verified.json`. The asynchronous artifact publisher uploads these alongside run metadata; network requests do not execute in the optimizer callback.
+
+The default reproduction uses local Trackio plus remote artifact storage. A public dashboard is a separate presentation service, not part of an environment Space. The recorded runs use the [shared comparison dashboard](https://huggingface.co/spaces/HuggingEnvs/data-agent-training-comparison-trackio); `hf/consolidate_async_runs.py` contains the historical collector. Use a distinct project/run identity for a new reproduction.
+
+For offline viewing, download a verified database backup, copy it to a local writable directory, and point `TRACKIO_DIR` there before running `trackio show --project PROJECT_NAME`. Avoid a writable SQLite database on a shared network filesystem. The pinned Trackio version is 0.33.0.
+
+Completed checkpoint scores are replayed only after fixed coverage, TiTO, harness-version and checkpoint checks. Evaluations that finish after training remain in the coordinator's score artifacts for later replay. The scalar payload includes reward, loss, gradient norm, staleness, token throughput, fork/row counts and pass@1 by harness/difficulty. Training reward and held-out pass@1 remain separate metrics.
+
+The current frozen results and full metrics snapshot are in [../results.md](../results.md). Historical operator notes and retired dashboard repair instructions are preserved locally under ignored `temp/historical-notes/`.
diff --git a/04-data-agent/train/_pypath/data_agent_env b/04-data-agent/train/_pypath/data_agent_env
new file mode 120000
index 0000000..84b66a2
--- /dev/null
+++ b/04-data-agent/train/_pypath/data_agent_env
@@ -0,0 +1 @@
+../../envs/blackbox-opencode
\ No newline at end of file
diff --git a/04-data-agent/train/_pypath/harbor_reward.py b/04-data-agent/train/_pypath/harbor_reward.py
new file mode 120000
index 0000000..19b7164
--- /dev/null
+++ b/04-data-agent/train/_pypath/harbor_reward.py
@@ -0,0 +1 @@
+../harbor_reward.py
\ No newline at end of file
diff --git a/04-data-agent/train/_pypath/whitebox_bash b/04-data-agent/train/_pypath/whitebox_bash
new file mode 120000
index 0000000..98792a1
--- /dev/null
+++ b/04-data-agent/train/_pypath/whitebox_bash
@@ -0,0 +1 @@
+../../envs/whitebox-bash
\ No newline at end of file
diff --git a/04-data-agent/train/atomic_rollouts.py b/04-data-agent/train/atomic_rollouts.py
new file mode 100644
index 0000000..cff332b
--- /dev/null
+++ b/04-data-agent/train/atomic_rollouts.py
@@ -0,0 +1,459 @@
+"""Single-GPU AsyncGRPO recipe that consumes every row of an admitted rollout.
+
+The reference uses four token-packed batches per update. A forked rollout can
+exceed one batch: keep it as one admission unit and stream its exact rows through
+bounded forwards before the optimizer changes weights. No TRL source patch or
+token rewriting is needed. The loss is a token mean over the entire update.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import json
+import queue
+import time
+from dataclasses import dataclass, replace
+from pathlib import Path
+
+import torch
+from torch.utils.data import DataLoader, IterableDataset
+from transformers import TrainerCallback
+from trl.experimental.async_grpo import AsyncGRPOTrainer
+from trl.experimental.async_grpo.async_grpo_trainer import DataCollatorForRollout
+from trl.experimental.async_grpo.openenv_harness import (
+ HarnessRolloutWorker,
+ _HarnessRolloutLoop,
+)
+
+
+@dataclass
+class RolloutBundle:
+ rows: list
+ rollout_id: str
+ enqueued_at: float | None = None
+
+ @property
+ def model_version(self):
+ return self.rows[0].model_version
+
+ @property
+ def group_id(self):
+ return self.rows[0].group_id
+
+ @property
+ def prompt(self):
+ return self.rows[0].prompt
+
+ @property
+ def completion(self):
+ return self.rows[0].completion
+
+ @property
+ def advantage(self):
+ return self.rows[0].advantage
+
+ @property
+ def metrics(self):
+ return self.rows[0].metrics
+
+ @property
+ def forwarded_tokens(self):
+ return sum(len(row.input_ids) for row in self.rows)
+
+
+class CreditQueue:
+ """Native queue plus a spawn-safe budget, passed through native worker IPC.
+
+ Credits cover generation, scoring and queued work until consumption. Putting
+ the counter beside the native queue avoids the worker's ordinary-pickle
+ validation of loop kwargs (shared values must use the multiprocessing spawn path).
+ """
+
+ def __init__(self, queue, credits, capacity):
+ self.queue, self.credits, self.capacity = queue, credits, capacity
+
+ def reserve_group(self, count):
+ # Reserve all generations together: individually competing waiters can
+ # otherwise occupy every credit with several incomplete GRPO groups.
+ with self.credits.get_lock():
+ if self.credits.value < count:
+ return False
+ self.credits.value -= count
+ return True
+
+ def release(self, count):
+ with self.credits.get_lock():
+ if self.credits.value + count > self.capacity:
+ raise RuntimeError("Rollout credit released more than once")
+ self.credits.value += count
+
+ def __getattr__(self, name):
+ queue = self.__dict__.get("queue")
+ if queue is None:
+ raise AttributeError(name)
+ return getattr(queue, name)
+
+
+class AtomicHarnessLoop(_HarnessRolloutLoop):
+ async def _reserve_group(self):
+ while not self.rollout_buffer.reserve_group(self.num_generations):
+ if self._stop_event.is_set():
+ return False
+ await asyncio.sleep(0.05)
+ return True
+
+ async def _generate_one(self, prompt, tool_dict, tools, group_id=0):
+ credits = getattr(self.rollout_buffer, "credits", None)
+ if credits is None:
+ return await super()._generate_one(prompt, tool_dict, tools, group_id)
+ if not hasattr(self, "_group_reservations"):
+ self._group_reservations = {}
+ if group_id not in self._group_reservations:
+ self._group_reservations[group_id] = asyncio.create_task(
+ self._reserve_group()
+ )
+ if not await self._group_reservations[group_id]:
+ return self._EMPTY_ROLLOUT
+ # Native groups can be created before credit becomes available. Record
+ # actual dispatch, never relabel an older sampled policy as a newer one.
+ version = self.model_version
+ try:
+ result = await super()._generate_one(prompt, tool_dict, tools, group_id)
+ except BaseException:
+ self.rollout_buffer.release(1)
+ raise
+ sequences = result[2]
+ if sequences:
+ if not hasattr(self, "_dispatch_versions"):
+ self._dispatch_versions = {}
+ self._dispatch_versions[sequences[0].rollout_id] = version
+ else:
+ self.rollout_buffer.release(1)
+ return result
+
+ async def _score_group(self, group):
+ getattr(self, "_group_reservations", {}).pop(group.group_id, None)
+ versions = getattr(self, "_dispatch_versions", {})
+ actual = [
+ versions.pop(sequences[0].rollout_id)
+ for sequences in group.completions_sequences
+ if sequences and sequences[0].rollout_id in versions
+ ]
+ if actual:
+ if len(actual) != sum(
+ bool(sequences) for sequences in group.completions_sequences
+ ):
+ raise RuntimeError("A scored rollout has no dispatch policy version")
+ group = replace(group, model_version=min(actual))
+ rows = await super()._score_group(group)
+ bundles, offset = [], 0
+ for sequences in group.completions_sequences:
+ count = len(sequences)
+ if count:
+ selected = rows[offset : offset + count]
+ assert len(selected) == count
+ rollout_id = sequences[0].rollout_id
+ assert all(seq.rollout_id == rollout_id for seq in sequences)
+ bundles.append(RolloutBundle(selected, rollout_id))
+ offset += count
+ assert offset == len(rows)
+ return bundles
+
+
+class AtomicHarnessWorker(HarnessRolloutWorker):
+ _loop_cls = AtomicHarnessLoop
+
+ def __init__(self, *, max_outstanding_rollouts=0, **kwargs):
+ super().__init__(**kwargs)
+ self.max_outstanding_rollouts = max_outstanding_rollouts
+ self.num_generations = kwargs["num_generations"]
+ if max_outstanding_rollouts:
+ if max_outstanding_rollouts < 2 * self.num_generations:
+ raise ValueError("Outstanding budget must allow two complete groups")
+ self.rollout_buffer = CreditQueue(
+ self.rollout_buffer,
+ self._mp_ctx.Value("i", max_outstanding_rollouts),
+ max_outstanding_rollouts,
+ )
+
+ def release_rollouts(self, count):
+ credits = getattr(self.rollout_buffer, "credits", None)
+ if credits is not None:
+ self.rollout_buffer.release(count)
+
+
+class AtomicRolloutDataset(IterableDataset):
+ def __init__(
+ self,
+ worker,
+ metrics,
+ target_tokens,
+ max_row_tokens,
+ max_staleness,
+ heartbeat_seconds,
+ max_rollouts_per_unit=None,
+ rejection_path=None,
+ group_offset=0,
+ ):
+ self.worker, self.metrics = worker, metrics
+ self.target_tokens, self.max_row_tokens = target_tokens, max_row_tokens
+ self.max_staleness, self.heartbeat_seconds = max_staleness, heartbeat_seconds
+ self.wait_s = 0.0
+ self.pending = None
+ self.max_rollouts_per_unit = max_rollouts_per_unit
+ self.rejection_path = rejection_path
+ self.group_offset = group_offset
+
+ def _next_bundle(self):
+ if self.pending is not None:
+ bundle, self.pending = self.pending, None
+ return bundle
+ started = time.monotonic()
+ while True:
+ try:
+ bundle = self.worker.rollout_buffer.get(timeout=5)
+ self.wait_s += time.monotonic() - started
+ return bundle
+ except queue.Empty:
+ self.worker.check_health(self.heartbeat_seconds)
+
+ def __iter__(self):
+ while True:
+ bundles, tokens = [], 0
+ while tokens < self.target_tokens and (
+ self.max_rollouts_per_unit is None
+ or len(bundles) < self.max_rollouts_per_unit
+ ):
+ bundle = self._next_bundle()
+ channel = self.worker.rollout_buffer
+ if isinstance(channel, CreditQueue):
+ self.metrics["admission/outstanding_rollouts_max"].append(
+ float(channel.capacity - channel.credits.value)
+ )
+ if not isinstance(bundle, RolloutBundle) or not bundle.rows:
+ raise RuntimeError(
+ "Atomic trainer requires nonempty rollout bundles"
+ )
+ staleness = self.worker.model_version - bundle.model_version
+ if staleness > self.max_staleness:
+ self.metrics["admission/stale_rollouts_dropped_total"].append(1.0)
+ self.metrics["sample/dropped_stale_total"].append(
+ float(len(bundle.rows))
+ )
+ if self.rejection_path:
+ self.rejection_path.parent.mkdir(parents=True, exist_ok=True)
+ with self.rejection_path.open("a") as stream:
+ stream.write(
+ json.dumps(
+ {
+ "rollout_id": bundle.rollout_id,
+ "group_id": bundle.group_id + self.group_offset,
+ "rows": len(bundle.rows),
+ "model_version": bundle.model_version,
+ "current_model_version": self.worker.model_version,
+ "reason": "whole_rollout_staleness_limit",
+ }
+ )
+ + "\n"
+ )
+ if hasattr(self.worker, "release_rollouts"):
+ self.worker.release_rollouts(1)
+ continue
+ if any(len(row.input_ids) > self.max_row_tokens for row in bundle.rows):
+ raise RuntimeError(
+ "Captured row exceeds the tested context limit; refusing to discard it"
+ )
+ if bundles and tokens + bundle.forwarded_tokens > self.target_tokens:
+ self.pending = bundle
+ break
+ # Recheck pending bundles above when they are actually admitted next time.
+ bundles.append(bundle)
+ tokens += bundle.forwarded_tokens
+ self.metrics["sample/staleness_mean"].append(float(staleness))
+ self.metrics["sample/staleness_max"].append(float(staleness))
+ self.metrics["sample/rollout_queue_size"].append(
+ float(self.worker.rollout_buffer.qsize())
+ )
+ if bundle.enqueued_at is not None:
+ self.metrics["sample/time_in_queue_s"].append(
+ time.time() - bundle.enqueued_at
+ )
+ yield {"rollouts": bundles}
+
+
+def pack_rows(bundles, target_tokens, max_row_tokens):
+ """Keep exact sequences intact; a long sequence gets one dedicated forward."""
+ packed, tokens = [], 0
+ for bundle in bundles:
+ for row in bundle.rows:
+ size = len(row.input_ids)
+ if size > max_row_tokens:
+ raise ValueError("Row exceeds maximum context")
+ if packed and tokens + size > target_tokens:
+ yield packed
+ packed, tokens = [], 0
+ packed.append(
+ {
+ key: getattr(row, key)
+ for key in (
+ "input_ids",
+ "completion_mask",
+ "old_log_probs",
+ "advantage",
+ "group_id",
+ "metrics",
+ )
+ }
+ )
+ tokens += size
+ if tokens >= target_tokens:
+ yield packed
+ packed, tokens = [], 0
+ if packed:
+ yield packed
+
+
+def identity(value):
+ return value
+
+
+class AtomicRolloutTrainer(AsyncGRPOTrainer):
+ def __init__(self, *args, max_row_tokens=131072, admission_dir=None, **kwargs):
+ self.max_row_tokens = max_row_tokens
+ self.admission_dir = Path(admission_dir) if admission_dir else None
+ self._atomic_finished = []
+ super().__init__(*args, **kwargs)
+ if self.accelerator.num_processes != 1 or self.aux_loss_enabled:
+ raise ValueError(
+ "Atomic recipe is validated only for a single-GPU dense trainer"
+ )
+ self.add_callback(AtomicAdmissionCallback(self))
+
+ def get_train_dataloader(self):
+ outstanding = getattr(self.rollout_worker, "max_outstanding_rollouts", 0)
+ max_per_unit = None
+ if outstanding:
+ # Leave capacity for an entire new GRPO group while accumulating.
+ # Otherwise a short partial batch can hold all credits and wait
+ # forever for a group whose final generation cannot start.
+ max_per_unit = (
+ outstanding - self.rollout_worker.num_generations
+ ) // self.args.gradient_accumulation_steps
+ if max_per_unit < 1:
+ raise ValueError("Outstanding budget cannot fill an optimizer update")
+ dataset = AtomicRolloutDataset(
+ self.rollout_worker,
+ self._metrics["train"],
+ self.args.token_budget,
+ self.max_row_tokens,
+ self.args.max_staleness,
+ self.args.heartbeat_stale_after_s,
+ max_rollouts_per_unit=max_per_unit,
+ rejection_path=(
+ self.admission_dir / "rejected_rollouts.jsonl"
+ if self.admission_dir
+ else None
+ ),
+ group_offset=self._groups_before_resume,
+ )
+ self._rollout_dataset = dataset
+ self._atomic_collator = DataCollatorForRollout(
+ self.processing_class.pad_token_id,
+ groups_trained=self._trained_groups,
+ metrics=self._metrics["train"],
+ token_budget=self.max_row_tokens,
+ )
+ # There is exactly one rank. Dispatcher prefetch/slicing would split the
+ # nested rollout container; inner tensors move to the GPU in training_step.
+ return DataLoader(dataset, batch_size=None, num_workers=0, collate_fn=identity)
+
+ def get_batch_samples(self, epoch_iterator, num_batches, device):
+ batches = [next(epoch_iterator) for _ in range(num_batches)]
+ count = sum(
+ sum(row.completion_mask[1:])
+ for batch in batches
+ for bundle in batch["rollouts"]
+ for row in bundle.rows
+ )
+ if count <= 0:
+ raise RuntimeError("Optimizer update has no supervised tokens")
+ for batch in batches:
+ batch["normalization_tokens"] = count
+ return batches, None
+
+ def compute_loss(
+ self, model, inputs, return_outputs=False, num_items_in_batch=None
+ ):
+ loss = super().compute_loss(model, inputs, return_outputs, num_items_in_batch)
+ # Native loss divides by this forward's token count and by GAS. Undo
+ # those factors and normalize once across all exact tokens in the update.
+ # Keep native counters untouched: they report actual forwarded/trained tokens.
+ return loss * (
+ self.current_gradient_accumulation_steps
+ * inputs["global_n_tokens"][0]
+ / self._atomic_normalization_tokens
+ )
+
+ def training_step(self, model, inputs, num_items_in_batch):
+ self._atomic_normalization_tokens = inputs["normalization_tokens"]
+ total_loss = torch.zeros((), device=self.args.device)
+ rows_done = tokens_done = 0
+ for rows in pack_rows(
+ inputs["rollouts"], self.args.token_budget, self.max_row_tokens
+ ):
+ tensors = self._atomic_collator([[rows]])
+ tokens_done += int(tensors["global_n_tokens"][0])
+ rows_done += len(rows)
+ total_loss += super().training_step(model, tensors, None)
+ expected_rows = sum(len(bundle.rows) for bundle in inputs["rollouts"])
+ expected_tokens = sum(
+ sum(row.completion_mask[1:])
+ for bundle in inputs["rollouts"]
+ for row in bundle.rows
+ )
+ assert rows_done == expected_rows and tokens_done == expected_tokens
+ self._atomic_finished.extend(
+ {
+ "rollout_id": bundle.rollout_id,
+ "local_group_id": bundle.group_id,
+ "group_id": bundle.group_id + self._groups_before_resume,
+ "model_version": bundle.model_version,
+ "rows": len(bundle.rows),
+ "supervised_tokens": sum(
+ sum(row.completion_mask[1:]) for row in bundle.rows
+ ),
+ }
+ for bundle in inputs["rollouts"]
+ )
+ return total_loss
+
+ def floating_point_ops(self, inputs):
+ # Native per-forward token/timing metrics cover the nested batches.
+ return 0
+
+
+class AtomicAdmissionCallback(TrainerCallback):
+ def __init__(self, trainer):
+ self.trainer = trainer
+
+ def on_step_end(self, args, state, control, **kwargs):
+ rows = self.trainer._atomic_finished
+ if self.trainer.admission_dir:
+ self.trainer.admission_dir.mkdir(parents=True, exist_ok=True)
+ with (self.trainer.admission_dir / "optimizer_rollouts.jsonl").open(
+ "a"
+ ) as stream:
+ stream.write(
+ json.dumps(
+ {
+ "step": state.global_step,
+ "rollouts": rows,
+ "all_admitted_rows_consumed": True,
+ "normalization": "update_supervised_token_mean",
+ }
+ )
+ + "\n"
+ )
+ self.trainer._atomic_finished = []
+ if hasattr(self.trainer.rollout_worker, "release_rollouts"):
+ self.trainer.rollout_worker.release_rollouts(len(rows))
diff --git a/04-data-agent/train/checkpoint_artifacts.py b/04-data-agent/train/checkpoint_artifacts.py
new file mode 100644
index 0000000..40acebc
--- /dev/null
+++ b/04-data-agent/train/checkpoint_artifacts.py
@@ -0,0 +1,187 @@
+"""Publish completed full-model checkpoints and stage read-only evaluation inputs."""
+import argparse
+import hashlib
+import json
+from pathlib import Path
+import shutil
+
+READY = 'checkpoint.ready.json'
+SAVED = 'checkpoint.saved.json'
+REQUIRED = ('config.json', 'trainer_state.json', 'tokenizer.json', 'tokenizer_config.json',
+ 'training_args.bin', 'optimizer.pt', 'scheduler.pt', 'rng_state.pth')
+METADATA = ('config.json', 'generation_config.json', 'tokenizer.json', 'tokenizer_config.json',
+ 'preprocessor_config.json', 'video_preprocessor_config.json', 'processor_config.json', 'chat_template.json',
+ 'chat_template.jinja', 'special_tokens_map.json', 'vocab.json', 'merges.txt',
+ 'added_tokens.json', 'model.safetensors.index.json')
+
+
+def digest(path):
+ h = hashlib.sha256()
+ with Path(path).open('rb') as stream:
+ for block in iter(lambda: stream.read(8 * 1024 * 1024), b''):
+ h.update(block)
+ return h.hexdigest()
+
+
+def write_json(path, data):
+ temporary = path.with_suffix(path.suffix + '.tmp')
+ temporary.write_text(json.dumps(data, indent=2) + '\n')
+ temporary.replace(path)
+
+
+def model_files(checkpoint):
+ from safetensors import safe_open
+ index = checkpoint / 'model.safetensors.index.json'
+ weight_map = json.loads(index.read_text())['weight_map'] if index.exists() else None
+ names = set(weight_map.values()) if weight_map else {'model.safetensors'}
+ if not names or any(Path(n).name != n or not n.endswith('.safetensors') for n in names):
+ raise ValueError('Invalid checkpoint shard index')
+ all_keys = set()
+ for name in sorted(names):
+ with safe_open(checkpoint / name, framework='numpy') as tensors:
+ keys = set(tensors.keys())
+ if not keys or all_keys.intersection(keys):
+ raise ValueError('Empty shard or duplicated tensor keys')
+ if weight_map and keys != {k for k, v in weight_map.items() if v == name}:
+ raise ValueError(f'Shard contents disagree with index: {name}')
+ all_keys.update(keys)
+ return sorted(names)
+
+
+def mark_saved(checkpoint, step, base_model, base_revision, *, final=False):
+ """Publish a small handoff after save; leave weight hashing to the CPU watcher."""
+ checkpoint = Path(checkpoint).resolve()
+ for name in REQUIRED:
+ if not (checkpoint / name).is_file() or (checkpoint / name).stat().st_size == 0:
+ raise ValueError(f'Incomplete checkpoint: {name}')
+ if json.loads((checkpoint / 'trainer_state.json').read_text())['global_step'] != step:
+ raise ValueError('Checkpoint step disagrees with trainer state')
+ names = set(REQUIRED) | set(model_files(checkpoint))
+ names.update(n for n in METADATA if (checkpoint / n).exists())
+ marker = {'schema_version': 1, 'checkpoint': str(checkpoint), 'step': step, 'final': final,
+ 'base_model': base_model, 'base_revision': base_revision,
+ 'file_stats': {n: [(checkpoint / n).stat().st_size, (checkpoint / n).stat().st_mtime_ns]
+ for n in sorted(names)}}
+ write_json(checkpoint / SAVED, marker)
+ return marker
+
+
+def finalize_saved(checkpoint):
+ checkpoint = Path(checkpoint).resolve()
+ marker = json.loads((checkpoint / SAVED).read_text())
+ if marker['checkpoint'] != str(checkpoint):
+ raise ValueError('Saved checkpoint path mismatch')
+ for name, expected in marker['file_stats'].items():
+ if Path(name).name != name:
+ raise ValueError('Invalid saved checkpoint filename')
+ stat = (checkpoint / name).stat()
+ if [stat.st_size, stat.st_mtime_ns] != expected:
+ raise ValueError(f'Checkpoint changed after save: {name}')
+ return mark_ready(checkpoint, marker['step'], marker['base_model'], marker['base_revision'],
+ final=marker.get('final', False))
+
+
+def mark_ready(checkpoint, step, base_model, base_revision, *, final=False):
+ checkpoint = Path(checkpoint).resolve()
+ for name in REQUIRED:
+ if not (checkpoint / name).is_file() or (checkpoint / name).stat().st_size == 0:
+ raise ValueError(f'Incomplete checkpoint: {name}')
+ if json.loads((checkpoint / 'trainer_state.json').read_text())['global_step'] != step:
+ raise ValueError('Checkpoint step disagrees with trainer state')
+ names = model_files(checkpoint) + [n for n in METADATA if (checkpoint / n).exists()]
+ names = sorted(set(names + ['trainer_state.json']))
+ before = {n: ((checkpoint / n).stat().st_size, (checkpoint / n).stat().st_mtime_ns) for n in names}
+ hashes = {n: digest(checkpoint / n) for n in names}
+ after = {n: ((checkpoint / n).stat().st_size, (checkpoint / n).stat().st_mtime_ns) for n in names}
+ if before != after:
+ raise ValueError('Checkpoint changed during finalization')
+ marker = {'schema_version': 1, 'step': step, 'checkpoint': str(checkpoint), 'final': final,
+ 'base_model': base_model, 'base_revision': base_revision,
+ 'files': hashes, 'file_stats': after, 'training_state_present': list(REQUIRED)}
+ write_json(checkpoint / READY, marker)
+ return marker
+
+
+def verify_ready(checkpoint):
+ checkpoint = Path(checkpoint).resolve()
+ marker = json.loads((checkpoint / READY).read_text())
+ if marker['checkpoint'] != str(checkpoint):
+ raise ValueError('Checkpoint path differs from its completion marker')
+ for name, expected in marker['files'].items():
+ if Path(name).name != name or digest(checkpoint / name) != expected:
+ raise ValueError(f'Checkpoint changed after completion: {name}')
+ model_files(checkpoint)
+ return marker
+
+
+def resume_info(checkpoint, base_model, base_revision):
+ """Validate a completed local Trainer checkpoint, including its rollout cursor."""
+ checkpoint = Path(checkpoint).resolve()
+ path = checkpoint / SAVED if (checkpoint / SAVED).exists() else checkpoint / READY
+ if not path.is_file():
+ raise ValueError('Resume requires a completed checkpoint marker')
+ marker = json.loads(path.read_text())
+ if (marker['checkpoint'] != str(checkpoint) or marker['base_model'] != base_model
+ or marker['base_revision'] != base_revision):
+ raise ValueError('Resume checkpoint path or base model/revision differs')
+ for name in REQUIRED + ('rollout_state.json',):
+ if not (checkpoint / name).is_file() or (checkpoint / name).stat().st_size == 0:
+ raise ValueError(f'Incomplete resume checkpoint: {name}')
+ for name, expected in marker['file_stats'].items():
+ if Path(name).name != name:
+ raise ValueError('Invalid checkpoint filename')
+ stat = (checkpoint / name).stat()
+ if [stat.st_size, stat.st_mtime_ns] != expected:
+ raise ValueError(f'Resume checkpoint changed after save: {name}')
+ model_files(checkpoint)
+ state = json.loads((checkpoint / 'trainer_state.json').read_text())
+ rollout = json.loads((checkpoint / 'rollout_state.json').read_text())
+ if state['global_step'] != marker['step'] or marker['step'] <= 0:
+ raise ValueError('Resume step disagrees with completion marker')
+ if any(type(rollout.get(k)) is not int or rollout[k] < 0 for k in ('prompt_index', 'model_version')):
+ raise ValueError('Invalid rollout cursor/model version')
+ return {'checkpoint': str(checkpoint), 'step': marker['step'],
+ 'group_offset': rollout['prompt_index'], 'model_version': rollout['model_version'],
+ 'rollout_state_sha256': digest(checkpoint / 'rollout_state.json')}
+
+
+def stage_model(checkpoint, target, base_metadata):
+ checkpoint, target, base_metadata = map(Path, (checkpoint, target, base_metadata))
+ marker = verify_ready(checkpoint)
+ target.mkdir(parents=True, exist_ok=False)
+ for name in model_files(checkpoint):
+ (target / name).symlink_to((checkpoint / name).resolve())
+ origins = {}
+ for name in METADATA:
+ source = checkpoint / name
+ if not source.exists():
+ if name == 'config.json' or name == 'model.safetensors.index.json':
+ continue
+ source = base_metadata / name
+ if source.exists():
+ shutil.copy2(source, target / name)
+ origins[name] = str(source.resolve())
+ assert (target / 'config.json').read_bytes() == (checkpoint / 'config.json').read_bytes()
+ record = {'checkpoint': marker, 'metadata_origins': origins,
+ 'files': {p.name: digest(p) for p in target.iterdir() if p.is_file()}}
+ write_json(target / 'checkpoint_source.json', record)
+ return record
+
+
+def verify_stage(target):
+ target = Path(target)
+ record = json.loads((target / 'checkpoint_source.json').read_text())
+ for name, expected in record['files'].items():
+ if Path(name).name != name or digest(target / name) != expected:
+ raise ValueError(f'Staged checkpoint changed: {name}')
+ model_files(target)
+ return record
+
+
+if __name__ == '__main__':
+ parser = argparse.ArgumentParser(description=__doc__)
+ parser.add_argument('command', choices=['verify'])
+ parser.add_argument('directory', type=Path)
+ args = parser.parse_args()
+ record = verify_stage(args.directory)
+ print(f"Verified checkpoint step {record['checkpoint']['step']}: {args.directory}")
diff --git a/04-data-agent/train/continue_allocation.py b/04-data-agent/train/continue_allocation.py
new file mode 100644
index 0000000..cbd7594
--- /dev/null
+++ b/04-data-agent/train/continue_allocation.py
@@ -0,0 +1,237 @@
+"""Resume a frozen multi-harness run after its allocation, without resetting training.
+
+Run ``execute`` in a CPU Slurm job with afterany: dependency. Preparation
+reuses the parent's immutable source and tasks; GPU submission uses its existing
+launcher. Explicit cancellation and STOP_AFTER_STEP are respected.
+"""
+from __future__ import annotations
+
+import argparse
+from copy import deepcopy
+from datetime import datetime, timezone
+import fcntl
+import importlib.util
+import json
+import os
+from pathlib import Path
+import shlex
+import shutil
+import subprocess
+import sys
+
+ROLES = ("training", "cleanup", "eval_watcher", "monitor", "logging")
+RESUMABLE_STATES = {"COMPLETED", "TIMEOUT", "NODE_FAIL", "BOOT_FAIL", "PREEMPTED"}
+
+
+def read(path):
+ return json.loads(Path(path).read_text())
+
+
+def save(path, value):
+ path = Path(path)
+ path.parent.mkdir(parents=True, exist_ok=True)
+ tmp = path.with_suffix(path.suffix + ".tmp")
+ tmp.write_text(json.dumps(value, indent=2) + "\n")
+ tmp.replace(path)
+
+
+def checkpoint_validator(parent):
+ source = parent / "source-snapshot/HuggingEnvs/04-data-agent/train/checkpoint_artifacts.py"
+ spec = importlib.util.spec_from_file_location("continuation_checkpoint_artifacts", source)
+ module = importlib.util.module_from_spec(spec)
+ spec.loader.exec_module(module)
+ return module.resume_info
+
+
+def select_checkpoint(parent, job, config, validate=None):
+ validate = validate or checkpoint_validator(parent)
+ candidates = [p for p in (parent / f"job-{job}/run").glob("checkpoint-*")
+ if p.is_dir() and p.name.removeprefix("checkpoint-").isdigit()]
+ rejected = []
+ for path in sorted(candidates, key=lambda p: int(p.name.split("-")[-1]), reverse=True):
+ try:
+ resume = validate(path, config["model"], config["model_revision"])
+ if resume["step"] != int(path.name.split("-")[-1]):
+ raise ValueError("Checkpoint directory and saved step disagree")
+ return resume, rejected
+ except (ValueError, OSError, KeyError) as exc:
+ rejected.append({"checkpoint": str(path), "reason": str(exc)})
+ raise ValueError(f"No valid full checkpoint found; rejected={rejected}")
+
+
+def may_continue(parent, job, state):
+ stopped = (parent / f"job-{job}/STOP_AFTER_STEP").exists()
+ stopped |= (parent / "operations/allocation-continuation/STOP").exists()
+ return not stopped and state in RESUMABLE_STATES
+
+
+def prepare(parent, output, resume, *, walltime="24:00:00", seconds=82200):
+ config = deepcopy(read(parent / "run_config.json"))
+ if seconds <= 0 or seconds >= 24 * 3600:
+ raise ValueError("The soft training limit must leave room before the 24-hour allocation ends")
+ output.mkdir(parents=True, exist_ok=False)
+ for name in ("manifest.json", "indices.txt", "harness_schedule.json", "pairs.jsonl",
+ "runtime_versions.json", "source_hashes.json"):
+ shutil.copy2(parent / name, output / name)
+ if (parent / "schedule_summary.json").exists():
+ shutil.copy2(parent / "schedule_summary.json", output / "schedule_summary.json")
+ # No working-tree overlays: the continuation runs the identical frozen code.
+ snapshot = output / "source-snapshot"
+ snapshot.symlink_to((parent / "source-snapshot").resolve(), target_is_directory=True)
+ evaluation = output / "checkpoint-evals/eval-source"
+ evaluation.parent.mkdir()
+ evaluation.symlink_to((parent / "checkpoint-evals/eval-source").resolve(), target_is_directory=True)
+ config.update(status="prepared", source_snapshot=str(snapshot), frozen_eval_source=str(evaluation),
+ restart_of=str(parent), resume_state=resume,
+ restart_reason="User-authorized allocation continuation; running-job extension denied by Slurm",
+ initialization="Full checkpoint: model, optimizer, scheduler, RNG and rollout cursor")
+ config.pop("job_id", None)
+ config.pop("replaced_by", None)
+ config.pop("allocation_continuation", None)
+ config["training"].update(resume_from_checkpoint=resume["checkpoint"], soft_max_train_seconds=seconds)
+ config["resources"]["slurm_walltime"] = walltime
+ config["dataset"]["schedule_file"] = str(output / "harness_schedule.json")
+ config["evaluation"]["protocol_file"] = str(evaluation / "protocol.json")
+ # Preserve the same Trackio project and evaluation curve; training-{job} records
+ # distinguish allocation histories while retaining global optimizer step numbers.
+ config["logging"].update(local_directory=str(output / "trackio"),
+ collector_file=str(snapshot / "tools/trackio_multi4.py"))
+ config["monitoring"]["stable_after_optimizer_step"] = resume["step"] + 3
+ config["monitoring"]["support_job_supervisor"].update(
+ host="Slurm CPU job", status_file=str(output / "supervisor/status.json"),
+ source_file=str(snapshot / "tools/supervise_multi4.py"))
+ save(output / "run_config.json", config)
+ save(output / "validation.json", {"prepared": True, "resume": resume,
+ "parent_validation_file": str(parent / "validation.json"),
+ "live_resume_validated": False, "frozen_source_reused": True})
+ save(output / "operations/ALLOCATION_CONTINUATION.json", {
+ "parent": str(parent), "resume": resume, "frozen_source": str(snapshot.resolve()),
+ "changed_training_fields": ["resume_from_checkpoint", "soft_max_train_seconds"],
+ "gpu_walltime": walltime, "target_step": config["training"]["max_steps"]})
+ return config
+
+
+def preflight(output):
+ env = {**os.environ, "TRAIN_RUN_ROOT": str(output), "MULTI4_PREFLIGHT_ONLY": "1",
+ "SLURM_JOB_ID": "preflight"}
+ log = output / "operations/continuation-preflight.log"
+ with log.open("w") as stream:
+ subprocess.run(["bash", str(output / "source-snapshot/tools/launch_multi4_long.sh")],
+ env=env, stdout=stream, stderr=subprocess.STDOUT, check=True)
+
+
+def parent_state(job):
+ result = subprocess.check_output(
+ ["sacct", "-X", "-n", "-P", "-j", str(job), "--format=JobIDRaw,State"], text=True)
+ for line in result.splitlines():
+ fields = line.split("|")
+ if fields[0] == str(job):
+ return fields[1].split()[0].rstrip("+")
+ return "UNKNOWN"
+
+
+def existing_submission(output):
+ path = output / "submission.json"
+ if not path.exists():
+ return None
+ record = read(path)
+ if not all(str(record.get(k, "")).isdigit() for k in ROLES):
+ raise RuntimeError("Incomplete submission record: reconcile Slurm before retrying; no duplicate GPU job submitted")
+ return record
+
+
+def execute(parent, output, expected_job):
+ operations = parent / "operations/allocation-continuation"
+ operations.mkdir(parents=True, exist_ok=True)
+ status_path = operations / "status.json"
+ with (operations / ".lock").open("w") as lock:
+ fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
+ status = read(status_path) if status_path.exists() else {}
+ try:
+ actual_job = str(read(parent / "submission.json")["training"])
+ if actual_job != str(expected_job):
+ raise ValueError("Parent training job changed")
+ state = parent_state(actual_job)
+ if not may_continue(parent, actual_job, state):
+ status.update(state="skipped", reason="Parent is not eligible for automatic allocation continuation",
+ parent_state=state)
+ save(status_path, status)
+ return status
+ config = read(parent / "run_config.json")
+ resume, rejected = select_checkpoint(parent, actual_job, config)
+ if resume["step"] >= config["training"]["max_steps"]:
+ status.update(state="complete", reason="Parent already reached target step", resume=resume)
+ save(status_path, status)
+ return status
+ if not output.exists():
+ prepare(parent, output, resume)
+ else:
+ prepared = read(output / "run_config.json")
+ if prepared.get("restart_of") != str(parent) or prepared.get("resume_state") != resume:
+ raise ValueError("Existing continuation directory has different provenance")
+ status.update(state="prepared", resume=resume, rejected_checkpoints=rejected,
+ output=str(output), parent_job=actual_job)
+ save(status_path, status)
+ submission = existing_submission(output)
+ if submission is None:
+ preflight(output)
+ env = {k: v for k, v in os.environ.items() if k not in {
+ "MULTI4_PREFLIGHT_ONLY", "TRAIN_RUN_ROOT", "TRAIN_JOB_ID", "CODE_ROOT", "TRAIN_AUDIT_TOOLS"}}
+ with (output / "operations/submission.log").open("a") as stream:
+ subprocess.run([sys.executable, str(output / "source-snapshot/tools/submit_multi4_long.py"),
+ "--run", str(output), "--submit"], env=env,
+ stdout=stream, stderr=subprocess.STDOUT, check=True)
+ submission = existing_submission(output)
+ status.update(state="submitted", submission=submission)
+ save(status_path, status)
+ if not status.get("supervisor_job"):
+ intent = operations / "supervisor-submission.json"
+ if intent.exists():
+ raise RuntimeError("Reconcile existing supervisor submission intent before retrying")
+ command = [sys.executable, "-u", str(output / "source-snapshot/tools/supervise_multi4.py"),
+ "--run", str(output)]
+ save(intent, {"state": "submitting", "command": command})
+ job = subprocess.check_output(["sbatch", "--parsable", "--partition=hopper-cpu",
+ "--ntasks=1", "--cpus-per-task=1", "--mem=2G", "--time=48:00:00",
+ "--job-name=multi4-support-supervisor", "--output=/fsx/%u/logs/%x-%j.out",
+ "--error=/fsx/%u/logs/%x-%j.err", "--wrap", shlex.join(command)], text=True).strip().split(";")[0]
+ if not job.isdigit():
+ raise RuntimeError("Unrecognized supervisor submission response")
+ status["supervisor_job"] = job
+ save(intent, {"state": "submitted", "job": job})
+ status.update(state="submitted", checked_at=datetime.now(timezone.utc).isoformat())
+ save(status_path, status)
+ return status
+ except Exception as exc:
+ status.update(state="error", error=str(exc), checked_at=datetime.now(timezone.utc).isoformat())
+ save(status_path, status)
+ raise
+
+
+def main():
+ parser = argparse.ArgumentParser(description=__doc__)
+ parser.add_argument("action", choices=["inspect", "prepare", "execute"])
+ parser.add_argument("--parent", type=Path, required=True)
+ parser.add_argument("--output", type=Path)
+ parser.add_argument("--parent-job", required=True)
+ args = parser.parse_args()
+ parent = args.parent.resolve()
+ if args.action == "execute":
+ if args.output is None:
+ parser.error("--output is required")
+ result = execute(parent, args.output.resolve(), args.parent_job)
+ else:
+ config = read(parent / "run_config.json")
+ resume, rejected = select_checkpoint(parent, args.parent_job, config)
+ result = {"resume": resume, "rejected_checkpoints": rejected}
+ if args.action == "prepare":
+ if args.output is None:
+ parser.error("--output is required")
+ prepare(parent, args.output.resolve(), resume)
+ preflight(args.output.resolve())
+ result.update(prepared=str(args.output.resolve()), preflight_passed=True, submitted=False)
+ print(json.dumps(result, indent=2))
+
+
+if __name__ == "__main__":
+ main()
diff --git a/04-data-agent/train/harbor_reward.py b/04-data-agent/train/harbor_reward.py
new file mode 100644
index 0000000..fb81c3a
--- /dev/null
+++ b/04-data-agent/train/harbor_reward.py
@@ -0,0 +1,128 @@
+"""Trainer-side reward: correctness from Harbor's verifier, efficiency from the trace.
+
+ reward = correctness x (1 + W_EFF * TOOL_BUDGET / (TOOL_BUDGET + n_tool_calls))
+
+MULTIPLICATIVE, NOT ADDITIVE-WITH-A-GATE. An efficiency bonus that is merely ADDED has a trapdoor:
+zero tool calls scores the MAXIMUM efficiency, so for a policy that cannot solve the task, doing
+nothing becomes the best move available (0.300 vs 0.030 for a real attempt that fails). The policy
+stops calling tools, `train_turn_fn=has_tool_call` then yields no trainable turns, and the group is
+empty. Jobs 72452 and 72473 wedged at step 7 and 10 of 100 exactly this way, spending 4,076 E2B
+sandboxes on 11 productive groups. A `if correct` gate patches that; multiplying STRUCTURALLY removes
+it -- `correctness == 0` zeroes the product, and reward is monotone non-decreasing in BOTH arguments,
+so efficiency can never be traded for correctness. The property survives refactoring; a gate may not.
+
+WHY IT IS COMPUTED HERE AND NOT IN THE VERIFIER. A reward belongs in the sandbox only if the sandbox
+is what makes it computable. Correctness needs the data, the gold answer and the tolerances. A
+tool-call count needs the TRACE, which lives in the capture proxy. The suite's own grader tried to
+read it from `/workdir/.n_tool_calls` and `$N_TOOL_CALLS`; nothing writes either, so it emitted `null`
+forever, Harbor's `dict[str, float|int]` rejected the whole dict, and `correctness` went down with it
+-- 86 of 250 tasks silently unscored until that was fixed at source (dataset rev 291c8e50).
+
+WHY 1/(1+n/B) AND NOT THE REFERENCE'S clip(1 - n/B). Measured over run 77284 (Qwen3.5-2B, opencode,
+118 logged steps), tool calls per rollout: p10 8.4, p50 14.7, p75 25.5, p90 56.6, max 125.5 -- a 15x
+range, drifting 13.1 -> 43.0 between the first and last 30 steps because nothing bounded it. A linear
+clamp cannot be both sensitive at 10 and unsaturated at 100:
+
+ n_tool_calls 5 8 15 30 66 95 125
+ clip(1-n/15) 0.667 0.467 0.000 0.000 0.000 0.000 0.000 <- inert above the MEDIAN
+ clip(1-n/60) 0.917 0.867 0.750 0.500 0.000 0.000 0.000 <- inert exactly where it is needed
+ 15/(15+n) 0.750 0.652 0.500 0.333 0.185 0.136 0.107 <- graded across the whole range
+
+The reciprocal keeps a gradient everywhere, and keeps it strongest near the budget, which is where we
+want the policy to land. `TOOL_BUDGET` stays the reference's 15 and now reads as a half-credit point
+rather than a cliff: efficiency is 0.5 at exactly 15 calls.
+
+THE SECOND REASON, WHICH IS THE LARGER ONE. 19 of 118 steps in run 77284 logged `reward_std == 0`,
+and ALL NINETEEN were groups where every generation SOLVED the task. Under pure-correctness reward an
+all-correct group has zero advantage for every member: 8 sandboxes, no gradient, 16% of the run. Those
+generations were not identical -- they differed in how long they took. Efficiency makes precisely
+those groups trainable. This term buys signal from rollouts already paid for.
+
+EFFICIENCY IS A TIE-BREAKER, NOT A COMPETING OBJECTIVE. Over the observed range the efficiency term
+moves the reward by at most 0.3 x (0.750 - 0.107) = 0.193, against the 1.0 swing of correctness. When
+a group disagrees about correctness, correctness dominates ~5:1; only when it agrees does efficiency
+decide. That ratio is the design, so keep `W_EFF` well under 1.
+
+A SOFT INCENTIVE IS NOT A BOUND. This shapes behaviour over many steps; it does not stop one rollout
+running 235 turns (77284's observed `turns_max`) and blowing the packed row, which grows with the
+SQUARE of the turn count -- `row_tokens_max` reached 40,598 against a 40,960 budget. The hard bound is
+`max_model_calls` in the capture proxy, the only harness-agnostic step limit, since only 1 of 29 seams
+honours `agent_step_limit` at all. Ship both; this one alone will not save the row.
+
+Reads `train/tools/call_frequency` in trackio, which is `float(n_calls)` per rollout
+(`async_rollout_worker.py:1052`) -- the exact quantity this reward acts on, already on the dashboard.
+"""
+
+from __future__ import annotations
+
+import logging
+import os
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING: # annotation-only, so this module imports (and unit-tests) without torch or vLLM
+ from trl.experimental.async_grpo.openenv_harness import HarnessRolloutOutcome
+
+logger = logging.getLogger(__name__)
+
+W_EFF = float(os.environ.get("REWARD_W_TOOL_EFFICIENCY", "0.3"))
+TOOL_BUDGET = float(os.environ.get("TOOL_BUDGET", "15"))
+
+_announced = False
+
+
+def tool_efficiency(n_tool_calls: int | None) -> float | None:
+ """`B / (B + n)` in `(0, 1]`, or `None` when the count is unknown.
+
+ Args:
+ n_tool_calls (`int`, *optional*):
+ Tool calls the agent made across every real turn, framework aux calls already dropped.
+
+ Returns:
+ `float` or `None`: `1.0` at zero calls, `0.5` at exactly `TOOL_BUDGET`, asymptotically `0` --
+ never actually `0`, which is what keeps a gradient in the 60-125 call regime where the
+ reference's `clip(1 - n/15)` is flat.
+ """
+ if n_tool_calls is None or TOOL_BUDGET <= 0:
+ return None
+ return TOOL_BUDGET / (TOOL_BUDGET + max(0, int(n_tool_calls)))
+
+
+def data_agent_reward(outcome: "HarnessRolloutOutcome") -> float | None:
+ """`correctness x (1 + W_EFF * efficiency)`, or `None` when the rollout is unscorable.
+
+ Args:
+ outcome (`HarnessRolloutOutcome`):
+ What the rollout produced -- verifier reward, transcript, tool-call counts, timeout flag.
+
+ Returns:
+ `float` or `None`: `None` means UNSCORABLE and DROPS the rollout from its group baseline.
+ Scoring it `0.0` would teach the policy that a crashed sandbox is as bad as a wrong
+ answer, and would poison the baseline with a value nothing produced.
+ """
+ global _announced
+
+ correctness = outcome.env_reward
+ if correctness is None:
+ logger.warning(
+ "verifier did not run (tool_calls=%d); rollout unscorable, dropped from the baseline",
+ outcome.tool_call_count,
+ )
+ return None
+
+ correctness = float(correctness)
+ if not _announced:
+ _announced = True
+ logger.warning(
+ "reward = correctness x (1 + %.2f * %.0f/(%.0f + tool_calls)); "
+ "max %.3f at 0 calls, %.3f at %.0f calls, ->1.0 as calls->inf",
+ W_EFF, TOOL_BUDGET, TOOL_BUDGET, 1.0 + W_EFF, 1.0 + W_EFF * 0.5, TOOL_BUDGET,
+ )
+ if outcome.timed_out:
+ # Kept, not zeroed: the verifier graded whatever work landed, and that is a measurement.
+ logger.warning("agent timed out; keeping the verifier's %.3f on the partial work", correctness)
+
+ eff = tool_efficiency(outcome.tool_call_count)
+ if eff is None or correctness <= 0.0:
+ # Nothing to scale, or scaling would invert the sign on a negative verifier score.
+ return correctness
+ return correctness * (1.0 + W_EFF * eff)
diff --git a/04-data-agent/train/harness_schedule.py b/04-data-agent/train/harness_schedule.py
new file mode 100644
index 0000000..75aeb18
--- /dev/null
+++ b/04-data-agent/train/harness_schedule.py
@@ -0,0 +1,82 @@
+"""Deterministic, difficulty-balanced task/harness rotation without changing TRL."""
+from collections import Counter
+import random
+
+
+def make_schedule(tasks, harnesses, *, seed=0, easy_start=32):
+ n, h = len(tasks), len(harnesses)
+ if not n or not h or n % h or len(set(harnesses)) != h:
+ raise ValueError('A balanced rotation requires distinct harnesses and a task count divisible by them')
+ if not 0 <= easy_start <= n or easy_start % h:
+ raise ValueError('The easy introduction must contain complete rounds of harnesses')
+ if any(t['difficulty'] != 'easy' for t in tasks[:easy_start]):
+ raise ValueError('The introduction must consist of easy tasks')
+ base, cursor = {}, 0
+ for tier in ['easy', 'medium', 'hard']:
+ for row, task in enumerate(tasks):
+ if task['difficulty'] == tier:
+ base[row] = cursor % h
+ cursor += 1
+ if len(base) != n:
+ raise ValueError('Unknown task difficulty')
+ groups = []
+ for pass_index in range(h):
+ rng = random.Random(seed + pass_index)
+ introduction = list(range(easy_start)) if pass_index == 0 else []
+ buckets = [[] for _ in harnesses]
+ for row in range(n):
+ if row not in introduction:
+ buckets[(base[row] + pass_index) % h].append(row)
+ for bucket in buckets:
+ rng.shuffle(bucket)
+ order = introduction + [bucket[i] for i in range(len(buckets[0])) for bucket in buckets]
+ for row in order:
+ groups.append({'group_in_cycle': len(groups), 'pass_index': pass_index,
+ 'task_row': row, 'task_index': tasks[row]['task_index'],
+ 'task_name': tasks[row]['name'], 'difficulty': tasks[row]['difficulty'],
+ 'harness': harnesses[(base[row] + pass_index) % h]})
+ result = {'schema_version': 1, 'mode': 'one_harness_per_task_per_pass', 'seed': seed,
+ 'harnesses': harnesses, 'tasks': tasks, 'task_count': n,
+ 'groups_per_pass': n, 'passes_per_cycle': h, 'groups_per_cycle': len(groups),
+ 'easy_start_task_count': easy_start, 'groups': groups}
+ validate_schedule(result)
+ return result
+
+
+def validate_schedule(schedule):
+ tasks, harnesses, groups = schedule['tasks'], schedule['harnesses'], schedule['groups']
+ n, h = len(tasks), len(harnesses)
+ if n == 0 or h == 0 or n % h or len(groups) != n * h or len(set(harnesses)) != h:
+ raise ValueError('Incomplete rotation cycle')
+ expected_metadata = {'schema_version': 1, 'mode': 'one_harness_per_task_per_pass',
+ 'task_count': n, 'groups_per_pass': n, 'passes_per_cycle': h,
+ 'groups_per_cycle': n * h}
+ if any(schedule.get(k) != v for k, v in expected_metadata.items()):
+ raise ValueError('Schedule metadata disagrees with the rotation')
+ if any(t['difficulty'] not in {'easy', 'medium', 'hard'} for t in tasks):
+ raise ValueError('Unknown task difficulty')
+ if len({t['name'] for t in tasks}) != n or len({t['task_index'] for t in tasks}) != n:
+ raise ValueError('Duplicate training tasks')
+ pairs = set()
+ for p in range(h):
+ section = groups[p * n:(p + 1) * n]
+ if {g['task_row'] for g in section} != set(range(n)):
+ raise ValueError('Each pass must contain every task exactly once')
+ if Counter(g['harness'] for g in section) != Counter({name: n // h for name in harnesses}):
+ raise ValueError('Harness counts are not balanced')
+ for tier in ['easy', 'medium', 'hard']:
+ counts = [sum(g['harness'] == name and g['difficulty'] == tier for g in section)
+ for name in harnesses]
+ if max(counts) - min(counts) > 1:
+ raise ValueError('Difficulty counts are not balanced')
+ for i, g in enumerate(section):
+ row = g['task_row']
+ task = tasks[row]
+ if (g['group_in_cycle'] != p * n + i or g['pass_index'] != p
+ or g['task_name'] != task['name'] or g['task_index'] != task['task_index']
+ or g['difficulty'] != task['difficulty']):
+ raise ValueError('Group identity disagrees with task metadata')
+ pairs.add((row, g['harness']))
+ if len(pairs) != n * h:
+ raise ValueError('Rotation repeats a task/harness pair')
+ return schedule
diff --git a/04-data-agent/train/launch.slurm b/04-data-agent/train/launch.slurm
new file mode 100755
index 0000000..69de3cf
--- /dev/null
+++ b/04-data-agent/train/launch.slurm
@@ -0,0 +1,207 @@
+#!/bin/bash
+#SBATCH --job-name=da-blackbox-2b
+#SBATCH --ntasks-per-node=1
+#SBATCH --gres=gpu:2
+#SBATCH --partition=hopper-extra
+#SBATCH --output=/fsx/%u/logs/%x-%j.out
+#SBATCH --error=/fsx/%u/logs/%x-%j.err
+#SBATCH --time=0-12:00:00
+set -euo pipefail
+
+# One job, three processes, in this order and for these reasons:
+#
+# GPU 0 vLLM, with the capture flags. The TRAINER's engine -- the agent calls the same weights the
+# optimizer updates, which is what makes the rollouts on-policy.
+# GPU 1 the trainer.
+# cpu the env server, which starts the capture proxy and publishes it.
+#
+# The sandboxes must reach CAPTURE, not vLLM: opencode is pointed at the proxy and its api key is the
+# capture session id, so vLLM is never exposed and never sees a sandbox. Capture is published with a
+# gradio tunnel because E2B runs off-cluster; `direct` only works for a sandbox on this host and fails
+# silently -- opencode cannot reach the engine, makes zero model calls, and the rollout returns an
+# empty answer that grades exactly like a model that could not do the task.
+
+REPO=/fsx/$USER/projects/trl_prod
+ENVDIR=$REPO/HuggingEnvs/04-data-agent/envs/blackbox-opencode
+TRAINDIR=$REPO/HuggingEnvs/04-data-agent/train
+
+# TWO INTERPRETERS, ON PURPOSE.
+# .venv312 torch / vLLM / TRL. The heavy stack, shared and reproducible via install.sh.
+# $ENVDIR/.venv the environment server only. Deliberately light -- no torch, no vLLM.
+# The trainer needs two LIGHT packages that .venv312 does not carry: the local `openenv` (the capture
+# stack is in no release) and `data_agent_env`. Both go on PYTHONPATH rather than being pip-installed,
+# because install.sh owns what is in .venv312 and an ad hoc install there stops being reproducible.
+# `data_agent_env` needs the symlink in _pypath/: the directory is named `blackbox-opencode`, which is
+# not a legal Python identifier, so PYTHONPATH alone cannot reach it.
+PY312=$REPO/.venv312/bin/python
+export PYTHONPATH="$TRAINDIR/_pypath:$REPO/OpenEnv/src${PYTHONPATH:+:$PYTHONPATH}"
+export TRL_EXPERIMENTAL_SILENCE=1
+
+cd "$ENVDIR"
+
+MODEL="${MODEL:-Qwen/Qwen3.5-2B}"
+
+# PORTS DERIVED PER JOB. Fixed ports are a trap on a shared node and it is not a theoretical one:
+# two of these jobs landed on ip-10-53-93-25 together, the second one's /server_info probe answered
+# 200 because it was talking to the FIRST job's vLLM, and the NCCL weight-transfer group then tried to
+# attach to another job's engine and died with "NCCL error: unhandled cuda error". Every preflight
+# check passed while pointing at the wrong process.
+#
+# Worse, had the models matched, it would have trained one job against the other's weights and looked
+# perfectly healthy. The reference launcher derives ports the same way and says so.
+_OFF=$(( ${SLURM_JOB_ID:-0} % 200 * 4 ))
+VLLM_PORT="${VLLM_PORT:-$(( 18000 + _OFF ))}"
+ENV_PORT="${ENV_PORT:-$(( 18001 + _OFF ))}"
+CAP_PORT="${CAP_PORT:-$(( 18002 + _OFF ))}"
+VLLM_URL="http://127.0.0.1:$VLLM_PORT"
+LOGS="$ENVDIR/logs/job-${SLURM_JOB_ID:-local}"
+mkdir -p "$LOGS/run"
+
+# BRIDGE THE CHECKPOINTS TO THE EVAL WATCHER.
+#
+# eval_watcher.py polls experiments/temp_asyncgrpo_code/logs/ckpt-/checkpoint-* . Without
+# this link it never sees a checkpoint from this run, and the 400-step job would finish with nothing
+# evaluated -- silently, because "no checkpoints yet" and "wrong directory" look identical to a poller.
+#
+# The link, rather than writing there directly: the run's artifacts belong with the environment, and
+# the eval stack is the one that produced the +0.2028 reference, so reusing it keeps the comparison
+# apples-to-apples instead of introducing a second evaluator.
+WATCHDIR=/fsx/$USER/projects/trl_prod/experiments/temp_asyncgrpo_code/logs/ckpt-${SLURM_JOB_ID:-local}
+ln -sfn "$LOGS/run" "$WATCHDIR"
+echo "checkpoints visible to eval_watcher at $WATCHDIR -> $LOGS/run"
+
+set -a; . "$REPO/experiments/.env"; set +a
+. "$ENVDIR/../blackbox-harbor/tools/hf_token.sh"
+
+cleanup() { pkill -P $$ 2>/dev/null || true; }
+trap cleanup EXIT
+
+# SAMPLING, FOR THE ENGINE AND THE TRAINER, FROM ONE VARIABLE SO THEY CANNOT DRIFT.
+# opencode sends NO sampling parameters, so whatever the engine defaults to is what actually
+# generated the actions -- and Qwen3.5-2B ships no generation_config.json, so that default is
+# vLLM's own 1.0/1.0, NOT the 0.8 the trainer divides logits by when it recomputes logprobs for
+# the importance ratio. Leaving the engine unpinned means the gradient is computed against a
+# distribution that never produced the samples. Measured, unpinned: entropy climbed 0.229 -> 0.587
+# over 24 steps while reward fell 0.592 -> 0.216, with turns/mean 10.4 against the reference's 6.1
+# (job 76577). The reference pins both from one variable (run_cluster.py:165) and holds entropy
+# flat at 0.17-0.21. top_k=-1 because top_k truncation is likewise unmodelled by the recomputation.
+TEMPERATURE="${TEMPERATURE:-0.8}"
+TOP_P="${TOP_P:-0.95}"
+
+echo "== vLLM on GPU 0 == (ports vllm=$VLLM_PORT env=$ENV_PORT capture=$CAP_PORT, derived from job ${SLURM_JOB_ID:-0})"
+# EVERY ONE OF THESE FLAGS IS LOad-BEARING. They are the set the +0.2028 run served with; a
+# hand-rolled `vllm serve` that omits any of them fails silently rather than loudly.
+#
+# --enable-auto-tool-choice / --tool-call-parser qwen3_xml
+# Without them vLLM never parses tool calls, so `has_tool_call` is False for EVERY turn and the
+# whole rollout is discarded with no error anywhere. The agent path is entirely tool calls.
+# --default-chat-template-kwargs {"enable_thinking": false}
+# Pins thinking OFF server-side. Qwen3.5-4B's template opens by default and -2B's
+# closes it -- INVERTED defaults between two models of the same family. Leaving it to the
+# default is what made a re-rendered prompt match the 2B and diverge on the 4B, forking 100% of
+# turn transitions. This is what `think_template_patch` used to monkeypatch; it belongs here.
+# --return-tokens-as-token-ids / --logprobs-mode processed_logprobs
+# The capture tier. Without them capture degrades to text and every rollout is untrainable --
+# and looks completely normal while being so.
+# --gdn-prefill-backend triton
+# Qwen3.5 is hybrid Gated-DeltaNet; this is the working prefill backend for it.
+# VLLM_USE_DEEP_GEMM=0 / VLLM_DEEP_GEMM_WARMUP=skip
+# On Hopper a stale importable deep_gemm kills startup with "DeepGEMM backend is not available
+# or outdated".
+# VLLM_SERVER_DEV_MODE=1
+# Gates /server_info, /pause and /init_weight_transfer_engine. TRL's weight transfer reads
+# /server_info for the server's dtype, so WITHOUT this the trainer dies at on_train_begin with
+# "404 Client Error: Not Found for url: .../server_info?config_format=json" -- after vLLM is
+# healthy, after the env server is up, after the dataset loads. Job 76488 died exactly there.
+# VLLM_USE_FLASHINFER_SAMPLER=0
+# flashinfer JIT-compiles its sampling kernel and needs nvcc; the PyTorch sampler does not.
+# --weight-transfer-config {"backend":"nccl"}
+# The trainer syncs weights into this engine over NCCL. Without it there is no transfer engine
+# to initialise.
+CUDA_VISIBLE_DEVICES=0 \
+ VLLM_SERVER_DEV_MODE=1 \
+ VLLM_USE_FLASHINFER_SAMPLER=0 \
+ VLLM_USE_DEEP_GEMM=0 VLLM_DEEP_GEMM_WARMUP=skip \
+ "$PY312" -m vllm.entrypoints.openai.api_server \
+ --model "$MODEL" --port "$VLLM_PORT" --served-model-name "$MODEL" \
+ --trust-remote-code \
+ --max-model-len "${MAX_MODEL_LEN:-131072}" \
+ --enable-auto-tool-choice --tool-call-parser qwen3_xml \
+ --reasoning-parser qwen3 \
+ --gdn-prefill-backend triton \
+ --default-chat-template-kwargs '{"enable_thinking": false}' \
+ --return-tokens-as-token-ids --logprobs-mode processed_logprobs \
+ --override-generation-config "{\"temperature\": $TEMPERATURE, \"top_p\": $TOP_P, \"top_k\": -1}" \
+ --weight-transfer-config '{"backend":"nccl"}' \
+ > "$LOGS/vllm.log" 2>&1 &
+
+# vLLM's /health returns 200 with an EMPTY body, so check the STATUS and parse nothing. A probe that
+# json.loads() it throws, and a caller treating that as "not ready" waits out the whole budget on a
+# server that came up minutes ago.
+echo "waiting for vLLM (up to 30 min: weights + torch.compile)"
+for _ in $(seq 1 900); do
+ code=$(curl -s -o /dev/null -w "%{http_code}" --max-time 3 "$VLLM_URL/health" || true)
+ [ "$code" = "200" ] && break
+ sleep 2
+done
+[ "$(curl -s -o /dev/null -w "%{http_code}" --max-time 5 "$VLLM_URL/health")" = "200" ] || {
+ echo "FATAL: vLLM never became healthy; its log follows"; tail -60 "$LOGS/vllm.log"; exit 1; }
+echo "vLLM healthy"
+# CHECK THE ENDPOINT THE TRAINER ACTUALLY NEEDS, not just /health. /health passing says nothing about
+# whether VLLM_SERVER_DEV_MODE took effect, and the trainer only finds out ~5 minutes later at
+# on_train_begin, after the env server and the whole dataset have loaded.
+si=$(curl -s -o /dev/null -w "%{http_code}" --max-time 10 "$VLLM_URL/server_info?config_format=json" || true)
+[ "$si" = "200" ] || { echo "FATAL: /server_info -> $si; VLLM_SERVER_DEV_MODE=1 did not take effect."; exit 1; }
+# Confirm the engine on this port is OURS. Reachability is not identity: a neighbouring job's vLLM
+# answers /health and /server_info exactly the same way, and the first sign of talking to the wrong
+# one is an NCCL failure minutes later -- or, if the models happen to match, no sign at all.
+served=$(curl -s --max-time 10 "$VLLM_URL/v1/models" | tr -d ' \n' | grep -o "\"id\":\"[^\"]*\"" | head -1)
+echo "vLLM /server_info ok; serving $served on :$VLLM_PORT"
+
+echo "== env server + capture =="
+PORT="$ENV_PORT" CAPTURE_PORT="$CAP_PORT" EXPOSE=gradio \
+ SPLITS="${SPLITS:-train}" SANDBOX="${SANDBOX:-e2b}" \
+ MAX_CONCURRENT="${MAX_CONCURRENT:-48}" \
+ LLM_URL="$VLLM_URL/v1" MODEL="$MODEL" \
+ ./serve.sh > "$LOGS/env.log" 2>&1 &
+
+for _ in $(seq 1 180); do
+ curl -s -o /dev/null --max-time 3 "http://127.0.0.1:$ENV_PORT/health" && break
+ sleep 2
+done
+curl -s -o /dev/null --max-time 5 "http://127.0.0.1:$ENV_PORT/health" || {
+ echo "FATAL: env server never became healthy; its log follows"; tail -60 "$LOGS/env.log"; exit 1; }
+grep -a "capture proxy ready" "$LOGS/env.log" || echo "WARNING: capture did not warm at boot"
+echo "env server healthy"
+
+# max_inflight 32 is the REFERENCE value, and it matters beyond throughput: at 16 this run
+# measured samples/step 7.2 against the reference's 16.1, i.e. less than half the training
+# signal per optimizer step, which over 400 steps is a different experiment rather than a
+# slower one. 16 was chosen after an EVAL collapsed at 64 concurrent MCP clients -- but that
+# eval ran ~16-turn rollouts (64 x 16 = ~1024 concurrent turn-slots) where training runs ~8.5
+# (32 x 8.5 = ~272), about a quarter of the load that broke it.
+echo "== trainer on GPU 1 =="
+# NOT from $REPO: the `trl/` submodule DIRECTORY shadows the `trl` package as a namespace package
+# there, and the import dies with "cannot import name '__version__' from trl (unknown location)".
+cd "$TRAINDIR"
+CUDA_VISIBLE_DEVICES=1 TRACKIO_STORAGE_MODE=sqlite \
+ "$PY312" -u train_blackbox_opencode.py \
+ --server "http://127.0.0.1:$ENV_PORT" \
+ --vllm-url "$VLLM_URL" \
+ --model "$MODEL" \
+ --split "${SPLITS:-train}" \
+ --sandbox "${SANDBOX:-e2b}" \
+ --learning-rate "${LR:-3e-6}" \
+ --temperature "$TEMPERATURE" \
+ --num-generations "${NUM_GENERATIONS:-8}" \
+ --max-inflight "${MAX_INFLIGHT:-32}" \
+ --grad-accum "${GRAD_ACCUM:-4}" \
+ --curriculum "${CURRICULUM:-warmup:125}" \
+ --agent-step-limit "${AGENT_STEP_LIMIT:-17}" \
+ --token-budget "${TOKEN_BUDGET:-40960}" \
+ --max-completion-length "${MAX_COMPLETION_LENGTH:-16384}" \
+ --heartbeat-stale-after-s "${HEARTBEAT_STALE_AFTER_S:-900}" \
+ --dtype "${TRAIN_DTYPE:-bfloat16}" \
+ --max-steps "${MAX_STEPS:-400}" \
+ --save-steps "${SAVE_STEPS:-100}" \
+ --output-dir "$LOGS/run"
diff --git a/04-data-agent/train/launch_harbor_multi.slurm b/04-data-agent/train/launch_harbor_multi.slurm
new file mode 100644
index 0000000..4fffbaf
--- /dev/null
+++ b/04-data-agent/train/launch_harbor_multi.slurm
@@ -0,0 +1,329 @@
+#!/bin/bash
+#SBATCH --job-name=agrpo-harbor-multi
+#SBATCH --ntasks-per-node=1
+#SBATCH --gres=gpu:2
+#SBATCH --partition=hopper-extra
+#SBATCH --output=/fsx/%u/logs/%x-%j.out
+#SBATCH --error=/fsx/%u/logs/%x-%j.err
+#SBATCH --time=0-12:00:00
+set -euo pipefail
+
+# One job, three processes, in this order and for these reasons:
+#
+# GPU 0 vLLM, with the capture flags. The TRAINER's engine -- the agent calls the same weights the
+# optimizer updates, which is what makes the rollouts on-policy.
+# GPU 1 the trainer.
+# cpu the env server, which starts the capture proxy and publishes it.
+#
+# The sandboxes must reach CAPTURE, not vLLM: opencode is pointed at the proxy and its api key is the
+# capture session id, so vLLM is never exposed and never sees a sandbox. Capture is published with a
+# gradio tunnel because E2B runs off-cluster; `direct` only works for a sandbox on this host and fails
+# silently -- opencode cannot reach the engine, makes zero model calls, and the rollout returns an
+# empty answer that grades exactly like a model that could not do the task.
+
+REPO=/fsx/$USER/projects/trl_prod
+CODE_ROOT="${CODE_ROOT:-$REPO}"
+ENVDIR=$REPO/HuggingEnvs/04-data-agent/envs/blackbox-opencode
+TRAINDIR=$CODE_ROOT/HuggingEnvs/04-data-agent/train
+
+# TWO INTERPRETERS, ON PURPOSE.
+# .venv312 torch / vLLM / TRL. The heavy stack, shared and reproducible via install.sh.
+# $ENVDIR/.venv the environment server only. Deliberately light -- no torch, no vLLM.
+# The trainer needs two LIGHT packages that .venv312 does not carry: the local `openenv` (the capture
+# stack is in no release) and `data_agent_env`. Both go on PYTHONPATH rather than being pip-installed,
+# because install.sh owns what is in .venv312 and an ad hoc install there stops being reproducible.
+# `data_agent_env` needs the symlink in _pypath/: the directory is named `blackbox-opencode`, which is
+# not a legal Python identifier, so PYTHONPATH alone cannot reach it.
+PY312=$REPO/.venv312/bin/python
+# `harbor_env` lives under OpenEnv/envs, not OpenEnv/src -- the blackbox launcher never needed it.
+export PYTHONPATH="$TRAINDIR:$TRAINDIR/_pypath:$CODE_ROOT/trl:$CODE_ROOT/OpenEnv/src:$CODE_ROOT/OpenEnv/envs${PYTHONPATH:+:$PYTHONPATH}"
+export TRL_EXPERIMENTAL_SILENCE=1
+
+cd "$ENVDIR"
+
+MODEL="${MODEL:-Qwen/Qwen3.5-2B}"
+
+# PORTS DERIVED PER JOB. Fixed ports are a trap on a shared node and it is not a theoretical one:
+# two of these jobs landed on ip-10-53-93-25 together, the second one's /server_info probe answered
+# 200 because it was talking to the FIRST job's vLLM, and the NCCL weight-transfer group then tried to
+# attach to another job's engine and died with "NCCL error: unhandled cuda error". Every preflight
+# check passed while pointing at the wrong process.
+#
+# Worse, had the models matched, it would have trained one job against the other's weights and looked
+# perfectly healthy. The reference launcher derives ports the same way and says so.
+_OFF=$(( ${SLURM_JOB_ID:-0} % 200 * 4 ))
+VLLM_PORT="${VLLM_PORT:-$(( 18000 + _OFF ))}"
+ENV_PORT="${ENV_PORT:-$(( 18001 + _OFF ))}"
+CAP_PORT="${CAP_PORT:-$(( 18002 + _OFF ))}"
+VLLM_URL="http://127.0.0.1:$VLLM_PORT"
+LOGS="${TRAIN_LOGS:-$ENVDIR/logs/job-${SLURM_JOB_ID:-local}}"
+mkdir -p "$LOGS/run"
+export OPENENV_HARBOR_TRIALS_DIR="${OPENENV_HARBOR_TRIALS_DIR:-$LOGS/trials}"
+IFS=',' read -ra JOB_GPUS <<< "${CUDA_VISIBLE_DEVICES:-0,1}"
+[ "${#JOB_GPUS[@]}" -ge 2 ] || { echo "FATAL: this launcher needs two allocated GPUs"; exit 1; }
+
+# BRIDGE THE CHECKPOINTS TO THE EVAL WATCHER.
+#
+# eval_watcher.py polls experiments/temp_asyncgrpo_code/logs/ckpt-/checkpoint-* . Without
+# this link it never sees a checkpoint from this run, and the 400-step job would finish with nothing
+# evaluated -- silently, because "no checkpoints yet" and "wrong directory" look identical to a poller.
+#
+# The link, rather than writing there directly: the run's artifacts belong with the environment, and
+# the eval stack is the one that produced the +0.2028 reference, so reusing it keeps the comparison
+# apples-to-apples instead of introducing a second evaluator.
+WATCHDIR=/fsx/$USER/projects/trl_prod/experiments/temp_asyncgrpo_code/logs/ckpt-${SLURM_JOB_ID:-local}
+ln -sfn "$LOGS/run" "$WATCHDIR"
+echo "checkpoints visible to eval_watcher at $WATCHDIR -> $LOGS/run"
+
+set -a; . "$REPO/experiments/.env"; set +a
+. "$ENVDIR/../blackbox-harbor/tools/hf_token.sh"
+
+VLLM_PID=""
+HARBOR_PID=""
+cleanup() {
+ [ -z "$HARBOR_PID" ] || kill -TERM -- "-$HARBOR_PID" 2>/dev/null || true
+ [ -z "$VLLM_PID" ] || kill -TERM -- "-$VLLM_PID" 2>/dev/null || true
+}
+trap cleanup EXIT
+
+# SAMPLING, FOR THE ENGINE AND THE TRAINER, FROM ONE VARIABLE SO THEY CANNOT DRIFT.
+# opencode sends NO sampling parameters, so whatever the engine defaults to is what actually
+# generated the actions -- and Qwen3.5-2B ships no generation_config.json, so that default is
+# vLLM's own 1.0/1.0, NOT the 0.8 the trainer divides logits by when it recomputes logprobs for
+# the importance ratio. Leaving the engine unpinned means the gradient is computed against a
+# distribution that never produced the samples. Measured, unpinned: entropy climbed 0.229 -> 0.587
+# over 24 steps while reward fell 0.592 -> 0.216, with turns/mean 10.4 against the reference's 6.1
+# (job 76577). The reference pins both from one variable (run_cluster.py:165) and holds entropy
+# flat at 0.17-0.21. top_k=-1 because top_k truncation is likewise unmodelled by the recomputation.
+TEMPERATURE="${TEMPERATURE:-0.8}"
+# 1.0, NOT the reference launcher's 0.95. `--logprobs-mode processed_logprobs` takes the logprob
+# AFTER truncation, so a truncating top_p renormalises every captured logprob over the kept set
+# while the trainer recomputes over the full vocabulary -- the step-0 importance ratio then
+# lands at kept_mass rather than 1, and the reordering is worse than a uniform shift. TRL's
+# AsyncGRPOConfig already defaults top_p to 1.0, so 0.95 here is an ENGINE/TRAINER MISMATCH,
+# not a policy choice. Measured: the reference sat at ratio 0.985-0.993 (the truncation
+# signature); at 1.0 it moves to 0.9984-0.9999.
+TOP_P="${TOP_P:-1.0}"
+
+echo "== vLLM on GPU 0 == (ports vllm=$VLLM_PORT env=$ENV_PORT capture=$CAP_PORT, derived from job ${SLURM_JOB_ID:-0})"
+# EVERY ONE OF THESE FLAGS IS LOad-BEARING. They are the set the +0.2028 run served with; a
+# hand-rolled `vllm serve` that omits any of them fails silently rather than loudly.
+#
+# --enable-auto-tool-choice / --tool-call-parser qwen3_xml
+# Without them vLLM never parses tool calls, so `has_tool_call` is False for EVERY turn and the
+# whole rollout is discarded with no error anywhere. The agent path is entirely tool calls.
+# --default-chat-template-kwargs {"enable_thinking": false}
+# Pins thinking OFF server-side. Qwen3.5-4B's template opens by default and -2B's
+# closes it -- INVERTED defaults between two models of the same family. Leaving it to the
+# default is what made a re-rendered prompt match the 2B and diverge on the 4B, forking 100% of
+# turn transitions. This is what `think_template_patch` used to monkeypatch; it belongs here.
+# --return-tokens-as-token-ids / --logprobs-mode processed_logprobs
+# The capture tier. Without them capture degrades to text and every rollout is untrainable --
+# and looks completely normal while being so.
+# --gdn-prefill-backend triton
+# Qwen3.5 is hybrid Gated-DeltaNet; this is the working prefill backend for it.
+# VLLM_USE_DEEP_GEMM=0 / VLLM_DEEP_GEMM_WARMUP=skip
+# On Hopper a stale importable deep_gemm kills startup with "DeepGEMM backend is not available
+# or outdated".
+# VLLM_SERVER_DEV_MODE=1
+# Gates /server_info, /pause and /init_weight_transfer_engine. TRL's weight transfer reads
+# /server_info for the server's dtype, so WITHOUT this the trainer dies at on_train_begin with
+# "404 Client Error: Not Found for url: .../server_info?config_format=json" -- after vLLM is
+# healthy, after the env server is up, after the dataset loads. Job 76488 died exactly there.
+# VLLM_USE_FLASHINFER_SAMPLER=0
+# flashinfer JIT-compiles its sampling kernel and needs nvcc; the PyTorch sampler does not.
+# --weight-transfer-config {"backend":"nccl"}
+# The trainer syncs weights into this engine over NCCL. Without it there is no transfer engine
+# to initialise.
+VLLM_EXECUTION_FLAGS=()
+if [[ "${VLLM_ENFORCE_EAGER:-1}" == 1 ]]; then
+ VLLM_EXECUTION_FLAGS+=(--enforce-eager)
+fi
+CUDA_VISIBLE_DEVICES="${JOB_GPUS[0]}" \
+ VLLM_SERVER_DEV_MODE=1 \
+ VLLM_USE_FLASHINFER_SAMPLER=0 \
+ VLLM_USE_DEEP_GEMM=0 VLLM_DEEP_GEMM_WARMUP=skip \
+ setsid "$PY312" -m vllm.entrypoints.openai.api_server \
+ --model "$MODEL" --port "$VLLM_PORT" --served-model-name "$MODEL" \
+ ${MODEL_REVISION:+--revision "$MODEL_REVISION"} \
+ --trust-remote-code \
+ --dtype "${TRAIN_DTYPE:-bfloat16}" --generation-config vllm \
+ --gpu-memory-utilization "${VLLM_GPU_MEMORY_UTILIZATION:-0.9}" \
+ "${VLLM_EXECUTION_FLAGS[@]}" --no-enable-prefix-caching \
+ --limit-mm-per-prompt '{"image":0,"video":0}' \
+ --max-model-len "${MAX_MODEL_LEN:-131072}" \
+ --enable-auto-tool-choice --tool-call-parser qwen3_xml \
+ --reasoning-parser qwen3 \
+ --gdn-prefill-backend triton \
+ --default-chat-template-kwargs '{"enable_thinking": false}' \
+ --return-tokens-as-token-ids --logprobs-mode processed_logprobs \
+ --override-generation-config "{\"temperature\": $TEMPERATURE, \"top_p\": $TOP_P, \"top_k\": -1}" \
+ --weight-transfer-config '{"backend":"nccl"}' \
+ > "$LOGS/vllm.log" 2>&1 &
+VLLM_PID=$!
+
+# vLLM's /health returns 200 with an EMPTY body, so check the STATUS and parse nothing. A probe that
+# json.loads() it throws, and a caller treating that as "not ready" waits out the whole budget on a
+# server that came up minutes ago.
+echo "waiting for vLLM (up to 30 min: weights + torch.compile)"
+for _ in $(seq 1 900); do
+ kill -0 "$VLLM_PID" 2>/dev/null || { tail -60 "$LOGS/vllm.log"; exit 1; }
+ code=$(curl -s -o /dev/null -w "%{http_code}" --max-time 3 "$VLLM_URL/health" || true)
+ [ "$code" = "200" ] && break
+ sleep 2
+done
+[ "$(curl -s -o /dev/null -w "%{http_code}" --max-time 5 "$VLLM_URL/health")" = "200" ] || {
+ echo "FATAL: vLLM never became healthy; its log follows"; tail -60 "$LOGS/vllm.log"; exit 1; }
+echo "vLLM healthy"
+# CHECK THE ENDPOINT THE TRAINER ACTUALLY NEEDS, not just /health. /health passing says nothing about
+# whether VLLM_SERVER_DEV_MODE took effect, and the trainer only finds out ~5 minutes later at
+# on_train_begin, after the env server and the whole dataset have loaded.
+si=$(curl -s -o /dev/null -w "%{http_code}" --max-time 10 "$VLLM_URL/server_info?config_format=json" || true)
+[ "$si" = "200" ] || { echo "FATAL: /server_info -> $si; VLLM_SERVER_DEV_MODE=1 did not take effect."; exit 1; }
+# Confirm the engine on this port is OURS. Reachability is not identity: a neighbouring job's vLLM
+# answers /health and /server_info exactly the same way, and the first sign of talking to the wrong
+# one is an NCCL failure minutes later -- or, if the models happen to match, no sign at all.
+served=$(curl -s --max-time 10 "$VLLM_URL/v1/models" | tr -d ' \n' | grep -o "\"id\":\"[^\"]*\"" | head -1)
+echo "vLLM /server_info ok; serving $served on :$VLLM_PORT"
+
+echo "== harbor server =="
+# OWNED BY THE JOB unless SERVER is passed. The reference launcher boots its own env server too, and
+# isolation matters more than the boot cost here: a shared server means a shared 400-session budget,
+# a shared gradio tunnel, and sessions that outlive the job. Measured tonight: two arms on one server
+# cost 6x throughput, and sessions leaked by a killed client filled it to 400/400 and dropped a run
+# from 82% graded to 18%. The dataset tree is cached, so booting our own is cheap.
+if [ -n "${SERVER:-}" ]; then
+ echo " using EXTERNAL server $SERVER (shared: its session budget and tunnel are not ours)"
+else
+ OPENENV_BIN="${OPENENV_BIN:-$REPO/OpenEnv/.venv/bin/openenv}"
+ [ -x "$OPENENV_BIN" ] || { echo "FATAL: no openenv CLI at $OPENENV_BIN"; exit 1; }
+ # Ports are derived from the job id further up, so two jobs on one node cannot collide.
+ MAX_CONCURRENT_ENVS="${MAX_CONCURRENT_ENVS:-400}" \
+ setsid "$OPENENV_BIN" harbor serve \
+ --dataset "${SPLIT:-AdithyaSK/data_agent_rl_environment_train}" \
+ --llm-url "$VLLM_URL/v1" --model "$MODEL" \
+ --port "$ENV_PORT" --capture-port "$CAP_PORT" --expose gradio \
+ --max-output-tokens "${MAX_OUTPUT_TOKENS:-12288}" \
+ > "$LOGS/env.log" 2>&1 &
+ HARBOR_PID=$!
+ SERVER="http://$(hostname -s):$ENV_PORT"
+ echo " booting our own on :$ENV_PORT (capture :$CAP_PORT) -> $SERVER"
+ for _ in $(seq 1 180); do
+ curl -s -o /dev/null --max-time 3 "http://127.0.0.1:$ENV_PORT/health" && break
+ sleep 2
+ done
+ curl -s -o /dev/null --max-time 5 "http://127.0.0.1:$ENV_PORT/health" || {
+ echo "FATAL: our harbor server never became healthy; log follows"; tail -60 "$LOGS/env.log"; exit 1; }
+ grep -a "capture" "$LOGS/env.log" | head -3
+fi
+export SERVER
+# IDENTITY, not just reachability: a server answering /health may host a different split, and every
+# rollout would then run tasks this job never asked for.
+SPLITS_JSON=$(curl -s --max-time 10 "$SERVER/harbor_env/splits")
+echo " splits $SPLITS_JSON"
+case "$SPLITS_JSON" in
+ *"${SPLIT:-AdithyaSK/data_agent_rl_environment_train}"*) echo " split OK" ;;
+ *) echo "FATAL: $SERVER does not host ${SPLIT:-AdithyaSK/data_agent_rl_environment_train}"; exit 1 ;;
+esac
+# The server probes the engine PER ROLLOUT from ITS host. If it cannot reach this node's vLLM the
+# tier grades `text`, every rollout returns zero trainable turns, and the run burns sandboxes while
+# looking healthy. Prove the path the workload actually uses.
+echo " engine from server host: $(curl -s -o /dev/null -w '%{http_code}' --max-time 8 "$VLLM_URL/health")"
+echo "harbor server healthy"
+
+# Reachability does not establish identity: gradio has reused a live URL across jobs.
+# Probe only during startup, before any rollouts; a busy /health endpoint is not a watchdog.
+if [ -n "$HARBOR_PID" ]; then
+"$PY312" - "$VLLM_URL" "$MODEL" "$CAP_PORT" "$LOGS/env.log" "$LOGS" <<'PY'
+import json
+import pathlib
+import re
+import sys
+import time
+
+import requests
+
+engine, model, cap_port, env_log, logs = sys.argv[1:]
+models = requests.get(engine + "/v1/models", timeout=15)
+models.raise_for_status()
+assert [entry["id"] for entry in models.json()["data"]] == [model], "wrong inference model"
+local = requests.get(f"http://127.0.0.1:{cap_port}/health", timeout=15)
+local.raise_for_status()
+identity = local.json()["instance"]
+match = re.search(r"^capture\s+:\d+\s+->\s+(https://\S+)", pathlib.Path(env_log).read_text(), re.M)
+if match is None:
+ raise RuntimeError("capture tunnel URL missing from the owned Harbor server log")
+url = match[1]
+for attempt in range(20):
+ try:
+ public = requests.get(url + "/health", timeout=10)
+ public.raise_for_status()
+ public_identity = public.json()["instance"]
+ break
+ except (requests.RequestException, ValueError, KeyError):
+ if attempt == 19:
+ raise
+ time.sleep(2)
+assert public_identity == identity, "capture tunnel points to a different job"
+pathlib.Path(logs, "endpoint_identity.json").write_text(json.dumps({
+ "model": model, "engine": engine, "capture_url": url, "capture_instance": identity,
+ "public_matches_local": True,
+}, indent=2) + "\n")
+print("Capture tunnel identity verified; private and public instance IDs match", flush=True)
+PY
+fi
+
+# max_inflight 32 is the REFERENCE value, and it matters beyond throughput: at 16 this run
+# measured samples/step 7.2 against the reference's 16.1, i.e. less than half the training
+# signal per optimizer step, which over 400 steps is a different experiment rather than a
+# slower one. 16 was chosen after an EVAL collapsed at 64 concurrent MCP clients -- but that
+# eval ran ~16-turn rollouts (64 x 16 = ~1024 concurrent turn-slots) where training runs ~8.5
+# (32 x 8.5 = ~272), about a quarter of the load that broke it.
+echo "== trainer on GPU 1 =="
+# NOT from $REPO: the `trl/` submodule DIRECTORY shadows the `trl` package as a namespace package
+# there, and the import dies with "cannot import name '__version__' from trl (unknown location)".
+cd "$TRAINDIR"
+# Keep the trainer independent of Hub/network availability. JSONL fragments are safe on FSx.
+unset TRACKIO_SPACE_ID TRACKIO_SERVER_URL TRACKIO_BUCKET_ID TRACKIO_DATASET_ID TRACKIO_WEBHOOK_URL
+export TRACKIO_DIR="$LOGS/trackio"
+export TRACKIO_STORAGE_MODE=jsonl
+CUDA_VISIBLE_DEVICES="${JOB_GPUS[1]}" \
+ "$PY312" -u train_harbor_multi.py \
+ --server "${SERVER:?harbor server was neither booted nor supplied}" \
+ --vllm-url "$VLLM_URL" \
+ --model "$MODEL" \
+ ${MODEL_REVISION:+--model-revision "$MODEL_REVISION"} \
+ ${RESUME_FROM_CHECKPOINT:+--resume-from-checkpoint "$RESUME_FROM_CHECKPOINT"} \
+ --split "${SPLIT:-AdithyaSK/data_agent_rl_environment_train}" \
+ --harnesses "${HARNESSES:-opencode+mini-swe-agent}" \
+ ${ALL_TASK_HARNESS_PAIRS:+--all-task-harness-pairs} \
+ ${HARNESS_SCHEDULE:+--harness-schedule "$HARNESS_SCHEDULE"} \
+ --sandbox "${SANDBOX:-e2b}" \
+ --learning-rate "${LR:-3e-6}" \
+ --temperature "$TEMPERATURE" \
+ --num-generations "${NUM_GENERATIONS:-8}" \
+ --max-inflight "${MAX_INFLIGHT:-32}" \
+ --max-staleness "${MAX_STALENESS:-4}" \
+ --grad-accum "${GRAD_ACCUM:-4}" \
+ ${ATOMIC_ROLLOUTS:+--atomic-rollouts} \
+ --max-outstanding-rollouts "${MAX_OUTSTANDING_ROLLOUTS:-0}" \
+ --max-row-tokens "${MAX_ROW_TOKENS:-131072}" \
+ --per-device-batch-size "${PER_DEVICE_BATCH_SIZE:-4}" \
+ ${TASK_INDICES:+--task-indices "$TASK_INDICES"} \
+ ${REWARD_KEY:+--reward-key "$REWARD_KEY"} \
+ ${AGENT_TURN_FILTER:+--agent-turn-filter "$AGENT_TURN_FILTER"} \
+ --agent-step-limit "${AGENT_STEP_LIMIT:-17}" \
+ --agent-timeout "${AGENT_TIMEOUT:-600}" \
+ --token-budget "${TOKEN_BUDGET:-40960}" \
+ --max-completion-length "${MAX_COMPLETION_LENGTH:-16384}" \
+ --heartbeat-stale-after-s "${HEARTBEAT_STALE_AFTER_S:-900}" \
+ --dtype "${TRAIN_DTYPE:-bfloat16}" \
+ --top-p "${TOP_P:-1.0}" \
+ --max-steps "${MAX_STEPS:-400}" \
+ --max-train-seconds "${MAX_TRAIN_SECONDS:-0}" \
+ --coverage-min-steps "${COVERAGE_MIN_STEPS:-0}" \
+ --audit-dir "$LOGS/audit" \
+ --project "${TRACKIO_PROJECT:-data-agent-harbor-multi}" \
+ --save-steps "${SAVE_STEPS:-100}" \
+ --checkpoint-max-seconds "${CHECKPOINT_MAX_SECONDS:-0}" \
+ --output-dir "$LOGS/run"
diff --git a/04-data-agent/train/launch_harbor_opencode.slurm b/04-data-agent/train/launch_harbor_opencode.slurm
new file mode 100644
index 0000000..af282e0
--- /dev/null
+++ b/04-data-agent/train/launch_harbor_opencode.slurm
@@ -0,0 +1,251 @@
+#!/bin/bash
+#SBATCH --job-name=agrpo-harbor-opencode
+#SBATCH --ntasks-per-node=1
+#SBATCH --gres=gpu:2
+#SBATCH --partition=hopper-extra
+#SBATCH --output=/fsx/%u/logs/%x-%j.out
+#SBATCH --error=/fsx/%u/logs/%x-%j.err
+#SBATCH --time=0-12:00:00
+set -euo pipefail
+
+# One job, three processes, in this order and for these reasons:
+#
+# GPU 0 vLLM, with the capture flags. The TRAINER's engine -- the agent calls the same weights the
+# optimizer updates, which is what makes the rollouts on-policy.
+# GPU 1 the trainer.
+# cpu the env server, which starts the capture proxy and publishes it.
+#
+# The sandboxes must reach CAPTURE, not vLLM: opencode is pointed at the proxy and its api key is the
+# capture session id, so vLLM is never exposed and never sees a sandbox. Capture is published with a
+# gradio tunnel because E2B runs off-cluster; `direct` only works for a sandbox on this host and fails
+# silently -- opencode cannot reach the engine, makes zero model calls, and the rollout returns an
+# empty answer that grades exactly like a model that could not do the task.
+
+REPO=/fsx/$USER/projects/trl_prod
+ENVDIR=$REPO/HuggingEnvs/04-data-agent/envs/blackbox-opencode
+TRAINDIR=$REPO/HuggingEnvs/04-data-agent/train
+
+# TWO INTERPRETERS, ON PURPOSE.
+# .venv312 torch / vLLM / TRL. The heavy stack, shared and reproducible via install.sh.
+# $ENVDIR/.venv the environment server only. Deliberately light -- no torch, no vLLM.
+# The trainer needs two LIGHT packages that .venv312 does not carry: the local `openenv` (the capture
+# stack is in no release) and `data_agent_env`. Both go on PYTHONPATH rather than being pip-installed,
+# because install.sh owns what is in .venv312 and an ad hoc install there stops being reproducible.
+# `data_agent_env` needs the symlink in _pypath/: the directory is named `blackbox-opencode`, which is
+# not a legal Python identifier, so PYTHONPATH alone cannot reach it.
+# `harbor_reward` is symlinked there too. The trainer imports it fine (Python puts the SCRIPT's dir on
+# sys.path[0], whatever the cwd), but `rollout_reward_fn` is pickled into the SPAWNED rollout child,
+# which must import it by name. Resting that on spawn's sys.path propagation would fail in the child
+# only -- after the sandboxes are already paid for. PYTHONPATH is inherited by any child either way.
+PY312=$REPO/.venv312/bin/python
+# `harbor_env` lives under OpenEnv/envs, not OpenEnv/src -- the blackbox launcher never needed it.
+export PYTHONPATH="$TRAINDIR/_pypath:$REPO/OpenEnv/src:$REPO/OpenEnv/envs${PYTHONPATH:+:$PYTHONPATH}"
+export TRL_EXPERIMENTAL_SILENCE=1
+
+cd "$ENVDIR"
+
+MODEL="${MODEL:-Qwen/Qwen3.5-2B}"
+
+# PORTS DERIVED PER JOB. Fixed ports are a trap on a shared node and it is not a theoretical one:
+# two of these jobs landed on ip-10-53-93-25 together, the second one's /server_info probe answered
+# 200 because it was talking to the FIRST job's vLLM, and the NCCL weight-transfer group then tried to
+# attach to another job's engine and died with "NCCL error: unhandled cuda error". Every preflight
+# check passed while pointing at the wrong process.
+#
+# Worse, had the models matched, it would have trained one job against the other's weights and looked
+# perfectly healthy. The reference launcher derives ports the same way and says so.
+_OFF=$(( ${SLURM_JOB_ID:-0} % 200 * 4 ))
+VLLM_PORT="${VLLM_PORT:-$(( 18000 + _OFF ))}"
+ENV_PORT="${ENV_PORT:-$(( 18001 + _OFF ))}"
+CAP_PORT="${CAP_PORT:-$(( 18002 + _OFF ))}"
+VLLM_URL="http://127.0.0.1:$VLLM_PORT"
+LOGS="$ENVDIR/logs/job-${SLURM_JOB_ID:-local}"
+mkdir -p "$LOGS/run"
+
+# BRIDGE THE CHECKPOINTS TO THE EVAL WATCHER.
+#
+# eval_watcher.py polls experiments/temp_asyncgrpo_code/logs/ckpt-/checkpoint-* . Without
+# this link it never sees a checkpoint from this run, and the 400-step job would finish with nothing
+# evaluated -- silently, because "no checkpoints yet" and "wrong directory" look identical to a poller.
+#
+# The link, rather than writing there directly: the run's artifacts belong with the environment, and
+# the eval stack is the one that produced the +0.2028 reference, so reusing it keeps the comparison
+# apples-to-apples instead of introducing a second evaluator.
+WATCHDIR=/fsx/$USER/projects/trl_prod/experiments/temp_asyncgrpo_code/logs/ckpt-${SLURM_JOB_ID:-local}
+ln -sfn "$LOGS/run" "$WATCHDIR"
+echo "checkpoints visible to eval_watcher at $WATCHDIR -> $LOGS/run"
+
+set -a; . "$REPO/experiments/.env"; set +a
+. "$ENVDIR/../blackbox-harbor/tools/hf_token.sh"
+
+cleanup() { pkill -P $$ 2>/dev/null || true; }
+trap cleanup EXIT
+
+# SAMPLING, FOR THE ENGINE AND THE TRAINER, FROM ONE VARIABLE SO THEY CANNOT DRIFT.
+# opencode sends NO sampling parameters, so whatever the engine defaults to is what actually
+# generated the actions -- and Qwen3.5-2B ships no generation_config.json, so that default is
+# vLLM's own 1.0/1.0, NOT the 0.8 the trainer divides logits by when it recomputes logprobs for
+# the importance ratio. Leaving the engine unpinned means the gradient is computed against a
+# distribution that never produced the samples. Measured, unpinned: entropy climbed 0.229 -> 0.587
+# over 24 steps while reward fell 0.592 -> 0.216, with turns/mean 10.4 against the reference's 6.1
+# (job 76577). The reference pins both from one variable (run_cluster.py:165) and holds entropy
+# flat at 0.17-0.21. top_k=-1 because top_k truncation is likewise unmodelled by the recomputation.
+TEMPERATURE="${TEMPERATURE:-0.8}"
+# 1.0, NOT the reference launcher's 0.95. `--logprobs-mode processed_logprobs` takes the logprob
+# AFTER truncation, so a truncating top_p renormalises every captured logprob over the kept set
+# while the trainer recomputes over the full vocabulary -- the step-0 importance ratio then
+# lands at kept_mass rather than 1, and the reordering is worse than a uniform shift. TRL's
+# AsyncGRPOConfig already defaults top_p to 1.0, so 0.95 here is an ENGINE/TRAINER MISMATCH,
+# not a policy choice. Measured: the reference sat at ratio 0.985-0.993 (the truncation
+# signature); at 1.0 it moves to 0.9984-0.9999.
+TOP_P="${TOP_P:-1.0}"
+
+echo "== vLLM on GPU 0 == (ports vllm=$VLLM_PORT env=$ENV_PORT capture=$CAP_PORT, derived from job ${SLURM_JOB_ID:-0})"
+# EVERY ONE OF THESE FLAGS IS LOad-BEARING. They are the set the +0.2028 run served with; a
+# hand-rolled `vllm serve` that omits any of them fails silently rather than loudly.
+#
+# --enable-auto-tool-choice / --tool-call-parser qwen3_xml
+# Without them vLLM never parses tool calls, so `has_tool_call` is False for EVERY turn and the
+# whole rollout is discarded with no error anywhere. The agent path is entirely tool calls.
+# --default-chat-template-kwargs {"enable_thinking": false}
+# Pins thinking OFF server-side. Qwen3.5-4B's template opens by default and -2B's
+# closes it -- INVERTED defaults between two models of the same family. Leaving it to the
+# default is what made a re-rendered prompt match the 2B and diverge on the 4B, forking 100% of
+# turn transitions. This is what `think_template_patch` used to monkeypatch; it belongs here.
+# --return-tokens-as-token-ids / --logprobs-mode processed_logprobs
+# The capture tier. Without them capture degrades to text and every rollout is untrainable --
+# and looks completely normal while being so.
+# --gdn-prefill-backend triton
+# Qwen3.5 is hybrid Gated-DeltaNet; this is the working prefill backend for it.
+# VLLM_USE_DEEP_GEMM=0 / VLLM_DEEP_GEMM_WARMUP=skip
+# On Hopper a stale importable deep_gemm kills startup with "DeepGEMM backend is not available
+# or outdated".
+# VLLM_SERVER_DEV_MODE=1
+# Gates /server_info, /pause and /init_weight_transfer_engine. TRL's weight transfer reads
+# /server_info for the server's dtype, so WITHOUT this the trainer dies at on_train_begin with
+# "404 Client Error: Not Found for url: .../server_info?config_format=json" -- after vLLM is
+# healthy, after the env server is up, after the dataset loads. Job 76488 died exactly there.
+# VLLM_USE_FLASHINFER_SAMPLER=0
+# flashinfer JIT-compiles its sampling kernel and needs nvcc; the PyTorch sampler does not.
+# --weight-transfer-config {"backend":"nccl"}
+# The trainer syncs weights into this engine over NCCL. Without it there is no transfer engine
+# to initialise.
+CUDA_VISIBLE_DEVICES=0 \
+ VLLM_SERVER_DEV_MODE=1 \
+ VLLM_USE_FLASHINFER_SAMPLER=0 \
+ VLLM_USE_DEEP_GEMM=0 VLLM_DEEP_GEMM_WARMUP=skip \
+ "$PY312" -m vllm.entrypoints.openai.api_server \
+ --model "$MODEL" --port "$VLLM_PORT" --served-model-name "$MODEL" \
+ --trust-remote-code \
+ --max-model-len "${MAX_MODEL_LEN:-131072}" \
+ --enable-auto-tool-choice --tool-call-parser qwen3_xml \
+ --reasoning-parser qwen3 \
+ --gdn-prefill-backend triton \
+ --default-chat-template-kwargs '{"enable_thinking": false}' \
+ --return-tokens-as-token-ids --logprobs-mode processed_logprobs \
+ --override-generation-config "{\"temperature\": $TEMPERATURE, \"top_p\": $TOP_P, \"top_k\": -1}" \
+ --weight-transfer-config '{"backend":"nccl"}' \
+ > "$LOGS/vllm.log" 2>&1 &
+
+# vLLM's /health returns 200 with an EMPTY body, so check the STATUS and parse nothing. A probe that
+# json.loads() it throws, and a caller treating that as "not ready" waits out the whole budget on a
+# server that came up minutes ago.
+echo "waiting for vLLM (up to 30 min: weights + torch.compile)"
+for _ in $(seq 1 900); do
+ code=$(curl -s -o /dev/null -w "%{http_code}" --max-time 3 "$VLLM_URL/health" || true)
+ [ "$code" = "200" ] && break
+ sleep 2
+done
+[ "$(curl -s -o /dev/null -w "%{http_code}" --max-time 5 "$VLLM_URL/health")" = "200" ] || {
+ echo "FATAL: vLLM never became healthy; its log follows"; tail -60 "$LOGS/vllm.log"; exit 1; }
+echo "vLLM healthy"
+# CHECK THE ENDPOINT THE TRAINER ACTUALLY NEEDS, not just /health. /health passing says nothing about
+# whether VLLM_SERVER_DEV_MODE took effect, and the trainer only finds out ~5 minutes later at
+# on_train_begin, after the env server and the whole dataset have loaded.
+si=$(curl -s -o /dev/null -w "%{http_code}" --max-time 10 "$VLLM_URL/server_info?config_format=json" || true)
+[ "$si" = "200" ] || { echo "FATAL: /server_info -> $si; VLLM_SERVER_DEV_MODE=1 did not take effect."; exit 1; }
+# Confirm the engine on this port is OURS. Reachability is not identity: a neighbouring job's vLLM
+# answers /health and /server_info exactly the same way, and the first sign of talking to the wrong
+# one is an NCCL failure minutes later -- or, if the models happen to match, no sign at all.
+served=$(curl -s --max-time 10 "$VLLM_URL/v1/models" | tr -d ' \n' | grep -o "\"id\":\"[^\"]*\"" | head -1)
+echo "vLLM /server_info ok; serving $served on :$VLLM_PORT"
+
+echo "== harbor server =="
+# OWNED BY THE JOB unless SERVER is passed. The reference launcher boots its own env server too, and
+# isolation matters more than the boot cost here: a shared server means a shared 400-session budget,
+# a shared gradio tunnel, and sessions that outlive the job. Measured tonight: two arms on one server
+# cost 6x throughput, and sessions leaked by a killed client filled it to 400/400 and dropped a run
+# from 82% graded to 18%. The dataset tree is cached, so booting our own is cheap.
+if [ -n "${SERVER:-}" ]; then
+ echo " using EXTERNAL server $SERVER (shared: its session budget and tunnel are not ours)"
+else
+ OPENENV_BIN="${OPENENV_BIN:-$REPO/OpenEnv/.venv/bin/openenv}"
+ [ -x "$OPENENV_BIN" ] || { echo "FATAL: no openenv CLI at $OPENENV_BIN"; exit 1; }
+ # Ports are derived from the job id further up, so two jobs on one node cannot collide.
+ MAX_CONCURRENT_ENVS="${MAX_CONCURRENT_ENVS:-400}" \
+ setsid "$OPENENV_BIN" harbor serve \
+ --dataset "${SPLIT:-AdithyaSK/data_agent_rl_environment_train}" \
+ --llm-url "$VLLM_URL/v1" --model "$MODEL" \
+ --port "$ENV_PORT" --capture-port "$CAP_PORT" --expose gradio \
+ --max-output-tokens "${MAX_OUTPUT_TOKENS:-12288}" \
+ > "$LOGS/env.log" 2>&1 &
+ SERVER="http://$(hostname -s):$ENV_PORT"
+ echo " booting our own on :$ENV_PORT (capture :$CAP_PORT) -> $SERVER"
+ for _ in $(seq 1 180); do
+ curl -s -o /dev/null --max-time 3 "http://127.0.0.1:$ENV_PORT/health" && break
+ sleep 2
+ done
+ curl -s -o /dev/null --max-time 5 "http://127.0.0.1:$ENV_PORT/health" || {
+ echo "FATAL: our harbor server never became healthy; log follows"; tail -60 "$LOGS/env.log"; exit 1; }
+ grep -a "capture" "$LOGS/env.log" | head -3
+fi
+export SERVER
+# IDENTITY, not just reachability: a server answering /health may host a different split, and every
+# rollout would then run tasks this job never asked for.
+SPLITS_JSON=$(curl -s --max-time 10 "$SERVER/harbor_env/splits")
+echo " splits $SPLITS_JSON"
+case "$SPLITS_JSON" in
+ *"${SPLIT:-AdithyaSK/data_agent_rl_environment_train}"*) echo " split OK" ;;
+ *) echo "FATAL: $SERVER does not host ${SPLIT:-AdithyaSK/data_agent_rl_environment_train}"; exit 1 ;;
+esac
+# The server probes the engine PER ROLLOUT from ITS host. If it cannot reach this node's vLLM the
+# tier grades `text`, every rollout returns zero trainable turns, and the run burns sandboxes while
+# looking healthy. Prove the path the workload actually uses.
+echo " engine from server host: $(curl -s -o /dev/null -w '%{http_code}' --max-time 8 "$VLLM_URL/health")"
+echo "harbor server healthy"
+
+# max_inflight 32 is the REFERENCE value, and it matters beyond throughput: at 16 this run
+# measured samples/step 7.2 against the reference's 16.1, i.e. less than half the training
+# signal per optimizer step, which over 400 steps is a different experiment rather than a
+# slower one. 16 was chosen after an EVAL collapsed at 64 concurrent MCP clients -- but that
+# eval ran ~16-turn rollouts (64 x 16 = ~1024 concurrent turn-slots) where training runs ~8.5
+# (32 x 8.5 = ~272), about a quarter of the load that broke it.
+echo "== trainer on GPU 1 =="
+# NOT from $REPO: the `trl/` submodule DIRECTORY shadows the `trl` package as a namespace package
+# there, and the import dies with "cannot import name '__version__' from trl (unknown location)".
+cd "$TRAINDIR"
+CUDA_VISIBLE_DEVICES=1 TRACKIO_STORAGE_MODE=sqlite \
+ "$PY312" -u train_harbor_opencode.py \
+ --server "${SERVER:?harbor server was neither booted nor supplied}" \
+ --vllm-url "$VLLM_URL" \
+ --model "$MODEL" \
+ --split "${SPLIT:-AdithyaSK/data_agent_rl_environment_train}" \
+ --sandbox "${SANDBOX:-e2b}" \
+ --learning-rate "${LR:-3e-6}" \
+ --temperature "$TEMPERATURE" \
+ --num-generations "${NUM_GENERATIONS:-8}" \
+ --max-inflight "${MAX_INFLIGHT:-32}" \
+ --grad-accum "${GRAD_ACCUM:-4}" \
+ ${TASK_INDICES:+--task-indices "$TASK_INDICES"} \
+ ${REWARD_KEY:+--reward-key "$REWARD_KEY"} \
+ ${AGENT_TURN_FILTER:+--agent-turn-filter "$AGENT_TURN_FILTER"} \
+ --agent-step-limit "${AGENT_STEP_LIMIT:-17}" \
+ --reward "${REWARD:-efficiency}" \
+ --token-budget "${TOKEN_BUDGET:-40960}" \
+ --max-completion-length "${MAX_COMPLETION_LENGTH:-16384}" \
+ --heartbeat-stale-after-s "${HEARTBEAT_STALE_AFTER_S:-900}" \
+ --dtype "${TRAIN_DTYPE:-bfloat16}" \
+ --top-p "${TOP_P:-1.0}" \
+ --max-steps "${MAX_STEPS:-400}" \
+ --save-steps "${SAVE_STEPS:-100}" \
+ --output-dir "$LOGS/run"
diff --git a/04-data-agent/train/multi_harness.py b/04-data-agent/train/multi_harness.py
new file mode 100644
index 0000000..bda797e
--- /dev/null
+++ b/04-data-agent/train/multi_harness.py
@@ -0,0 +1,142 @@
+"""Route each GRPO group to a harness, without changing TRL.
+
+WHY THIS SHAPE. `HarnessRolloutWorker` hands the factory only `(prompt, seed, episode_id)` --
+`async_rollout_worker.py` pulls `(group_id, row)` but calls `_generate_one(prompt, ..., group_id)`
+and drops the row. So a factory cannot read a harness off a dataset column. What it CAN read is
+`seed`, which `_run_session` sets to `group_id`, and `_repeat_iterator` yields the SAME group_id for
+all `num_generations` of a group.
+
+That is the load-bearing property: **harness is constant within a group.** Measured pass@4 across
+harnesses on this suite spans 0.320 to 0.020, so a group whose members ran under different harnesses
+would have a baseline averaging two competence levels, and the advantage would encode WHICH HARNESS
+rather than which action. Constant-within-group makes the spread a BETWEEN-group constant, which
+advantage normalisation removes entirely.
+
+Group -> row is `group_id % len(dataset)`. An explicit frozen schedule maps the same group ID
+to both its task and its harness: one harness per task per pass, rotating over later passes.
+The legacy modulo route pads rows to ensure that tasks rotate across harnesses over time.
+"""
+
+from __future__ import annotations
+
+import logging
+from math import gcd
+from typing import Any
+
+from harbor_env.harness import HarborSession, HarborSessionFactory
+
+logger = logging.getLogger(__name__)
+
+
+class MultiHarborSessionFactory(HarborSessionFactory):
+ """A HarborSessionFactory whose harness is chosen per GROUP, from `seed`."""
+
+ def __init__(self, *args: Any, harnesses: list[str], schedule=None, group_offset=0, **kw: Any) -> None:
+ super().__init__(*args, **kw)
+ if not harnesses:
+ raise ValueError("harnesses must be non-empty")
+ self.harnesses = list(harnesses)
+ self.schedule = schedule
+ if not isinstance(group_offset, int) or group_offset < 0:
+ raise ValueError('group_offset must be a nonnegative integer')
+ self.group_offset = group_offset
+ if schedule is not None:
+ from harness_schedule import validate_schedule
+ validate_schedule(schedule)
+ if schedule['harnesses'] != self.harnesses:
+ raise ValueError('Schedule harness order differs from the configured harnesses')
+ # group_id -> harness, so a violation is detectable rather than merely unlikely.
+ self._group_harness: dict[int, str] = {}
+
+ def __getstate__(self) -> dict[str, Any]:
+ # The factory is pickled into a spawned child; the parent's live client must not go with it.
+ state = super().__getstate__()
+ state["harnesses"] = self.harnesses
+ state["_group_harness"] = {}
+ return state
+
+ def harness_for(self, seed: int | None) -> str:
+ seed = (seed or 0) + getattr(self, 'group_offset', 0)
+ if getattr(self, 'schedule', None) is not None:
+ return self.schedule['groups'][(seed or 0) % len(self.schedule['groups'])]['harness']
+ return self.harnesses[(seed or 0) % len(self.harnesses)]
+
+ def create(self, task: Any, seed: int | None = None, episode_id: str | None = None) -> HarborSession:
+ harness = self.harness_for(seed)
+
+ # HARD failure, not a warning. If two generations of one group ran under different harnesses
+ # the group's central claim is void, and a warning in a log nobody reads is how that ships.
+ previous = self._group_harness.setdefault(int(seed or 0), harness)
+ if previous != harness:
+ raise RuntimeError(
+ f"group {seed} mixed harnesses ({previous} then {harness}). The GRPO baseline would "
+ f"average two competence levels (measured pass@4 spread 0.320-0.020 on this suite), "
+ f"so the advantage would encode which harness, not which action."
+ )
+
+ instruction = _instruction_of(task)
+ self.tasks() # builds the instruction -> index map
+ index = self._by_instruction.get(_instruction_id(instruction))
+ if index is None:
+ # Verbatim from HarborSessionFactory: a lookup failure must never silently run task 0.
+ raise KeyError(
+ "this prompt does not match any task on the server. Build the dataset from "
+ "`prompt_rows()` so the instruction the trainer sends is the one the server has."
+ )
+ if self.schedule is not None:
+ absolute_group = (seed or 0) + self.group_offset
+ expected = self.schedule['groups'][absolute_group % len(self.schedule['groups'])]
+ if index != expected['task_index']:
+ raise ValueError(f'Group {seed} received task {index}, expected {expected["task_index"]}')
+ return HarborSession(
+ env=self.new_client(), # one client PER SESSION: a shared MCP socket raises
+ owns_env=True, # ConcurrencyError on concurrent recv, making every rollout unscorable
+ split=self._split,
+ task_index=index,
+ instruction=instruction,
+ harness=harness, # <-- the only thing that varies
+ sandbox=self.sandbox,
+ llm_url=self.llm_url,
+ model=self.model,
+ sampling=self.sampling,
+ reward_key=self.reward_key,
+ api_key=self.api_key,
+ auth_header=self.auth_header,
+ agent_timeout_sec=self.agent_timeout_sec,
+ agent_step_limit=self.agent_step_limit,
+ )
+
+
+def _instruction_of(task: Any) -> str:
+ from harbor_env.harness import _instruction_of as f # reuse, never reimplement
+ return f(task)
+
+
+def _instruction_id(text: str) -> str:
+ from harbor_env.harness import instruction_id as f
+ return f(text)
+
+
+def pair_rows(factory: MultiHarborSessionFactory, *, all_pairs: bool = False) -> list[dict[str, Any]]:
+ """Align prompt rows with the explicit schedule, Cartesian mode, or legacy modulo route."""
+ rows = list(factory.prompt_rows())
+ h = len(factory.harnesses)
+ schedule = getattr(factory, 'schedule', None)
+ if schedule is not None:
+ if all_pairs:
+ raise ValueError('Choose either a rotating schedule or Cartesian scheduling')
+ expected = [(t['name'], t['task_index']) for t in schedule['tasks']]
+ if [(r['task_name'], r['task_index']) for r in rows] != expected:
+ raise ValueError('Server task identities/order differ from the frozen schedule')
+ return [dict(rows[g['task_row']]) for g in schedule['groups']]
+ if all_pairs:
+ # The worker repeats each row num_generations times with a fixed group ID.
+ # H consecutive rows per task align exactly with harness_for(group_id).
+ # This schedule repeats without coprime padding because its length is a multiple of H.
+ return [dict(row) for row in rows for _ in factory.harnesses]
+ if h > 1:
+ while len(rows) > 1 and gcd(len(rows), h) != 1:
+ rows.append(dict(rows[len(rows) % len(rows)])) # duplicate one row to break the common factor
+ assert gcd(len(rows), h) == 1, f"gcd({len(rows)},{h}) != 1"
+ logger.info("pair_rows: %d rows x %d harnesses, gcd=%d", len(rows), h, gcd(len(rows), h))
+ return rows
diff --git a/04-data-agent/train/prepare_harbor_opencode_run.py b/04-data-agent/train/prepare_harbor_opencode_run.py
new file mode 100644
index 0000000..9cb6e5e
--- /dev/null
+++ b/04-data-agent/train/prepare_harbor_opencode_run.py
@@ -0,0 +1,134 @@
+"""Prepare a fresh OpenCode-only ablation of the last stable Harbor recipe."""
+import argparse
+import copy
+import hashlib
+import json
+import os
+from pathlib import Path
+import shutil
+import subprocess
+import sys
+
+REPO = Path(__file__).resolve().parents[3]
+REFERENCE = REPO / 'experiments/async_grpo_harbor_data_agent/logs/multi4-long-prod-cont-20260915'
+TOOLS = REPO / 'experiments/async_grpo_harbor_data_agent/tools'
+DEFAULT_OUT = REPO / 'experiments/async_grpo_harbor_data_agent/logs/harbor-opencode-only-20260916'
+sys.path.insert(0, str(Path(__file__).parent))
+from harness_schedule import validate_schedule
+
+
+def read(path): return json.loads(path.read_text())
+def digest(path): return hashlib.sha256(path.read_bytes()).hexdigest()
+def write(path, value): path.write_text(json.dumps(value, indent=2) + '\n')
+
+
+def single_harness_schedule(reference):
+ validate_schedule(reference)
+ schedule = copy.deepcopy(reference)
+ n = schedule['task_count']
+ schedule.update(harnesses=['opencode'], passes_per_cycle=1, groups_per_cycle=n)
+ schedule['groups'] = schedule['groups'][:n]
+ for group in schedule['groups']:
+ group['harness'] = 'opencode'
+ validate_schedule(schedule)
+ assert all({k:v for k,v in a.items() if k != 'harness'} == {k:v for k,v in b.items() if k != 'harness'}
+ for a,b in zip(reference['groups'][:n], schedule['groups'], strict=True))
+ return schedule
+
+
+def prepare(root):
+ if root.exists():
+ raise ValueError('Run directory exists; inspect its submission record before retrying')
+ config = read(REFERENCE/'run_config.json')
+ prior = copy.deepcopy(config)
+ manifest = read(REFERENCE/'manifest.json')
+ schedule = single_harness_schedule(read(REFERENCE/'harness_schedule.json'))
+ root.mkdir(parents=True)
+ for name in ('indices.txt','runtime_versions.json'):
+ shutil.copyfile(REFERENCE/name,root/name)
+ manifest.update(harnesses=['opencode'],pairs_per_cycle=1000,passes_per_cycle=1,
+ groups_per_pass=1000,rollouts_per_scheduled_cycle=8000)
+ write(root/'manifest.json',manifest);write(root/'harness_schedule.json',schedule)
+ (root/'pairs.jsonl').write_text(''.join(json.dumps(g)+'\n' for g in schedule['groups']))
+ snap=root/'source-snapshot'
+ shutil.copytree(REFERENCE/'source-snapshot',snap,ignore=shutil.ignore_patterns('__pycache__','*.pyc'))
+ # Keep model/trainer/optimizer code identical to the completed stable run.
+ # Only transport receives the already-qualified keepalive fix used by eval800.
+ qualified=REFERENCE/'checkpoint-evals/step-000800/recovery-20260916-keepalive/source-snapshot'
+ assert read(REFERENCE/'checkpoint-evals/step-000800/scores.json')['comparison_ready']
+ transport_changes=[]
+ for name in ('server.py','sse.py'):
+ rel=Path('OpenEnv/src/openenv/core/harness/capture')/name
+ before=digest(snap/rel);shutil.copyfile(qualified/rel,snap/rel)
+ transport_changes.append({'file':str(rel),'before':before,'after':digest(snap/rel)})
+ for name in ('monitor_multi4.py','trackio_multi4.py','supervise_multi4.py'):
+ shutil.copyfile(TOOLS/name,snap/'tools'/name)
+ shutil.copyfile(REPO/'HuggingEnvs/04-data-agent/eval/checkpoint_evals.py',snap/'HuggingEnvs/04-data-agent/eval/checkpoint_evals.py')
+ launch=snap/'tools/launch_multi4_long.sh';text=launch.read_text()
+ text=text.replace("assert m['task_count']==1000 and m['pairs_per_cycle']==4000", "assert m['task_count']==1000 and m['pairs_per_cycle']==1000 * len(m['harnesses'])")
+ launch.write_text(text)
+ log=snap/'tools/launch_multi4_trackio.sh';log.write_text(log.read_text().replace('--watch --online','--watch'))
+ supervisor=snap/'tools/supervise_multi4.py';text=supervisor.read_text()
+ text=text.replace("'--train-job', train_job, '--watch', '--online']", "'--train-job', train_job, '--watch']\n if read_json(root / 'run_config.json', {}).get('logging', {}).get('online', True):\n command.append('--online')")
+ supervisor.write_text(text)
+ submit=snap/'tools/submit_multi4_long.py';text=submit.read_text()
+ text=text.replace("'--partition=hopper-extra'", "'--partition=' + config['resources']['partition']")
+ text=text.replace("'--job-name=multi4-long-2b'", "'--job-name=harbor-opencode-only'")
+ submit.write_text(text)
+ evaluator=root/'checkpoint-evals/eval-source';evaluator.parent.mkdir()
+ shutil.copytree(qualified,evaluator,ignore=shutil.ignore_patterns('__pycache__','*.pyc'))
+ write(evaluator/'source_hashes.json',{str(p.relative_to(evaluator)):digest(p) for p in evaluator.rglob('*') if p.is_file() and p!=evaluator/'source_hashes.json'})
+ watcher=snap/'tools/launch_multi4_eval_watcher.sh';text=watcher.read_text()
+ text=text.replace('--interval "$INTERVAL"', '--protocol "$TRAIN_RUN_ROOT/checkpoint-evals/eval-source/protocol.json" --interval "$INTERVAL"')
+ watcher.write_text(text)
+ config.update(status='prepared',harnesses=['opencode'],initialization='Fresh pinned Qwen3.5-2B base; independent optimizer and scheduler',
+ source_snapshot=str(snap),frozen_eval_source=str(evaluator),reference_run=str(REFERENCE),reference_job=80608)
+ for key in ('restart_of','restart_reason','resume_state','job_id','replaced_by'):
+ config.pop(key,None)
+ config['harness_versions']={'opencode':prior['harness_versions']['opencode']}
+ config['training'].pop('resume_from_checkpoint',None)
+ config['training'].pop('budget_reference',None)
+ config['training']['soft_max_train_seconds']=82200
+ config['resources'].update(partition='hopper-prod',slurm_walltime='24:00:00')
+ config['dataset'].update(pairs_per_cycle=1000,rollouts_per_scheduled_cycle=8000,passes_per_cycle=1,
+ harness_groups_per_pass={'opencode':1000},schedule_file=str(root/'harness_schedule.json'),
+ schedule_sha256=digest(root/'harness_schedule.json'),manifest_sha256=digest(root/'manifest.json'))
+ config['evaluation'].update(protocol_file=str(evaluator/'protocol.json'),partition='hopper-prod',
+ max_active_eval_jobs=1,interval_optimizer_steps=100)
+ config['logging'].update(project=root.name,space_id='HuggingEnvs/data-agent-training-comparison-trackio',
+ bucket_id='HuggingEnvs/data-agent-training-comparison-trackio',private=False,online=False,
+ online_via='Uniform comparison publisher',evaluation_sources=[],local_directory=str(root/'trackio'),
+ collector_file=str(snap/'tools/trackio_multi4.py'))
+ config['logging'].pop('parent_project',None)
+ config['monitoring'].update(stable_after_optimizer_step=10,startup_interval_seconds=120,stable_interval_seconds=600)
+ config['monitoring']['support_job_supervisor'].update(status_file=str(root/'supervisor/status.json'),source_file=str(snap/'tools/supervise_multi4.py'))
+ config['limitations']=[
+ 'Matches the last stable Harbor recipe, not every historical recipe used earlier in the resumed reference.',
+ 'The first pass preserves the reference task order exactly; the same 1000-task single-harness pass repeats if exhausted.',
+ '1000 is the optimizer-step target and task-pool size; it does not guarantee full task coverage.',
+ 'Worker ceiling32; total outstanding rollouts16; eight generations per task; staleness at most4.',
+ 'Separate eval GPUs, endpoints and node; FSx and E2B quota remain shared.',
+ 'Checkpoint capture transport uses the already-qualified streaming keepalive repair; sampled tokens and loss are unchanged.',
+ 'CPU monitoring writes local alerts every2minutes initially and every10minutes after stability; no automatic chat wakeups.',
+ 'Failed or incomplete evaluations are recorded as incomplete, never converted to final pass@1 scores.']
+ write(root/'run_config.json',config)
+ hashes={str(p.relative_to(snap)):digest(p) for p in snap.rglob('*') if p.is_file()}
+ write(root/'source_hashes.json',hashes)
+ excluded={'resume_from_checkpoint','budget_reference'}
+ assert {k:v for k,v in prior['training'].items() if k not in excluded} == config['training']
+ proof={'prepared':True,'fresh_base':True,'same_training_hyperparameters':True,'same_sampling':config['sampling']==prior['sampling'],
+ 'first_pass_task_order_identical':True,'training_harnesses':['opencode'],'evaluation_harnesses':read(evaluator/'protocol.json')['harnesses'],
+ 'reference_job':80608,'source_files':len(hashes),'transport_changes':transport_changes,'live_startup_verified':False}
+ write(root/'validation.json',proof)
+ env={**os.environ,'TRAIN_RUN_ROOT':str(root),'SLURM_JOB_ID':'preflight','MULTI4_PREFLIGHT_ONLY':'1'}
+ with (root/'preflight.log').open('w') as stream:
+ subprocess.run(['bash',str(launch)],env=env,stdout=stream,stderr=subprocess.STDOUT,check=True)
+ env.update(TRAIN_JOB_ID='PREFLIGHT',MULTI4_EVAL_PREFLIGHT_ONLY='1')
+ with (root/'eval-preflight.log').open('w') as stream:
+ subprocess.run(['bash',str(watcher)],env=env,stdout=stream,stderr=subprocess.STDOUT,check=True)
+ print(json.dumps({'run':str(root),'validation':proof},indent=2))
+
+
+if __name__=='__main__':
+ parser=argparse.ArgumentParser(description=__doc__);parser.add_argument('--out',type=Path,default=DEFAULT_OUT)
+ prepare(parser.parse_args().out.resolve())
diff --git a/04-data-agent/train/run_whitebox_bash.slurm b/04-data-agent/train/run_whitebox_bash.slurm
new file mode 100755
index 0000000..f46071a
--- /dev/null
+++ b/04-data-agent/train/run_whitebox_bash.slurm
@@ -0,0 +1,82 @@
+#!/usr/bin/env bash
+#SBATCH --job-name=wb-bash-run
+#SBATCH --partition=hopper-extra,hopper-dev
+#SBATCH --gres=gpu:1
+#SBATCH --cpus-per-task=24
+#SBATCH --output=/fsx/%u/logs/%x-%j.out
+#SBATCH --error=/fsx/%u/logs/%x-%j.err
+#SBATCH --time=0-04:00:00
+#
+# A REAL (small) run: one pass over the 100-task generated train split, checkpoints saved, so the
+# result can be evaluated against base on the DISJOINT 30-task test split.
+#
+# SPLIT DEFAULTS TO train:easy, AND THAT IS A GRADIENT DECISION, NOT A DIFFICULTY PREFERENCE.
+# GRPO needs variance WITHIN a group: if every rollout of a prompt scores the same, the advantages are
+# all zero and the step contributes nothing. Base pass@1 for this model by tier is easy 0.214,
+# medium 0.062, hard 0.060, so P(all `num_generations` rollouts fail) is
+# train:easy g=4 -> 0.38 g=8 -> 0.15 g=16 -> 0.02
+# train:medium g=4 -> 0.77 g=8 -> 0.60 g=16 -> 0.36
+# Measured: `train` at g=4 produced 0 reward and grad_norm 0 on 7 of 7 steps (job 76712). easy at
+# g=8 is the smallest configuration that actually learns on one GPU.
+#
+# Same 1-GPU colocate shape as the smoke. Everything here that looks incidental is load-bearing --
+# see smoke_whitebox_bash.slurm for why PATH, the ninja gate and the derived ports exist.
+set -euo pipefail
+
+TRL_PROD=/fsx/adithyaskolavi/projects/trl_prod
+cd "$TRL_PROD"
+PY="$TRL_PROD/.venv312/bin/python"
+export PATH="$TRL_PROD/.venv312/bin:$PATH"
+module load cuda/12.9 2>/dev/null || module load cuda/13.0 2>/dev/null || true
+command -v ninja >/dev/null || { echo "FATAL: ninja not on PATH; flashinfer's JIT dies at step 0"; exit 1; }
+
+set -a; source "$TRL_PROD/experiments/.env" 2>/dev/null || true; set +a
+export HF_TOKEN="${HF_TOKEN:-${HF_API_KEY:-}}"
+echo "E2B_API_KEY: ${E2B_API_KEY:+present}"
+[ -n "${E2B_API_KEY:-}" ] || { echo "FATAL: E2B_API_KEY unset"; exit 1; }
+
+export PYTHONPATH="$TRL_PROD/trl:$TRL_PROD/HuggingEnvs/04-data-agent/train:$TRL_PROD/HuggingEnvs/04-data-agent/train/_pypath:$TRL_PROD/OpenEnv/src"
+export TRL_EXPERIMENTAL_SILENCE=1 HF_HUB_ENABLE_HF_TRANSFER=1 TOKENIZERS_PARALLELISM=false
+export PYTORCH_CUDA_ALLOC_CONF="${PYTORCH_CUDA_ALLOC_CONF:-expandable_segments:True}"
+
+PORT=$(( 8400 + ${SLURM_JOB_ID:-0} % 150 ))
+export WHITE_BOX_BASH_URL="http://127.0.0.1:${PORT}"
+LOGS="$TRL_PROD/HuggingEnvs/04-data-agent/envs/whitebox-bash/logs/job-${SLURM_JOB_ID:-local}"
+mkdir -p "$LOGS"
+
+echo "== env server on :$PORT =="
+"$PY" -m uvicorn whitebox_bash.server.app:app --host 127.0.0.1 --port "$PORT" > "$LOGS/server.log" 2>&1 &
+SERVER_PID=$!
+trap 'kill $SERVER_PID 2>/dev/null || true' EXIT
+for _ in $(seq 1 60); do
+ curl -sf "http://127.0.0.1:${PORT}/health" >/dev/null 2>&1 && break
+ kill -0 $SERVER_PID 2>/dev/null || { echo "FATAL: env server died"; tail -30 "$LOGS/server.log"; exit 1; }
+ sleep 2
+done
+SPLIT="${SPLIT:-train:easy}"
+N=$(curl -sf -X POST "http://127.0.0.1:${PORT}/white_box_bash/num_tasks" -H 'content-type: application/json' \
+ -d "{\"split\":\"$SPLIT\"}" | "$PY" -c 'import json,sys; print(json.load(sys.stdin)["num_tasks"])')
+NT=$(curl -sf -X POST "http://127.0.0.1:${PORT}/white_box_bash/num_tasks" -H 'content-type: application/json' \
+ -d '{"split":"test"}' | "$PY" -c 'import json,sys; print(json.load(sys.stdin)["num_tasks"])')
+echo "== server healthy: $SPLIT=$N test=$NT =="
+[ "$N" -ge 100 ] || { echo "FATAL: split $SPLIT has only $N tasks"; exit 1; }
+# Tasks come from HuggingEnvs/data-agent, staged per task out of an HF bucket into the shared
+# E2B template. Both are required: the default E2B base has no huggingface_hub and no writable
+# /workdir, so staging dies before the agent starts.
+echo "== task source: ${WHITE_BOX_BASH_TASK_SOURCE:-data-agent} template: ${E2B_TEMPLATE:-data-agent-opencode} =="
+
+export MODEL="${MODEL:-Qwen/Qwen3.5-2B}"
+export VLLM_GPU_MEM_UTIL="${VLLM_GPU_MEM_UTIL:-0.3}"
+
+echo "== trainer: ${MAX_STEPS:-100} steps over $N tasks, saving every ${SAVE_STEPS:-50} =="
+"$PY" "$TRL_PROD/HuggingEnvs/04-data-agent/train/train_whitebox_bash.py" \
+ --model "$MODEL" --server "$WHITE_BOX_BASH_URL" --split "$SPLIT" \
+ --toolsets "${TOOLSETS:-bash,seta}" --output-dir "$LOGS/run" \
+ --max-steps "${MAX_STEPS:-100}" --save-steps "${SAVE_STEPS:-50}" \
+ --num-generations "${NUM_GENERATIONS:-8}" --per-device-train-batch-size "${PER_DEVICE_BS:-2}" \
+ --gradient-accumulation-steps "${GRAD_ACCUM:-4}" \
+ --max-tool-calling-iterations "${MAX_TOOL_ITERS:-8}" --step-limit "${STEP_LIMIT:-12}" \
+ --max-completion-length "${MAX_COMPLETION_LENGTH:-1024}" --report-to "${REPORT_TO:-none}"
+
+echo "== run finished; checkpoints: =="
+ls -d "$LOGS/run"/checkpoint-* 2>/dev/null || echo " (none)"
diff --git a/04-data-agent/train/smoke_whitebox_bash.slurm b/04-data-agent/train/smoke_whitebox_bash.slurm
new file mode 100755
index 0000000..d516539
--- /dev/null
+++ b/04-data-agent/train/smoke_whitebox_bash.slurm
@@ -0,0 +1,99 @@
+#!/usr/bin/env bash
+#SBATCH --job-name=wb-bash-smoke
+#SBATCH --partition=hopper-extra,hopper-dev
+#SBATCH --gres=gpu:1
+#SBATCH --cpus-per-task=24
+#SBATCH --output=/fsx/%u/logs/%x-%j.out
+#SBATCH --error=/fsx/%u/logs/%x-%j.err
+#SBATCH --time=0-02:00:00
+#
+# SMOKE: sync GRPO on the WHITE-BOX bash/SETA environment. Prove the loop runs end to end.
+#
+# ONE GPU. vLLM runs COLOCATE -- in-process on the training card -- so there is no separate serve and
+# no 2-GPU allocation. `vllm_gpu_memory_utilization` is what makes that fit: the trainer and the
+# engine share one card and the default 0.9 would leave nothing for the optimizer states.
+#
+# The env server runs on THIS node, on a port derived from the job id. Fixed ports are how one job
+# ends up talking to another job's server -- that happened on the async side and would have trained
+# against the wrong weights.
+set -euo pipefail
+
+TRL_PROD=/fsx/adithyaskolavi/projects/trl_prod
+cd "$TRL_PROD"
+PY="$TRL_PROD/.venv312/bin/python"
+
+# THE VENV'S bin/ MUST BE ON PATH, not just its python.
+# Qwen3.5 is hybrid Gated-DeltaNet, and vLLM routes its prefill through flashinfer, which JIT-COMPILES
+# a kernel at first generation. That compile shells out to `ninja` and `nvcc`. Calling
+# .venv312/bin/python directly (rather than activating) leaves .venv312/bin off PATH, so `ninja` is
+# not found and the run dies at step 0 with a bare
+# FileNotFoundError: [Errno 2] No such file or directory: 'ninja'
+# thirty frames below flashinfer, naming nothing that suggests PATH. The backend is chosen from vLLM's
+# `additional_config["gdn_prefill_backend"]`, which colocate mode does not expose, so PATH is the fix.
+export PATH="$TRL_PROD/.venv312/bin:$PATH"
+# nvcc for the same JIT. Already present at /usr/local/cuda-12.9/bin on these nodes; the module load
+# is belt-and-braces and must not fail the job when the module system has no such module.
+module load cuda/12.9 2>/dev/null || module load cuda/13.0 2>/dev/null || true
+command -v ninja >/dev/null || { echo "FATAL: ninja not on PATH; flashinfer's JIT will fail at step 0"; exit 1; }
+command -v nvcc >/dev/null || echo "WARN: nvcc not on PATH; flashinfer's JIT may fail"
+echo "ninja=$(command -v ninja) nvcc=$(command -v nvcc)"
+
+set -a; source "$TRL_PROD/experiments/.env" 2>/dev/null || true; set +a
+export HF_TOKEN="${HF_TOKEN:-${HF_API_KEY:-}}"
+# Presence only -- never the value.
+echo "E2B_API_KEY: ${E2B_API_KEY:+present}"
+[ -n "${E2B_API_KEY:-}" ] || { echo "FATAL: E2B_API_KEY unset; every episode would fail at sandbox creation"; exit 1; }
+
+export PYTHONPATH="$TRL_PROD/trl:$TRL_PROD/HuggingEnvs/04-data-agent/train:$TRL_PROD/HuggingEnvs/04-data-agent/train/_pypath:$TRL_PROD/OpenEnv/src"
+export TRL_EXPERIMENTAL_SILENCE=1
+export HF_HUB_ENABLE_HF_TRANSFER=1
+export PYTORCH_CUDA_ALLOC_CONF="${PYTORCH_CUDA_ALLOC_CONF:-expandable_segments:True}"
+export TOKENIZERS_PARALLELISM=false
+
+PORT=$(( 8400 + ${SLURM_JOB_ID:-0} % 150 ))
+export WHITE_BOX_BASH_URL="http://127.0.0.1:${PORT}"
+LOGS="$TRL_PROD/HuggingEnvs/04-data-agent/envs/whitebox-bash/logs/job-${SLURM_JOB_ID:-local}"
+mkdir -p "$LOGS"
+
+echo "== env server on :$PORT =="
+"$PY" -m uvicorn whitebox_bash.server.app:app --host 127.0.0.1 --port "$PORT" \
+ > "$LOGS/server.log" 2>&1 &
+SERVER_PID=$!
+# Kill the server on any exit, including failure: a survivor holds the port and the next job's
+# server cannot bind, which presents as "stuck loading forever" rather than as a port clash.
+trap 'kill $SERVER_PID 2>/dev/null || true' EXIT
+
+for _ in $(seq 1 60); do
+ curl -sf "http://127.0.0.1:${PORT}/health" >/dev/null 2>&1 && break
+ kill -0 $SERVER_PID 2>/dev/null || { echo "FATAL: env server died during startup"; tail -30 "$LOGS/server.log"; exit 1; }
+ sleep 2
+done
+curl -sf "http://127.0.0.1:${PORT}/health" >/dev/null || { echo "FATAL: env server never became healthy"; tail -30 "$LOGS/server.log"; exit 1; }
+
+# GATE: the server must actually serve tasks. A healthy server with an empty split would train on
+# nothing and report clean numbers doing it.
+N=$(curl -sf -X POST "http://127.0.0.1:${PORT}/white_box_bash/num_tasks" \
+ -H 'content-type: application/json' -d "{\"split\":\"${SPLIT:-train}\"}" | "$PY" -c 'import json,sys; print(json.load(sys.stdin)["num_tasks"])')
+echo "== server healthy, split=${SPLIT:-train} has $N tasks =="
+[ "$N" -gt 0 ] || { echo "FATAL: split has no tasks"; exit 1; }
+
+export MODEL="${MODEL:-Qwen/Qwen3.5-2B}"
+export VLLM_GPU_MEM_UTIL="${VLLM_GPU_MEM_UTIL:-0.3}"
+export VLLM_MAX_MODEL_LEN="${VLLM_MAX_MODEL_LEN:-16384}"
+
+echo "== trainer (colocate vLLM, 1 GPU) =="
+"$PY" "$TRL_PROD/HuggingEnvs/04-data-agent/train/train_whitebox_bash.py" \
+ --model "$MODEL" \
+ --server "$WHITE_BOX_BASH_URL" \
+ --split "${SPLIT:-train}" \
+ --toolsets "${TOOLSETS:-bash,seta}" \
+ --output-dir "$LOGS/run" \
+ --max-steps "${MAX_STEPS:-4}" \
+ --num-generations "${NUM_GENERATIONS:-4}" \
+ --per-device-train-batch-size "${PER_DEVICE_BS:-4}" \
+ --max-tool-calling-iterations "${MAX_TOOL_ITERS:-8}" \
+ --step-limit "${STEP_LIMIT:-12}" \
+ --max-completion-length "${MAX_COMPLETION_LENGTH:-1024}" \
+ --report-to "${REPORT_TO:-none}"
+
+echo "== smoke finished =="
diff --git a/04-data-agent/train/standalone_comparison.py b/04-data-agent/train/standalone_comparison.py
new file mode 100644
index 0000000..1c8ef60
--- /dev/null
+++ b/04-data-agent/train/standalone_comparison.py
@@ -0,0 +1,90 @@
+"""Native OpenCode sessions with the reference task schedule and binary reward.
+
+Only the session boundary differs from the multi-harness trainer. GPU loss,
+whole-rollout admission, backpressure and checkpoint callbacks are shared.
+"""
+from __future__ import annotations
+
+from data_agent_env import DataAgentSessionFactory
+from data_agent_env.harness import DataAgentSession, _instruction_of
+from data_agent_env.task import instruction_id
+from openenv.core.harness import VerifyResult
+
+
+class ComparisonSession(DataAgentSession):
+ @property
+ def result(self):
+ return self._result
+
+ @property
+ def _task_index(self):
+ return self._index
+
+ def verify(self, transcript, final_state=None):
+ native = super().verify(transcript, final_state)
+ correctness = self._result.correctness if self._result is not None else None
+ # The comparison trains binary task success, like the reference and SETA.
+ # Keep partial chat credit and efficiency bonuses in the raw artifact only.
+ reward = None if correctness is None else float(correctness >= 1.0)
+ return VerifyResult(env_reward=reward, done=True, metrics=native.metrics,
+ artifacts={**native.artifacts, "reward_policy": "binary_correctness"})
+
+
+class ScheduledOpenCodeFactory(DataAgentSessionFactory):
+ def __init__(self, server, *, harnesses, schedule, group_offset=0, split="train",
+ sandbox="daytona", llm_url, model, sampling, reward_key="",
+ api_key="", agent_timeout_sec=600, agent_step_limit=17,
+ indices=None, num_tasks=None):
+ from harness_schedule import validate_schedule
+ validate_schedule(schedule)
+ if harnesses != ["opencode"] or schedule["harnesses"] != harnesses:
+ raise ValueError("Standalone training requires an OpenCode-only schedule")
+ if sampling != {"temperature": 0.8, "top_p": 1.0, "top_k": 0}:
+ raise ValueError("Standalone comparison requires the pinned sampling policy")
+ if group_offset < 0:
+ raise ValueError("Negative schedule offset")
+ super().__init__(server, split="train", llm_url=llm_url, model=model,
+ sandbox=sandbox, api_key=api_key, agent_step_limit=agent_step_limit,
+ agent_timeout_s=agent_timeout_sec, sampling=sampling)
+ self.harnesses, self.schedule, self.group_offset = harnesses, schedule, group_offset
+ self._rows, self._by_instruction = None, None
+
+ def harness_for(self, seed):
+ return "opencode"
+
+ def _new_client(self):
+ from data_agent_env import DataAgentEnv
+ return DataAgentEnv(self._server, message_timeout_s=1800)
+
+ def prompt_rows(self):
+ if self._rows is None:
+ client = self._new_client()
+ try:
+ tasks = client.get_task_range("train")
+ finally:
+ client.close()
+ by_index = {task["index"]: task for task in tasks}
+ rows, lookup = [], {}
+ for expected in self.schedule["tasks"]:
+ task = by_index[expected["task_index"]]
+ if task["task_id"] != expected["name"]:
+ raise ValueError("Native Space task identity differs from the frozen schedule")
+ key = instruction_id(task["instruction"])
+ if key in lookup:
+ raise ValueError("Ambiguous training instruction")
+ lookup[key] = expected["task_index"]
+ rows.append({"prompt": [{"role": "user", "content": task["instruction"]}],
+ "task_name": expected["name"], "task_index": expected["task_index"]})
+ self._rows, self._by_instruction = rows, lookup
+ return self._rows
+
+ def create(self, task, seed=None, episode_id=None):
+ self.prompt_rows()
+ instruction = _instruction_of(task)
+ index = self._by_instruction.get(instruction_id(instruction))
+ absolute = (seed or 0) + self.group_offset
+ expected = self.schedule["groups"][absolute % len(self.schedule["groups"])]
+ if index != expected["task_index"]:
+ raise ValueError(f"Native schedule mismatch at group {absolute}: {index}")
+ return ComparisonSession(self._new_client(), "train", index, instruction,
+ **self._rollout_kwargs)
diff --git a/04-data-agent/train/test_atomic_rollouts.py b/04-data-agent/train/test_atomic_rollouts.py
new file mode 100644
index 0000000..615f9dd
--- /dev/null
+++ b/04-data-agent/train/test_atomic_rollouts.py
@@ -0,0 +1,431 @@
+import asyncio
+import pickle
+import queue
+from collections import defaultdict
+from types import SimpleNamespace
+
+import pytest
+import torch
+from atomic_rollouts import (
+ AtomicHarnessLoop,
+ AtomicRolloutDataset,
+ AtomicRolloutTrainer,
+ RolloutBundle,
+ pack_rows,
+)
+from transformers import Trainer, TrainingArguments
+from trl.experimental.async_grpo.async_grpo_trainer import DataCollatorForRollout
+from trl.experimental.async_grpo.async_rollout_worker import RolloutSample
+from trl.experimental.async_grpo.openenv_harness import _HarnessRolloutLoop
+
+
+def row(ids, mask=None, group=0, version=1, advantage=0.7):
+ return RolloutSample(
+ [],
+ [],
+ ids,
+ mask or [0] + [1] * (len(ids) - 1),
+ [-0.8] * len(ids),
+ advantage,
+ version,
+ group,
+ {"reward": 1.0},
+ )
+
+
+def worker(bundles, version=1):
+ q = queue.Queue()
+ for b in bundles:
+ q.put(b)
+ return SimpleNamespace(
+ rollout_buffer=q,
+ model_version=version,
+ check_health=lambda timeout: pytest.fail("Unexpected empty queue"),
+ )
+
+
+def test_bundle_survives_worker_process_pickle():
+ original = RolloutBundle([row([1, 2, 3]), row([1, 4])], "episode")
+ copy = pickle.loads(pickle.dumps(original))
+ assert copy.rollout_id == "episode" and len(copy.rows) == 2
+ assert copy.model_version == 1 and copy.group_id == 0
+
+
+def test_native_scored_rows_keep_their_rollout_and_advantage(monkeypatch):
+ original = [row([1, 2]), row([1, 3]), row([1, 4], advantage=-0.2)]
+
+ async def score(self, group):
+ return original
+
+ monkeypatch.setattr(_HarnessRolloutLoop, "_score_group", score)
+ group = SimpleNamespace(
+ group_id=0,
+ completions_sequences=[
+ [SimpleNamespace(rollout_id="a"), SimpleNamespace(rollout_id="a")],
+ [],
+ [SimpleNamespace(rollout_id="c")],
+ ],
+ )
+ bundles = asyncio.run(
+ AtomicHarnessLoop._score_group(object.__new__(AtomicHarnessLoop), group)
+ )
+ assert [b.rollout_id for b in bundles] == ["a", "c"]
+ assert bundles[0].rows == original[:2] and bundles[1].rows == original[2:]
+ assert bundles[1].advantage == -0.2
+
+
+def test_forked_rollout_is_admitted_whole_and_forwarded_without_token_changes():
+ bundle = RolloutBundle(
+ [row([1, 2, 3]), row([4, 5, 6, 7, 8, 9]), row([1, 4])], "fork"
+ )
+ data = AtomicRolloutDataset(worker([bundle]), defaultdict(list), 4, 8, 4, 60)
+ unit = next(iter(data))
+ assert unit["rollouts"] == [bundle]
+ packs = list(pack_rows(unit["rollouts"], 4, 8))
+ result = [r for pack in packs for r in pack]
+ assert [r["input_ids"] for r in result] == [r.input_ids for r in bundle.rows]
+ assert [r["completion_mask"] for r in result] == [
+ r.completion_mask for r in bundle.rows
+ ]
+ assert [r["old_log_probs"] for r in result] == [
+ r.old_log_probs for r in bundle.rows
+ ]
+
+
+def test_pending_rollout_is_rechecked_for_staleness_and_never_partially_dropped():
+ a = RolloutBundle([row([1, 2, 3])], "a")
+ b = RolloutBundle([row([1, 2, 3]), row([1, 4])], "b")
+ c = RolloutBundle([row([1, 2, 3, 4], version=6)], "c")
+ w = worker([a, b, c])
+ metrics = defaultdict(list)
+ data = AtomicRolloutDataset(w, metrics, 4, 8, 4, 60)
+ it = iter(data)
+ assert [x.rollout_id for x in next(it)["rollouts"]] == ["a"]
+ w.model_version = 6
+ assert [x.rollout_id for x in next(it)["rollouts"]] == ["c"]
+ assert metrics["admission/stale_rollouts_dropped_total"] == [1]
+ assert metrics["sample/dropped_stale_total"] == [2]
+
+
+def test_context_overflow_fails_instead_of_silently_losing_rows():
+ data = AtomicRolloutDataset(
+ worker([RolloutBundle([row(list(range(10)))], "long")]),
+ defaultdict(list),
+ 4,
+ 8,
+ 4,
+ 60,
+ )
+ with pytest.raises(RuntimeError, match="refusing to discard"):
+ next(iter(data))
+
+
+class ToyModel(torch.nn.Module):
+ def __init__(self):
+ super().__init__()
+ self.theta = torch.nn.Parameter(torch.tensor(0.02))
+
+ def forward(self, input_ids, **kwargs):
+ lp = self.theta - 0.8 + input_ids[:, 1:] * 0.003
+ return {"log_probs": lp, "entropy": torch.ones_like(lp)}
+
+
+def native_loss_trainer():
+ trainer = object.__new__(AtomicRolloutTrainer)
+ trainer.epsilon_low = trainer.epsilon_high = 0.2
+ trainer.aux_loss_enabled = False
+ trainer.accelerator = SimpleNamespace(
+ num_processes=1,
+ reduce=lambda value, reduction: value,
+ gather=lambda value: value,
+ )
+ trainer._metrics = {"train": defaultdict(list)}
+ trainer.current_gradient_accumulation_steps = 4
+ for name in [
+ "_step_forward_tokens",
+ "_step_trained_tokens",
+ "_step_seq_len_weighted",
+ "_step_samples",
+ "_step_forward_s",
+ ]:
+ setattr(trainer, name, 0.0)
+ return trainer
+
+
+@pytest.mark.parametrize("target", [4, 8, 100])
+def test_actual_trl_loss_gradient_is_invariant_to_fork_packing(target):
+ bundles = [
+ RolloutBundle(
+ [row([1, 2, 3], [0, 0, 1]), row([1, 4, 5, 6], advantage=-0.2)], "a"
+ ),
+ RolloutBundle([row([1, 7, 8, 9], [0, 0, 1, 1])], "b"),
+ ]
+ trainer = native_loss_trainer()
+ model = ToyModel()
+ denominator = sum(sum(r.completion_mask) for b in bundles for r in b.rows)
+ trainer._atomic_normalization_tokens = denominator
+ collator = DataCollatorForRollout(0)
+ total = torch.zeros(())
+ for packed in pack_rows(bundles, target, 16):
+ loss = trainer.compute_loss(model, collator([[packed]]))
+ loss.backward()
+ total += loss.detach()
+ expected_model = ToyModel()
+ expected = torch.zeros(())
+ for b in bundles:
+ for r in b.rows:
+ ids = torch.tensor(r.input_ids[1:])
+ mask = torch.tensor(r.completion_mask[1:])
+ expected += (
+ -torch.exp(expected_model.theta + ids * 0.003) * r.advantage * mask
+ ).sum()
+ expected /= denominator
+ expected.backward()
+ torch.testing.assert_close(total, expected.detach())
+ torch.testing.assert_close(model.theta.grad, expected_model.theta.grad)
+ assert trainer._step_trained_tokens == denominator
+
+
+@pytest.mark.parametrize("credit_limit", [0, 8])
+def test_real_hf_optimizer_loop_consumes_all_forks_before_updating(
+ tmp_path, credit_limit
+):
+ import json
+
+ from atomic_rollouts import AtomicAdmissionCallback
+
+ class CPUTrainer(AtomicRolloutTrainer):
+ _inner_training_loop = Trainer._inner_training_loop
+ log = Trainer.log
+
+ def __init__(self):
+ args = TrainingArguments(
+ output_dir=str(tmp_path),
+ use_cpu=True,
+ max_steps=2,
+ gradient_accumulation_steps=4,
+ learning_rate=1e-3,
+ report_to=[],
+ save_strategy="no",
+ logging_strategy="no",
+ disable_tqdm=True,
+ )
+ args.token_budget = 4
+ args.max_staleness = 4
+ args.heartbeat_stale_after_s = 60
+ Trainer.__init__(
+ self,
+ model=ToyModel(),
+ args=args,
+ compute_loss_func="native AsyncGRPO disables HF loss scaling",
+ )
+ self.model_accepts_loss_kwargs = False
+ self.processing_class = SimpleNamespace(pad_token_id=0)
+ self.rollout_worker = worker(
+ [
+ RolloutBundle(
+ [row([1, 2, 3], group=i // 4), row([1, 4, 5], group=i // 4)],
+ f"episode-{i}",
+ )
+ for i in range(8)
+ ]
+ )
+ self.released_rollouts = []
+ if credit_limit:
+ self.rollout_worker.max_outstanding_rollouts = credit_limit
+ self.rollout_worker.num_generations = 4
+ self.rollout_worker.release_rollouts = self.released_rollouts.append
+ self.max_row_tokens = 8
+ self._trained_groups = set()
+ self._groups_before_resume = 0
+ self._metrics = {"train": defaultdict(list)}
+ self.epsilon_low = self.epsilon_high = 0.2
+ self.aux_loss_enabled = False
+ self.admission_dir = tmp_path
+ self._atomic_finished = []
+ for name in [
+ "_step_forward_tokens",
+ "_step_trained_tokens",
+ "_step_seq_len_weighted",
+ "_step_samples",
+ "_step_forward_s",
+ "_step_microbatches",
+ "_current_train_step_time",
+ ]:
+ setattr(self, name, 0.0)
+ self.add_callback(AtomicAdmissionCallback(self))
+
+ trainer = CPUTrainer()
+ before = trainer.model.theta.detach().clone()
+ trainer.train()
+ assert trainer.state.global_step == 2
+ assert trainer._step_microbatches == 16
+ assert not torch.equal(before, trainer.model.theta.detach())
+ receipts = [
+ json.loads(line)
+ for line in (tmp_path / "optimizer_rollouts.jsonl").read_text().splitlines()
+ ]
+ assert [r["step"] for r in receipts] == [1, 2]
+ assert all(
+ len(r["rollouts"]) == 4 and r["all_admitted_rows_consumed"] for r in receipts
+ )
+ assert sum(x["rows"] for r in receipts for x in r["rollouts"]) == 16
+ assert len({x["rollout_id"] for r in receipts for x in r["rollouts"]}) == 8
+ assert trainer.released_rollouts == ([4, 4] if credit_limit else [])
+
+
+def _use_spawned_credits(channel, result):
+ first = channel.reserve_group(4)
+ second = channel.reserve_group(4)
+ channel.release(2)
+ result.put((first, second, channel.credits.value))
+
+
+def test_credit_queue_uses_native_spawn_ipc():
+ import multiprocessing
+
+ from atomic_rollouts import CreditQueue
+
+ ctx = multiprocessing.get_context("spawn")
+ channel = CreditQueue(ctx.Queue(), ctx.Value("i", 6), 6)
+ result = ctx.Queue()
+ child = ctx.Process(target=_use_spawned_credits, args=(channel, result))
+ child.start()
+ try:
+ assert result.get(timeout=150) == (True, False, 4)
+ child.join(timeout=15)
+ assert child.exitcode == 0 and channel.credits.value == 4
+ with pytest.raises(RuntimeError, match="more than once"):
+ channel.release(3)
+ finally:
+ if child.is_alive():
+ child.terminate()
+ child.join()
+ channel.close()
+ result.close()
+
+
+def test_generation_reserves_complete_groups_and_tags_actual_dispatch(monkeypatch):
+ import multiprocessing
+
+ from atomic_rollouts import CreditQueue
+ from trl.experimental.async_grpo.async_rollout_worker import RolloutGroup
+
+ ctx = multiprocessing.get_context("spawn")
+ channel = CreditQueue(queue.Queue(), ctx.Value("i", 4), 4)
+ loop = object.__new__(AtomicHarnessLoop)
+ loop.rollout_buffer = channel
+ loop.num_generations = 2
+ loop._model_version_value = SimpleNamespace(value=1)
+ loop._stop_event = asyncio.Event()
+ called = []
+
+ async def generate(self, prompt, tool_dict, tools, group_id=0):
+ called.append((group_id, self.model_version))
+ return ([], [], [SimpleNamespace(rollout_id=f"{group_id}-{prompt}")], 0, 0, 1.0)
+
+ async def score(self, group):
+ assert group.model_version == 5
+ return [row([1, 2], version=5) for _ in group.completions_sequences]
+
+ monkeypatch.setattr(_HarnessRolloutLoop, "_generate_one", generate)
+ monkeypatch.setattr(_HarnessRolloutLoop, "_score_group", score)
+
+ async def exercise():
+ tasks = [
+ asyncio.create_task(loop._generate_one(i, {}, [], i // 2)) for i in range(6)
+ ]
+ await asyncio.sleep(0.15)
+ assert len(called) == 4 and channel.credits.value == 0
+ assert sum(task.done() for task in tasks) == 4
+ loop._model_version_value.value = 5
+ channel.release(2) # two whole rollouts consumed by the optimizer
+ results = await asyncio.wait_for(asyncio.gather(*tasks), 2)
+ assert called[-2:] == [(2, 5), (2, 5)]
+ group = RolloutGroup(
+ [], {}, [], [], [r[2] for r in results[-2:]], [], [], 1, 2, [], []
+ )
+ bundles = await loop._score_group(group)
+ assert [b.model_version for b in bundles] == [5, 5]
+ assert channel.credits.value == 0 # scoring alone does not release credits
+
+ asyncio.run(exercise())
+
+
+def test_empty_generation_returns_its_reserved_credit(monkeypatch):
+ import multiprocessing
+
+ from atomic_rollouts import CreditQueue
+
+ ctx = multiprocessing.get_context("spawn")
+ loop = object.__new__(AtomicHarnessLoop)
+ loop.rollout_buffer = CreditQueue(queue.Queue(), ctx.Value("i", 4), 4)
+ loop.num_generations = 2
+ loop._model_version_value = SimpleNamespace(value=1)
+ loop._stop_event = asyncio.Event()
+
+ async def empty(*args, **kwargs):
+ return ([], [], [], 0, 0, None)
+
+ monkeypatch.setattr(_HarnessRolloutLoop, "_generate_one", empty)
+
+ async def exercise():
+ await asyncio.gather(
+ loop._generate_one(0, {}, [], 0), loop._generate_one(1, {}, [], 0)
+ )
+ assert loop.rollout_buffer.credits.value == 4
+
+ asyncio.run(exercise())
+
+
+def test_small_rows_cannot_hold_all_credits_waiting_for_token_target():
+ # Sixteen outstanding generations, G8 and GAS4: each unit may consume at
+ # most two rollouts, reserving room for a complete new group until update.
+ bundles = [RolloutBundle([row([1, 2])], f"episode-{i}") for i in range(8)]
+ data = AtomicRolloutDataset(
+ worker(bundles),
+ defaultdict(list),
+ 40960,
+ 131072,
+ 4,
+ 60,
+ max_rollouts_per_unit=2,
+ )
+ it = iter(data)
+ batches = [next(it) for _ in range(4)]
+ assert [len(b["rollouts"]) for b in batches] == [2, 2, 2, 2]
+
+
+def test_multiple_stale_rollouts_release_credits_and_log_exact_totals(tmp_path):
+ import json
+
+ from trl.experimental.async_grpo.async_grpo_trainer import _reduce_metric
+
+ old = [RolloutBundle([row([1, 2])] * n, f"old-{n}") for n in (2, 3)]
+ new = RolloutBundle([row([1, 2, 3, 4], version=6)], "new")
+ w = worker(old + [new], version=6)
+ released = []
+ w.release_rollouts = released.append
+ metrics = defaultdict(list)
+ path = tmp_path / "rejections.jsonl"
+ data = AtomicRolloutDataset(
+ w, metrics, 4, 8, 4, 60, rejection_path=path, group_offset=81
+ )
+ assert next(iter(data))["rollouts"] == [new]
+ assert released == [1, 1]
+ assert (
+ _reduce_metric(
+ "admission/stale_rollouts_dropped_total",
+ metrics["admission/stale_rollouts_dropped_total"],
+ )
+ == 2
+ )
+ assert (
+ _reduce_metric(
+ "sample/dropped_stale_total", metrics["sample/dropped_stale_total"]
+ )
+ == 5
+ )
+ records = [json.loads(line) for line in path.read_text().splitlines()]
+ assert [r["rows"] for r in records] == [2, 3]
+ assert all(r["group_id"] == 81 for r in records)
diff --git a/04-data-agent/train/test_checkpoint_artifacts.py b/04-data-agent/train/test_checkpoint_artifacts.py
new file mode 100644
index 0000000..caa9da3
--- /dev/null
+++ b/04-data-agent/train/test_checkpoint_artifacts.py
@@ -0,0 +1,99 @@
+"""A partial or changed checkpoint must never be evaluated as completed weights."""
+import json
+
+import numpy as np
+import pytest
+from safetensors.numpy import save_file
+
+from checkpoint_artifacts import REQUIRED, mark_ready, mark_saved, finalize_saved, stage_model, verify_stage, resume_info
+
+
+def test_trainer_handoff_does_not_hash_weights_and_cpu_finalizes(tmp_path, monkeypatch):
+ root = checkpoint(tmp_path / 'checkpoint-50', sharded=True)
+ with monkeypatch.context() as patch:
+ patch.setattr('checkpoint_artifacts.digest', lambda _: pytest.fail('Weight hashing ran on trainer'))
+ mark_saved(root, 50, 'base', 'revision')
+ assert (root / 'checkpoint.saved.json').exists()
+ assert not (root / 'checkpoint.ready.json').exists()
+ assert finalize_saved(root)['step'] == 50
+
+
+def test_cpu_rejects_checkpoint_modified_after_handoff(tmp_path):
+ root = checkpoint(tmp_path / 'checkpoint-50')
+ mark_saved(root, 50, 'base', 'revision')
+ (root / 'optimizer.pt').write_text('{"changed":true}')
+ with pytest.raises(ValueError, match='changed after save'):
+ finalize_saved(root)
+
+
+def checkpoint(root, *, sharded=False):
+ root.mkdir()
+ for name in REQUIRED:
+ (root / name).write_text('{}')
+ (root / 'config.json').write_text('{"trained_config":true}')
+ (root / 'trainer_state.json').write_text('{"global_step":50}')
+ if sharded:
+ save_file({'a': np.ones(2)}, root / 'model-1.safetensors')
+ save_file({'b': np.ones(2)}, root / 'model-2.safetensors')
+ (root / 'model.safetensors.index.json').write_text(json.dumps({
+ 'weight_map': {'a': 'model-1.safetensors', 'b': 'model-2.safetensors'}}))
+ else:
+ save_file({'weight': np.ones(2)}, root / 'model.safetensors')
+ return root
+
+
+def test_missing_optimizer_state_cannot_publish(tmp_path):
+ root = checkpoint(tmp_path / 'checkpoint-50')
+ (root / 'optimizer.pt').unlink()
+ with pytest.raises(ValueError, match='optimizer.pt'):
+ mark_ready(root, 50, 'base', 'revision')
+ assert not (root / 'checkpoint.ready.json').exists()
+
+
+def test_missing_shard_cannot_publish(tmp_path):
+ root = checkpoint(tmp_path / 'checkpoint-50', sharded=True)
+ (root / 'model-2.safetensors').unlink()
+ with pytest.raises(FileNotFoundError):
+ mark_ready(root, 50, 'base', 'revision')
+ assert not (root / 'checkpoint.ready.json').exists()
+
+
+def test_stage_preserves_trained_config_and_detects_changed_weights(tmp_path):
+ root = checkpoint(tmp_path / 'checkpoint-50', sharded=True)
+ mark_ready(root, 50, 'base', 'revision')
+ metadata = tmp_path / 'base-metadata'
+ metadata.mkdir()
+ (metadata / 'config.json').write_text('{"wrong_base_config":true}')
+ (metadata / 'video_preprocessor_config.json').write_text('{}')
+ target = tmp_path / 'staged'
+ stage_model(root, target, metadata)
+ assert json.loads((target / 'config.json').read_text()) == {'trained_config': True}
+ assert (target / 'video_preprocessor_config.json').exists()
+ assert (target / 'model-1.safetensors').is_symlink()
+ assert verify_stage(target)['checkpoint']['step'] == 50
+ save_file({'a': np.zeros(2)}, root / 'model-1.safetensors')
+ with pytest.raises(ValueError, match='Staged checkpoint changed'):
+ verify_stage(target)
+
+
+def test_wrong_optimizer_step_cannot_publish(tmp_path):
+ root = checkpoint(tmp_path / 'checkpoint-50')
+ with pytest.raises(ValueError, match='trainer state'):
+ mark_ready(root, 100, 'base', 'revision')
+
+
+def test_resume_requires_complete_unchanged_optimizer_and_cursor(tmp_path):
+ root = checkpoint(tmp_path / 'checkpoint-50')
+ with pytest.raises(ValueError, match='completed checkpoint'):
+ resume_info(root, 'base', 'revision')
+ mark_saved(root, 50, 'base', 'revision')
+ with pytest.raises(ValueError, match='rollout_state'):
+ resume_info(root, 'base', 'revision')
+ (root / 'rollout_state.json').write_text('{"prompt_index":37,"model_version":50}')
+ info = resume_info(root, 'base', 'revision')
+ assert info['step'] == 50 and info['group_offset'] == 37 and info['model_version'] == 50
+ with pytest.raises(ValueError, match='model/revision'):
+ resume_info(root, 'wrong-model', 'revision')
+ (root / 'optimizer.pt').write_text('{"changed":true}')
+ with pytest.raises(ValueError, match='changed after save'):
+ resume_info(root, 'base', 'revision')
diff --git a/04-data-agent/train/test_continue_allocation.py b/04-data-agent/train/test_continue_allocation.py
new file mode 100644
index 0000000..e222814
--- /dev/null
+++ b/04-data-agent/train/test_continue_allocation.py
@@ -0,0 +1,101 @@
+import json
+from pathlib import Path
+
+import pytest
+
+import continue_allocation as continuation
+
+
+def parent_run(tmp_path):
+ root = tmp_path / "parent"
+ root.mkdir()
+ config = {
+ "model": "model", "model_revision": "revision",
+ "training": {"max_steps": 1000, "save_steps": 50, "soft_max_train_seconds": 42000,
+ "learning_rate": 3e-6, "gradient_accumulation_steps": 4, "num_generations": 8},
+ "resources": {"slurm_walltime": "12:00:00"},
+ "dataset": {"schedule_sha256": "fixed"},
+ "evaluation": {"interval_optimizer_steps": 100, "concurrency": 100},
+ "logging": {"project": "existing-project", "space_id": "existing-space"},
+ "monitoring": {"support_job_supervisor": {}},
+ }
+ continuation.save(root / "run_config.json", config)
+ continuation.save(root / "submission.json", {"training": "123"})
+ for name in ("manifest.json", "indices.txt", "harness_schedule.json", "pairs.jsonl",
+ "runtime_versions.json", "source_hashes.json"):
+ (root / name).write_text("{}")
+ (root / "source-snapshot").mkdir()
+ (root / "checkpoint-evals/eval-source").mkdir(parents=True)
+ return root, config
+
+
+def test_latest_incomplete_checkpoint_is_never_selected(tmp_path):
+ root, config = parent_run(tmp_path)
+ for step in (450, 500):
+ (root / f"job-123/run/checkpoint-{step}").mkdir(parents=True)
+
+ def validate(path, model, revision):
+ if path.name == "checkpoint-500":
+ raise ValueError("optimizer save is incomplete")
+ return {"step": 450, "checkpoint": str(path)}
+
+ resume, rejected = continuation.select_checkpoint(root, "123", config, validate)
+ assert resume["step"] == 450
+ assert len(rejected) == 1
+
+
+def test_prepare_preserves_training_and_frozen_sources(tmp_path):
+ root, config = parent_run(tmp_path)
+ target = tmp_path / "next"
+ resume = {"step": 650, "checkpoint": str(root / "job-123/run/checkpoint-650")}
+ prepared = continuation.prepare(root, target, resume)
+ for key, value in config["training"].items():
+ if key != "soft_max_train_seconds":
+ assert prepared["training"][key] == value
+ assert prepared["training"]["soft_max_train_seconds"] == 82200
+ assert prepared["resources"]["slurm_walltime"] == "24:00:00"
+ assert prepared["logging"]["project"] == config["logging"]["project"]
+ assert prepared["evaluation"]["interval_optimizer_steps"] == 100
+ assert (target / "source-snapshot").resolve() == root / "source-snapshot"
+ assert (target / "checkpoint-evals/eval-source").resolve() == root / "checkpoint-evals/eval-source"
+ assert continuation.read(root / "run_config.json") == config
+
+
+def test_cancellation_and_explicit_stop_are_respected(tmp_path):
+ root, _ = parent_run(tmp_path)
+ assert continuation.may_continue(root, "123", "TIMEOUT")
+ assert not continuation.may_continue(root, "123", "CANCELLED")
+ assert not continuation.may_continue(root, "123", "RUNNING")
+ (root / "job-123").mkdir()
+ (root / "job-123/STOP_AFTER_STEP").touch()
+ assert not continuation.may_continue(root, "123", "COMPLETED")
+
+
+def test_partial_submission_requires_reconciliation(tmp_path):
+ continuation.save(tmp_path / "submission.json", {"training": "456"})
+ with pytest.raises(RuntimeError, match="no duplicate GPU"):
+ continuation.existing_submission(tmp_path)
+
+
+def test_completed_target_submits_nothing(tmp_path, monkeypatch):
+ root, _ = parent_run(tmp_path)
+ monkeypatch.setattr(continuation, "parent_state", lambda job: "COMPLETED")
+ monkeypatch.setattr(continuation, "select_checkpoint", lambda *args: ({"step": 1000}, []))
+ result = continuation.execute(root, tmp_path / "next", "123")
+ assert result["state"] == "complete"
+ assert not (tmp_path / "next").exists()
+
+
+def test_completed_submission_is_not_duplicated(tmp_path, monkeypatch):
+ root, _ = parent_run(tmp_path)
+ target = tmp_path / "next"
+ resume = {"step": 650, "checkpoint": str(root / "job-123/run/checkpoint-650")}
+ continuation.prepare(root, target, resume)
+ jobs = dict(zip(continuation.ROLES, map(str, range(456, 461))))
+ continuation.save(target / "submission.json", jobs)
+ continuation.save(root / "operations/allocation-continuation/status.json", {"supervisor_job": "461"})
+ monkeypatch.setattr(continuation, "parent_state", lambda job: "COMPLETED")
+ monkeypatch.setattr(continuation, "select_checkpoint", lambda *args: (resume, []))
+ monkeypatch.setattr(continuation.subprocess, "run", lambda *a, **kw: pytest.fail("duplicate submission"))
+ monkeypatch.setattr(continuation.subprocess, "check_output", lambda *a, **kw: pytest.fail("duplicate supervisor"))
+ assert continuation.execute(root, target, "123")["submission"] == jobs
diff --git a/04-data-agent/train/test_multi_harness_schedule.py b/04-data-agent/train/test_multi_harness_schedule.py
new file mode 100644
index 0000000..dd28427
--- /dev/null
+++ b/04-data-agent/train/test_multi_harness_schedule.py
@@ -0,0 +1,163 @@
+"""Guard complete task/harness dispatch and stopping at a saved optimizer boundary."""
+from types import SimpleNamespace
+from collections import Counter
+from copy import deepcopy
+
+import pytest
+
+from multi_harness import pair_rows, MultiHarborSessionFactory
+from training_audit import PairCoverageCallback, PeriodicCheckpointCallback, WallTimeCallback
+from harness_schedule import make_schedule, validate_schedule
+
+
+def test_balanced_rotation_routes_tasks_and_harnesses_across_four_passes(tmp_path):
+ harnesses = ['opencode', 'claude-code', 'codex', 'mini-swe-agent']
+ tiers = ['easy'] * 150 + ['medium'] * 600 + ['hard'] * 250
+ tasks = [{'name': f'task-{i}', 'task_index': 999 - i, 'difficulty': tier}
+ for i, tier in enumerate(tiers)]
+ schedule = make_schedule(tasks, harnesses)
+ assert schedule == make_schedule(tasks, harnesses)
+ factory = SimpleNamespace(harnesses=harnesses, schedule=schedule,
+ prompt_rows=lambda: [{'task_name': t['name'], 'task_index': t['task_index']} for t in tasks])
+ rows = pair_rows(factory)
+ assert len(rows) == 4000
+ from itertools import islice
+ from trl.experimental.async_grpo.async_rollout_worker import _AsyncRolloutLoop
+ worker = SimpleNamespace(dataset=rows, _dataset_iter=iter(rows), num_generations=4)
+ dispatched = list(islice(_AsyncRolloutLoop._repeat_iterator(worker), 16004))
+ for group_id, row in dispatched:
+ expected = schedule['groups'][group_id % 4000]
+ assert row['task_index'] == expected['task_index']
+ assert MultiHarborSessionFactory.harness_for(factory, group_id) == expected['harness']
+ for p in range(4):
+ groups = schedule['groups'][p * 1000:(p + 1) * 1000]
+ assert len({g['task_name'] for g in groups}) == 1000
+ assert Counter(g['harness'] for g in groups) == {h: 250 for h in harnesses}
+ assert [g['harness'] for g in groups] == harnesses * 250
+ for g in groups:
+ for _ in range(4): # GRPO generations share their group ID and route.
+ assert MultiHarborSessionFactory.harness_for(factory, g['group_in_cycle']) == g['harness']
+ assert rows[g['group_in_cycle']]['task_index'] == g['task_index']
+ assert [g['task_row'] for g in schedule['groups'][:32]] == list(range(32))
+ assert len({(g['task_name'], g['harness']) for g in schedule['groups']}) == 4000
+ trainer = SimpleNamespace(_trained_groups=set(range(1000)))
+ callback = PairCoverageCallback(trainer, len(rows), harnesses, 0, tmp_path, schedule=schedule)
+ control = SimpleNamespace(should_training_stop=False)
+ callback.on_step_end(SimpleNamespace(max_steps=1000), SimpleNamespace(global_step=200), control)
+ callback.on_step_end(SimpleNamespace(max_steps=1000), SimpleNamespace(global_step=201), control)
+ import json
+ coverage = json.loads((tmp_path / 'coverage.json').read_text())
+ assert coverage['unique_tasks_covered'] == 1000
+ assert not coverage['pair_coverage_complete']
+ assert coverage['harness_pair_counts'] == {h: 250 for h in harnesses}
+ with pytest.raises(ValueError, match='Cartesian'):
+ pair_rows(factory, all_pairs=True)
+ factory.prompt_rows = lambda: list(reversed(rows[:1000]))
+ with pytest.raises(ValueError, match='Server task identities'):
+ pair_rows(factory)
+ changed = deepcopy(schedule)
+ changed['task_count'] = 4000
+ with pytest.raises(ValueError, match='metadata'):
+ validate_schedule(changed)
+
+
+def test_cartesian_dispatch_reaches_each_pair_once_per_cycle(tmp_path):
+ harnesses = ['opencode', 'claude-code', 'codex', 'mini-swe-agent']
+ factory = SimpleNamespace(harnesses=harnesses, prompt_rows=lambda: [{'task': i} for i in range(5)])
+ rows = pair_rows(factory, all_pairs=True)
+ assert len(rows) == 20
+ for cycle in range(2):
+ pairs = [(rows[g % 20]['task'], MultiHarborSessionFactory.harness_for(factory, g))
+ for g in range(cycle * 20, (cycle + 1) * 20)]
+ assert len(set(pairs)) == 20
+ assert set(pairs) == {(i, h) for i in range(5) for h in harnesses}
+ trainer = SimpleNamespace(_trained_groups=set(range(20)))
+ callback = PairCoverageCallback(trainer, len(rows), harnesses, 20, tmp_path, all_pairs=True)
+ control = SimpleNamespace(should_training_stop=False)
+ callback.on_step_end(SimpleNamespace(max_steps=40), SimpleNamespace(global_step=20), control)
+ assert not control.should_training_stop # a prefetched row is not proof it was trained
+ callback.on_step_end(SimpleNamespace(max_steps=40), SimpleNamespace(global_step=21), control)
+ assert control.should_training_stop
+
+
+def test_wall_time_stop_requests_a_checkpoint(monkeypatch, tmp_path):
+ clock = [0]
+ monkeypatch.setattr('training_audit.time.monotonic', lambda: clock[0])
+ callback = WallTimeCallback(60)
+ control = SimpleNamespace(should_training_stop=False, should_save=False)
+ args = SimpleNamespace(output_dir=str(tmp_path / 'run'))
+ callback.on_train_begin(None, None, control)
+ clock[0] = 59
+ callback.on_step_end(args, None, control)
+ assert not control.should_training_stop
+ clock[0] = 60
+ callback.on_step_end(args, None, control)
+ assert control.should_training_stop and control.should_save
+
+
+def test_resume_keeps_task_and_harness_aligned_across_schedule_wrap(tmp_path):
+ from itertools import islice
+ import json
+ from trl.experimental.async_grpo.async_rollout_worker import _AsyncRolloutLoop
+ harnesses = ['opencode', 'claude-code', 'codex', 'mini-swe-agent']
+ tasks = [{'name': f'task-{i}', 'task_index': i, 'difficulty': 'easy'} for i in range(1000)]
+ schedule = make_schedule(tasks, harnesses)
+ factory = SimpleNamespace(harnesses=harnesses, schedule=schedule, group_offset=3997,
+ prompt_rows=lambda: [{'task_name': t['name'], 'task_index': t['task_index']} for t in tasks])
+ rows = pair_rows(factory)
+ worker = SimpleNamespace(dataset=rows, _dataset_iter=iter(rows[3997:]), num_generations=8)
+ for local_group, row in islice(_AsyncRolloutLoop._repeat_iterator(worker), 64):
+ expected = schedule['groups'][(3997 + local_group) % 4000]
+ assert row['task_index'] == expected['task_index']
+ assert MultiHarborSessionFactory.harness_for(factory, local_group) == expected['harness']
+ callback = PairCoverageCallback(SimpleNamespace(_trained_groups={0, 1, 2, 3}), len(rows),
+ harnesses, 0, tmp_path, schedule=schedule, group_offset=3997)
+ for step in [51, 52]:
+ callback.on_step_end(SimpleNamespace(max_steps=1000), SimpleNamespace(global_step=step),
+ SimpleNamespace(should_training_stop=False))
+ coverage = json.loads((tmp_path / 'coverage.json').read_text())
+ assert coverage['collated_group_ids'] == [3997, 3998, 3999, 4000]
+ expected = {(schedule['groups'][g % 4000]['task_row'], schedule['groups'][g % 4000]['harness'])
+ for g in coverage['collated_group_ids']}
+ assert {(p['task_row'], p['harness']) for p in coverage['covered_pairs']} == expected
+
+
+def test_operator_stop_saves_at_optimizer_boundary(tmp_path):
+ callback = WallTimeCallback(82800)
+ args = SimpleNamespace(output_dir=str(tmp_path / 'run'))
+ control = SimpleNamespace(should_training_stop=False, should_save=False)
+ callback.on_step_end(args, None, control)
+ assert not control.should_save
+ (tmp_path / 'STOP_AFTER_STEP').touch()
+ callback.on_step_end(args, None, control)
+ assert control.should_training_stop and control.should_save
+
+
+def test_hourly_checkpoint_saves_without_stopping_and_regular_save_resets_clock(monkeypatch):
+ clock = [0]
+ monkeypatch.setattr('training_audit.time.monotonic', lambda: clock[0])
+ callback = PeriodicCheckpointCallback(3600)
+ control = SimpleNamespace(should_training_stop=False, should_save=False)
+ callback.on_train_begin(None, None, control)
+ clock[0] = 3599
+ callback.on_step_end(None, None, control)
+ assert not control.should_save
+ clock[0] = 3700 # saves only after a complete optimizer update
+ callback.on_step_end(None, None, control)
+ assert control.should_save and not control.should_training_stop
+ callback.on_save(None, None, control)
+ control.should_save = False
+ clock[0] = 4000
+ callback.on_save(None, None, control) # regular checkpoint-50 resets the same timer
+ clock[0] = 7300
+ callback.on_step_end(None, None, control)
+ assert not control.should_save
+ clock[0] = 7600
+ callback.on_step_end(None, None, control)
+ assert control.should_save and not control.should_training_stop
+
+
+@pytest.mark.parametrize('seconds', [0, -1])
+def test_hourly_checkpoint_requires_positive_interval(seconds):
+ with pytest.raises(ValueError, match='positive'):
+ PeriodicCheckpointCallback(seconds)
diff --git a/04-data-agent/train/test_prepare_harbor_opencode_run.py b/04-data-agent/train/test_prepare_harbor_opencode_run.py
new file mode 100644
index 0000000..f993ac4
--- /dev/null
+++ b/04-data-agent/train/test_prepare_harbor_opencode_run.py
@@ -0,0 +1,32 @@
+from copy import deepcopy
+from types import SimpleNamespace
+
+from harness_schedule import make_schedule, validate_schedule
+from prepare_harbor_opencode_run import single_harness_schedule
+
+
+def test_single_harness_preserves_reference_order_without_reshuffling():
+ tasks=[{'name':f'task-{i}','task_index':999-i,'difficulty':tier}
+ for i,tier in enumerate(['easy']*150+['medium']*600+['hard']*250)]
+ reference=make_schedule(tasks,['opencode','claude-code','codex','mini-swe-agent'])
+ before=deepcopy(reference)
+ actual=single_harness_schedule(reference)
+ assert reference==before
+ assert actual['harnesses']==['opencode']
+ assert actual['groups_per_cycle']==1000 and actual['passes_per_cycle']==1
+ assert [g['task_index'] for g in actual['groups']]==[g['task_index'] for g in reference['groups'][:1000]]
+ assert {g['harness'] for g in actual['groups']}=={'opencode'}
+ validate_schedule(actual)
+
+
+def test_all_generations_route_through_opencode_across_cycle_boundary():
+ from multi_harness import pair_rows,MultiHarborSessionFactory
+ tasks=[{'name':f'task-{i}','task_index':i,'difficulty':'easy'} for i in range(40)]
+ schedule=single_harness_schedule(make_schedule(tasks,['opencode','claude-code','codex','mini-swe-agent']))
+ factory=SimpleNamespace(schedule=schedule,harnesses=['opencode'],group_offset=0,
+ prompt_rows=lambda:[{'task_name':t['name'],'task_index':t['task_index']} for t in tasks])
+ rows=pair_rows(factory)
+ for group in range(85):
+ for generation in range(8):
+ assert MultiHarborSessionFactory.harness_for(factory,group)=='opencode'
+ assert rows[group%40]['task_index']==schedule['groups'][group%40]['task_index']
diff --git a/04-data-agent/train/train_blackbox_opencode.py b/04-data-agent/train/train_blackbox_opencode.py
new file mode 100644
index 0000000..8d99598
--- /dev/null
+++ b/04-data-agent/train/train_blackbox_opencode.py
@@ -0,0 +1,212 @@
+# Copyright 2026 The HuggingFace Team. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+"""AsyncGRPO on `blackbox-opencode`. TRL hosts vLLM and orchestrates; the env does the rest.
+
+WHAT IS ABSENT IS THE POINT. The run this reproduces needed two monkeypatches and got one of them
+silently wrong:
+
+ * `prompt_ids_patch` rebound TRL's `_turns_from_trace` to prefer the engine's prompt ids. It never
+ took effect. The rollout loop runs in a multiprocessing child created with `spawn`, which
+ re-imports every module, so a parent-side rebind of a module function is simply lost -- the patch
+ logged "installed" and its counters stayed at zero across 12,000 rollouts. Both production runs
+ trained on re-tokenised prompts for a night.
+ * `think_template_patch` mutated the tokenizer OBJECT, which IS pickled into the child, so that one
+ survived. That asymmetry is the whole reason the first went unnoticed.
+
+Neither is here. The ids come from the environment on the wire, and TRL raises rather than falling
+back to a re-render. There is also no `chat_template_kwargs`: in loop-owning mode `_sample_turn`
+never runs, so nothing applies a chat template at all.
+
+The engine MUST be the trainer's own vLLM. That is what makes the rollouts on-policy -- the agent
+calls the same weights the optimizer is updating, through the capture proxy.
+"""
+
+from __future__ import annotations
+
+import argparse
+import os
+
+from data_agent_env import DataAgentSessionFactory, opencode_agent_turns
+from datasets import Dataset
+from transformers import AutoTokenizer
+
+from trl.experimental.async_grpo import AsyncGRPOConfig, AsyncGRPOTrainer
+from trl.experimental.async_grpo.openenv_harness import HarnessRolloutWorker, has_tool_call
+
+
+def main() -> None:
+ p = argparse.ArgumentParser(description=__doc__)
+ p.add_argument("--server", default="http://127.0.0.1:8200", help="a running blackbox-opencode")
+ p.add_argument("--vllm-url", required=True, help="the trainer's OWN vLLM; on-policy depends on it")
+ p.add_argument("--model", default="Qwen/Qwen3.5-2B")
+ p.add_argument("--split", default="train")
+ # The +0.2028 arm: 125 easy prompts first, then medium with hard sprinkled through. Without
+ # it the run meets the hard tiers at step 0, where a group of all-zero rollouts gives no
+ # gradient at all.
+ p.add_argument("--curriculum", default="warmup:125")
+ p.add_argument("--seed", type=int, default=0)
+ p.add_argument("--sandbox", default="e2b", choices=["e2b", "hf"])
+ # The arm that produced +0.2028 [+0.146,+0.259] at step 200 and held it at 400.
+ p.add_argument("--learning-rate", type=float, default=3e-6)
+ p.add_argument("--num-generations", type=int, default=8)
+ p.add_argument("--max-inflight", type=int, default=32)
+ p.add_argument("--grad-accum", type=int, default=4)
+ p.add_argument("--max-steps", type=int, default=400)
+ p.add_argument("--temperature", type=float, default=0.8)
+ p.add_argument("--max-staleness", type=int, default=4)
+ # 17: the value the reference +0.2028 run used, read off its own launch line --
+ # STEP_PERSIST_CAP=0 STEP_SOFT_CAP=0 STEP_HARD_CAP=17 STEP_FORCE_TOOL=
+ # -- where the hard cap fired 4,627 times, so it bound constantly rather than sitting unused.
+ #
+ # It has to sit at or below the reward's step_budget of 30: above it there is a band where the
+ # agent is allowed to act and punished for acting, and the policy escapes by not acting at all,
+ # which under train_turn_fn=has_tool_call yields no rows and therefore no gradient.
+ #
+ # 10 was far too tight -- 197 of 224 eval rollouts (88%) cut off mid-task, turns pinned at 9. But
+ # 25 is not the reference either, and the difference shows up in the DYNAMICS rather than as an
+ # error: longer rollouts mean fewer complete per optimizer step, measured at 9.8 samples/step
+ # against the reference's 16.1, which is a different effective batch and a different rate of
+ # consuming the curriculum.
+ #
+ # The reference deliberately ran with BOTH nudges off (SOFT_CAP=0, PERSIST_CAP=0), so the absence
+ # of prompt injection here matches it rather than departing from it.
+ p.add_argument("--agent-step-limit", type=int, default=17)
+ # PINNED, and not optional. Unset, token_budget defaults to the vLLM server's max_model_len --
+ # 131072 here -- which tripled the trained row and killed job 69906 with torch.OutOfMemoryError in
+ # fla/ops/gated_delta_rule/chunk.py before step 1. At 40960 the rows already reach 40,870 (99.8%),
+ # so this is the measured ceiling for a 4B-class model on one 80 GB card, not a safety margin.
+ p.add_argument("--token-budget", type=int, default=40960)
+ # 900, against agent_timeout_s=600. The default 300 killed job 69319 with "heartbeat stale: 302s >
+ # 300s; child is hung" on a worker that was not hung but BUSY: the worker ticks its heartbeat at
+ # the top of the dispatch loop, which does not re-iterate while every max_inflight slot is full.
+ p.add_argument("--heartbeat-stale-after-s", type=float, default=900.0)
+ # opencode asks for 32,000 output tokens and capture clamps it to 8192; the TRL default is 2048.
+ p.add_argument("--max-completion-length", type=int, default=16384)
+ p.add_argument("--per-device-batch-size", type=int, default=4)
+ p.add_argument("--optim", default="paged_adamw_8bit")
+ # bfloat16, to MATCH THE SERVER. AsyncGRPOConfig defaults dtype="float32" deliberately -- TRL
+ # prefers fp32 on the trainer because the training-inference mismatch is sensitive to it -- but its
+ # own docstring adds that closing that gap end to end "also requires serving the vLLM server in the
+ # same dtype", and a precision GAP BIASES THE IMPORTANCE RATIO
+ # (https://huggingface.co/papers/2510.26788).
+ #
+ # TRL's preferred direction, serving fp32, is impossible here and that was measured rather than
+ # assumed: Qwen3.5 is hybrid Gated-DeltaNet and vLLM asserts
+ # ChunkGatedDeltaRuleFunction does not support float32. Please use bfloat16.
+ # (qwen_gdn_linear_attn.py:1165, job 72978). So the match is made on the trainer's side.
+ #
+ # Left at the default, job 72939 warned "serves in bfloat16 but the weights sent to it are
+ # float32" with embed_tokens.weight at 2.54 GB against a 1 GB transfer buffer. Halving the
+ # optimizer state is a side benefit on a card this work has already OOMed.
+ p.add_argument("--dtype", default="bfloat16")
+ p.add_argument("--save-steps", type=int, default=200)
+ p.add_argument("--output-dir", default="")
+ p.add_argument("--run-name", default="")
+ p.add_argument("--project", default="data-agent-blackbox")
+ args = p.parse_args()
+
+ # Trackio keys a run by name inside a project, so two relaunches of one config land on top of each
+ # other and the earlier metrics read as part of the later run's history -- worst exactly when
+ # relaunching after a crash. Stamping with the job id keeps them apart.
+ stamp = os.environ.get("SLURM_JOB_ID", "local")
+ run_name = args.run_name or f"{args.model.split('/')[-1]}-lr{args.learning_rate:g}-{stamp}"
+ output_dir = args.output_dir or f"runs/{run_name}"
+
+ factory = DataAgentSessionFactory(
+ args.server,
+ split=args.split,
+ llm_url=args.vllm_url,
+ model=args.model,
+ sandbox=args.sandbox,
+ sampling={"temperature": args.temperature, "top_p": 1.0, "top_k": 0},
+ agent_step_limit=args.agent_step_limit,
+ curriculum=args.curriculum,
+ seed=args.seed,
+ )
+ # Built FROM THE FACTORY so the instruction the trainer sends is one the server can resolve back
+ # to a task. All `num_generations` rollouts of a group share a row, so they get the same task and
+ # the group baseline is well formed without any seed plumbing.
+ dataset = Dataset.from_list(factory.prompt_rows())
+ tokenizer = AutoTokenizer.from_pretrained(args.model)
+
+ print(f"server {args.server}")
+ print(f"vllm {args.vllm_url} model {args.model}")
+ print(f"tasks {len(dataset)} from {args.split} [{args.curriculum or 'shuffled'}], sandbox {args.sandbox}")
+ print(f"run {run_name} -> {output_dir}")
+ print(f"budgets token_budget={args.token_budget} max_completion={args.max_completion_length} "
+ f"heartbeat={args.heartbeat_stale_after_s:g}s agent_steps={args.agent_step_limit} "
+ f"dtype={args.dtype}")
+
+ worker = HarnessRolloutWorker(
+ harness_session_factory=factory,
+ harness_adapter=None, # loop-owning: the agent drives itself; we read what it did
+ # Reinforce turns that took an ACTION rather than prose -- right for an agent whose job is to
+ # inspect data and write a file. It works only because the env hands TRL tool calls in the
+ # nested OpenAI shape; flattened, `has_tool_call` is False for every turn and the whole
+ # rollout is silently discarded.
+ train_turn_fn=has_tool_call,
+ # Drop opencode's own title/summarizer calls. An earlier revision left this out on the theory
+ # that capture removes aux roots structurally -- it does not, and the run that assumed so
+ # collapsed: fork_frac 0.02-0.06 (reference: 0), drift_tokens_max 32,770 (reference: 0),
+ # samples_per_rollout up to 1.31 (reference: exactly 1.0), and the policy trained on title and
+ # summary tokens carrying the task's advantage. See `opencode_agent_turns` for the full
+ # measurement.
+ agent_turn_fn=opencode_agent_turns,
+ model_name=args.model,
+ dataset=dataset,
+ reward_funcs=[], # the environment's verify() is the reward
+ processing_class=tokenizer,
+ num_generations=args.num_generations,
+ max_inflight_tasks=args.max_inflight,
+ vllm_server_url=args.vllm_url,
+ max_tokens=args.max_completion_length,
+ temperature=args.temperature,
+ top_p=1.0,
+ top_k=0,
+ log_completions=True,
+ num_completions_to_print=2,
+ )
+
+ AsyncGRPOTrainer(
+ model=args.model,
+ args=AsyncGRPOConfig(
+ output_dir=output_dir,
+ save_strategy="steps",
+ save_steps=args.save_steps,
+ # Keep every checkpoint: the eval watcher picks them up asynchronously, and a
+ # save_total_limit would delete one out from under a queued evaluation.
+ save_total_limit=None,
+ num_generations=args.num_generations,
+ per_device_train_batch_size=args.per_device_batch_size,
+ gradient_accumulation_steps=args.grad_accum,
+ max_steps=args.max_steps,
+ max_completion_length=args.max_completion_length,
+ token_budget=args.token_budget,
+ heartbeat_stale_after_s=args.heartbeat_stale_after_s,
+ optim=args.optim,
+ dtype=args.dtype,
+ learning_rate=args.learning_rate,
+ temperature=args.temperature,
+ max_staleness=args.max_staleness,
+ vllm_server_base_url=args.vllm_url,
+ bf16=True,
+ gradient_checkpointing=True,
+ # Required: the reentrant checkpointer does not see inputs that reach a block through
+ # anything but positional args.
+ gradient_checkpointing_kwargs={"use_reentrant": False},
+ report_to="trackio",
+ project=args.project,
+ run_name=run_name,
+ # Every rollout costs a sandbox and minutes, so nothing is logged in arrears.
+ logging_steps=1,
+ log_completions=True,
+ ),
+ train_dataset=dataset,
+ processing_class=tokenizer,
+ rollout_worker=worker,
+ ).train()
+
+
+if __name__ == "__main__":
+ main()
diff --git a/04-data-agent/train/train_harbor_multi.py b/04-data-agent/train/train_harbor_multi.py
new file mode 100644
index 0000000..5b24524
--- /dev/null
+++ b/04-data-agent/train/train_harbor_multi.py
@@ -0,0 +1,321 @@
+"""Async GRPO on the data-agent task set via OpenEnv x Harbor, SEVERAL harnesses in one run.
+
+Identical to train_harbor_opencode.py in every training knob; the only change is that each GRPO
+GROUP is routed to a harness (see multi_harness.py). Harness is constant WITHIN a group and varies
+BETWEEN groups, which keeps the advantage encoding "which action" rather than "which harness" --
+measured pass@4 across harnesses on this suite spans 0.320 to 0.020.
+
+ADMISSION. Run the 10-harness smoke and inspect exact engine token ids, sampled logprobs,
+per-token masks, retained supervision, and sampling policy. Prefix drift increases packed context;
+it does not invalidate per-call TITO or rollout-level rewards. The loop-owning worker uses lossless
+reconciliation: exact prefixes merge, every rewritten history starts a new row.
+
+The default trains all retained agent turns. Some harnesses (for example Terminus) express actions
+as text, so a universal `has_tool_call` filter would silently remove their entire training signal.
+`--train-turn-filter tool_calls` is an explicit native-tool-call-only ablation. Auxiliary calls are
+already removed by Harbor's capture/ATIF reconciliation.
+
+THE TRAPDOOR. Jobs 72452/72473 wedged at step 7 and 10 of 100, spending 4,076 E2B sandboxes on 11
+productive groups, because an UNGATED efficiency term made zero tool calls the highest-scoring move
+while `train_turn_fn=has_tool_call` then yielded no trainable turns. The `_train` suite emits a
+single float with no efficiency term (verified across all 2,238 graders), so the first leg is absent
+here. The default all-agent-turn filter also avoids dropping text-action rollouts. Constant-reward
+groups can still have no advantage signal: band the task indices and watch reward_std from step 1.
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import os
+from typing import Any
+
+from datasets import Dataset
+from transformers import AutoTokenizer
+
+from trl.experimental.async_grpo import AsyncGRPOConfig, AsyncGRPOTrainer
+from trl.experimental.async_grpo.openenv_harness import HarnessRolloutWorker, has_tool_call
+
+TRAIN_SPLIT = "AdithyaSK/data_agent_rl_environment_train"
+
+
+def tool_calling_turns(trace: list[dict[str, Any]]) -> list[dict[str, Any]]:
+ """Optional filter for turns whose request offered native tools.
+
+ Harbor already removes auxiliary calls. This additional restriction is an ablation and must
+ not be used for text-action harnesses; prefix drift alone is not evidence of an auxiliary call.
+ """
+ return [e for e in trace if ((e.get("metadata") or {}).get("n_tools") or 0) > 0]
+
+
+def build(argv=None):
+ p = argparse.ArgumentParser(description=__doc__,
+ formatter_class=argparse.RawDescriptionHelpFormatter)
+ p.add_argument("--server", default="http://127.0.0.1:8200", help="a running `openenv harbor serve`")
+ p.add_argument("--vllm-url", required=True, help="the engine AsyncGRPO also syncs weights into")
+ p.add_argument("--model", default="Qwen/Qwen3.5-2B")
+ p.add_argument("--model-revision", default=None)
+ p.add_argument("--resume-from-checkpoint", default="", help="Completed local checkpoint including optimizer and rollout cursor")
+ p.add_argument("--split", default=TRAIN_SPLIT)
+ # '+'-separated, never commas: `sbatch --export=ALL,VAR=a,b,c` truncates at the first comma
+ # SILENTLY, and the job then runs a harness set it was never given.
+ p.add_argument("--harnesses", default="opencode+mini-swe-agent")
+ p.add_argument("--sandbox", default="e2b")
+ p.add_argument("--reward-key", default="", help="'' lets the server pick; required on a multi-reward suite")
+ p.add_argument("--task-indices", default="", help="comma-separated, or @file")
+ p.add_argument("--n-tasks", type=int, default=0, help="0 = the whole split")
+ p.add_argument("--all-task-harness-pairs", action="store_true",
+ help="Schedule every task under every harness before repeating the dataset")
+ p.add_argument("--harness-schedule", default="", help="Frozen one-harness-per-task rotation JSON")
+ p.add_argument("--agent-turn-filter", default="none", choices=["none", "tools"],
+ help="Optional tool-manifest filter; incompatible with harnesses that express actions as text")
+ p.add_argument("--train-turn-filter", default="all", choices=["all", "tool_calls"],
+ help="Train all selected agent turns, or explicitly restrict to native tool-call turns")
+
+ # ---- the reference's values, unchanged ---------------------------------------------------
+ p.add_argument("--learning-rate", type=float, default=3e-6)
+ p.add_argument("--num-generations", type=int, default=8)
+ p.add_argument("--max-inflight", type=int, default=32)
+ p.add_argument("--grad-accum", type=int, default=4)
+ p.add_argument("--atomic-rollouts", action="store_true",
+ help="Keep all rows of each admitted rollout in one update (single dense trainer GPU)")
+ p.add_argument("--max-outstanding-rollouts", type=int, default=0,
+ help="Atomic recipe: bound generating plus queued rollouts until optimizer consumption")
+ p.add_argument("--max-row-tokens", type=int, default=131072,
+ help="Hard context limit for atomic rollout forwards; token-budget is the packing target")
+ p.add_argument("--per-device-batch-size", type=int, default=4)
+ p.add_argument("--max-steps", type=int, default=400)
+ p.add_argument("--max-train-seconds", type=float, default=0,
+ help="If positive, save and stop at the first update boundary after this duration")
+ p.add_argument("--coverage-min-steps", type=int, default=0,
+ help="If positive, stop once this many updates and every task/harness pair are covered; max-steps remains a hard ceiling")
+ p.add_argument("--audit-dir", default="", help="Save per-rollout capture results and pair coverage locally")
+ p.add_argument("--max-staleness", type=int, default=4)
+ p.add_argument("--optim", default="paged_adamw_8bit")
+ # Pinned, and the SAME value must reach `vllm serve --override-generation-config`. opencode sends
+ # no sampling params and Qwen3.5 ships no generation_config.json, so an unpinned engine samples at
+ # 1.0 while the trainer divides logits by this -- gradients against a distribution that never
+ # produced the samples. Measured unpinned: entropy 0.229 -> 0.587 over 24 steps, reward 0.592 -> 0.216.
+ p.add_argument("--temperature", type=float, default=0.8)
+ # NEUTRAL, and this is a deliberate DEPARTURE from the reference's 0.95. processed_logprobs are
+ # taken AFTER truncation, so a truncating top_p renormalises every captured logprob over the kept
+ # set while the trainer recomputes full-vocab; the step-0 importance ratio then lands at
+ # kept_mass rather than 1. Validated: `ratio` moved from the reference's 0.985-0.993 signature to
+ # 0.9984-0.9999 once this was 1.0.
+ p.add_argument("--top-p", type=float, default=1.0)
+ p.add_argument("--top-k", type=int, default=0)
+ # 17, read off the reference run's own STEP_HARD_CAP, which fired 4,627 times. Must sit at or
+ # below the reward's step_budget, else there is a band where acting is allowed and punished and
+ # the policy escapes by not acting -- which under has_tool_call yields no rows and no gradient.
+ p.add_argument("--agent-step-limit", type=int, default=17)
+ p.add_argument("--agent-timeout", type=float, default=600.0)
+ # PINNED, not optional: unset, token_budget falls back to the engine's max_model_len, which
+ # tripled the trained row and OOMed job 69906 in fla/ops/gated_delta_rule/chunk.py before step 1.
+ p.add_argument("--token-budget", type=int, default=40960)
+ # 900 against agent_timeout 600. The 300 default killed job 69319 on a worker that was BUSY, not hung.
+ p.add_argument("--heartbeat-stale-after-s", type=float, default=900.0)
+ p.add_argument("--max-completion-length", type=int, default=16384)
+ # MATCH THE SERVER. AsyncGRPOConfig defaults to float32; a precision gap biases the importance ratio.
+ p.add_argument("--dtype", default="bfloat16")
+ p.add_argument("--save-steps", type=int, default=200)
+ p.add_argument("--checkpoint-max-seconds", type=float, default=0,
+ help="Also save at the first optimizer boundary after this interval; 0 disables")
+ p.add_argument("--output-dir", default="")
+ p.add_argument("--run-name", default="")
+ p.add_argument("--project", default="data-agent-harbor-multi")
+ p.add_argument("--seed", type=int, default=0)
+ return p.parse_args(argv)
+
+
+def indices_of(spec: str) -> list[int] | None:
+ """`@file` form exists because `sbatch --export=ALL,VAR=a,b,c` truncates at the first comma,
+ silently -- the job runs with a task list it was never given."""
+ if not spec:
+ return None
+ if spec.startswith("@"):
+ spec = open(spec[1:]).read()
+ out, seen = [], set()
+ for tok in spec.replace("\n", ",").split(","):
+ tok = tok.strip()
+ if tok and int(tok) not in seen:
+ seen.add(int(tok)); out.append(int(tok))
+ return out or None
+
+
+def main(argv=None, *, session_factory_class=None, agent_turn_selector=None) -> None:
+ args = build(argv)
+ from multi_harness import MultiHarborSessionFactory, pair_rows
+ resume = None
+ if args.resume_from_checkpoint:
+ from checkpoint_artifacts import resume_info
+ resume = resume_info(args.resume_from_checkpoint, args.model, args.model_revision)
+ group_offset = resume['group_offset'] if resume else 0
+
+ harnesses = [h.strip() for h in args.harnesses.replace(",", "+").split("+") if h.strip()]
+ if args.harness_schedule and args.all_task_harness_pairs:
+ raise ValueError('Choose either a rotating schedule or Cartesian scheduling')
+ schedule = None
+ if args.harness_schedule:
+ with open(args.harness_schedule) as stream:
+ schedule = json.load(stream)
+ factory_class = session_factory_class or MultiHarborSessionFactory
+ factory = factory_class(
+ args.server,
+ harnesses=harnesses,
+ schedule=schedule,
+ group_offset=group_offset,
+ split=args.split,
+ sandbox=args.sandbox,
+ # THE SAME engine the trainer syncs weights into. That is what makes the rollouts on-policy:
+ # the agent's calls and the weight updates go to one vLLM. It must be the node's ROUTABLE
+ # address -- the harbor server probes it from ANOTHER host, and with localhost the probe
+ # fails, the tier grades `text`, and every rollout comes back with no trainable turns.
+ llm_url=os.environ.get("ROLLOUT_LLM_URL", args.vllm_url),
+ api_key=os.environ.get("ROLLOUT_LLM_API_KEY", ""),
+ model=args.model,
+ sampling={"temperature": args.temperature, "top_p": args.top_p, "top_k": args.top_k},
+ reward_key=args.reward_key,
+ agent_timeout_sec=args.agent_timeout,
+ agent_step_limit=args.agent_step_limit,
+ indices=indices_of(args.task_indices),
+ num_tasks=args.n_tasks or None,
+ )
+ if args.coverage_min_steps and not 0 < args.coverage_min_steps <= args.max_steps:
+ raise ValueError("coverage-min-steps must be between 1 and max-steps")
+ if args.audit_dir:
+ from training_audit import AuditedFactory
+ factory = AuditedFactory(factory, args.audit_dir)
+
+ # Built FROM the factory so the instruction TRL sends is one the server can resolve: `create()`
+ # hashes the prompt back to a task index and RAISES on a miss rather than silently running task 0.
+ # pair_rows pads so gcd(len(rows), n_harnesses) == 1. Without it, group->row and group->harness
+ # stay in lockstep and each task meets only ONE harness: at 40 tasks and 2 harnesses, 0 of 40
+ # tasks meet both. The run looks multi-harness and is a disjoint partition.
+ rows = pair_rows(factory, all_pairs=args.all_task_harness_pairs)
+ dataset = Dataset.from_list(rows)
+ tokenizer = AutoTokenizer.from_pretrained(args.model, revision=args.model_revision, trust_remote_code=True)
+
+ implementation = "standalone-opencode" if session_factory_class else "harbor"
+ run_name = args.run_name or (
+ f"{args.model.split('/')[-1]}-multi{len(harnesses)}-{implementation}-{args.max_steps}steps"
+ # Stamped with the job id: trackio keys a run by name inside a project, so relaunches
+ # otherwise stack on top of each other.
+ f"-{os.environ.get('SLURM_JOB_ID', 'local')}"
+ )
+ out_dir = args.output_dir or f"/fsx/{os.environ.get('USER','x')}/runs/agrpo_harbor/{run_name}"
+ if resume:
+ from training_audit import write_json
+ write_json(os.path.join(args.audit_dir or out_dir, 'resume.json'), resume)
+ print(f"resume checkpoint step={resume['step']}, next schedule group={group_offset}", flush=True)
+
+ print(f"model {args.model}")
+ print(f"server {args.server} vllm {args.vllm_url}")
+ print(f"rollouts {'+'.join(harnesses)} on {args.sandbox}, {args.num_generations}x{args.max_inflight}")
+ print(f"routing {'frozen rotation' if schedule else 'modulo harness routing'}; constant within each group")
+ print(f"tasks {len(dataset)} from {args.split}")
+ print(f"sampling temperature={args.temperature} top_p={args.top_p} top_k={args.top_k}"
+ f" (explicit capture session policy; checked against trainer recompute)")
+ print(f"budgets token_budget={args.token_budget} max_completion={args.max_completion_length} "
+ f"heartbeat={args.heartbeat_stale_after_s:g}s agent_steps={args.agent_step_limit} dtype={args.dtype}")
+ print(f"admission atomic={args.atomic_rollouts} max_outstanding_rollouts={args.max_outstanding_rollouts}")
+ print(f"aux agent_turn_fn={agent_turn_selector.__name__ if agent_turn_selector else args.agent_turn_filter}; "
+ f"train_turn_filter={args.train_turn_filter}; implementation={implementation}")
+ print(f"output {out_dir}")
+
+ worker_class, trainer_class = HarnessRolloutWorker, AsyncGRPOTrainer
+ if args.atomic_rollouts:
+ from atomic_rollouts import AtomicHarnessWorker, AtomicRolloutTrainer
+ worker_class, trainer_class = AtomicHarnessWorker, AtomicRolloutTrainer
+ worker = worker_class(
+ **({"max_outstanding_rollouts": args.max_outstanding_rollouts} if args.atomic_rollouts else {}),
+ harness_session_factory=factory,
+ harness_adapter=None, # loop-owning: the agent drives itself; we read what it did
+ # Text-action harnesses have no native tool_calls. Keep their supervision by default.
+ train_turn_fn=has_tool_call if args.train_turn_filter == "tool_calls" else None,
+ lossless_capture=True,
+ fork_threshold_tokens=0,
+ agent_turn_fn=agent_turn_selector or (tool_calling_turns if args.agent_turn_filter == "tools" else None),
+ model_name=args.model,
+ dataset=dataset,
+ reward_funcs=[], # the environment's verify() IS the reward; None means UNSCORED, never 0.0
+ processing_class=tokenizer,
+ num_generations=args.num_generations,
+ max_inflight_tasks=args.max_inflight,
+ vllm_server_url=args.vllm_url,
+ max_tokens=args.max_completion_length,
+ temperature=args.temperature,
+ top_p=args.top_p,
+ top_k=args.top_k,
+ log_completions=True,
+ num_completions_to_print=2,
+ )
+
+ config = AsyncGRPOConfig(
+ output_dir=out_dir,
+ save_strategy="steps" if args.save_steps else "no",
+ save_steps=args.save_steps or 500,
+ save_total_limit=None, # never rob an eval watcher of a checkpoint
+ per_device_train_batch_size=args.per_device_batch_size,
+ gradient_accumulation_steps=args.grad_accum,
+ num_generations=args.num_generations,
+ max_completion_length=args.max_completion_length,
+ max_steps=args.max_steps,
+ learning_rate=args.learning_rate,
+ temperature=args.temperature,
+ top_p=args.top_p,
+ top_k=args.top_k,
+ max_staleness=args.max_staleness,
+ max_inflight_tasks=args.max_inflight,
+ fork_threshold_tokens=0,
+ vllm_server_base_url=args.vllm_url,
+ optim=args.optim,
+ bf16=True,
+ dtype=args.dtype,
+ trust_remote_code=True, # Qwen3_5ForConditionalGeneration is a custom arch
+ model_init_kwargs={"revision": args.model_revision} if args.model_revision else None,
+ token_budget=args.token_budget,
+ heartbeat_stale_after_s=args.heartbeat_stale_after_s,
+ gradient_checkpointing=True,
+ # Required: the reentrant checkpointer does not see inputs arriving through anything but
+ # positional args, and the hybrid-attention path passes state that way.
+ gradient_checkpointing_kwargs={"use_reentrant": False},
+ report_to="trackio",
+ project=args.project,
+ run_name=run_name,
+ trackio_space_id=None,
+ trackio_bucket_id=None,
+ trackio_static_space_id=False, # CPU logger owns online sync; never publish/freeze from trainer
+ log_completions=True,
+ logging_steps=1, # every rollout costs a sandbox and minutes; nothing is logged in arrears
+ seed=args.seed,
+ )
+
+ trainer_kwargs = ({"max_row_tokens": args.max_row_tokens, "admission_dir": args.audit_dir}
+ if args.atomic_rollouts else {})
+ trainer = trainer_class(
+ model=args.model, args=config, train_dataset=dataset, rollout_worker=worker,
+ **trainer_kwargs,
+ )
+ from training_audit import CheckpointReadyCallback
+ trainer.add_callback(CheckpointReadyCallback(args.model, args.model_revision))
+ if args.checkpoint_max_seconds > 0:
+ from training_audit import PeriodicCheckpointCallback
+ trainer.add_callback(PeriodicCheckpointCallback(args.checkpoint_max_seconds))
+ if args.max_train_seconds:
+ from training_audit import WallTimeCallback
+ trainer.add_callback(WallTimeCallback(args.max_train_seconds))
+ if args.audit_dir or args.coverage_min_steps:
+ from training_audit import PairCoverageCallback
+ trainer.add_callback(PairCoverageCallback(
+ trainer, len(rows), harnesses, args.coverage_min_steps, args.audit_dir or out_dir,
+ all_pairs=args.all_task_harness_pairs,
+ schedule=schedule,
+ group_offset=group_offset,
+ ))
+ trainer.train(resume_from_checkpoint=resume['checkpoint'] if resume else None)
+ trainer.save_state()
+ trainer.save_model(os.path.join(out_dir, "final"))
+
+
+if __name__ == "__main__":
+ main()
diff --git a/04-data-agent/train/train_harbor_opencode.py b/04-data-agent/train/train_harbor_opencode.py
new file mode 100644
index 0000000..3dc2f1f
--- /dev/null
+++ b/04-data-agent/train/train_harbor_opencode.py
@@ -0,0 +1,276 @@
+"""Async GRPO on the data-agent task set via OpenEnv x Harbor, opencode harness.
+
+A deliberate replication of HuggingEnvs/04-data-agent/train/train_blackbox_opencode.py -- the arm
+that reached +0.2343 (CI [+0.178, +0.291], p=0.0) on Qwen3.5-2B -- with ONE variable changed: the
+environment is Harbor through the OpenEnv capture proxy instead of the bespoke blackbox-opencode env.
+Every training knob below is the reference's value, so a difference in outcome is attributable to the
+environment and not to the recipe.
+
+TOKEN-IN-TOKEN-OUT, NO RE-RENDER. `to_trace_entries` (envs/harbor_env/harness.py) carries the
+engine's own `prompt_token_ids`, and TRL's `_turns_from_trace` reads them and RAISES if absent. A
+local re-render matched the engine on 0 of 28 measured turns, so this path is not optional.
+Verify from the metrics, not from reading this file: rollout/fork_frac == 0,
+rollout/samples_per_rollout == 1.00, rollout/drift_tokens_max == 0.
+
+WHY NO agent_turn_fn, AND HOW TO KNOW IF THAT IS WRONG. The reference passes `opencode_agent_turns`
+to strip opencode's title/summarizer calls, anchored on the first tool-enabled turn's SYSTEM PROMPT.
+That filter cannot be ported: Harbor's TraceEntry carries no `request`, and HarborTurn has no
+system_digest. It should not need to be. Harbor assigns roles structurally in capture/export.py --
+a path that never uses tools is AUXILIARY, `trainable` requires role == AGENT, and
+`to_trace_entries` skips anything not trainable. So the aux calls are dropped BEFORE TRL sees them.
+
+That is a claim about Harbor, and claims get checked: if `rollout/fork_frac` is non-zero AT STEP 1
+(structural, not a later collapse) or `samples_per_rollout` != 1.00, the drop did not happen and
+`--agent-turn-filter tools` supplies a fallback that keeps only turns that called a tool.
+
+THE TRAPDOOR THIS SCRIPT IS SAFE FROM, AND WHY IT IS WORTH KNOWING ANYWAY. Jobs 72452/72473 wedged
+at step 7 and 10 of 100, having spent 4,076 E2B sandboxes on 11 productive groups (294 and 201 EMPTY
+groups). Two individually reasonable settings combined:
+
+ reward = correctness + 0.1 * clamp(1 - n_tool_calls/15, 0, 1) # UNGATED efficiency
+ train_turn_fn = has_tool_call
+
+Zero tool calls scores the MAXIMUM efficiency bonus, so inaction is the best move available to a
+policy that cannot solve the task (0.100 vs 0.033 for a real attempt that fails). The policy learns
+to stop calling tools -- and `has_tool_call` then yields NO trainable turns, so the group is empty
+and the run starves while the logs keep moving.
+
+The first leg is absent here: the `_train` suite emits a single float from `grader.py` with no
+efficiency term at all (verified: zero occurrences across all 2,238 task graders). `--reward efficiency`
+(the default) adds one back TRAINER-side, from the trace, in a form that closes the first leg
+STRUCTURALLY rather than by a gate -- see `harbor_reward.py`. The SECOND leg is still present. A model that makes no tool calls for any
+reason -- including simply being too weak for the task -- still produces empty groups. That is what
+banded tasks are for: pick indices the model can SOMETIMES solve, so reward_std > 0 and the group
+carries gradient. Watch `reward_std` and the empty-group count from step 1.
+"""
+
+from __future__ import annotations
+
+import argparse
+import os
+from typing import Any
+
+from datasets import Dataset
+from transformers import AutoTokenizer
+
+from trl.experimental.async_grpo import AsyncGRPOConfig, AsyncGRPOTrainer
+from trl.experimental.async_grpo.openenv_harness import HarnessRolloutWorker, has_tool_call
+
+# Module-level import, not a lambda: the reward crosses the spawn boundary and must pickle.
+from harbor_reward import TOOL_BUDGET, W_EFF, data_agent_reward
+
+TRAIN_SPLIT = "AdithyaSK/data_agent_rl_environment_train"
+
+
+def tool_calling_turns(trace: list[dict[str, Any]]) -> list[dict[str, Any]]:
+ """Fallback `agent_turn_fn`: keep only turns that actually called a tool.
+
+ The Harbor-native stand-in for the reference's system-prompt anchor, which cannot be ported
+ (no `request` on a Harbor TraceEntry). opencode's bookkeeping calls -- the conversation-title
+ generator and the context summarizer -- use no tools, so `metadata.n_tools > 0` separates them
+ from real agent steps. Weaker than anchoring on the system prompt, which is why it is OFF by
+ default and gated on measured fork_frac rather than switched on out of caution.
+ """
+ return [e for e in trace if ((e.get("metadata") or {}).get("n_tools") or 0) > 0]
+
+
+def build(argv=None):
+ p = argparse.ArgumentParser(description=__doc__,
+ formatter_class=argparse.RawDescriptionHelpFormatter)
+ p.add_argument("--server", default="http://127.0.0.1:8200", help="a running `openenv harbor serve`")
+ p.add_argument("--vllm-url", required=True, help="the engine AsyncGRPO also syncs weights into")
+ p.add_argument("--model", default="Qwen/Qwen3.5-2B")
+ p.add_argument("--split", default=TRAIN_SPLIT)
+ p.add_argument("--harness", default="opencode")
+ p.add_argument("--sandbox", default="e2b")
+ p.add_argument("--reward-key", default="", help="'' lets the server pick; required on a multi-reward suite")
+ p.add_argument("--task-indices", default="", help="comma-separated, or @file")
+ p.add_argument("--n-tasks", type=int, default=0, help="0 = the whole split")
+ p.add_argument("--agent-turn-filter", default="none", choices=["none", "tools"],
+ help="'tools' keeps only turns with n_tools>0; use ONLY if fork_frac != 0 at step 1")
+ p.add_argument("--reward", choices=["efficiency", "correctness"], default="efficiency",
+ help="efficiency: correctness x (1 + W_EFF*B/(B+tool_calls)), tuned by "
+ "$REWARD_W_TOOL_EFFICIENCY and $TOOL_BUDGET. correctness: the verifier's "
+ "float untouched, which is the pure-correctness arm runs 76585/77284 used.")
+
+ # ---- the reference's values, unchanged ---------------------------------------------------
+ p.add_argument("--learning-rate", type=float, default=3e-6)
+ p.add_argument("--num-generations", type=int, default=8)
+ p.add_argument("--max-inflight", type=int, default=32)
+ p.add_argument("--grad-accum", type=int, default=4)
+ p.add_argument("--per-device-batch-size", type=int, default=4)
+ p.add_argument("--max-steps", type=int, default=400)
+ p.add_argument("--max-staleness", type=int, default=4)
+ p.add_argument("--optim", default="paged_adamw_8bit")
+ # Pinned, and the SAME value must reach `vllm serve --override-generation-config`. opencode sends
+ # no sampling params and Qwen3.5 ships no generation_config.json, so an unpinned engine samples at
+ # 1.0 while the trainer divides logits by this -- gradients against a distribution that never
+ # produced the samples. Measured unpinned: entropy 0.229 -> 0.587 over 24 steps, reward 0.592 -> 0.216.
+ p.add_argument("--temperature", type=float, default=0.8)
+ # NEUTRAL, and this is a deliberate DEPARTURE from the reference's 0.95. processed_logprobs are
+ # taken AFTER truncation, so a truncating top_p renormalises every captured logprob over the kept
+ # set while the trainer recomputes full-vocab; the step-0 importance ratio then lands at
+ # kept_mass rather than 1. Validated: `ratio` moved from the reference's 0.985-0.993 signature to
+ # 0.9984-0.9999 once this was 1.0.
+ p.add_argument("--top-p", type=float, default=1.0)
+ p.add_argument("--top-k", type=int, default=0)
+ # 17, read off the reference run's own STEP_HARD_CAP, which fired 4,627 times. Must sit at or
+ # below the reward's step_budget, else there is a band where acting is allowed and punished and
+ # the policy escapes by not acting -- which under has_tool_call yields no rows and no gradient.
+ p.add_argument("--agent-step-limit", type=int, default=17)
+ p.add_argument("--agent-timeout", type=float, default=600.0)
+ # PINNED, not optional: unset, token_budget falls back to the engine's max_model_len, which
+ # tripled the trained row and OOMed job 69906 in fla/ops/gated_delta_rule/chunk.py before step 1.
+ p.add_argument("--token-budget", type=int, default=40960)
+ # 900 against agent_timeout 600. The 300 default killed job 69319 on a worker that was BUSY, not hung.
+ p.add_argument("--heartbeat-stale-after-s", type=float, default=900.0)
+ p.add_argument("--max-completion-length", type=int, default=16384)
+ # MATCH THE SERVER. AsyncGRPOConfig defaults to float32; a precision gap biases the importance ratio.
+ p.add_argument("--dtype", default="bfloat16")
+ p.add_argument("--save-steps", type=int, default=200)
+ p.add_argument("--output-dir", default="")
+ p.add_argument("--run-name", default="")
+ p.add_argument("--project", default="data-agent-harbor-opencode")
+ p.add_argument("--seed", type=int, default=0)
+ return p.parse_args(argv)
+
+
+def indices_of(spec: str) -> list[int] | None:
+ """`@file` form exists because `sbatch --export=ALL,VAR=a,b,c` truncates at the first comma,
+ silently -- the job runs with a task list it was never given."""
+ if not spec:
+ return None
+ if spec.startswith("@"):
+ spec = open(spec[1:]).read()
+ out, seen = [], set()
+ for tok in spec.replace("\n", ",").split(","):
+ tok = tok.strip()
+ if tok and int(tok) not in seen:
+ seen.add(int(tok)); out.append(int(tok))
+ return out or None
+
+
+def main() -> None:
+ args = build()
+ from harbor_env.harness import HarborSessionFactory
+
+ factory = HarborSessionFactory(
+ args.server,
+ split=args.split,
+ harness=args.harness,
+ sandbox=args.sandbox,
+ # THE SAME engine the trainer syncs weights into. That is what makes the rollouts on-policy:
+ # the agent's calls and the weight updates go to one vLLM. It must be the node's ROUTABLE
+ # address -- the harbor server probes it from ANOTHER host, and with localhost the probe
+ # fails, the tier grades `text`, and every rollout comes back with no trainable turns.
+ llm_url=args.vllm_url,
+ model=args.model,
+ sampling={"temperature": args.temperature, "top_p": args.top_p, "top_k": args.top_k},
+ reward_key=args.reward_key,
+ agent_timeout_sec=args.agent_timeout,
+ agent_step_limit=args.agent_step_limit,
+ indices=indices_of(args.task_indices),
+ num_tasks=args.n_tasks or None,
+ )
+
+ # Built FROM the factory so the instruction TRL sends is one the server can resolve: `create()`
+ # hashes the prompt back to a task index and RAISES on a miss rather than silently running task 0.
+ rows = factory.prompt_rows()
+ dataset = Dataset.from_list(rows)
+ tokenizer = AutoTokenizer.from_pretrained(args.model, trust_remote_code=True)
+
+ run_name = args.run_name or (
+ f"{args.model.split('/')[-1]}-{args.harness}-harbor-{args.max_steps}steps"
+ # Stamped with the job id: trackio keys a run by name inside a project, so relaunches
+ # otherwise stack on top of each other.
+ f"-{os.environ.get('SLURM_JOB_ID', 'local')}"
+ )
+ out_dir = args.output_dir or f"/fsx/{os.environ.get('USER','x')}/runs/agrpo_harbor/{run_name}"
+
+ print(f"model {args.model}")
+ print(f"server {args.server} vllm {args.vllm_url}")
+ print(f"rollouts {args.harness} on {args.sandbox}, {args.num_generations}x{args.max_inflight}")
+ print(f"tasks {len(dataset)} from {args.split}")
+ print(f"sampling temperature={args.temperature} top_p={args.top_p} top_k={args.top_k}"
+ f" <-- the SAME values must be on `vllm serve --override-generation-config`")
+ print(f"budgets token_budget={args.token_budget} max_completion={args.max_completion_length} "
+ f"heartbeat={args.heartbeat_stale_after_s:g}s agent_steps={args.agent_step_limit} dtype={args.dtype}")
+ print(f"aux agent_turn_fn={args.agent_turn_filter} (Harbor drops AUXILIARY-role turns "
+ f"server-side; check rollout/fork_frac at STEP 1)")
+ print(f"output {out_dir}")
+
+ reward_fn = data_agent_reward if args.reward == "efficiency" else None
+ if reward_fn is None:
+ print("reward correctness only (the verifier's float, untouched)")
+ else:
+ print(f"reward correctness x (1 + {W_EFF:g} * {TOOL_BUDGET:g}/({TOOL_BUDGET:g} + tool_calls))"
+ f" max {1 + W_EFF:.3f} | {1 + W_EFF * 0.5:.3f} at {TOOL_BUDGET:g} calls | ->1.0 unbounded")
+ print(" watch train/tools/call_frequency -- that IS the penalised quantity")
+
+ worker = HarnessRolloutWorker(
+ harness_session_factory=factory,
+ lossless_capture=True,
+ fork_threshold_tokens=0,
+ harness_adapter=None, # loop-owning: the agent drives itself; we read what it did
+ # Reinforce turns that took an ACTION rather than prose. Works only because the env hands TRL
+ # tool calls in the NESTED OpenAI shape; flattened, this is False for every turn and the whole
+ # rollout is discarded with no error anywhere.
+ train_turn_fn=has_tool_call,
+ agent_turn_fn=tool_calling_turns if args.agent_turn_filter == "tools" else None,
+ model_name=args.model,
+ dataset=dataset,
+ reward_funcs=[], # the environment's verify() IS the reward; None means UNSCORED, never 0.0
+ # Replaces env_reward with correctness x efficiency. Returning None still means UNSCORED.
+ rollout_reward_fn=reward_fn,
+ processing_class=tokenizer,
+ num_generations=args.num_generations,
+ max_inflight_tasks=args.max_inflight,
+ vllm_server_url=args.vllm_url,
+ max_tokens=args.max_completion_length,
+ temperature=args.temperature,
+ log_completions=True,
+ num_completions_to_print=2,
+ )
+
+ config = AsyncGRPOConfig(
+ output_dir=out_dir,
+ save_strategy="steps" if args.save_steps else "no",
+ save_steps=args.save_steps or 500,
+ save_total_limit=None, # never rob an eval watcher of a checkpoint
+ per_device_train_batch_size=args.per_device_batch_size,
+ gradient_accumulation_steps=args.grad_accum,
+ num_generations=args.num_generations,
+ max_completion_length=args.max_completion_length,
+ max_steps=args.max_steps,
+ learning_rate=args.learning_rate,
+ temperature=args.temperature,
+ top_p=args.top_p,
+ top_k=args.top_k,
+ max_staleness=args.max_staleness,
+ fork_threshold_tokens=0,
+ vllm_server_base_url=args.vllm_url,
+ optim=args.optim,
+ bf16=True,
+ dtype=args.dtype,
+ trust_remote_code=True, # Qwen3_5ForConditionalGeneration is a custom arch
+ token_budget=args.token_budget,
+ heartbeat_stale_after_s=args.heartbeat_stale_after_s,
+ gradient_checkpointing=True,
+ # Required: the reentrant checkpointer does not see inputs arriving through anything but
+ # positional args, and the hybrid-attention path passes state that way.
+ gradient_checkpointing_kwargs={"use_reentrant": False},
+ report_to="trackio",
+ project=args.project,
+ run_name=run_name,
+ log_completions=True,
+ logging_steps=1, # every rollout costs a sandbox and minutes; nothing is logged in arrears
+ seed=args.seed,
+ )
+
+ AsyncGRPOTrainer(
+ model=args.model, args=config, train_dataset=dataset, rollout_worker=worker
+ ).train()
+
+
+if __name__ == "__main__":
+ main()
diff --git a/04-data-agent/train/train_standalone_comparison.py b/04-data-agent/train/train_standalone_comparison.py
new file mode 100644
index 0000000..ca49980
--- /dev/null
+++ b/04-data-agent/train/train_standalone_comparison.py
@@ -0,0 +1,7 @@
+"""Train the standalone blackbox-opencode environment with the reference async recipe."""
+from data_agent_env import opencode_agent_turns
+from standalone_comparison import ScheduledOpenCodeFactory
+from train_harbor_multi import main
+
+if __name__ == "__main__":
+ main(session_factory_class=ScheduledOpenCodeFactory, agent_turn_selector=opencode_agent_turns)
diff --git a/04-data-agent/train/train_whitebox_bash.py b/04-data-agent/train/train_whitebox_bash.py
new file mode 100644
index 0000000..a0a9a8f
--- /dev/null
+++ b/04-data-agent/train/train_whitebox_bash.py
@@ -0,0 +1,186 @@
+# Copyright 2026 The HuggingFace Team. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Train the white-box bash/SETA agent with TRL's SYNCHRONOUS GRPOTrainer.
+
+HOW THE DATASET SELECTS THE TASK
+TRL forwards the whole dataset row to `reset()` as kwargs
+(`grpo_trainer.py`: `reset_kwargs = x; environment.reset(**reset_kwargs)`), so a row carrying
+`split` and `index` lands on `WhiteBoxBashEnv.reset(split=..., index=...)` and starts that exact
+episode. That is the entire task-selection mechanism -- there is no side channel, and the row is the
+single place a task is chosen.
+
+`prompt` is the role framing only. The task text arrives from `reset()`, which TRL appends to the
+last user message; the gold answer never leaves the server, so the trainer cannot see it.
+
+WHY NO REWARD FUNCTION
+The environment owns its reward through `get_reward()`, so `reward_funcs` is empty. TRL exposes the
+env's reward as a column named after the env class. Passing a reward function as well would add a
+second, unweighted reward source and quietly change the objective.
+
+ONE GPU
+vLLM runs in COLOCATE mode, in-process on the training GPU, so the smoke needs no separate server and
+no 2-GPU allocation. `vllm_gpu_memory_utilization` is the knob that makes that fit -- the trainer and
+the engine share one card, and the default 0.9 would leave nothing for the optimizer states.
+"""
+
+from __future__ import annotations
+
+import argparse
+import os
+
+from datasets import Dataset
+from trl import GRPOConfig, GRPOTrainer
+
+from whitebox_bash import exposed_tool_names, white_box_bash_env
+
+
+SYSTEM = (
+ "You are a terminal agent working in a sandbox. Use the available tools to inspect the "
+ "filesystem and solve the task. Work step by step: look before you act. When you are confident, "
+ "call submit_solution with the final answer and nothing else -- not the command that would "
+ "produce it."
+)
+
+
+def build_dataset(server: str, split: str, limit: int = 0) -> Dataset:
+ """One row per task: the role framing plus the coordinates `reset()` needs.
+
+ Task text is deliberately NOT baked in here. It comes from `reset()` at rollout time, which keeps
+ one source of truth (the server) and means the trainer never holds anything the agent should have
+ had to discover.
+ """
+ import httpx
+
+ # Only the COUNT is needed: the row carries `split`/`index`, and the task text comes from
+ # `reset()`. The Task API routes are POST with a JSON body, not GET with query params.
+ r = httpx.post(f"{server.rstrip('/')}/white_box_bash/num_tasks",
+ json={"split": split}, timeout=60.0)
+ r.raise_for_status()
+ payload = r.json()
+ n = int(payload if isinstance(payload, int) else payload.get("num_tasks", payload.get("count", 0)))
+ if limit:
+ n = min(n, limit)
+ if n <= 0:
+ raise SystemExit(f"split {split!r} has no tasks; nothing to train on")
+ return Dataset.from_list([
+ {
+ "prompt": [{"role": "system", "content": SYSTEM},
+ {"role": "user", "content": "Solve the task."}],
+ # Read by `reset()` via TRL's row-as-kwargs forwarding. The names must match the
+ # signature exactly: a misspelling is silently dropped and every rollout gets index 0.
+ "split": split,
+ "index": i,
+ }
+ for i in range(n)
+ ])
+
+
+def main() -> None:
+ p = argparse.ArgumentParser()
+ p.add_argument("--model", default=os.environ.get("MODEL", "Qwen/Qwen3.5-2B"))
+ p.add_argument("--server", default=os.environ.get("WHITE_BOX_BASH_URL", "http://127.0.0.1:8412"))
+ p.add_argument("--split", default=os.environ.get("SPLIT", "train"))
+ p.add_argument("--toolsets", default=os.environ.get("TOOLSETS", "bash,seta"))
+ p.add_argument("--output-dir", default=os.environ.get("OUTPUT_DIR", "runs/whitebox-bash"))
+ p.add_argument("--max-steps", type=int, default=int(os.environ.get("MAX_STEPS", "4")))
+ p.add_argument("--num-generations", type=int, default=int(os.environ.get("NUM_GENERATIONS", "4")))
+ p.add_argument("--per-device-train-batch-size", type=int,
+ default=int(os.environ.get("PER_DEVICE_BS", "4")))
+ # 1e-6, not 3e-6, and beta 0.04 rather than 0. Both come from a previous SYNC GRPO run on this
+ # same dataset (experiments/rollout_control/harbor_trl, FAILURE_MODES.md §E2): with `beta=0` and
+ # LR 2e-6 BOTH Qwen3.5-2B runs collapsed -- entropy fell 0.4 -> 0.09 by step 40, the policy
+ # degenerated into malformed repetition, and reward went to 0. Nothing anchors a small policy
+ # without a KL term. The 4B tolerated it; the 2B does not.
+ p.add_argument("--learning-rate", type=float, default=float(os.environ.get("LR", "1e-6")))
+ p.add_argument("--kl-beta", type=float, default=float(os.environ.get("KL_BETA", "0.04")))
+ p.add_argument("--warmup-steps", type=int, default=int(os.environ.get("WARMUP_STEPS", "10")))
+ # The LOGITS batch, and the thing that OOMs. The loss materialises
+ # `per_device_batch x seq_len x vocab` in fp32, and Qwen3.5's vocab is ~152k: at batch 8 and a
+ # 4096 completion budget that is ~24 GB and the run dies in `logits / temperature` (job 76717).
+ # Keep this small and recover the effective batch with gradient accumulation -- GRPO only needs
+ # the GROUP intact, and `per_device_bs * grad_accum` still covers `num_generations`.
+ p.add_argument("--gradient-accumulation-steps", type=int,
+ default=int(os.environ.get("GRAD_ACCUM", "4")))
+ p.add_argument("--temperature", type=float, default=float(os.environ.get("TEMPERATURE", "0.8")))
+ # Bounds the WHOLE multi-turn completion, tool-result tokens included -- they sit in
+ # `completion_ids` (masked out of the loss and out of the length metric, but still counted here).
+ # 1024 was too small for real data: one clipped CSV read consumed it and the agent could never
+ # take a second turn.
+ p.add_argument("--max-completion-length", type=int,
+ default=int(os.environ.get("MAX_COMPLETION_LENGTH", "4096")))
+ # The multi-turn cap on the SYNC path. Each iteration is one generate + one tool call, so this is
+ # the real bound on episode length; the client's own step_limit backs it up from the other side.
+ p.add_argument("--max-tool-calling-iterations", type=int,
+ default=int(os.environ.get("MAX_TOOL_ITERS", "8")))
+ p.add_argument("--step-limit", type=int, default=int(os.environ.get("STEP_LIMIT", "12")))
+ p.add_argument("--limit-tasks", type=int, default=int(os.environ.get("LIMIT_TASKS", "0")))
+ p.add_argument("--report-to", default=os.environ.get("REPORT_TO", "none"))
+ # Saving is OFF by default because the 4-step smoke has nothing worth keeping; a real run turns
+ # it on. `save_steps` is in TRAINER steps, and with num_generations completions per prompt a step
+ # is one prompt, so 100 steps is one pass over a 100-task split.
+ p.add_argument("--save-steps", type=int, default=int(os.environ.get("SAVE_STEPS", "0")))
+ args = p.parse_args()
+
+ dataset = build_dataset(args.server, args.split, args.limit_tasks)
+ factory = white_box_bash_env(
+ args.server, split=args.split, toolsets=args.toolsets, step_limit=args.step_limit,
+ )
+ print(f"[whitebox-bash] model={args.model} server={args.server} split={args.split} "
+ f"tasks={len(dataset)} tools={exposed_tool_names(args.toolsets)}", flush=True)
+
+ config = GRPOConfig(
+ output_dir=args.output_dir,
+ learning_rate=args.learning_rate,
+ # KL anchor to the reference model. Costs a second model in memory, which is why
+ # vllm_gpu_memory_utilization is kept low.
+ beta=args.kl_beta,
+ warmup_steps=args.warmup_steps,
+ num_generations=args.num_generations,
+ per_device_train_batch_size=args.per_device_train_batch_size,
+ gradient_accumulation_steps=args.gradient_accumulation_steps,
+ max_steps=args.max_steps,
+ max_completion_length=args.max_completion_length,
+ max_tool_calling_iterations=args.max_tool_calling_iterations,
+ temperature=args.temperature,
+ # Qwen3.5's template opens by default on the 4B and closes it on the 2B -- INVERTED
+ # between two models of the same family. Pinning it here removes that as a variable.
+ chat_template_kwargs={"enable_thinking": False},
+ gradient_checkpointing=True,
+ gradient_checkpointing_kwargs={"use_reentrant": False},
+ log_completions=True,
+ num_completions_to_print=1,
+ logging_steps=1,
+ save_strategy="steps" if args.save_steps > 0 else "no",
+ **({"save_steps": args.save_steps, "save_total_limit": None} if args.save_steps > 0 else {}),
+ report_to=args.report_to,
+ # vLLM in-process on the training GPU. 0.3 leaves room for the optimizer states; the default
+ # 0.9 would OOM the trainer on the same card.
+ use_vllm=True,
+ vllm_mode="colocate",
+ vllm_gpu_memory_utilization=float(os.environ.get("VLLM_GPU_MEM_UTIL", "0.3")),
+ vllm_max_model_length=int(os.environ.get("VLLM_MAX_MODEL_LEN", "16384")),
+ )
+
+ GRPOTrainer(
+ model=args.model,
+ args=config,
+ train_dataset=dataset,
+ environment_factory=factory,
+ reward_funcs=[], # the environment owns the reward via get_reward()
+ ).train()
+
+
+if __name__ == "__main__":
+ main()
diff --git a/04-data-agent/train/training_audit.py b/04-data-agent/train/training_audit.py
new file mode 100644
index 0000000..5c8f6dc
--- /dev/null
+++ b/04-data-agent/train/training_audit.py
@@ -0,0 +1,182 @@
+"""Local capture artifacts and bounded coverage checks for the multiharness recipe."""
+
+from __future__ import annotations
+
+import json
+import math
+import time
+from pathlib import Path
+
+from transformers import TrainerCallback
+
+
+def write_json(path, value):
+ path = Path(path)
+ path.parent.mkdir(parents=True, exist_ok=True)
+ temporary = path.with_suffix(".tmp")
+ temporary.write_text(json.dumps(value, indent=2) + "\n")
+ temporary.replace(path)
+
+
+class AuditedSession:
+ def __init__(self, session, path, metadata):
+ self.session = session
+ self.path = path
+ self.metadata = metadata
+
+ def __getattr__(self, name):
+ return getattr(self.session, name)
+
+ def wait_for_completion(self, *args, **kwargs):
+ started = time.time()
+ try:
+ return self.session.wait_for_completion(*args, **kwargs)
+ finally:
+ result = self.session.result
+ write_json(self.path, {
+ **self.metadata, "started_at": started, "finished_at": time.time(),
+ "result": result.model_dump(mode="json") if result is not None else None,
+ })
+
+
+class AuditedFactory:
+ def __init__(self, factory, directory):
+ self.factory = factory
+ self.directory = str(directory)
+
+ def __getattr__(self, name):
+ # During unpickling the wrapped factory has not been assigned yet.
+ factory = self.__dict__.get("factory")
+ if factory is None:
+ raise AttributeError(name)
+ return getattr(factory, name)
+
+ def create(self, task, seed=None, episode_id=None):
+ session = self.factory.create(task, seed=seed, episode_id=episode_id)
+ return AuditedSession(
+ session, Path(self.directory) / "rollouts" / f"{episode_id}.json",
+ {"group_id": (seed or 0) + getattr(self.factory, 'group_offset', 0),
+ "local_group_id": seed, "episode_id": episode_id,
+ "harness": self.factory.harness_for(seed), "task_index": session._task_index},
+ )
+
+
+class PairCoverageCallback(TrainerCallback):
+ """Use TRL's existing collated group counter; require coverage to persist a full update.
+
+ The extra update avoids stopping on the dataloader's prefetched batch. Coverage counts a
+ group once any of its rows enters training, and does not claim every forked row was consumed.
+ """
+
+ def __init__(self, trainer, n_tasks, harnesses, min_steps, directory, *, all_pairs=False, schedule=None,
+ group_offset=0):
+ self.trainer = trainer
+ self.n_rows = n_tasks
+ self.all_pairs = all_pairs
+ self.schedule = schedule
+ self.group_offset = group_offset
+ if all_pairs and n_tasks % len(harnesses):
+ raise ValueError("Cartesian task/harness schedule has an incomplete task")
+ self.n_tasks = n_tasks // len(harnesses) if all_pairs else n_tasks
+ if schedule is not None:
+ self.n_tasks = schedule['task_count']
+ self.harnesses = harnesses
+ self.min_steps = min_steps
+ self.directory = Path(directory)
+ self.directory.mkdir(parents=True, exist_ok=True)
+ self.previous_pairs = set()
+
+ def on_step_end(self, args, state, control, **kwargs):
+ group_ids = sorted(g + self.group_offset for g in self.trainer._trained_groups)
+ pairs = {
+ ((g % self.n_rows) // len(self.harnesses) if self.all_pairs else g % self.n_tasks,
+ g % len(self.harnesses)) for g in group_ids
+ }
+ if self.schedule is not None:
+ groups = self.schedule['groups']
+ pairs = {(groups[g % len(groups)]['task_row'],
+ self.harnesses.index(groups[g % len(groups)]['harness'])) for g in group_ids}
+ stable_pairs = pairs & self.previous_pairs
+ complete = len(stable_pairs) == self.n_tasks * len(self.harnesses)
+ report = {
+ "optimizer_steps": state.global_step, "target_min_steps": self.min_steps,
+ "resumed_group_offset": self.group_offset,
+ "hard_max_steps": args.max_steps, "collated_group_ids": group_ids,
+ "covered_pairs": [{"task_row": t, "harness": self.harnesses[h]}
+ for t, h in sorted(stable_pairs)],
+ "pair_coverage_complete": complete,
+ "unique_tasks_covered": len({t for t, _ in stable_pairs}),
+ "target_unique_tasks": self.n_tasks,
+ "harness_pair_counts": {name: sum(h == i for _, h in stable_pairs)
+ for i, name in enumerate(self.harnesses)},
+ "coverage_semantics": "at least one row per pair; stable over two optimizer boundaries",
+ }
+ write_json(self.directory / "coverage.json", report)
+ print(f"PAIR_COVERAGE step={state.global_step} pairs={len(stable_pairs)}/"
+ f"{self.n_tasks * len(self.harnesses)}", flush=True)
+ self.previous_pairs = pairs
+ if self.min_steps and state.global_step >= self.min_steps and complete:
+ control.should_training_stop = True
+
+ def on_log(self, args, state, control, logs=None, **kwargs):
+ with (self.directory / "metrics.jsonl").open("a") as output:
+ output.write(json.dumps({"step": state.global_step, **(logs or {})}) + "\n")
+ for key in ("loss", "grad_norm", "ratio"):
+ value = (logs or {}).get(key)
+ if isinstance(value, (int, float)) and not math.isfinite(value):
+ raise RuntimeError(f"Non-finite {key}: stopping the training check")
+
+
+class CheckpointReadyCallback(TrainerCallback):
+ """Publish a completion marker after Trainer has finished writing a checkpoint."""
+
+ def __init__(self, base_model, base_revision):
+ self.base_model = base_model
+ self.base_revision = base_revision
+
+ def on_save(self, args, state, control, **kwargs):
+ if state.is_world_process_zero:
+ from checkpoint_artifacts import mark_saved
+ mark_saved(Path(args.output_dir) / f"checkpoint-{state.global_step}",
+ state.global_step, self.base_model, self.base_revision,
+ final=control.should_training_stop or state.global_step >= args.max_steps)
+
+
+class PeriodicCheckpointCallback(TrainerCallback):
+ """Bound recovery loss when optimizer updates are too slow for step-based saves."""
+
+ def __init__(self, seconds):
+ if seconds <= 0:
+ raise ValueError("Checkpoint interval must be positive")
+ self.seconds = seconds
+ self.last_saved = None
+
+ def on_train_begin(self, args, state, control, **kwargs):
+ self.last_saved = time.monotonic()
+
+ def on_step_end(self, args, state, control, **kwargs):
+ if self.last_saved is not None and time.monotonic() - self.last_saved >= self.seconds:
+ control.should_save = True
+
+ def on_save(self, args, state, control, **kwargs):
+ # A regular step-based checkpoint also resets the recovery interval.
+ self.last_saved = time.monotonic()
+
+
+class WallTimeCallback(TrainerCallback):
+ """Save and stop at an optimizer boundary before the allocation expires."""
+
+ def __init__(self, seconds):
+ if seconds <= 0:
+ raise ValueError("Training wall-time budget must be positive")
+ self.seconds = seconds
+ self.started = None
+
+ def on_train_begin(self, args, state, control, **kwargs):
+ self.started = time.monotonic()
+
+ def on_step_end(self, args, state, control, **kwargs):
+ requested = (Path(args.output_dir).parent / 'STOP_AFTER_STEP').exists()
+ if requested or (self.started is not None and time.monotonic() - self.started >= self.seconds):
+ control.should_save = True
+ control.should_training_stop = True
diff --git a/README.md b/README.md
index db28a9a..ce81c75 100644
--- a/README.md
+++ b/README.md
@@ -54,6 +54,7 @@ results and README, plus the Hub repos it owns. They read in order but stand alo
| **01** | **[LaTeX OCR](./01-latex-ocr/)** | Train Qwen3-VL-2B to read math images into LaTeX, with a verifiable reward. | 1 | 1 | 1 | ✅ stable |
| **02** | **[Watercolour](./02-watercolour/)** | Train Qwen3.5-35B-A3B to paint watercolours by writing p5.brush sketches, rewarded by an aesthetic preference model. | 1 | 1 | 0 | ✅ trained |
| **03** | **[GeoGuesser](./03-geoguesser/)** | Drop a VLM at a random street corner on Earth and score it on kilometres of error. | 1 | 1 | 1 | ✅ stable |
+| **04** | **[Data Agent](./04-data-agent/)** | Three agent-loop implementations with exact-token training, fixed pass@1 evaluations, and local or HF Jobs reproduction. | 3 | 1 | 3 | ✅ trained |
Generated from each project's `project.yaml` by `tools/build_index.py`. Adding a project means
diff --git a/content/README.md b/content/README.md
index c1cf2b2..2c5cc51 100644
--- a/content/README.md
+++ b/content/README.md
@@ -8,6 +8,7 @@ Each item ships to the Hub as a Space; the source of truth is here.
| Article | Source | Live |
|---|---|---|
| **The ultimate guide to RL environments** | [`articles/rl-environments-guide/`](./articles/rl-environments-guide/) | [▶️ Space](https://huggingface.co/spaces/AdithyaSK/rl-environments-guide) |
+| **The ultimate guide to multi-harness RL** | [`articles/multi-harness-rl/`](./articles/multi-harness-rl/) | 🚧 drafting |
Built with [research-article-template](https://huggingface.co/spaces/tfrere/research-article-template)
(Astro), served as a Docker Space.
diff --git a/content/articles/multi-harness-rl/.ai/skills/article-frontmatter/SKILL.md b/content/articles/multi-harness-rl/.ai/skills/article-frontmatter/SKILL.md
new file mode 100644
index 0000000..0d7c314
--- /dev/null
+++ b/content/articles/multi-harness-rl/.ai/skills/article-frontmatter/SKILL.md
@@ -0,0 +1,116 @@
+---
+name: article-frontmatter
+description: Configure article metadata via MDX frontmatter. Use when the user asks about titles, authors, affiliations, template variants, banner, DOI, PDF export, or any article.mdx frontmatter field.
+---
+
+# Article Frontmatter Reference
+
+All metadata lives in `app/src/content/article.mdx` frontmatter (YAML block).
+
+## Frontmatter fields
+
+| Field | Type | Default | Description |
+|-------|------|---------|-------------|
+| `title` | string | required | Article title. Supports `\n` for line breaks (rendered as ` `). |
+| `description` | string | `""` | Short description / subtitle shown below the title. |
+| `authors` | array | `[]` | List of authors (see below). |
+| `affiliations` | array | `[]` | List of affiliations (see below). |
+| `published` | string | — | Publication date, e.g. `"Apr. 04, 2026"`. |
+| `template` | `"article"` or `"paper"` | `"article"` | Layout variant (see below). |
+| `banner` | string | `"banner.html"` | Banner embed filename in `embeds/`. |
+| `doi` | string | — | DOI identifier, shown in footer. |
+| `showPdf` | boolean | `true` | Show PDF download button in metadata bar. |
+| `tableOfContentsAutoCollapse` | boolean | `false` | Auto-collapse TOC sections on scroll. |
+| `licence` | string | — | Licence text (HTML allowed), shown in footer. |
+| `pdfProOnly` | boolean | `false` | Gate PDF download behind HF Pro badge. |
+| `seoThumbImage` | string | — | Custom OG image URL for social sharing. |
+| `links` | array | `[]` | External links shown in paper template hero (see below). |
+
+## Title line breaks
+
+Long titles are automatically balanced across lines (`text-wrap: balance`). Titles longer than 60 characters are automatically downsized for readability (>100 chars: even smaller).
+
+To force a manual line break, use `\n` inside the title string:
+
+```yaml
+title: "Why Open-Source LLMs\nAre Reshaping the AI Landscape"
+```
+
+This renders as two lines in the Hero section. The plain-text version (for SEO / PDF) strips the break automatically.
+
+## Template variants
+
+| Value | Layout | Features |
+|-------|--------|----------|
+| `article` (default) | Full layout | Banner, sidebar TOC, figure numbering, citation block, DOI, PDF export |
+| `paper` | Single centered column | No TOC sidebar, no figure numbering, no citation/DOI block, lighter footer |
+
+## Authors and affiliations
+
+```yaml
+authors:
+ - name: "Alice Martin"
+ url: "https://example.com/alice"
+ affiliations: [1]
+ - name: "Bob Chen"
+ affiliations: [1, 2]
+affiliations:
+ - name: "Hugging Face"
+ url: "https://huggingface.co"
+ - name: "MIT"
+ url: "https://mit.edu"
+```
+
+Affiliation indices are 1-based and rendered as superscript numbers next to author names.
+
+## External links (paper template)
+
+The `links` field adds buttons below the author/affiliation line in the `paper` template hero. Each link has a `label` and a `url`:
+
+```yaml
+links:
+ - label: "Paper"
+ url: "https://arxiv.org/abs/..."
+ - label: "Code"
+ url: "https://github.com/..."
+ - label: "Demo"
+ url: "https://huggingface.co/spaces/..."
+ - label: "Data"
+ url: "https://huggingface.co/datasets/..."
+```
+
+Links are rendered as pill-shaped buttons and only visible in the `paper` template. They are hidden in the `article` template.
+
+## README tag (critical)
+
+The project `README.md` contains a YAML frontmatter block with a `tags` field:
+
+```yaml
+tags:
+ - research-article-template
+```
+
+**NEVER remove the `research-article-template` tag from the README.** This tag is used by the [Research Article Gallery](https://huggingface.co/spaces/tfrere/research-article-gallery) to discover and list all articles built with this template. Removing it will make the Space invisible in the gallery.
+
+## Complete example
+
+```yaml
+---
+title: "Scaling Laws for\nNeural Language Models"
+description: "An empirical study of scaling behavior across model size, data, and compute"
+authors:
+ - name: "Alice Martin"
+ url: "https://example.com/alice"
+ affiliations: [1]
+affiliations:
+ - name: "Hugging Face"
+ url: "https://huggingface.co"
+published: "Apr. 04, 2026"
+template: "article"
+banner: "banner.html"
+doi: "10.1234/example.2026"
+showPdf: true
+tableOfContentsAutoCollapse: true
+licence: "This work is licensed under CC BY 4.0."
+---
+```
diff --git a/content/articles/multi-harness-rl/.ai/skills/create-html-embed/SKILL.md b/content/articles/multi-harness-rl/.ai/skills/create-html-embed/SKILL.md
new file mode 100644
index 0000000..436aeb8
--- /dev/null
+++ b/content/articles/multi-harness-rl/.ai/skills/create-html-embed/SKILL.md
@@ -0,0 +1,133 @@
+---
+name: create-html-embed
+description: Create self-contained D3 HTML embed charts for the research article template. Use when the user asks to create a chart, visualization, embed, D3 chart, line chart, bar chart, scatter plot, sankey diagram, or any data visualization as an HTML embed file.
+---
+
+# Create HTML Embed
+
+Create self-contained D3.js chart embeds for the research article template.
+
+## Before you start
+
+**Read the full directives file** for all conventions, patterns, and checklists:
+
+- [directives.md](directives.md) — single source of truth for embed authoring rules
+
+This covers: colors & palettes, layout, SVG scope, mounting, theming, controls, tooltips, data loading, responsiveness, legends, accessibility, performance, error handling, printing, and the full agent checklist.
+
+## Workflow
+
+### Step 1: Understand the request
+
+Clarify with the user:
+- What type of chart? (line, bar, scatter, sankey, waffle, heatmap, custom)
+- What data source? (CSV path, JSON, inline data)
+- Interactive controls needed? (metric selector, filters)
+- Any specific design requirements?
+
+### Step 2: Create the HTML file
+
+- Location: `app/src/content/embeds/`
+- Naming: `d3-.html` (e.g., `d3-training-loss.html`)
+- Root class: `.d3-` (must match filename)
+
+### Step 3: Follow the mandatory structure
+
+Every embed must have this structure:
+
+```html
+
+
+
+```
+
+### Step 4: Integrate in MDX
+
+Import and use the `HtmlEmbed` component:
+
+```mdx
+import HtmlEmbed from '../../components/HtmlEmbed.astro';
+
+
+```
+
+#### HtmlEmbed props
+
+| Prop | Type | Description |
+|------|------|-------------|
+| `src` | string | Path to HTML file in `embeds/` (required) |
+| `title` | string | Title above the card |
+| `desc` | string | Description below (supports HTML) |
+| `frameless` | boolean | Removes card background/border |
+| `wide` | boolean | Wide layout (~1100px) |
+| `data` | string or string[] | Path(s) to data files |
+| `config` | object | JSON config passed via `data-config` attribute |
+
+#### Usage examples
+
+```mdx
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+```
+
+## Key conventions (quick reference)
+
+Full details in the directives file. The critical ones:
+
+1. **Colors**: Use `window.ColorPalettes.getColors('categorical', n)` — never hardcode palettes
+2. **CSS variables**: `--text-color`, `--surface-bg`, `--border-color`, `--axis-color`, `--tick-color`, `--grid-color`
+3. **Dark mode**: Check `document.documentElement.getAttribute('data-theme') === 'dark'`
+4. **Mount guard**: Always set `container.dataset.mounted = 'true'`
+5. **Data loading**: Try `/data/` first, then `./assets/data/` — use `fetchFirstAvailable()`
+6. **Responsiveness**: `ResizeObserver` on container, recompute on resize
+7. **Legend**: HTML-based, title "Legend", swatch 14x14px
+8. **Controls**: HTML only (no SVG UI), selects labeled "Metric" when applicable
+9. **Tooltip**: Single `.d3-tooltip` absolutely positioned inside container
+10. **No globals**: Everything in IIFE, nothing on `window`
+
+## Data files
+
+- Store data in: `app/src/content/assets/data/`
+- Served from: `/data/` (public) at build time
+- Formats: CSV (preferred for tabular), JSON (for nested/hierarchical)
+
+## Post-creation checklist
+
+After creating the embed, verify against the **Agent Checklist** (section 14.1) and **Definition of Done** (section 14.2) in [directives.md](directives.md).
diff --git a/content/articles/multi-harness-rl/.ai/skills/create-html-embed/directives.md b/content/articles/multi-harness-rl/.ai/skills/create-html-embed/directives.md
new file mode 100644
index 0000000..329833c
--- /dev/null
+++ b/content/articles/multi-harness-rl/.ai/skills/create-html-embed/directives.md
@@ -0,0 +1,504 @@
+## Embed Chart Authoring Guidelines
+
+### Quickstart (TL;DR)
+- Create a single self-contained HTML fragment: root div + scoped style + IIFE script.
+- Draw marks/axes in SVG; render UI (legend and controls) in HTML.
+- Place legend and controls BELOW the chart (header appended after the chart). Include a legend title "Legend" and a select labeled "Metric" when relevant.
+- Load data from public `/data` first, then fall back to `assets/data`.
+- Use `window.ColorPalettes` for colors; stick to CSS variables for theming.
+
+Minimal header markup:
+```html
+
+
Legend
+
+
+
+
+
+
+
+
+
+
+```
+
+See also: `d3-line-simple.html`, `d3-line-quad.html`, `d3-benchmark.html`.
+
+Authoring rules for creating a new interactive chart as a single self-contained `.html` file under `src/content/embeds/`. These conventions are derived from `d3-bar.html`, `d3-comparison.html`, `d3-neural.html`, `d3-line.html`, and `d3-pie.html`.
+
+### A) Colors & palettes (MANDATORY)
+- Always obtain color arrays from `window.ColorPalettes`; do not hardcode palettes.
+- Use the categorical/sequential/diverging helpers and the current primary color.
+- If you change `--primary-color` dynamically, call `window.ColorPalettes.refresh()` so listeners update.
+
+Usage:
+```js
+// Usage (with explicit counts)
+const cat = window.ColorPalettes.getColors('categorical', 8);
+const seq = window.ColorPalettes.getColors('sequential', 8);
+const div = window.ColorPalettes.getColors('diverging', 7);
+
+// For current primary color string
+const primaryHex = window.ColorPalettes.getPrimary();
+
+// If you change --primary-color dynamically, call refresh to notify listeners
+document.documentElement.style.setProperty('--primary-color', '#6D4AFF');
+window.ColorPalettes.refresh();
+```
+
+Notes:
+- Keep chart accents (lines, markers, selection) aligned with `--primary-color`.
+- Prefer CSS variables for fills/strokes when possible; derive series colors via `ColorPalettes`.
+- Provide a graceful fallback to CSS variables if `window.ColorPalettes` is unavailable.
+
+### B) Layout & form elements (HTML-only)
+- All UI controls (labels, selects, sliders, buttons, toggles) must be plain HTML inside the root container.
+- Do not draw controls with SVG; style them consistently (rounded 8px, custom caret, focus ring).
+- Use `