From d30e32c607cb7f8a58be6044f7e767190865cd3a Mon Sep 17 00:00:00 2001 From: neubig Date: Thu, 16 Jul 2026 18:48:49 +0000 Subject: [PATCH 01/18] Support proxy credentials for Pier mini-SWE runs Preserve nested agent kwargs and expose the existing LiteLLM proxy credential under the OpenAI-compatible aliases expected by mini-swe-agent. Co-authored-by: openhands --- benchmarks/harbor/run_infer.py | 32 +++++++++++++++++++++++----- tests/test_harbor_run_infer.py | 38 ++++++++++++++++++++++++++++++++++ 2 files changed, 65 insertions(+), 5 deletions(-) diff --git a/benchmarks/harbor/run_infer.py b/benchmarks/harbor/run_infer.py index bef91c423..4e9aa362b 100644 --- a/benchmarks/harbor/run_infer.py +++ b/benchmarks/harbor/run_infer.py @@ -154,12 +154,36 @@ def _split_json_values(raw: str | None) -> list[str]: return [] data = json.loads(raw) if isinstance(data, dict): - return [f"{key}={value}" for key, value in data.items()] + return [ + f"{key}={value if isinstance(value, str) else json.dumps(value, separators=(',', ':'))}" + for key, value in data.items() + ] if isinstance(data, list) and all(isinstance(item, str) for item in data): return data raise ValueError("Expected a JSON object or list of KEY=VALUE strings") +def _llm_agent_env(llm: LLM, harbor_agent: str) -> list[str]: + """Build credential env flags for the selected Harbor-compatible agent. + + The OpenHands SDK agent consumes the generic ``LLM_*`` variables. Pier's + mini-SWE agent resolves ``litellm_proxy/*`` through LiteLLM, which consumes + the OpenAI-compatible aliases instead. Both names carry the same proxy + credential and endpoint. + """ + values: list[str] = [] + if llm.api_key: + api_key = _secret_value(llm.api_key) + values.append(f"LLM_API_KEY={api_key}") + if harbor_agent == "mini-swe-agent": + values.append(f"OPENAI_API_KEY={api_key}") + if llm.base_url: + values.append(f"LLM_BASE_URL={llm.base_url}") + if harbor_agent == "mini-swe-agent": + values.append(f"OPENAI_BASE_URL={llm.base_url}") + return values + + def run_harbor( args: argparse.Namespace, llm: LLM, @@ -186,10 +210,8 @@ def run_harbor( str(args.num_workers), ] - if llm.api_key: - cmd.extend(["--ae", f"LLM_API_KEY={_secret_value(llm.api_key)}"]) - if llm.base_url: - cmd.extend(["--ae", f"LLM_BASE_URL={llm.base_url}"]) + for env_value in _llm_agent_env(llm, args.harbor_agent): + cmd.extend(["--ae", env_value]) for env_value in _parse_key_value( [*args.agent_env, *_split_json_values(args.agent_env_json)] ): diff --git a/tests/test_harbor_run_infer.py b/tests/test_harbor_run_infer.py index 7d2d4167f..57053aa0f 100644 --- a/tests/test_harbor_run_infer.py +++ b/tests/test_harbor_run_infer.py @@ -10,12 +10,14 @@ from benchmarks.harbor.run_infer import ( _is_sensitive_value, + _llm_agent_env, _load_task_ids, _parse_key_value, _resolve_target, _split_json_values, _target_args, ) +from openhands.sdk import LLM def test_load_task_ids_strips_and_ignores_comments(tmp_path: Path) -> None: @@ -58,6 +60,12 @@ def test_split_json_values_dict() -> None: assert _split_json_values('{"A": "1", "B": "2"}') == ["A=1", "B=2"] +def test_split_json_values_preserves_nested_json() -> None: + assert _split_json_values( + '{"model_kwargs": {"temperature": 1.0, "top_p": 0.95}}' + ) == ['model_kwargs={"temperature":1.0,"top_p":0.95}'] + + def test_split_json_values_list() -> None: assert _split_json_values('["A=1", "B=2"]') == ["A=1", "B=2"] @@ -67,6 +75,36 @@ def test_split_json_values_invalid() -> None: _split_json_values('"not-an-object-or-list"') +def test_llm_agent_env_uses_generic_names_for_openhands() -> None: + assert _llm_agent_env( + LLM( + model="litellm_proxy/test/model", + api_key="secret", + base_url="https://proxy.example.com", + ), + "openhands-sdk", + ) == [ + "LLM_API_KEY=secret", + "LLM_BASE_URL=https://proxy.example.com", + ] + + +def test_llm_agent_env_adds_openai_proxy_aliases_for_mini_swe() -> None: + assert _llm_agent_env( + LLM( + model="litellm_proxy/test/model", + api_key="secret", + base_url="https://proxy.example.com", + ), + "mini-swe-agent", + ) == [ + "LLM_API_KEY=secret", + "OPENAI_API_KEY=secret", + "LLM_BASE_URL=https://proxy.example.com", + "OPENAI_BASE_URL=https://proxy.example.com", + ] + + def _make_args(**kwargs: object) -> argparse.Namespace: defaults: dict[str, object] = dict( harbor_target=None, From 0770cbe9a9ca3c901e08f20efe242926239ab583 Mon Sep 17 00:00:00 2001 From: neubig Date: Thu, 16 Jul 2026 22:59:10 +0000 Subject: [PATCH 02/18] Add bounded-history model for MiniSWE evaluations --- extras/miniswe-context-model/README.md | 15 +++ extras/miniswe-context-model/pyproject.toml | 12 +++ .../__init__.py | 3 + .../history.py | 94 +++++++++++++++++++ .../openhands_miniswe_context_model/model.py | 35 +++++++ tests/test_miniswe_context_history.py | 63 +++++++++++++ 6 files changed, 222 insertions(+) create mode 100644 extras/miniswe-context-model/README.md create mode 100644 extras/miniswe-context-model/pyproject.toml create mode 100644 extras/miniswe-context-model/src/openhands_miniswe_context_model/__init__.py create mode 100644 extras/miniswe-context-model/src/openhands_miniswe_context_model/history.py create mode 100644 extras/miniswe-context-model/src/openhands_miniswe_context_model/model.py create mode 100644 tests/test_miniswe_context_history.py diff --git a/extras/miniswe-context-model/README.md b/extras/miniswe-context-model/README.md new file mode 100644 index 000000000..1b732de37 --- /dev/null +++ b/extras/miniswe-context-model/README.md @@ -0,0 +1,15 @@ +# MiniSWE context model + +This small optional package provides a `LitellmModel` subclass for long-running +MiniSWE evaluations. It retains the system prompt, task, and newest complete +assistant/tool turns while eliding old turns before a request exceeds the configured +serialized-history budget. + +Install it in the MiniSWE tool environment and select: + +```text +openhands_miniswe_context_model.model.ContextSafeLitellmModel +``` + +The `model.max_history_chars` config field defaults to 500,000 characters and can +be overridden by the benchmark's agent configuration. diff --git a/extras/miniswe-context-model/pyproject.toml b/extras/miniswe-context-model/pyproject.toml new file mode 100644 index 000000000..8363cd70f --- /dev/null +++ b/extras/miniswe-context-model/pyproject.toml @@ -0,0 +1,12 @@ +[build-system] +requires = ["setuptools>=68"] +build-backend = "setuptools.build_meta" + +[project] +name = "openhands-miniswe-context-model" +version = "0.1.0" +description = "Bounded-history LiteLLM model wrapper for mini-swe-agent evaluations" +requires-python = ">=3.11" + +[tool.setuptools.packages.find] +where = ["src"] diff --git a/extras/miniswe-context-model/src/openhands_miniswe_context_model/__init__.py b/extras/miniswe-context-model/src/openhands_miniswe_context_model/__init__.py new file mode 100644 index 000000000..e31b35ca1 --- /dev/null +++ b/extras/miniswe-context-model/src/openhands_miniswe_context_model/__init__.py @@ -0,0 +1,3 @@ +"""Context-safe model helpers for mini-swe-agent evaluations.""" + +__version__ = "0.1.0" diff --git a/extras/miniswe-context-model/src/openhands_miniswe_context_model/history.py b/extras/miniswe-context-model/src/openhands_miniswe_context_model/history.py new file mode 100644 index 000000000..0fa6cdacc --- /dev/null +++ b/extras/miniswe-context-model/src/openhands_miniswe_context_model/history.py @@ -0,0 +1,94 @@ +"""History bounding that preserves tool-call protocol boundaries.""" + +from __future__ import annotations + +import json +from collections.abc import Sequence +from typing import Any + + +ELISION_NOTICE = ( + "[Earlier assistant and tool interactions were omitted to stay within the " + "model context window. Their effects remain in the current workspace.]" +) + + +def _message_size(message: dict[str, Any]) -> int: + return len( + json.dumps( + message, + ensure_ascii=False, + separators=(",", ":"), + default=str, + ) + ) + + +def _history_units(messages: Sequence[dict[str, Any]]) -> list[list[dict[str, Any]]]: + """Group assistant messages with their following tool results.""" + units: list[list[dict[str, Any]]] = [] + for message in messages: + if ( + message.get("role") == "tool" + and units + and units[-1][0].get("role") == "assistant" + ): + units[-1].append(message) + else: + units.append([message]) + return units + + +def bound_message_history( + messages: Sequence[dict[str, Any]], + *, + max_chars: int, + preserve_first: int = 2, + min_recent_units: int = 8, +) -> list[dict[str, Any]]: + """Drop the oldest complete turns when serialized history exceeds a budget. + + The initial system/task messages are always retained. Assistant tool calls and + their tool results are grouped into atomic units, so trimming never leaves an + orphaned tool response in the request sent to the model. + """ + if max_chars <= 0: + raise ValueError("max_chars must be positive") + if preserve_first < 0: + raise ValueError("preserve_first cannot be negative") + if min_recent_units < 1: + raise ValueError("min_recent_units must be at least one") + + copied = [dict(message) for message in messages] + if sum(_message_size(message) for message in copied) <= max_chars: + return copied + + header = copied[:preserve_first] + units = _history_units(copied[preserve_first:]) + notice = {"role": "user", "content": ELISION_NOTICE} + remaining = ( + max_chars + - sum(_message_size(message) for message in header) + - _message_size(notice) + ) + + kept_reversed: list[list[dict[str, Any]]] = [] + used = 0 + for unit in reversed(units): + unit_size = sum(_message_size(message) for message in unit) + if ( + kept_reversed + and used + unit_size > remaining + and len(kept_reversed) >= min_recent_units + ): + break + kept_reversed.append(unit) + used += unit_size + + kept = [message for unit in reversed(kept_reversed) for message in unit] + dropped_count = len(copied) - len(header) - len(kept) + if dropped_count <= 0: + return copied + + notice["content"] += f" ({dropped_count} messages omitted.)" + return [*header, notice, *kept] diff --git a/extras/miniswe-context-model/src/openhands_miniswe_context_model/model.py b/extras/miniswe-context-model/src/openhands_miniswe_context_model/model.py new file mode 100644 index 000000000..7197ff72a --- /dev/null +++ b/extras/miniswe-context-model/src/openhands_miniswe_context_model/model.py @@ -0,0 +1,35 @@ +"""A LiteLLM mini-swe-agent model with bounded request history.""" + +from __future__ import annotations + +from typing import Any + +from minisweagent.models.litellm_model import LitellmModel, LitellmModelConfig +from pydantic import Field + +from openhands_miniswe_context_model.history import bound_message_history + + +class ContextSafeLitellmModelConfig(LitellmModelConfig): + max_history_chars: int = Field(default=500_000, gt=0) + """Maximum serialized characters retained in requests to the model.""" + + min_recent_history_units: int = Field(default=8, ge=1) + """Minimum number of recent assistant/tool units to retain.""" + + +class ContextSafeLitellmModel(LitellmModel): + """Retain task setup and recent complete turns below a context-safe budget.""" + + config: ContextSafeLitellmModelConfig + + def __init__(self, **kwargs: Any): + super().__init__(config_class=ContextSafeLitellmModelConfig, **kwargs) + + def _prepare_messages_for_api(self, messages: list[dict]) -> list[dict]: + prepared = super()._prepare_messages_for_api(messages) + return bound_message_history( + prepared, + max_chars=self.config.max_history_chars, + min_recent_units=self.config.min_recent_history_units, + ) diff --git a/tests/test_miniswe_context_history.py b/tests/test_miniswe_context_history.py new file mode 100644 index 000000000..b03c49cdf --- /dev/null +++ b/tests/test_miniswe_context_history.py @@ -0,0 +1,63 @@ +from __future__ import annotations + +import importlib.util +from pathlib import Path + + +_HISTORY_PATH = ( + Path(__file__).parents[1] + / "extras/miniswe-context-model/src/openhands_miniswe_context_model/history.py" +) +_SPEC = importlib.util.spec_from_file_location("miniswe_context_history", _HISTORY_PATH) +assert _SPEC and _SPEC.loader +_HISTORY = importlib.util.module_from_spec(_SPEC) +_SPEC.loader.exec_module(_HISTORY) +bound_message_history = _HISTORY.bound_message_history + + +def _turn(index: int, size: int = 80) -> list[dict]: + return [ + { + "role": "assistant", + "content": f"turn-{index}", + "tool_calls": [{"id": f"call-{index}", "function": {"name": "bash"}}], + }, + {"role": "tool", "tool_call_id": f"call-{index}", "content": "x" * size}, + ] + + +def test_short_history_is_unchanged() -> None: + messages = [ + {"role": "system", "content": "system"}, + {"role": "user", "content": "task"}, + *_turn(1), + ] + + assert bound_message_history(messages, max_chars=10_000) == messages + + +def test_long_history_keeps_header_and_complete_recent_turns() -> None: + messages = [ + {"role": "system", "content": "system"}, + {"role": "user", "content": "task"}, + *[message for index in range(12) for message in _turn(index, size=120)], + ] + + bounded = bound_message_history(messages, max_chars=1_200, min_recent_units=3) + + assert bounded[:2] == messages[:2] + assert "messages omitted" in bounded[2]["content"] + assert bounded[-2:] == messages[-2:] + for index, message in enumerate(bounded): + if message["role"] == "tool": + assert index > 0 + assert bounded[index - 1]["role"] in {"assistant", "tool"} + + +def test_history_budget_validation() -> None: + try: + bound_message_history([], max_chars=0) + except ValueError as exc: + assert "max_chars" in str(exc) + else: + raise AssertionError("Expected invalid max_chars to raise") From a50c04132004cf7ff6c77b3568185ee40f8e5154 Mon Sep 17 00:00:00 2001 From: neubig Date: Fri, 17 Jul 2026 06:46:48 +0000 Subject: [PATCH 03/18] Honor verifier rewards after agent failures --- benchmarks/utils/harbor.py | 15 ++++++++++++--- tests/test_terminalbench.py | 33 +++++++++++++++++++++++++++++++++ 2 files changed, 45 insertions(+), 3 deletions(-) diff --git a/benchmarks/utils/harbor.py b/benchmarks/utils/harbor.py index 4dda977ce..a6245e7a7 100644 --- a/benchmarks/utils/harbor.py +++ b/benchmarks/utils/harbor.py @@ -236,7 +236,16 @@ def convert_harbor_to_eval_output( instance_id = canonicalize(trial.get("task_name", result_file.parent.name)) - if trial.get("exception_info"): + verifier_result = trial.get("verifier_result") or {} + rewards = verifier_result.get("rewards") or {} + has_primary_reward = "reward" in rewards + + # Harbor can still run the verifier after an agent timeout or other + # non-zero exit. In that case the verifier is authoritative: a reward + # of 1 is a resolved task, and a reward of 0 is a completed unresolved + # task. Only emit an incomplete error when no primary verifier reward + # exists (for example, setup or verifier failures). + if trial.get("exception_info") and not has_primary_reward: errors.append( { "instance_id": instance_id, @@ -246,8 +255,6 @@ def convert_harbor_to_eval_output( ) continue - verifier_result = trial.get("verifier_result", {}) - rewards = verifier_result.get("rewards", {}) passed = rewards.get("reward", 0.0) > 0 agent_result = trial.get("agent_result", {}) @@ -270,6 +277,8 @@ def convert_harbor_to_eval_output( "total_cost_usd": agent_result.get("cost_usd") or 0.0, }, } + if trial.get("exception_info"): + eval_entry["test_result"]["agent_exception"] = trial["exception_info"] results.append(eval_entry) logger.info( f"Processed trial {instance_id}: reward={rewards.get('reward', 'N/A')}" diff --git a/tests/test_terminalbench.py b/tests/test_terminalbench.py index d9b62e07e..4da48b9e9 100644 --- a/tests/test_terminalbench.py +++ b/tests/test_terminalbench.py @@ -402,6 +402,39 @@ def test_trial_with_exception(self, tmp_path: Path) -> None: assert report["error_instances"] == 1 assert report["incomplete_ids"] == ["error-task"] + def test_trial_with_exception_and_verifier_reward_is_scored( + self, tmp_path: Path + ) -> None: + """A post-agent verifier result remains authoritative after a timeout.""" + trial_result = { + "task_name": "completed-before-timeout", + "trial_name": "completed-before-timeout__abc", + "trial_uri": "file:///path/to/trial", + "agent_result": { + "n_input_tokens": 500, + "n_output_tokens": 100, + "cost_usd": 0.01, + }, + "verifier_result": {"rewards": {"reward": 1.0}}, + "exception_info": { + "type": "AgentTimeoutError", + "message": "Agent execution timed out", + }, + } + + harbor_dir = self._create_harbor_structure( + tmp_path, [("completed-before-timeout__abc", trial_result)] + ) + output_file = tmp_path / "output.jsonl" + + convert_harbor_to_eval_output(harbor_dir, output_file) + + entry = json.loads(output_file.read_text()) + assert entry["error"] is None + assert entry["test_result"]["passed"] is True + assert entry["test_result"]["rewards"] == {"reward": 1.0} + assert entry["test_result"]["agent_exception"] == trial_result["exception_info"] + def test_mixed_valid_and_exception_trials(self, tmp_path: Path) -> None: """Test handling mix of successful and exception trials.""" trials = [ From 5e8c8118f2172d8cc2afa63da32d389aaf57209b Mon Sep 17 00:00:00 2001 From: neubig Date: Fri, 17 Jul 2026 07:01:49 +0000 Subject: [PATCH 04/18] Refresh Harbor results before evaluation --- benchmarks/harbor/eval_infer.py | 26 ++++++++++++ tests/test_harbor_eval_infer.py | 75 +++++++++++++++++++++++++++++++++ 2 files changed, 101 insertions(+) create mode 100644 tests/test_harbor_eval_infer.py diff --git a/benchmarks/harbor/eval_infer.py b/benchmarks/harbor/eval_infer.py index fe4c33473..7a9811859 100644 --- a/benchmarks/harbor/eval_infer.py +++ b/benchmarks/harbor/eval_infer.py @@ -9,6 +9,7 @@ from pathlib import Path from typing import Any +from benchmarks.utils.harbor import convert_harbor_to_eval_output from benchmarks.utils.laminar import LaminarService from benchmarks.utils.report_costs import generate_cost_report from openhands.sdk import get_logger @@ -17,6 +18,30 @@ logger = get_logger(__name__) +def refresh_eval_output_from_harbor(input_file: Path) -> bool: + """Rebuild converted output from raw Harbor results when they are available. + + Inference archives contain both ``output.jsonl`` and the authoritative + ``harbor_output`` tree. Re-running the converter in the evaluation phase + lets converter fixes take effect during a rescore and avoids permanently + classifying verifier-scored agent failures according to stale JSONL. + """ + harbor_output_dir = input_file.parent / "harbor_output" + if not harbor_output_dir.is_dir(): + logger.info( + "Raw Harbor output not found at %s; using existing converted output", + harbor_output_dir, + ) + return False + + convert_harbor_to_eval_output( + harbor_output_dir=harbor_output_dir, + eval_output_path=input_file, + ) + logger.info("Refreshed %s from raw Harbor results", input_file) + return True + + def _metric( data: dict[str, Any], test_result: dict[str, Any], key: str, default: Any ) -> Any: @@ -143,6 +168,7 @@ def main() -> None: else input_file.with_suffix(".report.json") ) try: + refresh_eval_output_from_harbor(input_file) process_harbor_results(str(input_file), str(output_file)) generate_cost_report(str(input_file)) except Exception as exc: diff --git a/tests/test_harbor_eval_infer.py b/tests/test_harbor_eval_infer.py new file mode 100644 index 000000000..687daebb4 --- /dev/null +++ b/tests/test_harbor_eval_infer.py @@ -0,0 +1,75 @@ +"""Tests for the generic Harbor evaluation phase.""" + +import json +from pathlib import Path + +from benchmarks.harbor.eval_infer import ( + process_harbor_results, + refresh_eval_output_from_harbor, +) + + +def test_refresh_reconverts_authoritative_harbor_results(tmp_path: Path) -> None: + output_file = tmp_path / "output.jsonl" + output_file.write_text( + json.dumps( + { + "instance_id": "finished-before-timeout", + "test_result": {}, + "error": "stale timeout classification", + } + ) + + "\n" + ) + + job_dir = tmp_path / "harbor_output" / "2026-01-01__00-00-00" + trial_dir = job_dir / "finished-before-timeout__abc" + trial_dir.mkdir(parents=True) + (job_dir / "result.json").write_text(json.dumps({"id": "job"})) + (trial_dir / "result.json").write_text( + json.dumps( + { + "task_name": "finished-before-timeout", + "trial_name": "finished-before-timeout__abc", + "agent_result": { + "n_input_tokens": 10, + "n_output_tokens": 2, + "cost_usd": 0.0, + }, + "verifier_result": {"rewards": {"reward": 1.0}}, + "exception_info": { + "exception_type": "AgentTimeoutError", + "exception_message": "Agent timed out after saving the answer", + }, + } + ) + ) + + assert refresh_eval_output_from_harbor(output_file) is True + + report_file = tmp_path / "output.report.json" + report = process_harbor_results(str(output_file), str(report_file)) + converted = json.loads(output_file.read_text()) + + assert report["completed_instances"] == 1 + assert report["resolved_instances"] == 1 + assert report["error_instances"] == 0 + assert converted["test_result"]["agent_exception"]["exception_type"] == ( + "AgentTimeoutError" + ) + + +def test_refresh_keeps_existing_output_without_raw_harbor_results( + tmp_path: Path, +) -> None: + output_file = tmp_path / "output.jsonl" + original = ( + json.dumps( + {"instance_id": "existing", "test_result": {"passed": False}, "error": None} + ) + + "\n" + ) + output_file.write_text(original) + + assert refresh_eval_output_from_harbor(output_file) is False + assert output_file.read_text() == original From 115bcdf35fd9c9ea55ef7f6f96604bb1739b28ee Mon Sep 17 00:00:00 2001 From: neubig Date: Fri, 17 Jul 2026 10:26:50 +0000 Subject: [PATCH 05/18] Keep optional MiniSWE import type-checkable --- .../src/openhands_miniswe_context_model/model.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/extras/miniswe-context-model/src/openhands_miniswe_context_model/model.py b/extras/miniswe-context-model/src/openhands_miniswe_context_model/model.py index 7197ff72a..292ca3e0c 100644 --- a/extras/miniswe-context-model/src/openhands_miniswe_context_model/model.py +++ b/extras/miniswe-context-model/src/openhands_miniswe_context_model/model.py @@ -4,7 +4,10 @@ from typing import Any -from minisweagent.models.litellm_model import LitellmModel, LitellmModelConfig +from minisweagent.models.litellm_model import ( # pyright: ignore[reportMissingImports] + LitellmModel, + LitellmModelConfig, +) from pydantic import Field from openhands_miniswe_context_model.history import bound_message_history From e53ac8a921ba1707f2113e7ebb18ef926a4be9db Mon Sep 17 00:00:00 2001 From: neubig Date: Thu, 30 Jul 2026 02:58:11 +0000 Subject: [PATCH 06/18] Add Perplexity-enabled TerminalBench agent Co-authored-by: openhands --- benchmarks/terminalbench/pplx_agent.py | 51 ++++++++++++++++++++++++++ benchmarks/terminalbench/run_infer.py | 24 +++++++++++- benchmarks/utils/harbor.py | 15 +++++++- 3 files changed, 88 insertions(+), 2 deletions(-) create mode 100644 benchmarks/terminalbench/pplx_agent.py diff --git a/benchmarks/terminalbench/pplx_agent.py b/benchmarks/terminalbench/pplx_agent.py new file mode 100644 index 000000000..69ffd6f4d --- /dev/null +++ b/benchmarks/terminalbench/pplx_agent.py @@ -0,0 +1,51 @@ +# pyright: reportMissingImports=false, reportGeneralTypeIssues=false +"""Perplexity-enabled OpenHands SDK agent for Terminal-Bench experiments.""" + +from __future__ import annotations + +from typing import Any, override + +from harbor.agents.installed.openhands_sdk import OpenHandsSDK +from harbor.environments.base import BaseEnvironment + + +class PplxOpenHandsSDK(OpenHandsSDK): + """OpenHands SDK with a pinned ``pplx`` binary and scoped API-key forwarding.""" + + PPLX_VERSION = "v0.2.2" + + @override + async def install(self, environment: BaseEnvironment) -> None: + await super().install(environment) + await self.exec_as_root( + environment, + command=( + "set -euo pipefail; " + "asset=pplx-x86_64-linux-gnu.bin; tmp=$(mktemp -d); " + "trap 'rm -rf \"$tmp\"' EXIT; " + f"base=https://github.com/perplexityai/perplexity-cli/releases/download/{self.PPLX_VERSION}; " + 'curl -fsSL --retry 3 "$base/SHA256SUMS" -o "$tmp/SHA256SUMS"; ' + 'curl -fsSL --retry 3 "$base/$asset" -o "$tmp/$asset"; ' + '(cd "$tmp" && grep " $asset$" SHA256SUMS | sha256sum -c -); ' + 'install -m 0755 "$tmp/$asset" /usr/local/bin/pplx; ' + "pplx --version" + ), + ) + + @override + async def exec_as_agent( + self, + environment: BaseEnvironment, + command: str, + env: dict[str, str] | None = None, + cwd: str | None = None, + timeout_sec: int | None = None, + ) -> Any: + """Forward the key only to commands executed as the task-solving agent.""" + api_key = self._get_env("PERPLEXITY_API_KEY") + if api_key is not None: + env = dict(env or {}) + env["PERPLEXITY_API_KEY"] = api_key + return await super().exec_as_agent( + environment, command, env=env, cwd=cwd, timeout_sec=timeout_sec + ) diff --git a/benchmarks/terminalbench/run_infer.py b/benchmarks/terminalbench/run_infer.py index 48e785c79..a52a407c8 100644 --- a/benchmarks/terminalbench/run_infer.py +++ b/benchmarks/terminalbench/run_infer.py @@ -64,12 +64,31 @@ def run_harbor_evaluation( Returns: Path to the harbor output directory. """ + pplx_enabled = os.environ.get("TERMINALBENCH_PPLX_ENABLED") == "true" + agent_name = HARBOR_DEFAULTS["agent_name"] + agent_env: dict[str, str] | None = None + agent_kwargs: dict[str, object] | None = None + skills: list[str] | None = None + if pplx_enabled: + api_key = os.environ.get("PERPLEXITY_API_KEY") + if not api_key: + raise RuntimeError( + "PERPLEXITY_API_KEY is required when TERMINALBENCH_PPLX_ENABLED=true" + ) + agent_name = "benchmarks.terminalbench.pplx_agent:PplxOpenHandsSDK" + agent_env = {"PERPLEXITY_API_KEY": api_key} + agent_kwargs = {"skill_paths": ["/harbor/skills"]} + skills = [ + "https://github.com/perplexityai/api-platform-developers/tree/" + "906630d8b9787b29afd693699fe34c1b86adf2de/skills/pplx-cli" + ] + return _run_harbor_evaluation( llm=llm, dataset=dataset, output_dir=output_dir, harbor_executable=HARBOR_DEFAULTS["harbor_executable"], - agent_name=HARBOR_DEFAULTS["agent_name"], + agent_name=agent_name, num_workers=num_workers, task_ids=task_ids, n_limit=n_limit, @@ -77,6 +96,9 @@ def run_harbor_evaluation( HARBOR_DEFAULTS["harbor_executable"] ), credential_mode=HarborCredentialMode.AGENT_ENV_FLAGS, + agent_env=agent_env, + agent_kwargs=agent_kwargs, + skills=skills, subprocess_run=subprocess.run, ) diff --git a/benchmarks/utils/harbor.py b/benchmarks/utils/harbor.py index a6245e7a7..a683692ed 100644 --- a/benchmarks/utils/harbor.py +++ b/benchmarks/utils/harbor.py @@ -99,6 +99,9 @@ def run_harbor_evaluation( task_filter_flag: str = "--task-name", normalize_task_id: Callable[[str], str] | None = None, credential_mode: HarborCredentialMode = HarborCredentialMode.AGENT_ENV_FLAGS, + agent_env: dict[str, str] | None = None, + agent_kwargs: dict[str, Any] | None = None, + skills: list[str] | None = None, retry_legacy_task_flag: bool = False, subprocess_run: Callable[..., Any] = subprocess.run, ) -> Path: @@ -138,6 +141,13 @@ def run_harbor_evaluation( if llm.base_url: env["LLM_BASE_URL"] = llm.base_url + for key, value in (agent_env or {}).items(): + cmd.extend(["--ae", f"{key}={value}"]) + for key, value in (agent_kwargs or {}).items(): + cmd.extend(["--agent-kwarg", f"{key}={json.dumps(value)}"]) + for skill in skills or []: + cmd.extend(["--skill", skill]) + if task_ids: normalize = normalize_task_id or (lambda task_id: task_id) for task_id in task_ids: @@ -147,7 +157,10 @@ def run_harbor_evaluation( cmd.extend(["--n-tasks", str(n_limit)]) safe_cmd = [ - "***" if prev == "--ae" and part.startswith("LLM_") else part + "***" + if prev == "--ae" + and (part.startswith("LLM_") or part.startswith("PERPLEXITY_")) + else part for prev, part in zip([""] + cmd, cmd) ] logger.info(f"Running harbor command: {' '.join(safe_cmd)}") From e9b7f79fe99c329c3cbab2201c9db4e14d8f9954 Mon Sep 17 00:00:00 2001 From: neubig Date: Thu, 30 Jul 2026 03:49:31 +0000 Subject: [PATCH 07/18] Use resolvable Perplexity skill ref --- benchmarks/terminalbench/run_infer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/benchmarks/terminalbench/run_infer.py b/benchmarks/terminalbench/run_infer.py index a52a407c8..a3f2c7f5d 100644 --- a/benchmarks/terminalbench/run_infer.py +++ b/benchmarks/terminalbench/run_infer.py @@ -80,7 +80,7 @@ def run_harbor_evaluation( agent_kwargs = {"skill_paths": ["/harbor/skills"]} skills = [ "https://github.com/perplexityai/api-platform-developers/tree/" - "906630d8b9787b29afd693699fe34c1b86adf2de/skills/pplx-cli" + "main/skills/pplx-cli" ] return _run_harbor_evaluation( From d940ac0d50dc4739108e75872442ee6cadbb2b83 Mon Sep 17 00:00:00 2001 From: neubig Date: Thu, 30 Jul 2026 03:55:46 +0000 Subject: [PATCH 08/18] Load Perplexity Harbor agent outside benchmarks package --- benchmarks/terminalbench/run_infer.py | 2 +- harbor_agents/__init__.py | 1 + .../pplx_agent.py => harbor_agents/pplx_openhands_sdk.py | 0 3 files changed, 2 insertions(+), 1 deletion(-) create mode 100644 harbor_agents/__init__.py rename benchmarks/terminalbench/pplx_agent.py => harbor_agents/pplx_openhands_sdk.py (100%) diff --git a/benchmarks/terminalbench/run_infer.py b/benchmarks/terminalbench/run_infer.py index a3f2c7f5d..51b23d950 100644 --- a/benchmarks/terminalbench/run_infer.py +++ b/benchmarks/terminalbench/run_infer.py @@ -75,7 +75,7 @@ def run_harbor_evaluation( raise RuntimeError( "PERPLEXITY_API_KEY is required when TERMINALBENCH_PPLX_ENABLED=true" ) - agent_name = "benchmarks.terminalbench.pplx_agent:PplxOpenHandsSDK" + agent_name = "harbor_agents.pplx_openhands_sdk:PplxOpenHandsSDK" agent_env = {"PERPLEXITY_API_KEY": api_key} agent_kwargs = {"skill_paths": ["/harbor/skills"]} skills = [ diff --git a/harbor_agents/__init__.py b/harbor_agents/__init__.py new file mode 100644 index 000000000..58a4f76a8 --- /dev/null +++ b/harbor_agents/__init__.py @@ -0,0 +1 @@ +"""Custom Harbor agent adapters that must not import the benchmarks package.""" diff --git a/benchmarks/terminalbench/pplx_agent.py b/harbor_agents/pplx_openhands_sdk.py similarity index 100% rename from benchmarks/terminalbench/pplx_agent.py rename to harbor_agents/pplx_openhands_sdk.py From 37c38e7f507b5b72cf4ec5d7031dd4a8cb2c617f Mon Sep 17 00:00:00 2001 From: neubig Date: Fri, 31 Jul 2026 16:09:32 +0000 Subject: [PATCH 09/18] Require Perplexity search in TerminalBench runs --- benchmarks/terminalbench/run_infer.py | 8 +++++++- harbor_agents/skills/pplx-required/SKILL.md | 21 +++++++++++++++++++++ 2 files changed, 28 insertions(+), 1 deletion(-) create mode 100644 harbor_agents/skills/pplx-required/SKILL.md diff --git a/benchmarks/terminalbench/run_infer.py b/benchmarks/terminalbench/run_infer.py index 51b23d950..def5fb474 100644 --- a/benchmarks/terminalbench/run_infer.py +++ b/benchmarks/terminalbench/run_infer.py @@ -80,7 +80,13 @@ def run_harbor_evaluation( agent_kwargs = {"skill_paths": ["/harbor/skills"]} skills = [ "https://github.com/perplexityai/api-platform-developers/tree/" - "main/skills/pplx-cli" + "main/skills/pplx-cli", + str( + Path(__file__).resolve().parents[2] + / "harbor_agents" + / "skills" + / "pplx-required" + ), ] return _run_harbor_evaluation( diff --git a/harbor_agents/skills/pplx-required/SKILL.md b/harbor_agents/skills/pplx-required/SKILL.md new file mode 100644 index 000000000..83b5ef990 --- /dev/null +++ b/harbor_agents/skills/pplx-required/SKILL.md @@ -0,0 +1,21 @@ +--- +name: pplx-required +description: Require a Perplexity web search before solving every evaluation task. +--- + +# Required Perplexity research + +For every task, you must use Perplexity web search before making any changes or +attempting a solution. This requirement applies even if the task appears fully +self-contained. + +1. Extract a short, task-specific search query from the user instruction. Use + distinctive technical terms, filenames, error messages, APIs, or concepts + from the task; do not use a generic query. +2. Run at least one `pplx search web ""` command in the + terminal. +3. Read the result and use it to inform your solution. If it is not useful, + briefly state why in your reasoning and continue with the task. + +`PERPLEXITY_API_KEY` is already available in the agent environment. Do not run +interactive authentication commands and never print or expose the key. From e272429f7a677ee6120470e1599db5ca3fb32504 Mon Sep 17 00:00:00 2001 From: neubig Date: Fri, 31 Jul 2026 18:23:50 +0000 Subject: [PATCH 10/18] Allow PPLX CLI API egress in TerminalBench --- benchmarks/terminalbench/run_infer.py | 6 ++++++ benchmarks/utils/harbor.py | 3 +++ tests/test_harbor.py | 12 ++++++++++++ 3 files changed, 21 insertions(+) diff --git a/benchmarks/terminalbench/run_infer.py b/benchmarks/terminalbench/run_infer.py index def5fb474..0e83ec2b8 100644 --- a/benchmarks/terminalbench/run_infer.py +++ b/benchmarks/terminalbench/run_infer.py @@ -68,6 +68,7 @@ def run_harbor_evaluation( agent_name = HARBOR_DEFAULTS["agent_name"] agent_env: dict[str, str] | None = None agent_kwargs: dict[str, object] | None = None + agent_allowed_hosts: list[str] | None = None skills: list[str] | None = None if pplx_enabled: api_key = os.environ.get("PERPLEXITY_API_KEY") @@ -78,6 +79,10 @@ def run_harbor_evaluation( agent_name = "harbor_agents.pplx_openhands_sdk:PplxOpenHandsSDK" agent_env = {"PERPLEXITY_API_KEY": api_key} agent_kwargs = {"skill_paths": ["/harbor/skills"]} + # Harbor enforces egress per task. The DNS resolver sidecar enables + # resolution of allowlisted hosts, while this explicit grant permits + # the CLI's API request during the agent phase. + agent_allowed_hosts = ["api.perplexity.ai"] skills = [ "https://github.com/perplexityai/api-platform-developers/tree/" "main/skills/pplx-cli", @@ -104,6 +109,7 @@ def run_harbor_evaluation( credential_mode=HarborCredentialMode.AGENT_ENV_FLAGS, agent_env=agent_env, agent_kwargs=agent_kwargs, + agent_allowed_hosts=agent_allowed_hosts, skills=skills, subprocess_run=subprocess.run, ) diff --git a/benchmarks/utils/harbor.py b/benchmarks/utils/harbor.py index a683692ed..39c25fa6d 100644 --- a/benchmarks/utils/harbor.py +++ b/benchmarks/utils/harbor.py @@ -101,6 +101,7 @@ def run_harbor_evaluation( credential_mode: HarborCredentialMode = HarborCredentialMode.AGENT_ENV_FLAGS, agent_env: dict[str, str] | None = None, agent_kwargs: dict[str, Any] | None = None, + agent_allowed_hosts: list[str] | None = None, skills: list[str] | None = None, retry_legacy_task_flag: bool = False, subprocess_run: Callable[..., Any] = subprocess.run, @@ -145,6 +146,8 @@ def run_harbor_evaluation( cmd.extend(["--ae", f"{key}={value}"]) for key, value in (agent_kwargs or {}).items(): cmd.extend(["--agent-kwarg", f"{key}={json.dumps(value)}"]) + for host in agent_allowed_hosts or []: + cmd.extend(["--allow-agent-host", host]) for skill in skills or []: cmd.extend(["--skill", skill]) diff --git a/tests/test_harbor.py b/tests/test_harbor.py index 6eac84d90..e4cdecfab 100644 --- a/tests/test_harbor.py +++ b/tests/test_harbor.py @@ -187,6 +187,18 @@ def test_process_env_mode_sets_env_vars(self, tmp_path: Path) -> None: assert env["LLM_API_KEY"] == "my-key" assert env["LLM_BASE_URL"] == "https://proxy.example.com" + def test_agent_allowed_hosts_adds_harbor_flags(self, tmp_path: Path) -> None: + run = _fake_run() + run_harbor_evaluation( + llm=LLM(model="test/model"), + dataset="my-dataset", + output_dir=str(tmp_path), + agent_allowed_hosts=["api.perplexity.ai"], + subprocess_run=run, + ) + cmd = run.captured["cmds"][0] + assert cmd[cmd.index("--allow-agent-host") + 1] == "api.perplexity.ai" + class TestRunHarborEvaluationTaskFiltering: """Tests for task_ids, n_limit, and fallback-retry in run_harbor_evaluation.""" From 5b639f3fd84531eeb8a55d9aa20c0a9efe1f1675 Mon Sep 17 00:00:00 2001 From: neubig Date: Mon, 3 Aug 2026 02:33:31 +0000 Subject: [PATCH 11/18] Allow longer PPLX Harbor agent setup --- benchmarks/terminalbench/run_infer.py | 11 +++++++++++ benchmarks/utils/harbor.py | 8 ++++++++ tests/test_harbor.py | 14 ++++++++++++++ 3 files changed, 33 insertions(+) diff --git a/benchmarks/terminalbench/run_infer.py b/benchmarks/terminalbench/run_infer.py index 0e83ec2b8..925f37389 100644 --- a/benchmarks/terminalbench/run_infer.py +++ b/benchmarks/terminalbench/run_infer.py @@ -70,6 +70,7 @@ def run_harbor_evaluation( agent_kwargs: dict[str, object] | None = None agent_allowed_hosts: list[str] | None = None skills: list[str] | None = None + agent_setup_timeout_multiplier: float | None = None if pplx_enabled: api_key = os.environ.get("PERPLEXITY_API_KEY") if not api_key: @@ -93,6 +94,15 @@ def run_harbor_evaluation( / "pplx-required" ), ] + raw_setup_multiplier = os.environ.get( + "TERMINALBENCH_PPLX_AGENT_SETUP_TIMEOUT_MULTIPLIER" + ) + if raw_setup_multiplier: + agent_setup_timeout_multiplier = float(raw_setup_multiplier) + if agent_setup_timeout_multiplier <= 0: + raise ValueError( + "TERMINALBENCH_PPLX_AGENT_SETUP_TIMEOUT_MULTIPLIER must be positive" + ) return _run_harbor_evaluation( llm=llm, @@ -111,6 +121,7 @@ def run_harbor_evaluation( agent_kwargs=agent_kwargs, agent_allowed_hosts=agent_allowed_hosts, skills=skills, + agent_setup_timeout_multiplier=agent_setup_timeout_multiplier, subprocess_run=subprocess.run, ) diff --git a/benchmarks/utils/harbor.py b/benchmarks/utils/harbor.py index 39c25fa6d..74e7e8a1e 100644 --- a/benchmarks/utils/harbor.py +++ b/benchmarks/utils/harbor.py @@ -103,6 +103,7 @@ def run_harbor_evaluation( agent_kwargs: dict[str, Any] | None = None, agent_allowed_hosts: list[str] | None = None, skills: list[str] | None = None, + agent_setup_timeout_multiplier: float | None = None, retry_legacy_task_flag: bool = False, subprocess_run: Callable[..., Any] = subprocess.run, ) -> Path: @@ -150,6 +151,13 @@ def run_harbor_evaluation( cmd.extend(["--allow-agent-host", host]) for skill in skills or []: cmd.extend(["--skill", skill]) + if agent_setup_timeout_multiplier is not None: + cmd.extend( + [ + "--agent-setup-timeout-multiplier", + str(agent_setup_timeout_multiplier), + ] + ) if task_ids: normalize = normalize_task_id or (lambda task_id: task_id) diff --git a/tests/test_harbor.py b/tests/test_harbor.py index e4cdecfab..7e1d75b08 100644 --- a/tests/test_harbor.py +++ b/tests/test_harbor.py @@ -199,6 +199,20 @@ def test_agent_allowed_hosts_adds_harbor_flags(self, tmp_path: Path) -> None: cmd = run.captured["cmds"][0] assert cmd[cmd.index("--allow-agent-host") + 1] == "api.perplexity.ai" + def test_agent_setup_timeout_multiplier_adds_harbor_flag( + self, tmp_path: Path + ) -> None: + run = _fake_run() + run_harbor_evaluation( + llm=LLM(model="test/model"), + dataset="my-dataset", + output_dir=str(tmp_path), + agent_setup_timeout_multiplier=2.0, + subprocess_run=run, + ) + cmd = run.captured["cmds"][0] + assert cmd[cmd.index("--agent-setup-timeout-multiplier") + 1] == "2.0" + class TestRunHarborEvaluationTaskFiltering: """Tests for task_ids, n_limit, and fallback-retry in run_harbor_evaluation.""" From a5712d1c69dbee848e60b89862d5c01b9a6cac9b Mon Sep 17 00:00:00 2001 From: neubig Date: Wed, 5 Aug 2026 15:33:41 +0000 Subject: [PATCH 12/18] Pass TerminalBench Harbor runtime mounts --- benchmarks/utils/harbor.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/benchmarks/utils/harbor.py b/benchmarks/utils/harbor.py index 74e7e8a1e..59e707383 100644 --- a/benchmarks/utils/harbor.py +++ b/benchmarks/utils/harbor.py @@ -158,6 +158,9 @@ def run_harbor_evaluation( str(agent_setup_timeout_multiplier), ] ) + mounts_json = os.environ.get("TERMINALBENCH_HARBOR_MOUNTS_JSON") + if mounts_json: + cmd.extend(["--mounts", mounts_json]) if task_ids: normalize = normalize_task_id or (lambda task_id: task_id) From 51916bf34e400e57bf8c38b6e3fc846281055fad Mon Sep 17 00:00:00 2001 From: neubig Date: Wed, 5 Aug 2026 15:45:48 +0000 Subject: [PATCH 13/18] Support mounted runtime in PPLX Harbor agent --- harbor_agents/pplx_openhands_sdk.py | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/harbor_agents/pplx_openhands_sdk.py b/harbor_agents/pplx_openhands_sdk.py index 69ffd6f4d..04cb53a35 100644 --- a/harbor_agents/pplx_openhands_sdk.py +++ b/harbor_agents/pplx_openhands_sdk.py @@ -3,6 +3,7 @@ from __future__ import annotations +from pathlib import Path from typing import Any, override from harbor.agents.installed.openhands_sdk import OpenHandsSDK @@ -16,7 +17,22 @@ class PplxOpenHandsSDK(OpenHandsSDK): @override async def install(self, environment: BaseEnvironment) -> None: - await super().install(environment) + # A read-only evaluator-built venv is bind-mounted for batch evals. + # Harbor's stock installer unnecessarily chowns that mount after its + # existence probe, so install just the runner when it is available. + mounted_runtime = await environment.exec( + command="/opt/openhands-sdk-venv/bin/python -c 'import openhands.sdk'", + ) + if mounted_runtime.return_code == 0: + import harbor.agents.installed.openhands_sdk as adapter + + runner_path = Path(adapter.__file__).parent / "openhands_sdk_runner.py" + local_copy = self.logs_dir / "run_agent.py" + local_copy.write_text(runner_path.read_text()) + await environment.upload_file(source_path=local_copy, target_path="/installed-agent/run_agent.py") + await environment.exec(command="chmod +x /installed-agent/run_agent.py", user="root") + else: + await super().install(environment) await self.exec_as_root( environment, command=( From 77e727d263521322f5d1700224850f15f74273a9 Mon Sep 17 00:00:00 2001 From: neubig Date: Wed, 5 Aug 2026 19:26:47 +0000 Subject: [PATCH 14/18] Require PPLX bootstrap search in Harbor agent --- harbor_agents/pplx_openhands_sdk.py | 63 ++++++++++++++++++++++++++++- 1 file changed, 61 insertions(+), 2 deletions(-) diff --git a/harbor_agents/pplx_openhands_sdk.py b/harbor_agents/pplx_openhands_sdk.py index 04cb53a35..a6e135d1c 100644 --- a/harbor_agents/pplx_openhands_sdk.py +++ b/harbor_agents/pplx_openhands_sdk.py @@ -3,11 +3,15 @@ from __future__ import annotations +import json +import re +import shlex from pathlib import Path from typing import Any, override from harbor.agents.installed.openhands_sdk import OpenHandsSDK from harbor.environments.base import BaseEnvironment +from harbor.models.agent.context import AgentContext class PplxOpenHandsSDK(OpenHandsSDK): @@ -15,6 +19,11 @@ class PplxOpenHandsSDK(OpenHandsSDK): PPLX_VERSION = "v0.2.2" + @staticmethod + def _bootstrap_query(instruction: str) -> str: + """Build a bounded, task-specific query without another model call.""" + return re.sub(r"\s+", " ", instruction).strip()[:300] + @override async def install(self, environment: BaseEnvironment) -> None: # A read-only evaluator-built venv is bind-mounted for batch evals. @@ -29,8 +38,12 @@ async def install(self, environment: BaseEnvironment) -> None: runner_path = Path(adapter.__file__).parent / "openhands_sdk_runner.py" local_copy = self.logs_dir / "run_agent.py" local_copy.write_text(runner_path.read_text()) - await environment.upload_file(source_path=local_copy, target_path="/installed-agent/run_agent.py") - await environment.exec(command="chmod +x /installed-agent/run_agent.py", user="root") + await environment.upload_file( + source_path=local_copy, target_path="/installed-agent/run_agent.py" + ) + await environment.exec( + command="chmod +x /installed-agent/run_agent.py", user="root" + ) else: await super().install(environment) await self.exec_as_root( @@ -48,6 +61,52 @@ async def install(self, environment: BaseEnvironment) -> None: ), ) + @override + async def run( + self, + instruction: str, + environment: BaseEnvironment, + context: AgentContext, + ) -> None: + """Require a successful CLI search and place its evidence in the prompt.""" + query = self._bootstrap_query(instruction) + search = await self.exec_as_agent( + environment, + command=( + "mkdir -p /tmp/pplx-bootstrap && " + f"pplx search web {shlex.quote(query)} -n 5 " + "--output-dir /tmp/pplx-bootstrap --stdout-preview=500" + ), + timeout_sec=120, + ) + if search.return_code != 0: + raise RuntimeError( + "Required Perplexity bootstrap search failed: " + f"{(search.stderr or '').strip()[:2000]}" + ) + + try: + payload = json.loads(search.stdout or "") + except json.JSONDecodeError as exc: + raise RuntimeError( + "Perplexity bootstrap search returned invalid JSON" + ) from exc + if not isinstance(payload, dict) or not isinstance(payload.get("hits"), list): + raise RuntimeError("Perplexity bootstrap search JSON did not contain hits") + + research = json.dumps(payload, ensure_ascii=False)[:16000] + augmented_instruction = f"""REQUIRED PERPLEXITY RESEARCH +The agent wrapper has already run a task-specific `pplx search web` command. +Use the results below when solving the task. You may run additional `pplx` +searches when useful. Do not expose the API key. + +Search query: {query} +Search result JSON: {research} + +ORIGINAL TASK +{instruction}""" + await super().run(augmented_instruction, environment, context) + @override async def exec_as_agent( self, From 80665e4ba8762a6a25905c7a8b0c694d39cdfe71 Mon Sep 17 00:00:00 2001 From: neubig Date: Wed, 5 Aug 2026 19:36:01 +0000 Subject: [PATCH 15/18] Install PPLX CLI without task image dependencies --- harbor_agents/pplx_openhands_sdk.py | 41 +++++++++++++++++++++-------- 1 file changed, 30 insertions(+), 11 deletions(-) diff --git a/harbor_agents/pplx_openhands_sdk.py b/harbor_agents/pplx_openhands_sdk.py index a6e135d1c..35184ed27 100644 --- a/harbor_agents/pplx_openhands_sdk.py +++ b/harbor_agents/pplx_openhands_sdk.py @@ -3,6 +3,7 @@ from __future__ import annotations +import base64 import json import re import shlex @@ -24,6 +25,34 @@ def _bootstrap_query(instruction: str) -> str: """Build a bounded, task-specific query without another model call.""" return re.sub(r"\s+", " ", instruction).strip()[:300] + @classmethod + def _install_command(cls) -> str: + """Install the CLI using only the portable Python runtime.""" + script = f"""import hashlib +import os +import urllib.request + +asset = "pplx-x86_64-linux-gnu.bin" +base = "https://github.com/perplexityai/perplexity-cli/releases/download/{cls.PPLX_VERSION}" +with urllib.request.urlopen(f"{{base}}/SHA256SUMS", timeout=60) as response: + checksum_lines = response.read().decode().splitlines() +expected = next(line.split()[0] for line in checksum_lines if line.split()[-1] == asset) +with urllib.request.urlopen(f"{{base}}/{{asset}}", timeout=120) as response: + binary = response.read() +actual = hashlib.sha256(binary).hexdigest() +if actual != expected: + raise RuntimeError(f"pplx checksum mismatch: expected {{expected}}, got {{actual}}") +with open("/usr/local/bin/pplx", "wb") as output: + output.write(binary) +os.chmod("/usr/local/bin/pplx", 0o755) +""" + encoded = base64.b64encode(script.encode()).decode() + return ( + "/opt/openhands-sdk-venv/bin/python -c " + f"{shlex.quote(f'import base64; exec(base64.b64decode({encoded!r}))')}" + " && pplx --version" + ) + @override async def install(self, environment: BaseEnvironment) -> None: # A read-only evaluator-built venv is bind-mounted for batch evals. @@ -48,17 +77,7 @@ async def install(self, environment: BaseEnvironment) -> None: await super().install(environment) await self.exec_as_root( environment, - command=( - "set -euo pipefail; " - "asset=pplx-x86_64-linux-gnu.bin; tmp=$(mktemp -d); " - "trap 'rm -rf \"$tmp\"' EXIT; " - f"base=https://github.com/perplexityai/perplexity-cli/releases/download/{self.PPLX_VERSION}; " - 'curl -fsSL --retry 3 "$base/SHA256SUMS" -o "$tmp/SHA256SUMS"; ' - 'curl -fsSL --retry 3 "$base/$asset" -o "$tmp/$asset"; ' - '(cd "$tmp" && grep " $asset$" SHA256SUMS | sha256sum -c -); ' - 'install -m 0755 "$tmp/$asset" /usr/local/bin/pplx; ' - "pplx --version" - ), + command=self._install_command(), ) @override From 29766e341ff71b0070a3e810b84f2b58f7295625 Mon Sep 17 00:00:00 2001 From: neubig Date: Wed, 5 Aug 2026 20:38:23 +0000 Subject: [PATCH 16/18] Use PPLX binary from mounted Harbor runtime --- harbor_agents/pplx_openhands_sdk.py | 25 +++++++++++++++++-------- 1 file changed, 17 insertions(+), 8 deletions(-) diff --git a/harbor_agents/pplx_openhands_sdk.py b/harbor_agents/pplx_openhands_sdk.py index 35184ed27..5bccca695 100644 --- a/harbor_agents/pplx_openhands_sdk.py +++ b/harbor_agents/pplx_openhands_sdk.py @@ -30,18 +30,27 @@ def _install_command(cls) -> str: """Install the CLI using only the portable Python runtime.""" script = f"""import hashlib import os +import ssl import urllib.request +import certifi + asset = "pplx-x86_64-linux-gnu.bin" base = "https://github.com/perplexityai/perplexity-cli/releases/download/{cls.PPLX_VERSION}" -with urllib.request.urlopen(f"{{base}}/SHA256SUMS", timeout=60) as response: - checksum_lines = response.read().decode().splitlines() -expected = next(line.split()[0] for line in checksum_lines if line.split()[-1] == asset) -with urllib.request.urlopen(f"{{base}}/{{asset}}", timeout=120) as response: - binary = response.read() -actual = hashlib.sha256(binary).hexdigest() -if actual != expected: - raise RuntimeError(f"pplx checksum mismatch: expected {{expected}}, got {{actual}}") +mounted_binary = "/opt/openhands-sdk-venv/bin/pplx" +if os.path.isfile(mounted_binary): + with open(mounted_binary, "rb") as source: + binary = source.read() +else: + context = ssl.create_default_context(cafile=certifi.where()) + with urllib.request.urlopen(f"{{base}}/SHA256SUMS", timeout=60, context=context) as response: + checksum_lines = response.read().decode().splitlines() + expected = next(line.split()[0] for line in checksum_lines if line.split()[-1] == asset) + with urllib.request.urlopen(f"{{base}}/{{asset}}", timeout=120, context=context) as response: + binary = response.read() + actual = hashlib.sha256(binary).hexdigest() + if actual != expected: + raise RuntimeError(f"pplx checksum mismatch: expected {{expected}}, got {{actual}}") with open("/usr/local/bin/pplx", "wb") as output: output.write(binary) os.chmod("/usr/local/bin/pplx", 0o755) From ca38ea72dd130e70f20297c8089c4c7683e0e69c Mon Sep 17 00:00:00 2001 From: neubig Date: Wed, 5 Aug 2026 23:12:48 +0000 Subject: [PATCH 17/18] Checkpoint Harbor trials across Kubernetes retries --- benchmarks/terminalbench/run_infer.py | 153 ++++++++++++++++++++++++-- benchmarks/utils/harbor.py | 52 ++++++++- 2 files changed, 191 insertions(+), 14 deletions(-) diff --git a/benchmarks/terminalbench/run_infer.py b/benchmarks/terminalbench/run_infer.py index 925f37389..707922e4a 100644 --- a/benchmarks/terminalbench/run_infer.py +++ b/benchmarks/terminalbench/run_infer.py @@ -13,6 +13,8 @@ import os import subprocess import sys +import tempfile +import threading from datetime import datetime, timezone from pathlib import Path @@ -21,6 +23,7 @@ from benchmarks.utils.harbor import ( HarborCredentialMode, check_harbor_installed as _check_harbor_installed, + completed_harbor_task_ids, convert_harbor_to_eval_output, get_supported_task_filter_flag, run_harbor_evaluation as _run_harbor_evaluation, @@ -33,6 +36,106 @@ # Output filename for results OUTPUT_FILENAME = "output.jsonl" +CHECKPOINT_INTERVAL_SECONDS = 60 + + +def _checkpoint_gcs_uri() -> str | None: + bucket = os.environ.get("RESULTS_BUCKET") + model_slug = os.environ.get("MODEL_SLUG") + run_id = os.environ.get("GITHUB_RUN_ID") + if not bucket or not model_slug or not run_id: + return None + return f"gs://{bucket}/terminalbench/{model_slug}/{run_id}/checkpoint.tar.gz" + + +def restore_harbor_checkpoint(structured_output_dir: Path) -> bool: + """Restore incremental Harbor results after a Kubernetes pod retry.""" + checkpoint_uri = _checkpoint_gcs_uri() + if checkpoint_uri is None: + return False + stat = subprocess.run( + ["gsutil", "-q", "stat", checkpoint_uri], + capture_output=True, + text=True, + ) + if stat.returncode != 0: + return False + + structured_output_dir.parent.mkdir(parents=True, exist_ok=True) + with tempfile.TemporaryDirectory(prefix="terminalbench-checkpoint-") as tmpdir: + archive = Path(tmpdir) / "checkpoint.tar.gz" + subprocess.run(["gsutil", "cp", checkpoint_uri, str(archive)], check=True) + subprocess.run( + [ + "tar", + "-xzf", + str(archive), + "-C", + str(structured_output_dir.parent), + ], + check=True, + ) + logger.info("Restored Harbor checkpoint from %s", checkpoint_uri) + return True + + +def upload_harbor_checkpoint( + harbor_output_dir: Path, + output_path: Path, +) -> bool: + """Convert and upload all complete Harbor trials seen so far.""" + checkpoint_uri = _checkpoint_gcs_uri() + if checkpoint_uri is None or not harbor_output_dir.exists(): + return False + if not completed_harbor_task_ids(harbor_output_dir): + return False + + convert_harbor_to_eval_output( + harbor_output_dir=harbor_output_dir, + eval_output_path=output_path, + ) + with tempfile.TemporaryDirectory(prefix="terminalbench-checkpoint-") as tmpdir: + archive = Path(tmpdir) / "checkpoint.tar.gz" + subprocess.run( + [ + "tar", + "-czf", + str(archive), + "-C", + str(output_path.parent.parent), + output_path.parent.name, + ], + check=True, + ) + subprocess.run( + [ + "gsutil", + "-h", + "Cache-Control:no-cache, no-store, must-revalidate", + "cp", + str(archive), + checkpoint_uri, + ], + check=True, + ) + logger.info( + "Uploaded resumable Harbor checkpoint with %d tasks to %s", + len(completed_harbor_task_ids(harbor_output_dir)), + checkpoint_uri, + ) + return True + + +def _checkpoint_loop( + stop_event: threading.Event, + harbor_output_dir: Path, + output_path: Path, +) -> None: + while not stop_event.wait(CHECKPOINT_INTERVAL_SECONDS): + try: + upload_harbor_checkpoint(harbor_output_dir, output_path) + except Exception: + logger.exception("Failed to upload Harbor checkpoint; will retry") def check_harbor_installed() -> bool: @@ -238,6 +341,11 @@ def main() -> None: ) logger.info(f"Output directory: {structured_output_dir}") + structured_output_path = Path(structured_output_dir) + try: + restore_harbor_checkpoint(structured_output_path) + except Exception: + logger.exception("Failed to restore Harbor checkpoint; starting without it") os.makedirs(structured_output_dir, exist_ok=True) # Save metadata @@ -267,20 +375,49 @@ def main() -> None: if not args.skip_harbor: # Run harbor evaluation try: - harbor_output_dir = run_harbor_evaluation( - llm=llm, - dataset=args.dataset, - output_dir=structured_output_dir, - num_workers=args.num_workers, - task_ids=task_ids, - n_limit=args.n_limit, - ) + harbor_output_dir = Path(structured_output_dir) / "harbor_output" + if task_ids and harbor_output_dir.exists(): + completed_ids = completed_harbor_task_ids(harbor_output_dir) + if completed_ids: + task_ids = [ + task_id for task_id in task_ids if task_id not in completed_ids + ] + logger.info( + "Restored %d completed Harbor tasks; %d selected tasks remain", + len(completed_ids), + len(task_ids), + ) + + if task_ids is None or task_ids: + stop_event = threading.Event() + checkpoint_thread = threading.Thread( + target=_checkpoint_loop, + args=(stop_event, harbor_output_dir, output_path), + name="terminalbench-checkpoint", + daemon=True, + ) + checkpoint_thread.start() + try: + harbor_output_dir = run_harbor_evaluation( + llm=llm, + dataset=args.dataset, + output_dir=structured_output_dir, + num_workers=args.num_workers, + task_ids=task_ids, + n_limit=args.n_limit, + ) + finally: + stop_event.set() + checkpoint_thread.join() + else: + logger.info("All selected tasks were restored; skipping Harbor execution") # Convert harbor output to standard format convert_harbor_to_eval_output( harbor_output_dir=harbor_output_dir, eval_output_path=output_path, ) + upload_harbor_checkpoint(harbor_output_dir, output_path) except Exception as e: logger.error(f"Evaluation failed: {e}") diff --git a/benchmarks/utils/harbor.py b/benchmarks/utils/harbor.py index 59e707383..3fb0de48a 100644 --- a/benchmarks/utils/harbor.py +++ b/benchmarks/utils/harbor.py @@ -231,6 +231,27 @@ def _find_job_dir(harbor_output_dir: Path) -> Path: return sorted(candidates)[-1] +def find_harbor_trial_result_files(harbor_output_dir: Path) -> list[Path]: + """Return trial results from every timestamped Harbor attempt.""" + return sorted(harbor_output_dir.glob("*/*/result.json")) + + +def completed_harbor_task_ids(harbor_output_dir: Path) -> set[str]: + """Return task IDs with a complete, parseable Harbor trial result.""" + completed: set[str] = set() + for result_file in find_harbor_trial_result_files(harbor_output_dir): + try: + with result_file.open() as stream: + trial = json.load(stream) + except (json.JSONDecodeError, OSError): + continue + task_name = trial.get("task_name") + rewards = (trial.get("verifier_result") or {}).get("rewards") or {} + if isinstance(task_name, str) and task_name and "reward" in rewards: + completed.add(task_name) + return completed + + def convert_harbor_to_eval_output( harbor_output_dir: Path, eval_output_path: Path, @@ -241,17 +262,36 @@ def convert_harbor_to_eval_output( logger.info(f"Converting harbor output from {harbor_output_dir}") canonicalize = canonicalize_instance_id or (lambda instance_id: instance_id) - job_dir = _find_job_dir(harbor_output_dir) - logger.info(f"Using harbor job directory: {job_dir}") - - result_files = [f for f in job_dir.glob("*/result.json") if f.parent != job_dir] + result_files = find_harbor_trial_result_files(harbor_output_dir) if not result_files: + # Preserve the established distinction between no Harbor job at all + # and a completed job that contains no trial results. + job_dir = _find_job_dir(harbor_output_dir) raise RuntimeError( f"No trial result files found in {job_dir}. " - f"Expected result.json files in trial subdirectories." + "Expected result.json files in timestamp/trial subdirectories." ) - logger.info(f"Found {len(result_files)} trial results in {job_dir}") + logger.info( + f"Found {len(result_files)} trial results across Harbor attempts in " + f"{harbor_output_dir}" + ) + + # A retry normally filters restored IDs, but a crash can occur between a + # result write and the next checkpoint. Prefer the later timestamped copy. + latest_result_by_instance: dict[str, Path] = {} + unreadable_result_files: list[Path] = [] + for result_file in result_files: + try: + with result_file.open() as stream: + trial = json.load(stream) + instance_id = canonicalize( + trial.get("task_name", result_file.parent.name) + ) + latest_result_by_instance[instance_id] = result_file + except (json.JSONDecodeError, OSError): + unreadable_result_files.append(result_file) + result_files = list(latest_result_by_instance.values()) + unreadable_result_files results: list[dict] = [] errors: list[dict] = [] From b09429c2b33858ea8d5d4f4f393c267324368814 Mon Sep 17 00:00:00 2001 From: neubig Date: Thu, 6 Aug 2026 22:29:04 +0000 Subject: [PATCH 18/18] chore: trigger CI Co-authored-by: openhands