From d30e32c607cb7f8a58be6044f7e767190865cd3a Mon Sep 17 00:00:00 2001 From: neubig Date: Thu, 16 Jul 2026 18:48:49 +0000 Subject: [PATCH 1/6] Support proxy credentials for Pier mini-SWE runs Preserve nested agent kwargs and expose the existing LiteLLM proxy credential under the OpenAI-compatible aliases expected by mini-swe-agent. Co-authored-by: openhands --- benchmarks/harbor/run_infer.py | 32 +++++++++++++++++++++++----- tests/test_harbor_run_infer.py | 38 ++++++++++++++++++++++++++++++++++ 2 files changed, 65 insertions(+), 5 deletions(-) diff --git a/benchmarks/harbor/run_infer.py b/benchmarks/harbor/run_infer.py index bef91c423..4e9aa362b 100644 --- a/benchmarks/harbor/run_infer.py +++ b/benchmarks/harbor/run_infer.py @@ -154,12 +154,36 @@ def _split_json_values(raw: str | None) -> list[str]: return [] data = json.loads(raw) if isinstance(data, dict): - return [f"{key}={value}" for key, value in data.items()] + return [ + f"{key}={value if isinstance(value, str) else json.dumps(value, separators=(',', ':'))}" + for key, value in data.items() + ] if isinstance(data, list) and all(isinstance(item, str) for item in data): return data raise ValueError("Expected a JSON object or list of KEY=VALUE strings") +def _llm_agent_env(llm: LLM, harbor_agent: str) -> list[str]: + """Build credential env flags for the selected Harbor-compatible agent. + + The OpenHands SDK agent consumes the generic ``LLM_*`` variables. Pier's + mini-SWE agent resolves ``litellm_proxy/*`` through LiteLLM, which consumes + the OpenAI-compatible aliases instead. Both names carry the same proxy + credential and endpoint. + """ + values: list[str] = [] + if llm.api_key: + api_key = _secret_value(llm.api_key) + values.append(f"LLM_API_KEY={api_key}") + if harbor_agent == "mini-swe-agent": + values.append(f"OPENAI_API_KEY={api_key}") + if llm.base_url: + values.append(f"LLM_BASE_URL={llm.base_url}") + if harbor_agent == "mini-swe-agent": + values.append(f"OPENAI_BASE_URL={llm.base_url}") + return values + + def run_harbor( args: argparse.Namespace, llm: LLM, @@ -186,10 +210,8 @@ def run_harbor( str(args.num_workers), ] - if llm.api_key: - cmd.extend(["--ae", f"LLM_API_KEY={_secret_value(llm.api_key)}"]) - if llm.base_url: - cmd.extend(["--ae", f"LLM_BASE_URL={llm.base_url}"]) + for env_value in _llm_agent_env(llm, args.harbor_agent): + cmd.extend(["--ae", env_value]) for env_value in _parse_key_value( [*args.agent_env, *_split_json_values(args.agent_env_json)] ): diff --git a/tests/test_harbor_run_infer.py b/tests/test_harbor_run_infer.py index 7d2d4167f..57053aa0f 100644 --- a/tests/test_harbor_run_infer.py +++ b/tests/test_harbor_run_infer.py @@ -10,12 +10,14 @@ from benchmarks.harbor.run_infer import ( _is_sensitive_value, + _llm_agent_env, _load_task_ids, _parse_key_value, _resolve_target, _split_json_values, _target_args, ) +from openhands.sdk import LLM def test_load_task_ids_strips_and_ignores_comments(tmp_path: Path) -> None: @@ -58,6 +60,12 @@ def test_split_json_values_dict() -> None: assert _split_json_values('{"A": "1", "B": "2"}') == ["A=1", "B=2"] +def test_split_json_values_preserves_nested_json() -> None: + assert _split_json_values( + '{"model_kwargs": {"temperature": 1.0, "top_p": 0.95}}' + ) == ['model_kwargs={"temperature":1.0,"top_p":0.95}'] + + def test_split_json_values_list() -> None: assert _split_json_values('["A=1", "B=2"]') == ["A=1", "B=2"] @@ -67,6 +75,36 @@ def test_split_json_values_invalid() -> None: _split_json_values('"not-an-object-or-list"') +def test_llm_agent_env_uses_generic_names_for_openhands() -> None: + assert _llm_agent_env( + LLM( + model="litellm_proxy/test/model", + api_key="secret", + base_url="https://proxy.example.com", + ), + "openhands-sdk", + ) == [ + "LLM_API_KEY=secret", + "LLM_BASE_URL=https://proxy.example.com", + ] + + +def test_llm_agent_env_adds_openai_proxy_aliases_for_mini_swe() -> None: + assert _llm_agent_env( + LLM( + model="litellm_proxy/test/model", + api_key="secret", + base_url="https://proxy.example.com", + ), + "mini-swe-agent", + ) == [ + "LLM_API_KEY=secret", + "OPENAI_API_KEY=secret", + "LLM_BASE_URL=https://proxy.example.com", + "OPENAI_BASE_URL=https://proxy.example.com", + ] + + def _make_args(**kwargs: object) -> argparse.Namespace: defaults: dict[str, object] = dict( harbor_target=None, From 0770cbe9a9ca3c901e08f20efe242926239ab583 Mon Sep 17 00:00:00 2001 From: neubig Date: Thu, 16 Jul 2026 22:59:10 +0000 Subject: [PATCH 2/6] Add bounded-history model for MiniSWE evaluations --- extras/miniswe-context-model/README.md | 15 +++ extras/miniswe-context-model/pyproject.toml | 12 +++ .../__init__.py | 3 + .../history.py | 94 +++++++++++++++++++ .../openhands_miniswe_context_model/model.py | 35 +++++++ tests/test_miniswe_context_history.py | 63 +++++++++++++ 6 files changed, 222 insertions(+) create mode 100644 extras/miniswe-context-model/README.md create mode 100644 extras/miniswe-context-model/pyproject.toml create mode 100644 extras/miniswe-context-model/src/openhands_miniswe_context_model/__init__.py create mode 100644 extras/miniswe-context-model/src/openhands_miniswe_context_model/history.py create mode 100644 extras/miniswe-context-model/src/openhands_miniswe_context_model/model.py create mode 100644 tests/test_miniswe_context_history.py diff --git a/extras/miniswe-context-model/README.md b/extras/miniswe-context-model/README.md new file mode 100644 index 000000000..1b732de37 --- /dev/null +++ b/extras/miniswe-context-model/README.md @@ -0,0 +1,15 @@ +# MiniSWE context model + +This small optional package provides a `LitellmModel` subclass for long-running +MiniSWE evaluations. It retains the system prompt, task, and newest complete +assistant/tool turns while eliding old turns before a request exceeds the configured +serialized-history budget. + +Install it in the MiniSWE tool environment and select: + +```text +openhands_miniswe_context_model.model.ContextSafeLitellmModel +``` + +The `model.max_history_chars` config field defaults to 500,000 characters and can +be overridden by the benchmark's agent configuration. diff --git a/extras/miniswe-context-model/pyproject.toml b/extras/miniswe-context-model/pyproject.toml new file mode 100644 index 000000000..8363cd70f --- /dev/null +++ b/extras/miniswe-context-model/pyproject.toml @@ -0,0 +1,12 @@ +[build-system] +requires = ["setuptools>=68"] +build-backend = "setuptools.build_meta" + +[project] +name = "openhands-miniswe-context-model" +version = "0.1.0" +description = "Bounded-history LiteLLM model wrapper for mini-swe-agent evaluations" +requires-python = ">=3.11" + +[tool.setuptools.packages.find] +where = ["src"] diff --git a/extras/miniswe-context-model/src/openhands_miniswe_context_model/__init__.py b/extras/miniswe-context-model/src/openhands_miniswe_context_model/__init__.py new file mode 100644 index 000000000..e31b35ca1 --- /dev/null +++ b/extras/miniswe-context-model/src/openhands_miniswe_context_model/__init__.py @@ -0,0 +1,3 @@ +"""Context-safe model helpers for mini-swe-agent evaluations.""" + +__version__ = "0.1.0" diff --git a/extras/miniswe-context-model/src/openhands_miniswe_context_model/history.py b/extras/miniswe-context-model/src/openhands_miniswe_context_model/history.py new file mode 100644 index 000000000..0fa6cdacc --- /dev/null +++ b/extras/miniswe-context-model/src/openhands_miniswe_context_model/history.py @@ -0,0 +1,94 @@ +"""History bounding that preserves tool-call protocol boundaries.""" + +from __future__ import annotations + +import json +from collections.abc import Sequence +from typing import Any + + +ELISION_NOTICE = ( + "[Earlier assistant and tool interactions were omitted to stay within the " + "model context window. Their effects remain in the current workspace.]" +) + + +def _message_size(message: dict[str, Any]) -> int: + return len( + json.dumps( + message, + ensure_ascii=False, + separators=(",", ":"), + default=str, + ) + ) + + +def _history_units(messages: Sequence[dict[str, Any]]) -> list[list[dict[str, Any]]]: + """Group assistant messages with their following tool results.""" + units: list[list[dict[str, Any]]] = [] + for message in messages: + if ( + message.get("role") == "tool" + and units + and units[-1][0].get("role") == "assistant" + ): + units[-1].append(message) + else: + units.append([message]) + return units + + +def bound_message_history( + messages: Sequence[dict[str, Any]], + *, + max_chars: int, + preserve_first: int = 2, + min_recent_units: int = 8, +) -> list[dict[str, Any]]: + """Drop the oldest complete turns when serialized history exceeds a budget. + + The initial system/task messages are always retained. Assistant tool calls and + their tool results are grouped into atomic units, so trimming never leaves an + orphaned tool response in the request sent to the model. + """ + if max_chars <= 0: + raise ValueError("max_chars must be positive") + if preserve_first < 0: + raise ValueError("preserve_first cannot be negative") + if min_recent_units < 1: + raise ValueError("min_recent_units must be at least one") + + copied = [dict(message) for message in messages] + if sum(_message_size(message) for message in copied) <= max_chars: + return copied + + header = copied[:preserve_first] + units = _history_units(copied[preserve_first:]) + notice = {"role": "user", "content": ELISION_NOTICE} + remaining = ( + max_chars + - sum(_message_size(message) for message in header) + - _message_size(notice) + ) + + kept_reversed: list[list[dict[str, Any]]] = [] + used = 0 + for unit in reversed(units): + unit_size = sum(_message_size(message) for message in unit) + if ( + kept_reversed + and used + unit_size > remaining + and len(kept_reversed) >= min_recent_units + ): + break + kept_reversed.append(unit) + used += unit_size + + kept = [message for unit in reversed(kept_reversed) for message in unit] + dropped_count = len(copied) - len(header) - len(kept) + if dropped_count <= 0: + return copied + + notice["content"] += f" ({dropped_count} messages omitted.)" + return [*header, notice, *kept] diff --git a/extras/miniswe-context-model/src/openhands_miniswe_context_model/model.py b/extras/miniswe-context-model/src/openhands_miniswe_context_model/model.py new file mode 100644 index 000000000..7197ff72a --- /dev/null +++ b/extras/miniswe-context-model/src/openhands_miniswe_context_model/model.py @@ -0,0 +1,35 @@ +"""A LiteLLM mini-swe-agent model with bounded request history.""" + +from __future__ import annotations + +from typing import Any + +from minisweagent.models.litellm_model import LitellmModel, LitellmModelConfig +from pydantic import Field + +from openhands_miniswe_context_model.history import bound_message_history + + +class ContextSafeLitellmModelConfig(LitellmModelConfig): + max_history_chars: int = Field(default=500_000, gt=0) + """Maximum serialized characters retained in requests to the model.""" + + min_recent_history_units: int = Field(default=8, ge=1) + """Minimum number of recent assistant/tool units to retain.""" + + +class ContextSafeLitellmModel(LitellmModel): + """Retain task setup and recent complete turns below a context-safe budget.""" + + config: ContextSafeLitellmModelConfig + + def __init__(self, **kwargs: Any): + super().__init__(config_class=ContextSafeLitellmModelConfig, **kwargs) + + def _prepare_messages_for_api(self, messages: list[dict]) -> list[dict]: + prepared = super()._prepare_messages_for_api(messages) + return bound_message_history( + prepared, + max_chars=self.config.max_history_chars, + min_recent_units=self.config.min_recent_history_units, + ) diff --git a/tests/test_miniswe_context_history.py b/tests/test_miniswe_context_history.py new file mode 100644 index 000000000..b03c49cdf --- /dev/null +++ b/tests/test_miniswe_context_history.py @@ -0,0 +1,63 @@ +from __future__ import annotations + +import importlib.util +from pathlib import Path + + +_HISTORY_PATH = ( + Path(__file__).parents[1] + / "extras/miniswe-context-model/src/openhands_miniswe_context_model/history.py" +) +_SPEC = importlib.util.spec_from_file_location("miniswe_context_history", _HISTORY_PATH) +assert _SPEC and _SPEC.loader +_HISTORY = importlib.util.module_from_spec(_SPEC) +_SPEC.loader.exec_module(_HISTORY) +bound_message_history = _HISTORY.bound_message_history + + +def _turn(index: int, size: int = 80) -> list[dict]: + return [ + { + "role": "assistant", + "content": f"turn-{index}", + "tool_calls": [{"id": f"call-{index}", "function": {"name": "bash"}}], + }, + {"role": "tool", "tool_call_id": f"call-{index}", "content": "x" * size}, + ] + + +def test_short_history_is_unchanged() -> None: + messages = [ + {"role": "system", "content": "system"}, + {"role": "user", "content": "task"}, + *_turn(1), + ] + + assert bound_message_history(messages, max_chars=10_000) == messages + + +def test_long_history_keeps_header_and_complete_recent_turns() -> None: + messages = [ + {"role": "system", "content": "system"}, + {"role": "user", "content": "task"}, + *[message for index in range(12) for message in _turn(index, size=120)], + ] + + bounded = bound_message_history(messages, max_chars=1_200, min_recent_units=3) + + assert bounded[:2] == messages[:2] + assert "messages omitted" in bounded[2]["content"] + assert bounded[-2:] == messages[-2:] + for index, message in enumerate(bounded): + if message["role"] == "tool": + assert index > 0 + assert bounded[index - 1]["role"] in {"assistant", "tool"} + + +def test_history_budget_validation() -> None: + try: + bound_message_history([], max_chars=0) + except ValueError as exc: + assert "max_chars" in str(exc) + else: + raise AssertionError("Expected invalid max_chars to raise") From a50c04132004cf7ff6c77b3568185ee40f8e5154 Mon Sep 17 00:00:00 2001 From: neubig Date: Fri, 17 Jul 2026 06:46:48 +0000 Subject: [PATCH 3/6] Honor verifier rewards after agent failures --- benchmarks/utils/harbor.py | 15 ++++++++++++--- tests/test_terminalbench.py | 33 +++++++++++++++++++++++++++++++++ 2 files changed, 45 insertions(+), 3 deletions(-) diff --git a/benchmarks/utils/harbor.py b/benchmarks/utils/harbor.py index 4dda977ce..a6245e7a7 100644 --- a/benchmarks/utils/harbor.py +++ b/benchmarks/utils/harbor.py @@ -236,7 +236,16 @@ def convert_harbor_to_eval_output( instance_id = canonicalize(trial.get("task_name", result_file.parent.name)) - if trial.get("exception_info"): + verifier_result = trial.get("verifier_result") or {} + rewards = verifier_result.get("rewards") or {} + has_primary_reward = "reward" in rewards + + # Harbor can still run the verifier after an agent timeout or other + # non-zero exit. In that case the verifier is authoritative: a reward + # of 1 is a resolved task, and a reward of 0 is a completed unresolved + # task. Only emit an incomplete error when no primary verifier reward + # exists (for example, setup or verifier failures). + if trial.get("exception_info") and not has_primary_reward: errors.append( { "instance_id": instance_id, @@ -246,8 +255,6 @@ def convert_harbor_to_eval_output( ) continue - verifier_result = trial.get("verifier_result", {}) - rewards = verifier_result.get("rewards", {}) passed = rewards.get("reward", 0.0) > 0 agent_result = trial.get("agent_result", {}) @@ -270,6 +277,8 @@ def convert_harbor_to_eval_output( "total_cost_usd": agent_result.get("cost_usd") or 0.0, }, } + if trial.get("exception_info"): + eval_entry["test_result"]["agent_exception"] = trial["exception_info"] results.append(eval_entry) logger.info( f"Processed trial {instance_id}: reward={rewards.get('reward', 'N/A')}" diff --git a/tests/test_terminalbench.py b/tests/test_terminalbench.py index d9b62e07e..4da48b9e9 100644 --- a/tests/test_terminalbench.py +++ b/tests/test_terminalbench.py @@ -402,6 +402,39 @@ def test_trial_with_exception(self, tmp_path: Path) -> None: assert report["error_instances"] == 1 assert report["incomplete_ids"] == ["error-task"] + def test_trial_with_exception_and_verifier_reward_is_scored( + self, tmp_path: Path + ) -> None: + """A post-agent verifier result remains authoritative after a timeout.""" + trial_result = { + "task_name": "completed-before-timeout", + "trial_name": "completed-before-timeout__abc", + "trial_uri": "file:///path/to/trial", + "agent_result": { + "n_input_tokens": 500, + "n_output_tokens": 100, + "cost_usd": 0.01, + }, + "verifier_result": {"rewards": {"reward": 1.0}}, + "exception_info": { + "type": "AgentTimeoutError", + "message": "Agent execution timed out", + }, + } + + harbor_dir = self._create_harbor_structure( + tmp_path, [("completed-before-timeout__abc", trial_result)] + ) + output_file = tmp_path / "output.jsonl" + + convert_harbor_to_eval_output(harbor_dir, output_file) + + entry = json.loads(output_file.read_text()) + assert entry["error"] is None + assert entry["test_result"]["passed"] is True + assert entry["test_result"]["rewards"] == {"reward": 1.0} + assert entry["test_result"]["agent_exception"] == trial_result["exception_info"] + def test_mixed_valid_and_exception_trials(self, tmp_path: Path) -> None: """Test handling mix of successful and exception trials.""" trials = [ From 5e8c8118f2172d8cc2afa63da32d389aaf57209b Mon Sep 17 00:00:00 2001 From: neubig Date: Fri, 17 Jul 2026 07:01:49 +0000 Subject: [PATCH 4/6] Refresh Harbor results before evaluation --- benchmarks/harbor/eval_infer.py | 26 ++++++++++++ tests/test_harbor_eval_infer.py | 75 +++++++++++++++++++++++++++++++++ 2 files changed, 101 insertions(+) create mode 100644 tests/test_harbor_eval_infer.py diff --git a/benchmarks/harbor/eval_infer.py b/benchmarks/harbor/eval_infer.py index fe4c33473..7a9811859 100644 --- a/benchmarks/harbor/eval_infer.py +++ b/benchmarks/harbor/eval_infer.py @@ -9,6 +9,7 @@ from pathlib import Path from typing import Any +from benchmarks.utils.harbor import convert_harbor_to_eval_output from benchmarks.utils.laminar import LaminarService from benchmarks.utils.report_costs import generate_cost_report from openhands.sdk import get_logger @@ -17,6 +18,30 @@ logger = get_logger(__name__) +def refresh_eval_output_from_harbor(input_file: Path) -> bool: + """Rebuild converted output from raw Harbor results when they are available. + + Inference archives contain both ``output.jsonl`` and the authoritative + ``harbor_output`` tree. Re-running the converter in the evaluation phase + lets converter fixes take effect during a rescore and avoids permanently + classifying verifier-scored agent failures according to stale JSONL. + """ + harbor_output_dir = input_file.parent / "harbor_output" + if not harbor_output_dir.is_dir(): + logger.info( + "Raw Harbor output not found at %s; using existing converted output", + harbor_output_dir, + ) + return False + + convert_harbor_to_eval_output( + harbor_output_dir=harbor_output_dir, + eval_output_path=input_file, + ) + logger.info("Refreshed %s from raw Harbor results", input_file) + return True + + def _metric( data: dict[str, Any], test_result: dict[str, Any], key: str, default: Any ) -> Any: @@ -143,6 +168,7 @@ def main() -> None: else input_file.with_suffix(".report.json") ) try: + refresh_eval_output_from_harbor(input_file) process_harbor_results(str(input_file), str(output_file)) generate_cost_report(str(input_file)) except Exception as exc: diff --git a/tests/test_harbor_eval_infer.py b/tests/test_harbor_eval_infer.py new file mode 100644 index 000000000..687daebb4 --- /dev/null +++ b/tests/test_harbor_eval_infer.py @@ -0,0 +1,75 @@ +"""Tests for the generic Harbor evaluation phase.""" + +import json +from pathlib import Path + +from benchmarks.harbor.eval_infer import ( + process_harbor_results, + refresh_eval_output_from_harbor, +) + + +def test_refresh_reconverts_authoritative_harbor_results(tmp_path: Path) -> None: + output_file = tmp_path / "output.jsonl" + output_file.write_text( + json.dumps( + { + "instance_id": "finished-before-timeout", + "test_result": {}, + "error": "stale timeout classification", + } + ) + + "\n" + ) + + job_dir = tmp_path / "harbor_output" / "2026-01-01__00-00-00" + trial_dir = job_dir / "finished-before-timeout__abc" + trial_dir.mkdir(parents=True) + (job_dir / "result.json").write_text(json.dumps({"id": "job"})) + (trial_dir / "result.json").write_text( + json.dumps( + { + "task_name": "finished-before-timeout", + "trial_name": "finished-before-timeout__abc", + "agent_result": { + "n_input_tokens": 10, + "n_output_tokens": 2, + "cost_usd": 0.0, + }, + "verifier_result": {"rewards": {"reward": 1.0}}, + "exception_info": { + "exception_type": "AgentTimeoutError", + "exception_message": "Agent timed out after saving the answer", + }, + } + ) + ) + + assert refresh_eval_output_from_harbor(output_file) is True + + report_file = tmp_path / "output.report.json" + report = process_harbor_results(str(output_file), str(report_file)) + converted = json.loads(output_file.read_text()) + + assert report["completed_instances"] == 1 + assert report["resolved_instances"] == 1 + assert report["error_instances"] == 0 + assert converted["test_result"]["agent_exception"]["exception_type"] == ( + "AgentTimeoutError" + ) + + +def test_refresh_keeps_existing_output_without_raw_harbor_results( + tmp_path: Path, +) -> None: + output_file = tmp_path / "output.jsonl" + original = ( + json.dumps( + {"instance_id": "existing", "test_result": {"passed": False}, "error": None} + ) + + "\n" + ) + output_file.write_text(original) + + assert refresh_eval_output_from_harbor(output_file) is False + assert output_file.read_text() == original From 115bcdf35fd9c9ea55ef7f6f96604bb1739b28ee Mon Sep 17 00:00:00 2001 From: neubig Date: Fri, 17 Jul 2026 10:26:50 +0000 Subject: [PATCH 5/6] Keep optional MiniSWE import type-checkable --- .../src/openhands_miniswe_context_model/model.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/extras/miniswe-context-model/src/openhands_miniswe_context_model/model.py b/extras/miniswe-context-model/src/openhands_miniswe_context_model/model.py index 7197ff72a..292ca3e0c 100644 --- a/extras/miniswe-context-model/src/openhands_miniswe_context_model/model.py +++ b/extras/miniswe-context-model/src/openhands_miniswe_context_model/model.py @@ -4,7 +4,10 @@ from typing import Any -from minisweagent.models.litellm_model import LitellmModel, LitellmModelConfig +from minisweagent.models.litellm_model import ( # pyright: ignore[reportMissingImports] + LitellmModel, + LitellmModelConfig, +) from pydantic import Field from openhands_miniswe_context_model.history import bound_message_history From 652d92dcd1186cb719aaadb222a31c93d7cd2c1c Mon Sep 17 00:00:00 2001 From: openhands Date: Sun, 19 Jul 2026 08:47:32 +0000 Subject: [PATCH 6/6] Record Harbor live validation Co-authored-by: openhands