Improve generic Harbor and MiniSWE evaluation resilience

benchmarks/harbor/eval_infer.py

@@ -9,6 +9,7 @@
 from pathlib import Path
 from typing import Any
 
+from benchmarks.utils.harbor import convert_harbor_to_eval_output
 from benchmarks.utils.laminar import LaminarService
 from benchmarks.utils.report_costs import generate_cost_report
 from openhands.sdk import get_logger
@@ -17,6 +18,30 @@
 logger = get_logger(__name__)
 
 
+def refresh_eval_output_from_harbor(input_file: Path) -> bool:
+    """Rebuild converted output from raw Harbor results when they are available.
+
+    Inference archives contain both ``output.jsonl`` and the authoritative
+    ``harbor_output`` tree. Re-running the converter in the evaluation phase
+    lets converter fixes take effect during a rescore and avoids permanently
+    classifying verifier-scored agent failures according to stale JSONL.
+    """
+    harbor_output_dir = input_file.parent / "harbor_output"
+    if not harbor_output_dir.is_dir():
+        logger.info(
+            "Raw Harbor output not found at %s; using existing converted output",
+            harbor_output_dir,
+        )
+        return False
+
+    convert_harbor_to_eval_output(
+        harbor_output_dir=harbor_output_dir,
+        eval_output_path=input_file,
+    )
+    logger.info("Refreshed %s from raw Harbor results", input_file)
+    return True
+
+
 def _metric(
     data: dict[str, Any], test_result: dict[str, Any], key: str, default: Any
 ) -> Any:
@@ -143,6 +168,7 @@ def main() -> None:
         else input_file.with_suffix(".report.json")
     )
     try:
+        refresh_eval_output_from_harbor(input_file)
         process_harbor_results(str(input_file), str(output_file))
         generate_cost_report(str(input_file))
     except Exception as exc:

benchmarks/harbor/run_infer.py

@@ -154,12 +154,36 @@ def _split_json_values(raw: str | None) -> list[str]:
         return []
     data = json.loads(raw)
     if isinstance(data, dict):
-        return [f"{key}={value}" for key, value in data.items()]
+        return [
+            f"{key}={value if isinstance(value, str) else json.dumps(value, separators=(',', ':'))}"
+            for key, value in data.items()
+        ]
     if isinstance(data, list) and all(isinstance(item, str) for item in data):
         return data
     raise ValueError("Expected a JSON object or list of KEY=VALUE strings")
 
 
+def _llm_agent_env(llm: LLM, harbor_agent: str) -> list[str]:
+    """Build credential env flags for the selected Harbor-compatible agent.
+
+    The OpenHands SDK agent consumes the generic ``LLM_*`` variables. Pier's
+    mini-SWE agent resolves ``litellm_proxy/*`` through LiteLLM, which consumes
+    the OpenAI-compatible aliases instead. Both names carry the same proxy
+    credential and endpoint.
+    """
+    values: list[str] = []
+    if llm.api_key:
+        api_key = _secret_value(llm.api_key)
+        values.append(f"LLM_API_KEY={api_key}")
+        if harbor_agent == "mini-swe-agent":
+            values.append(f"OPENAI_API_KEY={api_key}")
+    if llm.base_url:
+        values.append(f"LLM_BASE_URL={llm.base_url}")
+        if harbor_agent == "mini-swe-agent":
+            values.append(f"OPENAI_BASE_URL={llm.base_url}")
+    return values
+
+
 def run_harbor(
     args: argparse.Namespace,
     llm: LLM,
@@ -186,10 +210,8 @@ def run_harbor(
         str(args.num_workers),
     ]
 
-    if llm.api_key:
-        cmd.extend(["--ae", f"LLM_API_KEY={_secret_value(llm.api_key)}"])
-    if llm.base_url:
-        cmd.extend(["--ae", f"LLM_BASE_URL={llm.base_url}"])
+    for env_value in _llm_agent_env(llm, args.harbor_agent):
+        cmd.extend(["--ae", env_value])
     for env_value in _parse_key_value(
         [*args.agent_env, *_split_json_values(args.agent_env_json)]
     ):

benchmarks/utils/harbor.py

@@ -236,7 +236,16 @@ def convert_harbor_to_eval_output(
 
             instance_id = canonicalize(trial.get("task_name", result_file.parent.name))
 
-            if trial.get("exception_info"):
+            verifier_result = trial.get("verifier_result") or {}
+            rewards = verifier_result.get("rewards") or {}
+            has_primary_reward = "reward" in rewards
+
+            # Harbor can still run the verifier after an agent timeout or other
+            # non-zero exit. In that case the verifier is authoritative: a reward
+            # of 1 is a resolved task, and a reward of 0 is a completed unresolved
+            # task. Only emit an incomplete error when no primary verifier reward
+            # exists (for example, setup or verifier failures).
+            if trial.get("exception_info") and not has_primary_reward:
                 errors.append(
                     {
                         "instance_id": instance_id,
@@ -246,8 +255,6 @@ def convert_harbor_to_eval_output(
                 )
                 continue
 
-            verifier_result = trial.get("verifier_result", {})
-            rewards = verifier_result.get("rewards", {})
             passed = rewards.get("reward", 0.0) > 0
             agent_result = trial.get("agent_result", {})
 
@@ -270,6 +277,8 @@ def convert_harbor_to_eval_output(
                     "total_cost_usd": agent_result.get("cost_usd") or 0.0,
                 },
             }
+            if trial.get("exception_info"):
+                eval_entry["test_result"]["agent_exception"] = trial["exception_info"]
             results.append(eval_entry)
             logger.info(
                 f"Processed trial {instance_id}: reward={rewards.get('reward', 'N/A')}"

extras/miniswe-context-model/README.md

@@ -0,0 +1,15 @@
+# MiniSWE context model
+
+This small optional package provides a `LitellmModel` subclass for long-running
+MiniSWE evaluations. It retains the system prompt, task, and newest complete
+assistant/tool turns while eliding old turns before a request exceeds the configured
+serialized-history budget.
+
+Install it in the MiniSWE tool environment and select:
+
+```text
+openhands_miniswe_context_model.model.ContextSafeLitellmModel
+```
+
+The `model.max_history_chars` config field defaults to 500,000 characters and can
+be overridden by the benchmark's agent configuration.

extras/miniswe-context-model/pyproject.toml

@@ -0,0 +1,12 @@
+[build-system]
+requires = ["setuptools>=68"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "openhands-miniswe-context-model"
+version = "0.1.0"
+description = "Bounded-history LiteLLM model wrapper for mini-swe-agent evaluations"
+requires-python = ">=3.11"
+
+[tool.setuptools.packages.find]
+where = ["src"]

extras/miniswe-context-model/src/openhands_miniswe_context_model/__init__.py

@@ -0,0 +1,3 @@
+"""Context-safe model helpers for mini-swe-agent evaluations."""
+
+__version__ = "0.1.0"

extras/miniswe-context-model/src/openhands_miniswe_context_model/history.py

@@ -0,0 +1,94 @@
+"""History bounding that preserves tool-call protocol boundaries."""
+
+from __future__ import annotations
+
+import json
+from collections.abc import Sequence
+from typing import Any
+
+
+ELISION_NOTICE = (
+    "[Earlier assistant and tool interactions were omitted to stay within the "
+    "model context window. Their effects remain in the current workspace.]"
+)
+
+
+def _message_size(message: dict[str, Any]) -> int:
+    return len(
+        json.dumps(
+            message,
+            ensure_ascii=False,
+            separators=(",", ":"),
+            default=str,
+        )
+    )
+
+
+def _history_units(messages: Sequence[dict[str, Any]]) -> list[list[dict[str, Any]]]:
+    """Group assistant messages with their following tool results."""
+    units: list[list[dict[str, Any]]] = []
+    for message in messages:
+        if (
+            message.get("role") == "tool"
+            and units
+            and units[-1][0].get("role") == "assistant"
+        ):
+            units[-1].append(message)
+        else:
+            units.append([message])
+    return units
+
+
+def bound_message_history(
+    messages: Sequence[dict[str, Any]],
+    *,
+    max_chars: int,
+    preserve_first: int = 2,
+    min_recent_units: int = 8,
+) -> list[dict[str, Any]]:
+    """Drop the oldest complete turns when serialized history exceeds a budget.
+
+    The initial system/task messages are always retained. Assistant tool calls and
+    their tool results are grouped into atomic units, so trimming never leaves an
+    orphaned tool response in the request sent to the model.
+    """
+    if max_chars <= 0:
+        raise ValueError("max_chars must be positive")
+    if preserve_first < 0:
+        raise ValueError("preserve_first cannot be negative")
+    if min_recent_units < 1:
+        raise ValueError("min_recent_units must be at least one")
+
+    copied = [dict(message) for message in messages]
+    if sum(_message_size(message) for message in copied) <= max_chars:
+        return copied
+
+    header = copied[:preserve_first]
+    units = _history_units(copied[preserve_first:])
+    notice = {"role": "user", "content": ELISION_NOTICE}
+    remaining = (
+        max_chars
+        - sum(_message_size(message) for message in header)
+        - _message_size(notice)
+    )
+
+    kept_reversed: list[list[dict[str, Any]]] = []
+    used = 0
+    for unit in reversed(units):
+        unit_size = sum(_message_size(message) for message in unit)
+        if (
+            kept_reversed
+            and used + unit_size > remaining
+            and len(kept_reversed) >= min_recent_units
+        ):
+            break
+        kept_reversed.append(unit)
+        used += unit_size
+
+    kept = [message for unit in reversed(kept_reversed) for message in unit]
+    dropped_count = len(copied) - len(header) - len(kept)
+    if dropped_count <= 0:
+        return copied
+
+    notice["content"] += f" ({dropped_count} messages omitted.)"
+    return [*header, notice, *kept]

extras/miniswe-context-model/src/openhands_miniswe_context_model/model.py

@@ -0,0 +1,38 @@
+"""A LiteLLM mini-swe-agent model with bounded request history."""
+
+from __future__ import annotations
+
+from typing import Any
+
+from minisweagent.models.litellm_model import (  # pyright: ignore[reportMissingImports]
+    LitellmModel,
+    LitellmModelConfig,
+)
+from pydantic import Field
+
+from openhands_miniswe_context_model.history import bound_message_history
+
+
+class ContextSafeLitellmModelConfig(LitellmModelConfig):
+    max_history_chars: int = Field(default=500_000, gt=0)
+    """Maximum serialized characters retained in requests to the model."""
+
+    min_recent_history_units: int = Field(default=8, ge=1)
+    """Minimum number of recent assistant/tool units to retain."""
+
+
+class ContextSafeLitellmModel(LitellmModel):
+    """Retain task setup and recent complete turns below a context-safe budget."""
+
+    config: ContextSafeLitellmModelConfig
+
+    def __init__(self, **kwargs: Any):
+        super().__init__(config_class=ContextSafeLitellmModelConfig, **kwargs)
+
+    def _prepare_messages_for_api(self, messages: list[dict]) -> list[dict]:
+        prepared = super()._prepare_messages_for_api(messages)
+        return bound_message_history(
+            prepared,
+            max_chars=self.config.max_history_chars,
+            min_recent_units=self.config.min_recent_history_units,
+        )

tests/test_harbor_eval_infer.py

@@ -0,0 +1,75 @@
+"""Tests for the generic Harbor evaluation phase."""
+
+import json
+from pathlib import Path
+
+from benchmarks.harbor.eval_infer import (
+    process_harbor_results,
+    refresh_eval_output_from_harbor,
+)
+
+
+def test_refresh_reconverts_authoritative_harbor_results(tmp_path: Path) -> None:
+    output_file = tmp_path / "output.jsonl"
+    output_file.write_text(
+        json.dumps(
+            {
+                "instance_id": "finished-before-timeout",
+                "test_result": {},
+                "error": "stale timeout classification",
+            }
+        )
+        + "\n"
+    )
+
+    job_dir = tmp_path / "harbor_output" / "2026-01-01__00-00-00"
+    trial_dir = job_dir / "finished-before-timeout__abc"
+    trial_dir.mkdir(parents=True)
+    (job_dir / "result.json").write_text(json.dumps({"id": "job"}))
+    (trial_dir / "result.json").write_text(
+        json.dumps(
+            {
+                "task_name": "finished-before-timeout",
+                "trial_name": "finished-before-timeout__abc",
+                "agent_result": {
+                    "n_input_tokens": 10,
+                    "n_output_tokens": 2,
+                    "cost_usd": 0.0,
+                },
+                "verifier_result": {"rewards": {"reward": 1.0}},
+                "exception_info": {
+                    "exception_type": "AgentTimeoutError",
+                    "exception_message": "Agent timed out after saving the answer",
+                },
+            }
+        )
+    )
+
+    assert refresh_eval_output_from_harbor(output_file) is True
+
+    report_file = tmp_path / "output.report.json"
+    report = process_harbor_results(str(output_file), str(report_file))
+    converted = json.loads(output_file.read_text())
+
+    assert report["completed_instances"] == 1
+    assert report["resolved_instances"] == 1
+    assert report["error_instances"] == 0
+    assert converted["test_result"]["agent_exception"]["exception_type"] == (
+        "AgentTimeoutError"
+    )
+
+
+def test_refresh_keeps_existing_output_without_raw_harbor_results(
+    tmp_path: Path,
+) -> None:
+    output_file = tmp_path / "output.jsonl"
+    original = (
+        json.dumps(
+            {"instance_id": "existing", "test_result": {"passed": False}, "error": None}
+        )
+        + "\n"
+    )
+    output_file.write_text(original)
+
+    assert refresh_eval_output_from_harbor(output_file) is False
+    assert output_file.read_text() == original