From e857df86e7e4f37462d1d348f6d1c72b9f4af77f Mon Sep 17 00:00:00 2001 From: Adam Cohen Hillel Date: Fri, 3 Jul 2026 19:17:13 -0700 Subject: [PATCH] Add structured failure reasons to agent benchmark summary Each failed task in results.jsonl now carries a failure_reasons list covering harness failures, missing task.finished status, missing expected text, activity mismatches, missing final UI dumps, and missing trajectories or trajectory summaries. summary.json also gains a top-level failures list so contributors can see why tasks failed without digging through task directories. Existing fields are unchanged. Co-Authored-By: Claude Opus 4.7 --- scripts/run-agent-benchmark.sh | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/scripts/run-agent-benchmark.sh b/scripts/run-agent-benchmark.sh index adeca15..a40a164 100755 --- a/scripts/run-agent-benchmark.sh +++ b/scripts/run-agent-benchmark.sh @@ -196,6 +196,24 @@ passed = ( and not missing_text and activity_ok ) +failure_reasons = [] +if not passed: + if status != "ran": + failure_reasons.append(f"harness failed (status: {status})") + trajectory_dir = task_dir / "trajectory" + if not trajectory_dir.exists(): + failure_reasons.append("trajectory missing") + elif not summary_path.exists(): + failure_reasons.append("trajectory summary missing") + if assistant_status != "task.finished": + reported = assistant_status or "missing" + failure_reasons.append(f"assistant did not report task.finished (status: {reported})") + if missing_text: + failure_reasons.append("expected text missing: " + ", ".join(missing_text)) + if not activity_ok: + failure_reasons.append(f"expected activity missing or different: {expected_activity}") + if not final_window_text: + failure_reasons.append("final UI dump missing") print(json.dumps({ "task_id": task.get("id", ""), "goal": task.get("goal", ""), @@ -206,6 +224,7 @@ print(json.dumps({ "expected_activity": expected_activity, "activity_ok": activity_ok, "final_ui_captured": bool(final_window_text), + "failure_reasons": failure_reasons, "tool_sequence": tool_sequence, "task_dir": str(task_dir), })) @@ -220,6 +239,14 @@ summary = { "total": len(rows), "passed": sum(1 for row in rows if row["status"] == "pass"), "failed": sum(1 for row in rows if row["status"] != "pass"), + "failures": [ + { + "task_id": row.get("task_id", ""), + "failure_reasons": row.get("failure_reasons", []), + } + for row in rows + if row["status"] != "pass" + ], "results": rows, } json.dump(summary, open(sys.argv[2], "w"), indent=2)