From 62504db89d3337582d449abebe3fbefbc6381b09 Mon Sep 17 00:00:00 2001 From: surlymochan Date: Sun, 15 Nov 2026 07:54:12 +0000 Subject: [PATCH] feat(bench): per-cell cost field + uncensored wall-clock protocol (M16 slice 1, #205) - run.py records cost_usd + cost_price_row on every cell, priced at the new prices.json minimax-m31-flash row (MiniMax-M3.1-Flash list prices, fetched 2026-11-15; closest published PAYG price for the bench model MiniMax-M3.1-Flash-Preview). Computed, not billed (Token Plan). - report.py prefers the recorded cost, falls back to recomputation for older rows; default --price is now minimax-m31-flash (M3 row retained for comparability with pilot2 medians). - Wall clock: keep the 1200s cap (see #205 for the evidence); report.py now reports the median over uncensored cells only, with n, alongside the truncated count. Median over all cells is a cap artifact. - test_cost_field.py: formula, price row, report preference, uncensored wall stats (stdlib only, 5 tests). --- bench/swebench/prices.json | 10 +++ bench/swebench/report.py | 21 ++++- bench/swebench/run.py | 23 ++++++ bench/swebench/test_cost_field.py | 125 ++++++++++++++++++++++++++++++ 4 files changed, 177 insertions(+), 2 deletions(-) create mode 100644 bench/swebench/test_cost_field.py diff --git a/bench/swebench/prices.json b/bench/swebench/prices.json index fbd5020..c748945 100644 --- a/bench/swebench/prices.json +++ b/bench/swebench/prices.json @@ -22,6 +22,16 @@ "note": "MiniMax publishes no pay-as-you-go price for MiniMax-M3.1-Flash-Preview; it is sold only through M Plan / MiniMax Code. The bench uses the current MiniMax-M3 Standard list price (the 'permanent 50% off' price shown on the page, <=512k input). No cache-write price is listed for M3; Pi reported 0 cache-write tokens on these runs unless the record says otherwise.", "computed_not_billed": false }, + { + "id": "minimax-m31-flash", + "label": "MiniMax M3.1-Flash list price (cost basis for M16)", + "model": "MiniMax-M3.1-Flash, Standard tier", + "in": 0.15, "out": 0.60, "cache_read": 0.015, "cache_write": 0.0, + "source": "https://platform.minimax.io/docs/pricing/overview", + "fetched": "2026-11-15", + "note": "Closest published pay-as-you-go price for the bench model MiniMax-M3.1-Flash-Preview: the pricing page lists MiniMax-M3.1-Flash (no separate Preview price; Preview is sold via M Plan / MiniMax Code). Computed, not billed — actual usage runs on the Token Plan. The older 'minimax' row (MiniMax-M3 Standard, $0.30/$1.20/$0.06) is retained so earlier medians (e.g. pilot2 baseline $0.1964) stay reproducible.", + "computed_not_billed": false + }, { "id": "sonnet5", "label": "Claude Sonnet 5 list price (computed)", diff --git a/bench/swebench/report.py b/bench/swebench/report.py index 408f37a..0e5e941 100644 --- a/bench/swebench/report.py +++ b/bench/swebench/report.py @@ -85,7 +85,7 @@ def main(): ap.add_argument("path") ap.add_argument("--out") ap.add_argument("--raw") - ap.add_argument("--price", default="minimax") + ap.add_argument("--price", default="minimax-m31-flash") ap.add_argument("--model-label", default="") ap.add_argument("--only-instances", default="") ap.add_argument("--exclude") @@ -132,6 +132,14 @@ def is_excluded(r): s = {"n": n, "resolved": res, "resolved_rate": res / n if n else None, "instances": len({r["instance_id"] for r in rs}), "wall_s_q1_med_q3": quantiles(walls), "timed_out": sum(1 for r in rs if r.get("timed_out")), + # M16 slice 1 (#205): the 1200s cap truncates ~93% of cells, so + # the median over ALL cells is a cap artifact (~1203s in every + # arm). The wall leg is reported over uncensored cells only + # (exited before the cap), always with n; the <=1.3x median + # ratio test applies only when the median is computable + # (truncation <50% in both arms), else the leg is 无法判定. + "wall_s_uncensored_n": len([r for r in rs if not r.get("timed_out")]), + "wall_s_uncensored_q1_med_q3": quantiles([r["wall_s"] for r in rs if not r.get("timed_out")]), "tokens_in_out_median": statistics.median(io), "tokens_in_out_q1_med_q3": quantiles(io), "tokens_total_incl_cache_median": statistics.median(tot), @@ -144,7 +152,11 @@ def is_excluded(r): s["runs_with_rate_limit_retries"] = sum(1 for x in rl if x["retries"]) s["runs_ended_in_error"] = sum(1 for x in rl if x["ended_in_error"]) for row in prices: - cs = [cost(r["tokens"], row) for r in rs] + # M16 slice 1 (#205): prefer the cost recorded at run time when + # it matches this row; older rows without the field fall back to + # recomputation from tokens (identical formula). + cs = [r["cost_usd"] if r.get("cost_price_row") == row["id"] and "cost_usd" in r + else cost(r["tokens"], row) for r in rs] s["cost"][row["id"]] = {"median_per_run": statistics.median(cs), "total": sum(cs), "q1_med_q3": quantiles(cs), "per_resolved": (sum(cs) / res) if res else None} @@ -177,6 +189,11 @@ def is_excluded(r): for iid in sorted(per_inst): L.append(f"| `{iid}` | " + " | ".join("{}/{}".format(*per_inst[iid].get(a, [0, 0])) for a in arms) + " |") L += ["", f"Timed out (20 min cap): " + ", ".join(f"{a} {summary['arms'][a]['timed_out']}" for a in arms), + "Wall-clock, uncensored cells only (exited before the 1200s cap; median over all cells is a cap artifact): " + "; ".join( + (f"{a} median {summary['arms'][a]['wall_s_uncensored_q1_med_q3'][1]:.0f}s " + f"(n={summary['arms'][a]['wall_s_uncensored_n']})") + if summary['arms'][a]['wall_s_uncensored_q1_med_q3'][1] is not None + else f"{a} no uncensored cells" for a in arms), f"Guard blocks: " + ", ".join(f"{a} {summary['arms'][a]['guard_blocked']}" for a in arms), "Rate-limit (429) retries inside Pi: " + "; ".join( f"{a} {summary['arms'][a]['rate_limit_retries_total']} in " diff --git a/bench/swebench/run.py b/bench/swebench/run.py index 563c1aa..b455188 100644 --- a/bench/swebench/run.py +++ b/bench/swebench/run.py @@ -27,6 +27,25 @@ LOCK = threading.Lock() PREDICTION_LOCK = threading.Lock() +# M16 slice 1 (#205): per-cell $/task. Cost is computed at MiniMax list +# prices (prices.json) from the recorded token counts, so the runs file +# carries the $/task leg of P1-2 instead of leaving it to report time. +# Basis: the minimax-m31-flash row — the closest published pay-as-you-go +# price for the bench model MiniMax-M3.1-Flash-Preview (the pricing page +# lists M3.1-Flash; Preview itself is sold via M Plan / MiniMax Code with +# no PAYG price of its own). Computed, not billed: actual usage runs on +# the Token Plan. +PRICE_ROWS = json.loads((ROOT / "prices.json").read_text())["rows"] +COST_ROW_ID = "minimax-m31-flash" +COST_ROW = next(r for r in PRICE_ROWS if r["id"] == COST_ROW_ID) + + +def cost_usd(tok, row=COST_ROW): + """USD cost of one cell at a prices.json row (USD per 1M tokens).""" + return round((tok["input"] * row["in"] + tok["output"] * row["out"] + + tok["cache_read"] * row["cache_read"] + + tok["cache_write"] * row["cache_write"]) / 1e6, 6) + def instances(): return json.loads((ROOT / "instances.json").read_text())["instances"] @@ -90,6 +109,10 @@ def cell(inst, arm, rep, out, stamp, raw_root, label, predictions_dir, predictio rec["model"] = sorted(models) tot = {k: sum((c.get("usage") or {}).get(k, 0) for c in calls) for k in ("input", "output", "cache_read", "cache_write")} rec["tokens"] = tot + # M16 slice 1 (#205): $/task leg of P1-2 — record the cost with the + # cell, priced at COST_ROW_ID (see prices.json for the source). + rec["cost_usd"] = cost_usd(tot) + rec["cost_price_row"] = COST_ROW_ID rec["timed_out"] = any((c.get("record") or {}).get("timed_out") or c.get("rc") in (124, 137) for c in calls) or rec["wall_s"] >= A.CAP_S rec["errors"] = [e for c in calls for e in (c.get("usage") or {}).get("errors", [])][:5] g = E.grade(inst, repo, venv, events / "grade") diff --git a/bench/swebench/test_cost_field.py b/bench/swebench/test_cost_field.py new file mode 100644 index 0000000..d0bc596 --- /dev/null +++ b/bench/swebench/test_cost_field.py @@ -0,0 +1,125 @@ +#!/usr/bin/env python3 +"""Tests for the M16 slice 1 (#205) per-cell $/task cost field and the +wall-clock truncation protocol. + +- run.py records cost_usd + cost_price_row on every cell, computed from the + recorded token counts at the prices.json minimax-m31-flash row + (MiniMax-M3.1-Flash list prices, fetched 2026-11-15; closest published + pay-as-you-go price for the bench model MiniMax-M3.1-Flash-Preview). +- report.py prefers the recorded cost for the matching row and falls back + to recomputation for older rows without the field. +- report.py reports the wall-clock median over uncensored cells only, with + n, alongside the truncated count (CAP_S 1200s truncates ~93% of cells, + so the median over all cells is a cap artifact). + +Stdlib only, like test_planner_prompt.py: run.py's top-level imports are +stdlib-only (arms/swe_env are imported lazily inside main), so it can be +imported directly. +""" +import importlib.util +import json +import subprocess +import sys +import tempfile +import unittest +from pathlib import Path + +HERE = Path(__file__).resolve().parent +PRICES = json.loads((HERE / "prices.json").read_text()) + + +def _load_run(): + spec = importlib.util.spec_from_file_location("bench_run", HERE / "run.py") + mod = importlib.util.module_from_spec(spec) + spec.loader.exec_module(mod) + return mod + + +RUN = _load_run() + + +def _row(instance_id, arm, rep, wall_s, timed_out, resolved, tokens, cost_usd=None): + r = {"instance_id": instance_id, "arm": arm, "rep": rep, "wall_s": wall_s, + "timed_out": timed_out, "resolved": resolved, "tokens": tokens, + "calls": [], "errors": [], "model": ["minimax-cn/MiniMax-M3.1-Flash-Preview"]} + if cost_usd is not None: + r["cost_usd"] = cost_usd + r["cost_price_row"] = "minimax-m31-flash" + return r + + +class CostFieldTests(unittest.TestCase): + def test_price_row_present(self): + row = next(r for r in PRICES["rows"] if r["id"] == "minimax-m31-flash") + self.assertEqual((row["in"], row["out"], row["cache_read"], row["cache_write"]), + (0.15, 0.60, 0.015, 0.0)) + self.assertEqual(row["source"], "https://platform.minimax.io/docs/pricing/overview") + self.assertEqual(row["fetched"], "2026-11-15") + + def test_cost_formula(self): + # 1,000,000 in + 500,000 out + 2,000,000 cache_read at 0.15/0.60/0.015 + # = 0.15 + 0.30 + 0.03 = $0.48 + tok = {"input": 1_000_000, "output": 500_000, "cache_read": 2_000_000, "cache_write": 0} + self.assertAlmostEqual(RUN.cost_usd(tok), 0.48, places=6) + + def test_cost_row_id_matches_prices(self): + self.assertEqual(RUN.COST_ROW_ID, "minimax-m31-flash") + self.assertEqual(RUN.COST_ROW["in"], 0.15) + + def test_report_prefers_recorded_cost(self): + # One row carries a recorded cost that deliberately differs from what + # its tokens imply: the report must use the recorded value. + rows = [ + _row("sympy__sympy-17630", "baseline", 1, 1203.0, True, True, + {"input": 100_000, "output": 50_000, "cache_read": 2_000_000, "cache_write": 0}, + cost_usd=9.99), + _row("sympy__sympy-14248", "baseline", 1, 889.0, False, False, + {"input": 100_000, "output": 50_000, "cache_read": 2_000_000, "cache_write": 0}), + ] + with tempfile.TemporaryDirectory() as td: + td = Path(td) + runs = td / "runs.jsonl" + runs.write_text("\n".join(json.dumps(r) for r in rows) + "\n") + raw = td / "raw" + raw.mkdir() + out = td / "out" + cp = subprocess.run([sys.executable, str(HERE / "report.py"), str(runs), + "--out", str(out), "--raw", str(raw)], + capture_output=True, text=True) + self.assertEqual(cp.returncode, 0, cp.stderr) + summary = json.loads((out / "summary.json").read_text()) + s = summary["arms"]["baseline"] + # recorded 9.99 + recomputed 0.075 (0.015+0.03+0.03) -> median 5.0325 + self.assertAlmostEqual(s["cost"]["minimax-m31-flash"]["median_per_run"], 5.0325, places=4) + + def test_report_uncensored_wall_stats(self): + rows = [ + _row("sympy__sympy-17630", "baseline", 1, 1203.0, True, True, + {"input": 1, "output": 1, "cache_read": 1, "cache_write": 0}), + _row("sympy__sympy-14248", "baseline", 1, 800.0, False, False, + {"input": 1, "output": 1, "cache_read": 1, "cache_write": 0}), + _row("pytest-dev__pytest-5787", "baseline", 1, 1000.0, False, True, + {"input": 1, "output": 1, "cache_read": 1, "cache_write": 0}), + ] + with tempfile.TemporaryDirectory() as td: + td = Path(td) + runs = td / "runs.jsonl" + runs.write_text("\n".join(json.dumps(r) for r in rows) + "\n") + raw = td / "raw" + raw.mkdir() + out = td / "out" + cp = subprocess.run([sys.executable, str(HERE / "report.py"), str(runs), + "--out", str(out), "--raw", str(raw)], + capture_output=True, text=True) + self.assertEqual(cp.returncode, 0, cp.stderr) + summary = json.loads((out / "summary.json").read_text()) + tables = (out / "tables.md").read_text() + s = summary["arms"]["baseline"] + self.assertEqual(s["timed_out"], 1) + self.assertEqual(s["wall_s_uncensored_n"], 2) + self.assertEqual(s["wall_s_uncensored_q1_med_q3"][1], 900.0) + self.assertIn("uncensored cells only", tables) + + +if __name__ == "__main__": + unittest.main()