Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 10 additions & 0 deletions bench/swebench/prices.json
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,16 @@
"note": "MiniMax publishes no pay-as-you-go price for MiniMax-M3.1-Flash-Preview; it is sold only through M Plan / MiniMax Code. The bench uses the current MiniMax-M3 Standard list price (the 'permanent 50% off' price shown on the page, <=512k input). No cache-write price is listed for M3; Pi reported 0 cache-write tokens on these runs unless the record says otherwise.",
"computed_not_billed": false
},
{
"id": "minimax-m31-flash",
"label": "MiniMax M3.1-Flash list price (cost basis for M16)",
"model": "MiniMax-M3.1-Flash, Standard tier",
"in": 0.15, "out": 0.60, "cache_read": 0.015, "cache_write": 0.0,
"source": "https://platform.minimax.io/docs/pricing/overview",
"fetched": "2026-11-15",
"note": "Closest published pay-as-you-go price for the bench model MiniMax-M3.1-Flash-Preview: the pricing page lists MiniMax-M3.1-Flash (no separate Preview price; Preview is sold via M Plan / MiniMax Code). Computed, not billed — actual usage runs on the Token Plan. The older 'minimax' row (MiniMax-M3 Standard, $0.30/$1.20/$0.06) is retained so earlier medians (e.g. pilot2 baseline $0.1964) stay reproducible.",
"computed_not_billed": false
},
{
"id": "sonnet5",
"label": "Claude Sonnet 5 list price (computed)",
Expand Down
21 changes: 19 additions & 2 deletions bench/swebench/report.py
Original file line number Diff line number Diff line change
Expand Up @@ -85,7 +85,7 @@ def main():
ap.add_argument("path")
ap.add_argument("--out")
ap.add_argument("--raw")
ap.add_argument("--price", default="minimax")
ap.add_argument("--price", default="minimax-m31-flash")
ap.add_argument("--model-label", default="")
ap.add_argument("--only-instances", default="")
ap.add_argument("--exclude")
Expand Down Expand Up @@ -132,6 +132,14 @@ def is_excluded(r):
s = {"n": n, "resolved": res, "resolved_rate": res / n if n else None,
"instances": len({r["instance_id"] for r in rs}),
"wall_s_q1_med_q3": quantiles(walls), "timed_out": sum(1 for r in rs if r.get("timed_out")),
# M16 slice 1 (#205): the 1200s cap truncates ~93% of cells, so
# the median over ALL cells is a cap artifact (~1203s in every
# arm). The wall leg is reported over uncensored cells only
# (exited before the cap), always with n; the <=1.3x median
# ratio test applies only when the median is computable
# (truncation <50% in both arms), else the leg is 无法判定.
"wall_s_uncensored_n": len([r for r in rs if not r.get("timed_out")]),
"wall_s_uncensored_q1_med_q3": quantiles([r["wall_s"] for r in rs if not r.get("timed_out")]),
"tokens_in_out_median": statistics.median(io),
"tokens_in_out_q1_med_q3": quantiles(io),
"tokens_total_incl_cache_median": statistics.median(tot),
Expand All @@ -144,7 +152,11 @@ def is_excluded(r):
s["runs_with_rate_limit_retries"] = sum(1 for x in rl if x["retries"])
s["runs_ended_in_error"] = sum(1 for x in rl if x["ended_in_error"])
for row in prices:
cs = [cost(r["tokens"], row) for r in rs]
# M16 slice 1 (#205): prefer the cost recorded at run time when
# it matches this row; older rows without the field fall back to
# recomputation from tokens (identical formula).
cs = [r["cost_usd"] if r.get("cost_price_row") == row["id"] and "cost_usd" in r
else cost(r["tokens"], row) for r in rs]
s["cost"][row["id"]] = {"median_per_run": statistics.median(cs), "total": sum(cs),
"q1_med_q3": quantiles(cs),
"per_resolved": (sum(cs) / res) if res else None}
Expand Down Expand Up @@ -177,6 +189,11 @@ def is_excluded(r):
for iid in sorted(per_inst):
L.append(f"| `{iid}` | " + " | ".join("{}/{}".format(*per_inst[iid].get(a, [0, 0])) for a in arms) + " |")
L += ["", f"Timed out (20 min cap): " + ", ".join(f"{a} {summary['arms'][a]['timed_out']}" for a in arms),
"Wall-clock, uncensored cells only (exited before the 1200s cap; median over all cells is a cap artifact): " + "; ".join(
(f"{a} median {summary['arms'][a]['wall_s_uncensored_q1_med_q3'][1]:.0f}s "
f"(n={summary['arms'][a]['wall_s_uncensored_n']})")
if summary['arms'][a]['wall_s_uncensored_q1_med_q3'][1] is not None
else f"{a} no uncensored cells" for a in arms),
f"Guard blocks: " + ", ".join(f"{a} {summary['arms'][a]['guard_blocked']}" for a in arms),
"Rate-limit (429) retries inside Pi: " + "; ".join(
f"{a} {summary['arms'][a]['rate_limit_retries_total']} in "
Expand Down
23 changes: 23 additions & 0 deletions bench/swebench/run.py
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,25 @@
LOCK = threading.Lock()
PREDICTION_LOCK = threading.Lock()

# M16 slice 1 (#205): per-cell $/task. Cost is computed at MiniMax list
# prices (prices.json) from the recorded token counts, so the runs file
# carries the $/task leg of P1-2 instead of leaving it to report time.
# Basis: the minimax-m31-flash row — the closest published pay-as-you-go
# price for the bench model MiniMax-M3.1-Flash-Preview (the pricing page
# lists M3.1-Flash; Preview itself is sold via M Plan / MiniMax Code with
# no PAYG price of its own). Computed, not billed: actual usage runs on
# the Token Plan.
PRICE_ROWS = json.loads((ROOT / "prices.json").read_text())["rows"]
COST_ROW_ID = "minimax-m31-flash"
COST_ROW = next(r for r in PRICE_ROWS if r["id"] == COST_ROW_ID)


def cost_usd(tok, row=COST_ROW):
"""USD cost of one cell at a prices.json row (USD per 1M tokens)."""
return round((tok["input"] * row["in"] + tok["output"] * row["out"]
+ tok["cache_read"] * row["cache_read"]
+ tok["cache_write"] * row["cache_write"]) / 1e6, 6)


def instances():
return json.loads((ROOT / "instances.json").read_text())["instances"]
Expand Down Expand Up @@ -90,6 +109,10 @@ def cell(inst, arm, rep, out, stamp, raw_root, label, predictions_dir, predictio
rec["model"] = sorted(models)
tot = {k: sum((c.get("usage") or {}).get(k, 0) for c in calls) for k in ("input", "output", "cache_read", "cache_write")}
rec["tokens"] = tot
# M16 slice 1 (#205): $/task leg of P1-2 — record the cost with the
# cell, priced at COST_ROW_ID (see prices.json for the source).
rec["cost_usd"] = cost_usd(tot)
rec["cost_price_row"] = COST_ROW_ID
rec["timed_out"] = any((c.get("record") or {}).get("timed_out") or c.get("rc") in (124, 137) for c in calls) or rec["wall_s"] >= A.CAP_S
rec["errors"] = [e for c in calls for e in (c.get("usage") or {}).get("errors", [])][:5]
g = E.grade(inst, repo, venv, events / "grade")
Expand Down
125 changes: 125 additions & 0 deletions bench/swebench/test_cost_field.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,125 @@
#!/usr/bin/env python3
"""Tests for the M16 slice 1 (#205) per-cell $/task cost field and the
wall-clock truncation protocol.

- run.py records cost_usd + cost_price_row on every cell, computed from the
recorded token counts at the prices.json minimax-m31-flash row
(MiniMax-M3.1-Flash list prices, fetched 2026-11-15; closest published
pay-as-you-go price for the bench model MiniMax-M3.1-Flash-Preview).
- report.py prefers the recorded cost for the matching row and falls back
to recomputation for older rows without the field.
- report.py reports the wall-clock median over uncensored cells only, with
n, alongside the truncated count (CAP_S 1200s truncates ~93% of cells,
so the median over all cells is a cap artifact).

Stdlib only, like test_planner_prompt.py: run.py's top-level imports are
stdlib-only (arms/swe_env are imported lazily inside main), so it can be
imported directly.
"""
import importlib.util
import json
import subprocess
import sys
import tempfile
import unittest
from pathlib import Path

HERE = Path(__file__).resolve().parent
PRICES = json.loads((HERE / "prices.json").read_text())


def _load_run():
spec = importlib.util.spec_from_file_location("bench_run", HERE / "run.py")
mod = importlib.util.module_from_spec(spec)
spec.loader.exec_module(mod)
return mod


RUN = _load_run()


def _row(instance_id, arm, rep, wall_s, timed_out, resolved, tokens, cost_usd=None):
r = {"instance_id": instance_id, "arm": arm, "rep": rep, "wall_s": wall_s,
"timed_out": timed_out, "resolved": resolved, "tokens": tokens,
"calls": [], "errors": [], "model": ["minimax-cn/MiniMax-M3.1-Flash-Preview"]}
if cost_usd is not None:
r["cost_usd"] = cost_usd
r["cost_price_row"] = "minimax-m31-flash"
return r


class CostFieldTests(unittest.TestCase):
def test_price_row_present(self):
row = next(r for r in PRICES["rows"] if r["id"] == "minimax-m31-flash")
self.assertEqual((row["in"], row["out"], row["cache_read"], row["cache_write"]),
(0.15, 0.60, 0.015, 0.0))
self.assertEqual(row["source"], "https://platform.minimax.io/docs/pricing/overview")
self.assertEqual(row["fetched"], "2026-11-15")

def test_cost_formula(self):
# 1,000,000 in + 500,000 out + 2,000,000 cache_read at 0.15/0.60/0.015
# = 0.15 + 0.30 + 0.03 = $0.48
tok = {"input": 1_000_000, "output": 500_000, "cache_read": 2_000_000, "cache_write": 0}
self.assertAlmostEqual(RUN.cost_usd(tok), 0.48, places=6)

def test_cost_row_id_matches_prices(self):
self.assertEqual(RUN.COST_ROW_ID, "minimax-m31-flash")
self.assertEqual(RUN.COST_ROW["in"], 0.15)

def test_report_prefers_recorded_cost(self):
# One row carries a recorded cost that deliberately differs from what
# its tokens imply: the report must use the recorded value.
rows = [
_row("sympy__sympy-17630", "baseline", 1, 1203.0, True, True,
{"input": 100_000, "output": 50_000, "cache_read": 2_000_000, "cache_write": 0},
cost_usd=9.99),
_row("sympy__sympy-14248", "baseline", 1, 889.0, False, False,
{"input": 100_000, "output": 50_000, "cache_read": 2_000_000, "cache_write": 0}),
]
with tempfile.TemporaryDirectory() as td:
td = Path(td)
runs = td / "runs.jsonl"
runs.write_text("\n".join(json.dumps(r) for r in rows) + "\n")
raw = td / "raw"
raw.mkdir()
out = td / "out"
cp = subprocess.run([sys.executable, str(HERE / "report.py"), str(runs),
"--out", str(out), "--raw", str(raw)],
capture_output=True, text=True)
self.assertEqual(cp.returncode, 0, cp.stderr)
summary = json.loads((out / "summary.json").read_text())
s = summary["arms"]["baseline"]
# recorded 9.99 + recomputed 0.075 (0.015+0.03+0.03) -> median 5.0325
self.assertAlmostEqual(s["cost"]["minimax-m31-flash"]["median_per_run"], 5.0325, places=4)

def test_report_uncensored_wall_stats(self):
rows = [
_row("sympy__sympy-17630", "baseline", 1, 1203.0, True, True,
{"input": 1, "output": 1, "cache_read": 1, "cache_write": 0}),
_row("sympy__sympy-14248", "baseline", 1, 800.0, False, False,
{"input": 1, "output": 1, "cache_read": 1, "cache_write": 0}),
_row("pytest-dev__pytest-5787", "baseline", 1, 1000.0, False, True,
{"input": 1, "output": 1, "cache_read": 1, "cache_write": 0}),
]
with tempfile.TemporaryDirectory() as td:
td = Path(td)
runs = td / "runs.jsonl"
runs.write_text("\n".join(json.dumps(r) for r in rows) + "\n")
raw = td / "raw"
raw.mkdir()
out = td / "out"
cp = subprocess.run([sys.executable, str(HERE / "report.py"), str(runs),
"--out", str(out), "--raw", str(raw)],
capture_output=True, text=True)
self.assertEqual(cp.returncode, 0, cp.stderr)
summary = json.loads((out / "summary.json").read_text())
tables = (out / "tables.md").read_text()
s = summary["arms"]["baseline"]
self.assertEqual(s["timed_out"], 1)
self.assertEqual(s["wall_s_uncensored_n"], 2)
self.assertEqual(s["wall_s_uncensored_q1_med_q3"][1], 900.0)
self.assertIn("uncensored cells only", tables)


if __name__ == "__main__":
unittest.main()