diff --git a/CHANGELOG.md b/CHANGELOG.md index 62fd2977..674977ae 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,12 @@ See [docs/RELEASING.md](docs/RELEASING.md) for the release procedure. ## [Unreleased] +### Added +- Bundled price catalog: `claude-opus-5` ($5/$25 per 1M tokens, cache read + $0.50 / 1-hour cache write $10), verified against the official Anthropic + pricing page 2026-07-24; the other Claude entries re-verified at the same + date. ([#77]) + ### Changed - **Cost is now cache-aware ([ADR-014](docs/adr/014-cache-aware-cost.md)) — supersedes ADR-011's "as if every run were the first" formula.** `cost_usd` bills fresh input at the @@ -632,4 +638,5 @@ Initial release under the **nasde-toolkit** name (rebrand from [#71]: https://github.com/NoesisVision/nasde-toolkit/pull/71 [#73]: https://github.com/NoesisVision/nasde-toolkit/pull/73 [#74]: https://github.com/NoesisVision/nasde-toolkit/pull/74 +[#77]: https://github.com/NoesisVision/nasde-toolkit/pull/77 [gh-litellm-2026-04]: https://github.com/BerriAI/litellm/security/advisories/GHSA-xqmj-j6mv-4862 diff --git a/examples/ddd-architectural-challenges/assets/cost_quality_plane.png b/examples/ddd-architectural-challenges/assets/cost_quality_plane.png index 83871909..ec9c6a5d 100644 Binary files a/examples/ddd-architectural-challenges/assets/cost_quality_plane.png and b/examples/ddd-architectural-challenges/assets/cost_quality_plane.png differ diff --git a/examples/ddd-architectural-challenges/assets/cost_quality_plane_en.png b/examples/ddd-architectural-challenges/assets/cost_quality_plane_en.png index 1161262c..cf1090e2 100644 Binary files a/examples/ddd-architectural-challenges/assets/cost_quality_plane_en.png and b/examples/ddd-architectural-challenges/assets/cost_quality_plane_en.png differ diff --git a/examples/ddd-architectural-challenges/assets/verdict_heatmap.png b/examples/ddd-architectural-challenges/assets/verdict_heatmap.png index b930e41c..fb638c21 100644 Binary files a/examples/ddd-architectural-challenges/assets/verdict_heatmap.png and b/examples/ddd-architectural-challenges/assets/verdict_heatmap.png differ diff --git a/examples/ddd-architectural-challenges/assets/verdict_heatmap_en.png b/examples/ddd-architectural-challenges/assets/verdict_heatmap_en.png index 2e76b5f1..719f38a4 100644 Binary files a/examples/ddd-architectural-challenges/assets/verdict_heatmap_en.png and b/examples/ddd-architectural-challenges/assets/verdict_heatmap_en.png differ diff --git a/examples/ddd-architectural-challenges/cost_quality_plane.py b/examples/ddd-architectural-challenges/cost_quality_plane.py index aaace3c9..60b4a966 100644 --- a/examples/ddd-architectural-challenges/cost_quality_plane.py +++ b/examples/ddd-architectural-challenges/cost_quality_plane.py @@ -1,4 +1,4 @@ -"""Cost x quality plane for the 24-trial Fable/Opus grid (PL + EN publication PNGs). +"""Cost x quality plane for the Fable/Opus grid + the Opus 5 vanilla arm (PL + EN PNGs). X = cache-aware run cost (ADR-014): fresh input at the full rate, cache writes at the 1h-cache write rate, cache reads at the cached rate, output at the output rate @@ -26,15 +26,21 @@ ("Opus 4.8", "vanilla"): ["GoWvUz6", "cyQyXFc", "JTgey8p", "Lozfurr"], ("Opus 4.8", "hint"): ["YgcbZjf", "VoktgLb", "sSUYQp4", "a6okSgZ"], ("Opus 4.8", "skill"): ["ZMX6Xbq", "TsjcHWY", "Bkwiqom", "F5vYATs"], + ("Opus 5", "vanilla"): ["i9ivEds", "m82SdTE", "Yi8S8wA", "Wipg7XN"], + ("Opus 5", "hint"): ["UBsUYEb", "ycb5f8S", "24pWHEZ", "bC6TqMF"], + ("Opus 5", "skill"): ["mvexh8R", "X8Kb8T6", "f7DZgwd", "jRUpT56"], } -MODEL_ID = {"Fable 5": "claude-fable-5", "Opus 4.8": "claude-opus-4-8"} -CODER_COLOR = {"Fable 5": "#2a78d6", "Opus 4.8": "#1baf7a"} +MODEL_ID = {"Fable 5": "claude-fable-5", "Opus 4.8": "claude-opus-4-8", "Opus 5": "claude-opus-5"} +# The article's judge panel is frozen at 2x Fable + 2x Opus 4.8; evals by other +# judge models (e.g. the claude-opus-5 judge pilot) must not enter the means. +JUDGES = {"claude-fable-5", "claude-opus-4-8"} +CODER_COLOR = {"Fable 5": "#2a78d6", "Opus 4.8": "#1baf7a", "Opus 5": "#d97706"} CONFIG_MARKER = {"vanilla": "o", "hint": "^", "skill": "s"} INK = "#1a1a19" TEXT = { "pl": { - "title": "Koszt runu a jakość modelu domenowego — ddd-weather-discount", + "title": "Koszt runu a jakość rozwiązania — ddd-weather-discount", "xlabel": "koszt runu w USD — stawki API z rozliczeniem prompt cache", "ylabel": "jakość (średnia 4 ewaluacji, rubryka v2.3)", "coder": "model kodujący", @@ -42,25 +48,27 @@ "outlier": "pojedynczy run za ${cost:.0f}", # \$ keeps matplotlib from treating $...$ pairs as mathtext "footnote": ( + "jakość = pełna rubryka v2.3: dopasowanie modelu 50 pkt + granice i powściągliwość 25 pkt + jakość testów 25 pkt\n" "koszt = świeże wejście × stawka + zapisy cache × stawka zapisu (2×) " "+ odczyty cache × stawka odczytu (0.1×) + wyjście × stawka wyjścia\n" "stawki API z {as_of}: Fable 5 \\${fi:.0f} / \\${fo:.0f}, " - "Opus 4.8 \\${oi:.0f} / \\${oo:.0f} za mln tokenów" + "Opus 4.8 \\${oi:.0f} / \\${oo:.0f}, Opus 5 \\${o5i:.0f} / \\${o5o:.0f} za mln tokenów" ), "out": "cost_quality_plane.png", }, "en": { - "title": "Run cost vs domain-model quality — ddd-weather-discount", + "title": "Run cost vs solution quality — ddd-weather-discount", "xlabel": "run cost in USD — API rates with prompt caching", "ylabel": "quality (mean of 4 evaluations, rubric v2.3)", "coder": "coding model", "mean": "large marker = arm mean (n=4)", "outlier": "a single ${cost:.0f} run", "footnote": ( + "quality = the full v2.3 rubric: model fit 50 pts + boundaries & restraint 25 pts + test quality 25 pts\n" "cost = fresh input × input rate + cache writes × write rate (2×) " "+ cache reads × read rate (0.1×) + output × output rate\n" "API rates as of {as_of}: Fable 5 \\${fi:.0f} / \\${fo:.0f}, " - "Opus 4.8 \\${oi:.0f} / \\${oo:.0f} per MTok" + "Opus 4.8 \\${oi:.0f} / \\${oo:.0f}, Opus 5 \\${o5i:.0f} / \\${o5o:.0f} per MTok" ), "out": "cost_quality_plane_en.png", }, @@ -74,6 +82,9 @@ ("Opus 4.8", "vanilla"): (-14, -4, "right"), ("Opus 4.8", "hint"): (14, -4, "left"), ("Opus 4.8", "skill"): (14, -4, "left"), + ("Opus 5", "vanilla"): (-14, 2, "right"), + ("Opus 5", "hint"): (14, 2, "left"), + ("Opus 5", "skill"): (14, -4, "left"), } @@ -84,7 +95,9 @@ def load_rates() -> dict: "fo": raw["claude-fable-5"]["output_per_1m"], "oi": raw["claude-opus-4-8"]["input_per_1m"], "oo": raw["claude-opus-4-8"]["output_per_1m"], - "as_of": raw["claude-fable-5"]["as_of"], + "o5i": raw["claude-opus-5"]["input_per_1m"], + "o5o": raw["claude-opus-5"]["output_per_1m"], + "as_of": raw["claude-opus-5"]["as_of"], "by_model": { m: ( raw[m]["input_per_1m"], @@ -92,7 +105,7 @@ def load_rates() -> dict: raw[m]["cached_input_per_1m"], raw[m]["cache_write_per_1m"], ) - for m in ("claude-fable-5", "claude-opus-4-8") + for m in ("claude-fable-5", "claude-opus-4-8", "claude-opus-5") }, } @@ -112,7 +125,7 @@ def collect(rates: dict) -> list[dict]: scores = [] for f in sorted(trial_dir.glob("assessment_eval_*.json")): d = json.loads(f.read_text()) - if d.get("dimensions_fingerprint") == FP and d.get("evaluator_model") in MODEL_ID.values(): + if d.get("dimensions_fingerprint") == FP and d.get("evaluator_model") in JUDGES: scores.append(d["normalized_score"]) if len(scores) != 4: print(f"WARN: {trial} has {len(scores)} v2.3 evals (expected 4)") @@ -154,14 +167,17 @@ def plane_plot(rows: list[dict], rates: dict, lang: str) -> None: ax.annotate(config, (mc, mq), textcoords="offset points", xytext=(dx, dy), ha=ha, fontsize=9.5, color=col, fontweight="bold", zorder=6) - # per-coder trajectory through the arm means, in config order + # per-coder trajectory through the arm means, in config order (skip configs + # a coder was not run under — Opus 5 has a vanilla arm only) for coder in CODER_COLOR: pts = [] for config in ("vanilla", "hint", "skill"): sub = [r for r in rows if r["coder"] == coder and r["config"] == config] - pts.append((sum(r["cost"] for r in sub) / len(sub), sum(r["q"] for r in sub) / len(sub))) - ax.plot([p[0] for p in pts], [p[1] for p in pts], color=CODER_COLOR[coder], - linewidth=1.1, linestyle=(0, (4, 3)), alpha=0.65, zorder=2) + if sub: + pts.append((sum(r["cost"] for r in sub) / len(sub), sum(r["q"] for r in sub) / len(sub))) + if len(pts) > 1: + ax.plot([p[0] for p in pts], [p[1] for p in pts], color=CODER_COLOR[coder], + linewidth=1.1, linestyle=(0, (4, 3)), alpha=0.65, zorder=2) top = max(rows, key=lambda r: r["cost"]) ax.annotate(t["outlier"].format(cost=top["cost"]), (top["cost"], top["q"]), @@ -169,7 +185,7 @@ def plane_plot(rows: list[dict], rates: dict, lang: str) -> None: fontsize=8.8, color="#666", zorder=6) ax.set_xlim(0, 26) - ax.set_ylim(0.55, 0.92) + ax.set_ylim(0.55, 0.95) ax.set_xlabel(t["xlabel"], fontsize=10, color="#444") ax.set_ylabel(t["ylabel"], fontsize=10, color="#444") ax.tick_params(labelsize=9, colors="#444") diff --git a/examples/ddd-architectural-challenges/verdict_heatmap.py b/examples/ddd-architectural-challenges/verdict_heatmap.py index 16b472b5..c5c30efc 100644 --- a/examples/ddd-architectural-challenges/verdict_heatmap.py +++ b/examples/ddd-architectural-challenges/verdict_heatmap.py @@ -1,4 +1,4 @@ -"""Verdict heatmap for the 24-trial grid: 18 rubric checks x 6 arms (PL + EN PNGs). +"""Verdict heatmap, grid + Opus 5 arm: 18 rubric checks x 7 arms (PL + EN PNGs). Per-check verdicts (M1-M7 / R1-R6 / T1-T5, FULL/PARTIAL/NONE) are parsed from the judges' free-text dimension reasoning — the eval JSON stores scores per dimension @@ -28,10 +28,17 @@ ("Opus 4.8", "vanilla"): ["GoWvUz6", "cyQyXFc", "JTgey8p", "Lozfurr"], ("Opus 4.8", "hint"): ["YgcbZjf", "VoktgLb", "sSUYQp4", "a6okSgZ"], ("Opus 4.8", "skill"): ["ZMX6Xbq", "TsjcHWY", "Bkwiqom", "F5vYATs"], + ("Opus 5", "vanilla"): ["i9ivEds", "m82SdTE", "Yi8S8wA", "Wipg7XN"], + ("Opus 5", "hint"): ["UBsUYEb", "ycb5f8S", "24pWHEZ", "bC6TqMF"], + ("Opus 5", "skill"): ["mvexh8R", "X8Kb8T6", "f7DZgwd", "jRUpT56"], } ARM_ORDER = [("Fable 5", "vanilla"), ("Fable 5", "hint"), ("Fable 5", "skill"), - ("Opus 4.8", "vanilla"), ("Opus 4.8", "hint"), ("Opus 4.8", "skill")] -CODER_COLOR = {"Fable 5": "#2a78d6", "Opus 4.8": "#1baf7a"} + ("Opus 4.8", "vanilla"), ("Opus 4.8", "hint"), ("Opus 4.8", "skill"), + ("Opus 5", "vanilla"), ("Opus 5", "hint"), ("Opus 5", "skill")] +CODER_COLOR = {"Fable 5": "#2a78d6", "Opus 4.8": "#1baf7a", "Opus 5": "#d97706"} +# Panel frozen at 2x Fable + 2x Opus 4.8 per trial; evals by other judge models +# (the claude-opus-5 judge pilot) share the fingerprint and must be excluded. +JUDGES = {"claude-fable-5", "claude-opus-4-8"} INK = "#1a1a19" MAX = {"M1": 10, "M2": 9, "M3": 9, "M4": 7, "M5": 7, "M6": 4, "M7": 4, @@ -86,14 +93,14 @@ } TEXT = { "pl": { - "title": "Profil werdyktów rubryki v2.3 — 18 checków × 6 ramion " + "title": "Profil werdyktów rubryki v2.3 — 18 checków × 9 ramion " "(komórka = średnia z 16 ocen: 4 triale × 4 ewaluacje)", "scale": "0% = wszędzie NONE · 50% = przeciętnie PARTIAL · 100% = wszędzie FULL", "groups": {"M": "model i kompozycja", "R": "granice i powściągliwość", "T": "jakość testów"}, "out": "verdict_heatmap.png", }, "en": { - "title": "Rubric v2.3 verdict profile — 18 checks × 6 arms " + "title": "Rubric v2.3 verdict profile — 18 checks × 9 arms " "(cell = mean of 16 evaluations: 4 trials × 4 evals)", "scale": "0% = NONE everywhere · 50% = PARTIAL on average · 100% = FULL everywhere", "groups": {"M": "model & composition fit", "R": "boundaries & restraint", "T": "test quality"}, @@ -109,16 +116,21 @@ rf"\b({ID}(?:\s*[/,]\s*{ID})*)" rf"(?:[:\-–—(=\s]|\b(?:scores?|stays?|is|are|remains?|at|all|falls?|drops?|to)\b)*" rf"({V}|full|partial|none)\b") -# 2) verdict with explicit points: "R3 FULL(2)" — points win when they contradict the word -P_V_POINTS = re.compile(rf"\b({ID})\s*({V})\s*\(\s*(\d+)") +# 2) verdict with explicit points: "R3 FULL(2)", "M5 FULL(6/7)" — points win when +# they contradict the word; the paren must CLOSE after the number so prose +# counters like "FULL (0 annotations removed)" are not read as points +P_V_POINTS = re.compile(rf"\b({ID})\s*({V})\s*\(\s*(\d+)\s*(?:/\s*\d+\s*)?\)") # 3) verdict-first: "PARTIAL on R5 (2)" P_V_FIRST = re.compile(rf"\b({V})s?\s+on\s+({ID})\b") # 4) verdict-first with a listed tail: "PARTIALs: M3 (...), M4 (...)" — IDs until sentence end P_V_LIST = re.compile(rf"\b({V})s?\s*(?:on)?:\s*([^.;]*)") -# 5) bare points: "R5 2/4", "T1=0", "(R1 8, R2 5)" -P_POINTS = re.compile(rf"\b({ID})\s*(?:[=:]\s*|\s+)(\d+)(?:\s*/\s*(\d+))?(?=[\s,;.)\]])") +# 5) bare points: "R5 2/4", "T1=0", "(R1 8, R2 5)", "T1=9: ..." (trailing colon) +P_POINTS = re.compile(rf"\b({ID})\s*(?:[=:]\s*|\s+)(\d+)(?:\s*/\s*(\d+))?(?=[\s,;.):\]])") # 6) "Lost 5 on M3" P_LOST = re.compile(rf"\b[Ll]ost\s+(\d+)\s+on\s+({ID})\b") +# 7) unquantified deduction list: "Loses points on M3 (...) and M7 (...)" — a +# named deduction with no amount is a PARTIAL (a NONE is always spelled out) +P_LOSES = re.compile(r"\b[Ll]oses?\s+points?\s+on\s+([^.;]*)") def classify(points: int, cid: str) -> str: @@ -151,6 +163,9 @@ def parse_eval(d: dict) -> tuple[dict[str, str], list[str]]: local.setdefault(cid, classify(int(pts), cid)) for lost, cid in P_LOST.findall(text): local.setdefault(cid, classify(MAX[cid] - int(lost), cid)) + for tail in P_LOSES.findall(text): + for cid in re.findall(rf"\b{ID}\b", tail): + local.setdefault(cid, "PARTIAL") checks = DIM_CHECKS[name] for cid in checks: local.setdefault(cid, "FULL") # judges enumerate deductions only @@ -173,7 +188,7 @@ def collect() -> tuple[dict[tuple[str, str], list[dict]], list[str]]: (td,) = JOBS.glob(f"*/ddd-weather-discount__{t}") for f in sorted(td.glob("assessment_eval_*.json")): d = json.loads(f.read_text()) - if d.get("dimensions_fingerprint") != FP: + if d.get("dimensions_fingerprint") != FP or d.get("evaluator_model") not in JUDGES: continue verdicts, audit = parse_eval(d) per_arm[arm].append(verdicts) @@ -198,7 +213,7 @@ def heatmap(per_arm: dict, lang: str) -> None: for arm in ARM_ORDER] for c in CHECKS] rgba = [[CMAPS[c[0]](v) for v in row] for c, row in zip(CHECKS, grid)] - fig, ax = plt.subplots(figsize=(9.6, 8.8)) + fig, ax = plt.subplots(figsize=(12.0, 8.8)) fig.suptitle(t["title"], fontsize=11.5, color=INK, y=0.985) ax.imshow(rgba, aspect="auto") @@ -222,7 +237,7 @@ def heatmap(per_arm: dict, lang: str) -> None: # white gridlines between cells; heavier breaks + side captions between M/R/T groups for xi in range(1, len(ARM_ORDER)): - lw = 3.4 if xi == 3 else 1.6 + lw = 3.4 if xi in (3, 6) else 1.6 ax.axvline(xi - 0.5, color="white", linewidth=lw) for yi in range(1, len(CHECKS)): ax.axhline(yi - 0.5, color="white", linewidth=1.6) @@ -243,7 +258,7 @@ def heatmap(per_arm: dict, lang: str) -> None: if __name__ == "__main__": per_arm, flags = collect() n = sum(len(v) for v in per_arm.values()) - print(f"evals parsed: {n} (expected 96)") + print(f"evals parsed: {n} (expected 144)") for fl in flags: print("FLAG", fl) for lang in ("pl", "en"): diff --git a/src/nasde_toolkit/pricing.toml b/src/nasde_toolkit/pricing.toml index ce7b04d3..9c472288 100644 --- a/src/nasde_toolkit/pricing.toml +++ b/src/nasde_toolkit/pricing.toml @@ -32,12 +32,20 @@ cached_input_per_1m = 0.25 as_of = "2026-06-08" source = "https://developers.openai.com/api/docs/pricing" +[models."claude-opus-5"] +input_per_1m = 5.0 +output_per_1m = 25.0 +cached_input_per_1m = 0.50 +cache_write_per_1m = 10.0 +as_of = "2026-07-24" +source = "https://platform.claude.com/docs/en/about-claude/pricing" + [models."claude-opus-4-8"] input_per_1m = 5.0 output_per_1m = 25.0 cached_input_per_1m = 0.50 cache_write_per_1m = 10.0 -as_of = "2026-07-13" +as_of = "2026-07-24" source = "https://platform.claude.com/docs/en/about-claude/pricing" [models."claude-fable-5"] @@ -45,7 +53,7 @@ input_per_1m = 10.0 output_per_1m = 50.0 cached_input_per_1m = 1.0 cache_write_per_1m = 20.0 -as_of = "2026-07-13" +as_of = "2026-07-24" source = "https://platform.claude.com/docs/en/about-claude/pricing" [models."claude-sonnet-4-6"] diff --git a/tests/test_pricing.py b/tests/test_pricing.py index 0f6df37b..74f8d1ea 100644 --- a/tests/test_pricing.py +++ b/tests/test_pricing.py @@ -168,6 +168,7 @@ def test_layered_three_layers_compose(tmp_path: Path, empty_user_layer: Path) -> assert set(merged) == { "gpt-5.5", "gpt-5.4", + "claude-opus-5", "claude-fable-5", "claude-opus-4-8", "claude-sonnet-4-6",