Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
18 changes: 16 additions & 2 deletions examples/ddd-architectural-challenges/verdict_heatmap.py
Original file line number Diff line number Diff line change
Expand Up @@ -207,7 +207,7 @@ def collect() -> tuple[dict[tuple[str, str], list[dict]], list[str]]:
for g, ramp in GROUP_RAMP.items()}


def heatmap(per_arm: dict, lang: str) -> None:
def heatmap(per_arm: dict, lang: str, highlight: str | None = None) -> None:
t = TEXT[lang]
grid = [[sum(VAL[evals[c]] for evals in per_arm[arm]) / len(per_arm[arm])
for arm in ARM_ORDER] for c in CHECKS]
Expand Down Expand Up @@ -248,9 +248,20 @@ def heatmap(per_arm: dict, lang: str) -> None:
xycoords="axes fraction", ha="left", va="center", fontsize=8.6,
color=GROUP_RAMP[group][1], rotation=270)

if highlight:
yi = CHECKS.index(highlight)
ax.add_patch(plt.Rectangle((-0.5, yi - 0.5), len(ARM_ORDER), 1.0,
fill=False, edgecolor=INK, linewidth=2.4,
zorder=6, clip_on=False))
ax.get_yticklabels()[yi].set_fontweight("bold")
ax.get_yticklabels()[yi].set_color(INK)

fig.text(0.5, 0.015, t["scale"], ha="center", fontsize=8.2, color="#777")
fig.tight_layout(rect=(0, 0.035, 0.97, 0.905))
out = HERE / "assets" / t["out"]
name = t["out"]
if highlight:
name = name.replace(".png", f"_{highlight.lower()}.png")
out = HERE / "assets" / name
fig.savefig(out, dpi=160, facecolor="white")
print("saved:", out)

Expand All @@ -263,3 +274,6 @@ def heatmap(per_arm: dict, lang: str) -> None:
print("FLAG", fl)
for lang in ("pl", "en"):
heatmap(per_arm, lang)
# article/social variant: the bounty row outlined (T2 - the check
# no model met for two generations until Opus 5)
heatmap(per_arm, lang, highlight="T2")
22 changes: 16 additions & 6 deletions src/nasde_toolkit/pricing.toml
Original file line number Diff line number Diff line change
Expand Up @@ -15,8 +15,10 @@
# per-step cost accounting on the 2026-07 grid. OpenAI bills no write premium,
# so gpt entries omit the field (writes fall back to the base input rate).
#
# Verified 2026-07-13 (Claude models re-checked against the official pricing page
# before publishing) — CONFIRM before publishing any cost figures.
# Claude models re-verified 2026-08-24 against the official pricing page (Fable 5,
# Opus 5, Opus 4.8, Sonnet 4.6 unchanged since 2026-07-13; Sonnet 5 added) —
# CONFIRM before publishing any cost figures. gpt entries not re-checked since
# 2026-06-08.

[models."gpt-5.5"]
input_per_1m = 5.0
Expand All @@ -37,29 +39,37 @@ input_per_1m = 5.0
output_per_1m = 25.0
cached_input_per_1m = 0.50
cache_write_per_1m = 10.0
as_of = "2026-07-24"
as_of = "2026-08-24"
source = "https://platform.claude.com/docs/en/about-claude/pricing"

[models."claude-opus-4-8"]
input_per_1m = 5.0
output_per_1m = 25.0
cached_input_per_1m = 0.50
cache_write_per_1m = 10.0
as_of = "2026-07-24"
as_of = "2026-08-24"
source = "https://platform.claude.com/docs/en/about-claude/pricing"

[models."claude-fable-5"]
input_per_1m = 10.0
output_per_1m = 50.0
cached_input_per_1m = 1.0
cache_write_per_1m = 20.0
as_of = "2026-07-24"
as_of = "2026-08-24"
source = "https://platform.claude.com/docs/en/about-claude/pricing"

[models."claude-sonnet-5"]
input_per_1m = 2.0
output_per_1m = 10.0
cached_input_per_1m = 0.20
cache_write_per_1m = 4.0
as_of = "2026-08-24"
source = "https://platform.claude.com/docs/en/about-claude/pricing"

[models."claude-sonnet-4-6"]
input_per_1m = 3.0
output_per_1m = 15.0
cached_input_per_1m = 0.30
cache_write_per_1m = 6.0
as_of = "2026-06-08"
as_of = "2026-08-24"
source = "https://platform.claude.com/docs/en/about-claude/pricing"
2 changes: 1 addition & 1 deletion tests/test_evaluator.py
Original file line number Diff line number Diff line change
Expand Up @@ -282,7 +282,7 @@ def test_assessment_summary_includes_economics(tmp_path: Path) -> None:
assert summary.token_usage["total_tokens"] == 1_060_000
# sonnet $3/$15/$0.30 cached (ADR-014): fresh 0.2M*3 + cached 0.8M*0.30 + 0.06M*15 = 1.74
assert summary.cost_usd == pytest.approx(1.74)
assert summary.pricing_as_of == "2026-06-08"
assert summary.pricing_as_of == "2026-08-24"
assert not hasattr(summary, "cost_efficiency") # removed: arbitrary zero → use Pareto front
assert not hasattr(summary, "token_efficiency")

Expand Down
1 change: 1 addition & 0 deletions tests/test_pricing.py
Original file line number Diff line number Diff line change
Expand Up @@ -171,6 +171,7 @@ def test_layered_three_layers_compose(tmp_path: Path, empty_user_layer: Path) ->
"claude-opus-5",
"claude-fable-5",
"claude-opus-4-8",
"claude-sonnet-5",
"claude-sonnet-4-6",
"azure-gpt5",
"enterprise-claude",
Expand Down
2 changes: 1 addition & 1 deletion tests/test_results_exporter.py
Original file line number Diff line number Diff line change
Expand Up @@ -174,7 +174,7 @@ def test_export_includes_token_cost_economics(job_dir: Path, tmp_path: Path) ->
assert usage["total_tokens"] == 1_060_000
# sonnet $3/$15/$0.30 cached (ADR-014): fresh 0.2M*3 + cached 0.8M*0.30 + 0.06M*15 = 1.74
assert metrics["cost_usd"] == pytest.approx(1.74)
assert metrics["pricing_as_of"] == "2026-06-08"
assert metrics["pricing_as_of"] == "2026-08-24"
assert "cost_efficiency" not in metrics # removed: arbitrary zero → use Pareto front
assert "token_efficiency" not in metrics
assert metrics["reasoning_effort"] == "" # fixture set no override
Expand Down