plot_precision.py

"""precision@20 — the inspection budget, drawn as the budget.

  (a) 20 dots per model = the 20 highest-ranked lots on the temporal test set. Filled =
      that inspection found a real failure. This is the operational claim, at the size
      it actually is.
  (b) precision@20 temporal vs random. Unlike AUC, the gap here goes the OTHER way for
      three of five models — and with 26 failures in the test set none of these
      differences are resolvable. That is the point of drawing it.

Reads phase4_temporal.csv and phase4_splits.csv (phase4.py).
"""
import numpy as np, pandas as pd
import matplotlib
matplotlib.use("Agg")
import matplotlib.pyplot as plt

HIT, MISS, TEMP, RAND = "#eb6834", "#d9d8d3", "#4a3aa7", "#8a8a85"
INK, MUTED, SURF = "#0b0b0b", "#52514e", "#fcfcfb"
TOPK = 20
SHORT = {
    "A  all 590 signals": "A · all 590 signals",
    "B  shortlist-5 (FDR ∩ stability)": "B · shortlist-5",
    "B' tier-1 only (Bonferroni ∩ stab)": "B' · tier-1 (3 signals)",
    "C  train-fold-only shortlist": "C · train-only reselection",
    "D  BH q=0.05 set (28 signals)": "D · BH q=0.05 (28)",
}

# dtype=str or pandas reads the 0/1 mask as an int and eats the leading zeros
T = pd.read_csv("phase4_temporal.csv", index_col=0, dtype={"topk_mask": str})
S = pd.read_csv("phase4_splits.csv")
t = S[S.split == "temporal"].set_index("model").prec20
r = S[S.split == "random"].groupby("model").prec20.agg(["mean", "std"])
nfail, ntest, base = int(T.nfail.iloc[0]), int(T.ntest.iloc[0]), float(T.base.iloc[0])
order = t.sort_values().index.tolist()

fig, (ax1, ax2) = plt.subplots(1, 2, figsize=(11.8, 4.9),
                               gridspec_kw={"width_ratios": [1.3, 1]})
fig.patch.set_facecolor(SURF)

# --- (a) the inspection budget ------------------------------------------
for i, mdl in enumerate(order):
    mask = T.topk_mask[mdl].zfill(TOPK)          # true rank order, not hits-first
    hits = mask.count("1")
    for k in range(TOPK):
        ax1.plot(k, i, "o", ms=11, color=HIT if mask[k] == "1" else MISS,
                 markeredgecolor=SURF, markeredgewidth=1.3)
    ax1.annotate(f"{hits} of {nfail} failures caught" if hits else "no failures caught",
                 xy=(1.02, i), xycoords=("axes fraction", "data"), ha="left", va="center",
                 fontsize=8.5, color=INK if hits >= 4 else MUTED)
ax1.set_yticks(range(len(order)))
ax1.set_yticklabels([SHORT.get(m, m) for m in order], fontsize=9.5)
ax1.set_xticks([])
ax1.set_xlim(-.8, TOPK-.2)
ax1.set_ylim(-.7, len(order)-.3)
ax1.set_xlabel(f"the {TOPK} highest-ranked lots of {ntest} in the test period "
               f"(left = ranked first)", color=MUTED)
ax1.set_title(f"Inspect {TOPK} lots: filled = a real failure found", fontsize=11,
              color=INK, loc="left", pad=26)
ax1.annotate(f"temporal split   ·   {nfail} failures among {ntest} lots ({base:.1%})   ·   "
             f"random inspection would find ~{base*TOPK:.1f}",
             xy=(0, 1.012), xycoords="axes fraction", ha="left", va="bottom",
             fontsize=8.5, color=MUTED)

# --- (b) temporal vs random --------------------------------------------
for i, mdl in enumerate(order):
    ax2.plot([t[mdl], r.loc[mdl, "mean"]], [i, i], color=MUTED, lw=1.3, zorder=1)
    ax2.errorbar(r.loc[mdl, "mean"], i, xerr=r.loc[mdl, "std"], fmt="o", ms=8,
                 color=RAND, ecolor=RAND, elinewidth=1.6, capsize=3,
                 markeredgecolor=SURF, markeredgewidth=1.2, zorder=3)
    ax2.plot(t[mdl], i, "o", ms=9, color=TEMP, markeredgecolor=SURF,
             markeredgewidth=1.2, zorder=4)
ax2.axvline(base, color=INK, ls=":", lw=1.1)
ax2.annotate("random\ninspection", xy=(base, len(order)-.75), xytext=(5, 0),
             textcoords="offset points", fontsize=8, color=INK, va="center", linespacing=1.3)
ax2.set_yticks(range(len(order)))
ax2.set_yticklabels([])
ax2.set_xlim(-.02, .52)
ax2.set_ylim(-.7, len(order)-.3)
ax2.xaxis.set_major_formatter(lambda v, _: f"{v:.0%}")
ax2.set_xlabel("precision@20", color=MUTED)
ax2.set_title("Temporal vs random — and not resolvable", fontsize=11,
              color=INK, loc="left", pad=26)
ax2.annotate("● temporal     ● random, mean ± sd over 20 seeds",
             xy=(0, 1.012), xycoords="axes fraction", ha="left", va="bottom",
             fontsize=8.5, color=MUTED)
ax2.annotate("● ", xy=(0, 1.012), xycoords="axes fraction", ha="left", va="bottom",
             fontsize=8.5, color=TEMP)

for ax in (ax1, ax2):
    ax.set_facecolor(SURF)
    ax.set_axisbelow(True)
    for s in ("top", "right", "left"):
        ax.spines[s].set_visible(False)
    ax.spines["bottom"].set_color("#d5d4cf")
    ax.tick_params(colors=MUTED, labelsize=9)
ax2.grid(axis="x", color="#e6e5e1", lw=.8)

fig.suptitle("precision@20 — what an inspection budget of 20 lots actually buys",
             fontsize=13, color=INK, x=.008, ha="left", y=.975)
fig.text(.008, .012,
         f"Bootstrap 95% CI on the shortlist's {t.max():.0%} is [15%, 60%] — every model here sits inside "
         f"every other model's interval, so panel (b) ranks nothing.\n"
         f"Three of five models score HIGHER on the temporal split than on random ones, the reverse of the "
         f"AUC gap: at {nfail} test failures precision@20 moves in steps of 5 points.\n"
         f"Dots are in true rank order. Model C — the only protocol without selection leakage — finds its "
         f"one failure at rank 11. See REPORT.md.",
         fontsize=7.5, color=MUTED, ha="left", linespacing=1.5)
fig.tight_layout(rect=[0, .095, .995, .935])
fig.savefig("figures/precision_at_20.png", dpi=200, facecolor=fig.get_facecolor())

out = pd.DataFrame({"temporal": t, "hits": (t*TOPK).round().astype(int),
                    "random_mean": r["mean"], "random_sd": r["std"], "gap": r["mean"]-t})
print(out.round(3).to_string())
print(f"test: {nfail} failures / {ntest} lots, base {base:.4f}, random-inspection hits {base*TOPK:.2f}")
print("  -> figures/precision_at_20.png")