Add an unactivated v2 household prompt contract for issue 165

.github/workflows/ci.yml

@@ -23,6 +23,10 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v6
+        with:
+          # The release gates read the September 22c base files from git
+          # (scripts/finish_adds0928.py BASE_COMMIT), so fetch full history.
+          fetch-depth: 0
       - uses: actions/setup-python@v6
         with:
           python-version: "3.12"

app/next.config.ts

@@ -1,7 +1,21 @@
 import type { NextConfig } from "next";
 
+// Note URLs that moved. The BBCE note, first published September 23, was
+// republished as of October 5 on release dashboard-data-20260930; its old URL
+// stays a permanent link to it.
+export const noteRedirects = [
+  {
+    source: "/notes/2026-09-23-five-snap-households-bbce",
+    destination: "/notes/2026-10-05-five-snap-households-bbce",
+    permanent: true,
+  },
+];
+
 const nextConfig: NextConfig = {
   reactStrictMode: true,
+  async redirects() {
+    return noteRedirects;
+  },
 };
 
 export default nextConfig;

app/package.json

@@ -20,6 +20,7 @@
     "@types/react-dom": "^19.2.3",
     "eslint": "^9.38.0",
     "eslint-config-next": "^16.2.6",
+    "fast-check": "^4.10.2",
     "next": "^16.2.6",
     "tailwindcss": "^4.2.4",
     "typescript": "~5.9.3"

app/src/app/paper/page.tsx

@@ -3,11 +3,11 @@ import type { Metadata } from "next";
 import Link from "next/link";
 
 import SiteHeader from "../../components/SiteHeader";
+import paperSnapshot from "../../paperSnapshot.json";
 
-const SNAPSHOT_DATE_LABEL = "Snapshot 2026-09-05";
+const SNAPSHOT_DATE_LABEL = `Snapshot ${paperSnapshot.snapshotDate}`;
 
-const PAPER_DESCRIPTION =
-  "PolicyBench paper: the 2026-09-05 manuscript snapshot reporting the household-impact-weighted exact-match rate, with model responses collected from June 12 through September 1 and PolicyEngine reference outputs on the US populace microdata.";
+const PAPER_DESCRIPTION = `PolicyBench paper: the ${paperSnapshot.snapshotDate} manuscript snapshot reporting the household-impact-weighted exact-match rate, with model responses collected between ${paperSnapshot.responseWindow}, against PolicyEngine reference outputs on the US populace microdata.`;
 
 export const metadata: Metadata = {
   title: "Paper",
@@ -38,9 +38,11 @@ export const metadata: Metadata = {
   },
 };
 
+// paperSnapshot.json is written by scripts/freeze_snapshot.py from the paper
+// snapshot manifest; the cache keys follow the rendered files' hashes.
 const manuscriptPaths = {
-  pdf: "/paper/policybench.pdf",
-  web: "/paper/web/index.html?v=20260901-refreeze",
+  pdf: `/paper/policybench.pdf?v=${paperSnapshot.pdfVersion}`,
+  web: `/paper/web/index.html?v=${paperSnapshot.webVersion}`,
 };
 const ssrnUrl = process.env.NEXT_PUBLIC_POLICYBENCH_SSRN_URL;
 
@@ -49,9 +51,10 @@ export default function PaperPage() {
     <>
       <p className="max-w-2xl text-sm leading-relaxed text-text-secondary sm:text-base">
         Benchmarking no-tool tax-and-benefit estimation in frontier language
-        models. This page embeds the 2026-09-05 scored manuscript snapshot: a
-        100-household public preview reporting the household-impact-weighted
-        exact-match rate against PolicyEngine reference outputs.
+        models. This page embeds the {paperSnapshot.snapshotDate} scored
+        manuscript snapshot: a 100-household public preview reporting the
+        household-impact-weighted exact-match rate against PolicyEngine
+        reference outputs.
       </p>
       <div className="mt-4 flex flex-wrap items-center gap-3">
         <span className="inline-flex items-center gap-1.5 rounded-full border border-border bg-card px-2.5 py-1 text-[10px] font-medium uppercase tracking-[0.12em] text-text-secondary">

app/src/components/ExclusionNote.tsx

@@ -32,10 +32,14 @@ export default function ExclusionNote({
   isBinary: boolean;
   currencySymbol: "$" | "£";
 }) {
-  const unlistedInput = exclusion?.unlistedInput ?? pred.excludedInput;
-  const reason = describeExclusionReason(
-    exclusion?.reasonCode ?? pred.excludedReason,
-  );
+  const reasonCode = exclusion?.reasonCode ?? pred.excludedReason;
+  const isDefect = reasonCode === "reference_engine_defect";
+  const isLaterLaw = reasonCode === "reference_law_published_after_freeze";
+  const unlistedInput =
+    isDefect || isLaterLaw
+      ? undefined
+      : (exclusion?.unlistedInput ?? pred.excludedInput);
+  const reason = describeExclusionReason(reasonCode);
   return (
     <section
       className="mt-4 rounded-lg border border-border-subtle bg-surface px-4 py-3"
@@ -59,7 +63,52 @@ export default function ExclusionNote({
         . The prediction above is shown for inspection and counts neither for
         nor against this model.
       </p>
-      {exclusion ? (
+      {exclusion && isLaterLaw ? (
+        <>
+          <p className="mt-2 text-sm text-text-secondary leading-relaxed">
+            {exclusion.published ? `${exclusion.published}. ` : ""}
+            {exclusion.alternativeReading} Under the published figure the value
+            is{" "}
+            <span className="font-[family-name:var(--font-mono)]">
+              {formatValue(exclusion.alternativeValue, isBinary, currencySymbol)}
+            </span>
+            ; the frozen{" "}
+            <span className="font-[family-name:var(--font-mono)]">
+              {formatValue(exclusion.frozenValue, isBinary, currencySymbol)}
+            </span>{" "}
+            used the engine&rsquo;s projection.
+          </p>
+          <p className="mt-2 text-xs text-text-muted leading-relaxed">
+            {exclusion.law ? `Law: ${exclusion.law}. ` : ""}
+            Both values computed with {exclusion.engineVersion}; excluded by the
+            developers on {exclusion.decidedOn}.
+            {exclusion.note ? ` ${exclusion.note}` : ""}
+          </p>
+        </>
+      ) : exclusion && isDefect ? (
+        <>
+          <p className="mt-2 text-sm text-text-secondary leading-relaxed">
+            {exclusion.defect ? `${exclusion.defect}. ` : ""}
+            {exclusion.alternativeReading} Applying that rule gives{" "}
+            <span className="font-[family-name:var(--font-mono)]">
+              {formatValue(exclusion.alternativeValue, isBinary, currencySymbol)}
+            </span>{" "}
+            rather than the frozen{" "}
+            <span className="font-[family-name:var(--font-mono)]">
+              {formatValue(exclusion.frozenValue, isBinary, currencySymbol)}
+            </span>
+            .
+          </p>
+          <p className="mt-2 text-xs text-text-muted leading-relaxed">
+            {exclusion.law ? `Law: ${exclusion.law}. ` : ""}
+            Corrected value computed with {exclusion.engineVersion} and a
+            sandbox fix of the rule; excluded by the developers on{" "}
+            {exclusion.decidedOn}.
+            {exclusion.upstream ? ` Upstream: ${exclusion.upstream}.` : ""}
+            {exclusion.note ? ` ${exclusion.note}` : ""}
+          </p>
+        </>
+      ) : exclusion ? (
         <>
           <p className="mt-2 text-sm text-text-secondary leading-relaxed">
             {exclusion.alternativeReading} Under that reading the reference is{" "}

app/src/components/Methodology.tsx

@@ -13,6 +13,12 @@ import {
   AUDIT_SELECTION_RULE,
   summarizeAuditUniverse,
 } from "../lib/auditUniverse";
+import { excludedOutputEngineSentence } from "../lib/referenceEngine";
+import {
+  capitalizeFirst,
+  chunkedServingModels,
+  numberWord,
+} from "../lib/servingConfig";
 
 const versionRegistry = parseDataVersionRegistry(versionRegistryJson);
 
@@ -94,6 +100,22 @@ export default function Methodology({
     country === "uk" ? "UK transfer households" : "populace households";
   const referenceOutputSource =
     country === "uk" ? "PolicyEngine-UK" : "PolicyEngine-US";
+  // The engine release behind this board's references, as its payload records
+  // it; archived boards name their own.
+  const referenceEngineVersion =
+    benchData.policyengineBundles?.[country]?.model_version ?? null;
+  // The current board's chunked rows, from the frozen serving configuration,
+  // against the board's own model count.
+  const chunkedRowsPhrase = `${capitalizeFirst(
+    numberWord(chunkedServingModels().length),
+  )} of the ${noToolsModels.length} models`;
+  const excludedOutputsSentence =
+    currentBoard && country === "us"
+      ? excludedOutputEngineSentence(
+          benchData.referenceExclusions,
+          referenceEngineVersion,
+        )
+      : null;
   const benchmarkDescription = currentBoard
     ? country === "uk"
       ? "This app shows the current no-tools UK benchmark on a fixed test set, with PolicyEngine reference outputs computed by PolicyEngine-UK for fiscal year 2026-27."
@@ -156,13 +178,13 @@ export default function Methodology({
           Each model sees the same household facts and requested outputs and
           must return every scored output plus a short explanation for each,
           with no external tools — no calculator, search, or PolicyEngine
-          access. The answer instructions follow the provider's transport:
-          a forced answer-schema tool call where the provider accepts one, and a
-          JSON object where the provider rejects a forced tool or the model card
-          selects JSON for the family (the older Gemini and DeepSeek rows).{" "}
+          access. The answer instructions follow each row's transport: a
+          forced answer-schema tool call, or a JSON object where the model card
+          or its family default selects JSON, for most such rows because the
+          provider rejects a forced tool call.{" "}
           {currentBoard ? (
             <>
-              Ten of the 39 models answer the same facts in subsets of one or
+              {chunkedRowsPhrase} answer the same facts in subsets of one or
               three outputs per request, an accommodation that predates the
               whole-scenario rule. The per-model transport and request shape are
               recorded in the paper&apos;s serving-configuration table and the
@@ -194,7 +216,7 @@ export default function Methodology({
         <SectionCard title="Reference outputs">
           {country === "uk"
             ? "PolicyEngine-UK computes the PolicyEngine reference output for every household-variable pair in UK fiscal year 2026-27. The displayed variables define the benchmark scope for this snapshot."
-            : "PolicyEngine-US computes the PolicyEngine reference output for every household-variable pair in tax year 2026. The displayed variables define the benchmark scope for this snapshot."}
+            : `PolicyEngine-US${referenceEngineVersion ? ` (policyengine-us ${referenceEngineVersion})` : ""} computes each scored PolicyEngine reference output in tax year 2026.${excludedOutputsSentence ? ` ${excludedOutputsSentence}` : ""} The displayed variables define the benchmark scope for this snapshot.`}
         </SectionCard>
 
         <SectionCard title="Output selection">
@@ -244,9 +266,10 @@ export default function Methodology({
         </SectionCard>
 
         <SectionCard title="Cost basis">
-          Each frozen row uses its recorded per-call cost: provider-reported
-          where the provider returns one, otherwise reconstructed at the
-          configured list price at request time. List-price overrides apply at
+          Each frozen row uses its recorded per-call cost: reconstructed from
+          token counts at the list price configured at request time, or the
+          provider-reported charge where no reconstruction was available.
+          List-price overrides apply at
           request time, not retroactively to recorded costs. Models without
           per-call costs use the frozen release-metadata cost.
         </SectionCard>

app/src/components/ModelLeaderboard.tsx

@@ -24,12 +24,19 @@ import ServingSensitivityChip from "./ServingSensitivityChip";
 import {
   NEXT_BOARD_HREF,
   SENSITIVITY_DOC_HREF,
+  servingSensitivityCounts,
   servingSensitivityFor,
 } from "../lib/servingSensitivity";
 import {
   rankWithFallbackScore,
   rankWithRecomputedScores,
 } from "../lib/leaderboardRows";
+import {
+  capitalizeFirst,
+  joinWithAnd,
+  jsonContractClaudeModels,
+  numberWord,
+} from "../lib/servingConfig";
 import {
   SENSITIVITY_VIEWS,
   modelScoresForView,
@@ -240,6 +247,14 @@ export default function ModelLeaderboard({
   // metric, weighting or program filter the reader has selected.
   const showSensitivity =
     selectedView === "us" && isCurrentBoard(versionId, liveVersionId);
+  // The Claude rows the live board serves as JSON, from the frozen serving
+  // configuration: each one's API rejects a forced tool call.
+  const forcedToolRejectingClaudeLabels = joinWithAnd(
+    jsonContractClaudeModels().map((model) => MODEL_LABELS[model] ?? model),
+  );
+  // The auto re-runs, and the board rows among them that ran without
+  // extended thinking (src/lib/servingSensitivity.ts).
+  const sensitivityCounts = servingSensitivityCounts();
   const unfilteredExact = (model: string): number => {
     const row = baseNoTools.find((entry) => entry.model === model);
     return row?.exact ?? row?.score ?? 0;
@@ -306,19 +321,22 @@ export default function ModelLeaderboard({
             Serving sensitivity · August 2026
           </div>
           <p className="mt-2 text-sm leading-relaxed text-text-secondary">
-            Three Claude rows ran without extended thinking: this board forces
+            {capitalizeFirst(numberWord(sensitivityCounts.thinkingSuppressed))}{" "}
+            Claude rows ran without extended thinking: this board forces
             the answer tool call for every row whose model card selects the
-            tool contract, which switches Claude&apos;s thinking off. Claude
-            Fable 5.1 rejects forced calls, so its row selects JSON, answers as
-            a JSON object and reasons either way; its marker compares
-            transports. The{" "}
-            <code>tool_choice: auto</code> re-runs are marked on the four rows
+            tool contract, which switches Claude&apos;s thinking off.{" "}
+            {forcedToolRejectingClaudeLabels} reject forced calls, so their
+            rows select JSON, answer as a JSON object and reason either way;
+            Fable 5.1&apos;s marker compares transports.
+            The{" "}
+            <code>tool_choice: auto</code> re-runs are marked on the{" "}
+            {numberWord(sensitivityCounts.reruns)} rows
             (open a marker for the score and why); the board itself is
             unchanged. The{" "}
             <a href={SENSITIVITY_DOC_HREF} className="text-primary hover:underline">
               sensitivity note
             </a>{" "}
-            has all four runs, and the{" "}
+            has all {numberWord(sensitivityCounts.reruns)} runs, and the{" "}
             <a href={NEXT_BOARD_HREF} className="text-primary hover:underline">
               next board version
             </a>{" "}