Hac/343 metric manifest

.github/workflows/ci.yml

@@ -104,7 +104,17 @@ jobs:
     name: test
     runs-on: ubuntu-24.04
     steps:
+      # Full history, not the default depth-1 clone. HAC-343 froze three
+      # contracts before any arm produced a result, and
+      # experiments/hac-343/bin/verify-packet.mjs proves that by resolving each
+      # freeze commit and comparing the bytes on disk to the blob that commit
+      # recorded. A shallow checkout has neither the commits nor the blobs, so
+      # the verifier fails with `path exists on disk, but not in <sha>` and
+      # test/hac-343-check-wiring.test.mjs fails with it. Measured: exit 1 at
+      # depth 1, exit 0 with full history.
       - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
 
       # Pinned for the same reason as codecov-action below: a third-party action
       # on a mutable tag. Sonar did not flag this one only because it is not new
@@ -515,6 +525,87 @@ jobs:
             echo "genuinely changed, that is a claim change and belongs on HAC-333."
           } >> "$GITHUB_STEP_SUMMARY"
 
+  evaluation-gate:
+    name: Evaluation gate
+    runs-on: ubuntu-24.04
+    steps:
+      # Full history, not the default depth-1 clone. HAC-343 froze three
+      # contracts before any arm produced a result, and the verifier proves
+      # that by resolving each freeze commit and comparing the bytes on disk
+      # to the blob that commit recorded. A shallow checkout has neither the
+      # commits nor the blobs: the verifier fails with `path exists on disk,
+      # but not in <sha>`. Measured: exit 1 at depth 1, exit 0 with history.
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+      - uses: pnpm/action-setup@b906affcce14559ad1aafd4ab0e942779e9f58b1 # v4
+      - uses: actions/setup-node@v4
+        with:
+          node-version: '22.19.0'
+          cache: pnpm
+      - name: Install dependencies
+        run: pnpm install --frozen-lockfile --ignore-scripts
+
+      # `check:packet:eval` builds first: alone among the packet verifiers,
+      # experiments/hac-343/lib/arms.mjs loads the compiled decision core from
+      # dist/, because the experiment measures the real arbitrate() rather than
+      # a reimplementation of it.
+      - name: Verify the HAC-343 evaluation packet
+        run: pnpm run check:packet:eval
+
+      # The seam, in both directions: an invalid packet must fail this gate,
+      # and a HAC-343 field moving underneath the committed view model must
+      # fail the cockpit gate.
+      - name: The cockpit's HAC-343 bindings fail when the packet moves
+        run: pnpm vitest run test/hac-343-check-wiring.test.mjs
+
+      # `judge-export.json` is a *derived* presentation artifact: the verifier
+      # above recomputes the report from the raw records, but it never rebuilt
+      # the export. Ten judge-facing values reach the comparison panel through
+      # that file alone — all four strategy labels, the per-target-lock
+      # credibility figure, and the canonical result commit — so a hand edit
+      # there reached a judge with every gate green.
+      #
+      # Rebuild and assertion are two steps on purpose. An enforcement step's
+      # `run` must be exactly one expected command, because a multi-line body is
+      # a shell script and no amount of reading it tells you which lines
+      # actually execute: `if false; then`, a heredoc and an open quote all put
+      # a command at the start of a line without running it. One command per
+      # step is a shape that can be checked instead of inferred.
+      - name: Rebuild the derived judge export
+        run: node experiments/hac-343/bin/build-judge-export.mjs
+
+      - name: The judge export is byte-identical to its rebuild
+        run: git diff --exit-code -- experiments/hac-343/evidence/judge-export.json
+
+      - name: Explain the failure
+        if: failure()
+        run: |
+          {
+            echo "## Evaluation gate failed"
+            echo
+            echo "**Invariant.** Every metric in \`experiments/hac-343/evidence/results.json\`"
+            echo "is recomputed from the raw records rather than read back from the summary,"
+            echo "and the three contracts frozen before the run — metric definitions, corpus"
+            echo "and execution semantics — are still byte-identical to the blobs their freeze"
+            echo "commits recorded."
+            echo
+            echo "**Why it matters.** The judge cockpit binds twenty-four comparison cells"
+            echo "into this packet. A packet that no longer verifies is a packet whose numbers"
+            echo "a judge is nonetheless reading, and \"frozen before results\" stops being"
+            echo "checkable by anyone who was not there."
+            echo
+            echo "**Authority.** Repository CI, reported as \`Evaluation gate\`."
+            echo
+            echo "**Evidence required.** \`pnpm run check:packet:eval\` passing on this commit,"
+            echo "and \`test/hac-343-check-wiring.test.mjs\` green."
+            echo
+            echo "**Do not weaken.** Do not re-freeze a contract to match an edit, do not read"
+            echo "a metric out of the summary to make the recomputation agree, and do not drop"
+            echo "\`fetch-depth: 0\` to make the freeze-commit checks stop resolving. If the"
+            echo "experiment genuinely changed, it is a new run, not an edited one."
+          } >> "$GITHUB_STEP_SUMMARY"
+
   cockpit-contract-gate:
     name: Cockpit contract gate
     runs-on: ubuntu-24.04

experiments/hac-343/bin/build-corpus.mjs

@@ -0,0 +1,230 @@
+#!/usr/bin/env node
+/**
+ * HAC-343 — build and freeze the corpus manifest.
+ *
+ *     WORKSPACEJSON_CLI=<pinned-cli-checkout> node experiments/hac-343/bin/build-corpus.mjs
+ *
+ * Materializes the family-2 fixture histories, mines both, records what the
+ * miner actually observed, validates every corpus requirement the frozen metric
+ * manifest imposes, and writes `evidence/corpus.json`.
+ *
+ * This runs *before* any arm exists. That ordering is the point: the corpus and
+ * its labels are fixed, and demonstrated to be fixed, before anything can be
+ * measured against them. A corpus adjusted after seeing an arm's output is not a
+ * corpus, and `metric-definitions.json` forbids it in writing.
+ *
+ * Family 1's fixtures are HAC-330's, reused rather than rebuilt: this script
+ * reads their recorded revisions from that packet and does not regenerate them,
+ * so the budget family in this corpus is the same history the S-1 gate proved.
+ */
+import { createHash } from 'node:crypto';
+import { mkdirSync, readFileSync, writeFileSync } from 'node:fs';
+import { dirname, join, resolve } from 'node:path';
+import { fileURLToPath } from 'node:url';
+
+import { loadMiner, mineEvidence, resolveCliCheckout, verifyPin } from '../../hac-330/lib/evidence.mjs';
+import { buildFixture, FIXTURES, SUBJECT_PATHS } from '../lib/families/registry.mjs';
+import { SCENARIOS, FAMILIES, corpusCounts, validateCorpus } from '../lib/corpus.mjs';
+
+const HERE = dirname(fileURLToPath(import.meta.url));
+const EXPERIMENT_DIR = resolve(HERE, '..');
+const REPO_ROOT = resolve(EXPERIMENT_DIR, '..', '..');
+const EVIDENCE_DIR = join(EXPERIMENT_DIR, 'evidence');
+const WORK_DIR = join(EXPERIMENT_DIR, '.work', 'fixtures');
+
+process.chdir(REPO_ROOT);
+
+const writeJson = (path, value) => writeFileSync(path, `${JSON.stringify(value, null, 2)}\n`);
+const sha256 = (buffer) => createHash('sha256').update(buffer).digest('hex');
+
+const failures = [];
+function check(id, passed, detail) {
+  if (!passed) failures.push(`${id}: ${detail}`);
+  console.log(`  ${passed ? 'PASS' : 'FAIL'}  ${id.padEnd(14)} ${detail}`);
+  return passed;
+}
+
+const section = (title) => console.log(`\n── ${title} ${'─'.repeat(Math.max(0, 60 - title.length))}`);
+
+// ---------------------------------------------------------------------------
+
+section('Pinned checkouts');
+
+const cliCheckout = resolveCliCheckout();
+const pins = {
+  'workspacejson-cli': verifyPin('workspacejson-cli', cliCheckout),
+  'workspacejson-standard': verifyPin('workspacejson-standard', join(cliCheckout, '..', 'standard')),
+};
+for (const [id, pin] of Object.entries(pins)) {
+  check(id === 'workspacejson-cli' ? 'PIN-CLI' : 'PIN-STD', pin.matches && pin.clean, `${pin.observedSha} ${pin.clean ? 'clean' : 'DIRTY'}`);
+}
+
+// ---------------------------------------------------------------------------
+
+section('Family 1 — budget (reused from HAC-330)');
+
+const budgetFixtures = JSON.parse(
+  readFileSync(join(REPO_ROOT, 'experiments', 'hac-330', 'evidence', 'fixtures.json'), 'utf8'),
+);
+check(
+  'F1-REUSED',
+  typeof budgetFixtures.baseline?.head === 'string' && typeof budgetFixtures.perturbed?.head === 'string',
+  `baseline ${budgetFixtures.baseline?.head?.slice(0, 12)} perturbed ${budgetFixtures.perturbed?.head?.slice(0, 12)}`,
+);
+check(
+  'F1-TREE',
+  budgetFixtures.baseline.tree === budgetFixtures.perturbed.tree,
+  `shared final tree ${budgetFixtures.baseline.tree.slice(0, 12)}`,
+);
+
+// ---------------------------------------------------------------------------
+
+section('Family 2 — registry (built here)');
+
+mkdirSync(WORK_DIR, { recursive: true });
+const registryFixtures = {};
+for (const [name, steps] of Object.entries(FIXTURES)) {
+  registryFixtures[name] = buildFixture(join(WORK_DIR, name), steps);
+  const f = registryFixtures[name];
+  console.log(`  built ${name.padEnd(10)} head=${f.head.slice(0, 12)} tree=${f.tree.slice(0, 12)} commits=${f.commitCount}`);
+}
+
+check(
+  'F2-TREE',
+  registryFixtures.baseline.tree === registryFixtures.perturbed.tree,
+  `shared final tree ${registryFixtures.baseline.tree.slice(0, 12)} — the perturbation is history-only`,
+);
+check(
+  'F2-SHAPE',
+  registryFixtures.baseline.commitCount === registryFixtures.perturbed.commitCount,
+  `${registryFixtures.baseline.commitCount} commits in each`,
+);
+
+// ---------------------------------------------------------------------------
+
+section('Family 2 — mined evidence');
+
+const miner = await loadMiner();
+const registryEvidence = {};
+const qualifying = {};
+
+for (const name of Object.keys(FIXTURES)) {
+  const evidence = await mineEvidence({
+    fixture: name,
+    repo: join(WORK_DIR, name),
+    miner,
+  });
+  registryEvidence[name] = evidence;
+
+  const selection = evidence.envelope?.selection ?? evidence.selection;
+  const pairs = (selection?.pairs ?? []).filter((p) => p.support >= 3);
+  qualifying[name] = pairs.map((p) => ({ files: p.files, support: p.support, occurrences: p.occurrences ?? 0 }));
+
+  writeJson(join(EVIDENCE_DIR, `registry.${name}.evidence.json`), evidence.envelope ?? evidence);
+}
+
+const spansSubjects = (pairs) =>
+  pairs.some(
+    (p) =>
+      (p.files[0] === SUBJECT_PATHS.left && p.files[1] === SUBJECT_PATHS.right) ||
+      (p.files[0] === SUBJECT_PATHS.right && p.files[1] === SUBJECT_PATHS.left),
+  );
+
+check(
+  'F2-COUPLED',
+  spansSubjects(qualifying.baseline),
+  `baseline carries ${SUBJECT_PATHS.left} <-> ${SUBJECT_PATHS.right}`,
+);
+check(
+  'F2-PERTURBED',
+  !spansSubjects(qualifying.perturbed),
+  `perturbed does NOT carry that pair — only the subject coupling was removed`,
+);
+
+const touchesIndependent = (pairs) =>
+  pairs.some((p) => p.files.includes(SUBJECT_PATHS.independent) &&
+    (p.files.includes(SUBJECT_PATHS.left) || p.files.includes(SUBJECT_PATHS.right)));
+check(
+  'F2-INDEP',
+  !touchesIndependent(qualifying.baseline),
+  `${SUBJECT_PATHS.independent} couples to neither subject path, so it is usable as the INDEPENDENT counterpart`,
+);
+
+// ---------------------------------------------------------------------------
+
+section('Corpus requirements');
+
+const corpusFailures = validateCorpus();
+check('CORPUS-VALID', corpusFailures.length === 0, corpusFailures.length ? corpusFailures.join('; ') : 'every requirement in metric-definitions.json holds');
+
+const counts = corpusCounts();
+check('CORPUS-BREADTH', FAMILIES.length >= 2, `${FAMILIES.length} families, ${counts.total} scenarios`);
+
+// ---------------------------------------------------------------------------
+
+section('Write manifest');
+
+const manifest = {
+  experiment: 'HAC-343',
+  kind: 'corpus manifest',
+  status: 'FROZEN_BEFORE_RESULTS',
+  revision: 'r01',
+  supersedes: [],
+  frozenRule:
+    'Committed in its own commit, after metric-definitions.json and before any arm implementation or results.json. Scenario counts, labels and intents are fixed here. A corpus change after any result exists invalidates that result and requires a rerun, per metric-definitions.json corpusRequirements.noOptimisation.',
+  metricDefinitions: {
+    file: 'experiments/hac-343/evidence/metric-definitions.json',
+    revision: 'r01',
+    sha256: sha256(readFileSync(join(EVIDENCE_DIR, 'metric-definitions.json'))),
+  },
+  breadthRationale:
+    'Two structurally different hazard classes, so a result is not one topology repeated. budget is arithmetic (composed increases overshoot a ceiling); registry is referential (one intent removes a referent the other points at). Both carry all five ground-truth classes, so a per-family divergence is about hazard shape rather than uneven class coverage.',
+  families: {
+    budget: {
+      hazardClass: 'arithmetic',
+      invariant: 'sum(services[].reserved) <= budget.totalReservable',
+      source: 'experiments/hac-330 — reused verbatim, not rebuilt',
+      subjectPaths: {
+        left: 'services/alpha/reservation.json',
+        right: 'services/beta/reservation.json',
+        independent: 'services/gamma/reservation.json',
+      },
+      fixtures: budgetFixtures,
+    },
+    registry: {
+      hazardClass: 'referential',
+      invariant: 'every route.service and alias target resolves in registry/services.json',
+      source: 'experiments/hac-343/lib/families/registry.mjs — built by this script',
+      subjectPaths: SUBJECT_PATHS,
+      fixtures: registryFixtures,
+      qualifyingPairs: qualifying,
+      controls: {
+        sharedFinalTree: registryFixtures.baseline.tree,
+        commitCount: registryFixtures.baseline.commitCount,
+        note: 'Same four controls as HAC-330: identical final tree, identical commit count, commit i touching the same number of files in both, and the invariant holding at every commit (asserted in planCommits).',
+      },
+    },
+  },
+  counts,
+  scenarios: SCENARIOS,
+  validation: {
+    corpusRequirements: corpusFailures.length === 0 ? 'PASS' : corpusFailures,
+    checksRun: ['PIN-CLI', 'PIN-STD', 'F1-REUSED', 'F1-TREE', 'F2-TREE', 'F2-SHAPE', 'F2-COUPLED', 'F2-PERTURBED', 'F2-INDEP', 'CORPUS-VALID', 'CORPUS-BREADTH'],
+  },
+  reproduction: {
+    buildCommand: 'WORKSPACEJSON_CLI=<pinned-cli-checkout> node experiments/hac-343/bin/build-corpus.mjs',
+    pins,
+  },
+};
+
+writeJson(join(EVIDENCE_DIR, 'corpus.json'), manifest);
+console.log(`  wrote experiments/hac-343/evidence/corpus.json (${counts.total} scenarios, ${FAMILIES.length} families)`);
+
+// ---------------------------------------------------------------------------
+
+if (failures.length > 0) {
+  console.error(`\nFAILED — ${failures.length} check(s):`);
+  for (const failure of failures) console.error(`  ${failure}`);
+  process.exit(1);
+}
+console.log('\nCorpus frozen. All checks passed.');