Add decisions-v1 corpus of real GitHub decisions

.gitignore

@@ -46,3 +46,9 @@ vite.config.ts.timestamp-*
 # Temporary local design jigs.
 /apps/web/design-jig/
 /apps/web/src/design-jig/
+
+# Raw GitHub responses for evals/datasets/decisions-v1 (never commit)
+/.cache/decisions/
+
+# Python bytecode from dataset tooling
+__pycache__/

evals/datasets/README.md

@@ -35,6 +35,14 @@ Likewise, GitHub's [site terms](https://docs.github.com/en/site-policy/github-te
 do not by themselves clearly permit republication of complete third-party
 issue and discussion comments outside GitHub, so v2 uses attributed summaries.
 
+## decisions-v1
+
+[decisions-v1](decisions-v1/README.md) is a larger corpus of 127 adapted GitHub
+decision threads in the same `async-discussion` format. Each thread has a
+decision record annotated for Chopin's questions design. It passed the holdout
+check on 2026-10-19. `bun test evals/datasets` runs its structural validator;
+the verbatim-leak check needs the collector's raw cache.
+
 ## Offline use
 
 ```ts

evals/datasets/decisions-v1.test.ts

@@ -0,0 +1,43 @@
+import { describe, expect, test } from "bun:test";
+import { createHash } from "node:crypto";
+import { readFile } from "node:fs/promises";
+import manifest from "./decisions-v1/manifest.json";
+
+let root = new URL("./decisions-v1/", import.meta.url);
+
+// The leak check needs the collector's git-ignored raw cache, so CI runs the
+// structural checks only.
+function validate(script: string) {
+	let run = Bun.spawnSync(["python3", "-I", new URL(script, root).pathname, "--no-leaks"]);
+	return { code: run.exitCode, output: run.stdout.toString() + run.stderr.toString() };
+}
+
+describe("decisions-v1 corpus", () => {
+	test("passes its validator", () => {
+		let result = validate("validate.py");
+		expect(result.output).toContain(" 0 problems");
+		expect(result.code).toBe(0);
+	});
+
+	test("cuts every case at its checkpoints the way v2 does", async () => {
+		expect(manifest.cases.length).toBeGreaterThan(0);
+		for (let entry of manifest.cases) {
+			let bytes = await readFile(new URL(entry.file.path, root));
+			expect(bytes.byteLength).toBe(entry.file.bytes);
+			expect(createHash("sha256").update(bytes).digest("hex")).toBe(entry.file.sha256);
+			let document = JSON.parse(new TextDecoder().decode(bytes));
+			expect(document).toMatchObject({ schemaVersion: 2, origin: "adapted", id: entry.id });
+			expect(document.cutoffs.map((cutoff: { id: string }) => cutoff.id)).toEqual(
+				entry.checkpoints,
+			);
+			for (let cutoff of document.cutoffs) {
+				let at = Date.parse(cutoff.at);
+				let visible = document.events.filter(
+					(event: { createdAt: string; visibleAt?: string }) =>
+						Date.parse(event.visibleAt ?? event.createdAt) <= at,
+				);
+				expect(visible[0]?.kind).toBe("opening");
+			}
+		}
+	});
+});

evals/datasets/decisions-v1/README.md

@@ -0,0 +1,200 @@
+# decisions-v1
+
+A corpus of 127 real software decisions debated on public GitHub threads. It
+has two jobs:
+
+1. **Coverage critique** for Chopin's questions feature. Each decision record
+   says which question shape it needed (explore, choice, tune, check), which
+   flows it exercised, and what it needed that the design does not offer.
+2. **Evals.** Each case file uses the v2 `async-discussion` format from
+   [`../v2/discussions`](../v2/discussions/README.md), with 2–4 cutoffs at
+   meaningful moments.
+
+| File                                 | Contents                                                                      |
+| ------------------------------------ | ----------------------------------------------------------------------------- |
+| [`cases/*.json`](cases)              | 127 adapted threads, 8,367 paraphrased events                                 |
+| [`decisions.jsonl`](decisions.jsonl) | 132 decision records; a thread may hold more than one                         |
+| [`manifest.json`](manifest.json)     | Path, SHA-256, bytes, source URL, retrieval time and cutoff purposes per case |
+| [`stats.json`](stats.json)           | Shape, flow and gap distributions, plus process shares                        |
+| [`candidates.json`](candidates.json) | Every one of the 192 screened candidates and its outcome                      |
+| [`validate.py`](validate.py)         | Schema, enum, identity, manifest and licensing-leak checks                    |
+| [`tools/`](tools)                    | The fetcher, normaliser, assembler, packager and annotation guide             |
+
+## Provenance
+
+The 192 candidates came from two sources:
+
+- **24 seeds** from an earlier cloud session's scratch corpus.
+- **168 search hits** from `gh search issues`, `gh search prs` and GraphQL
+  discussion search, targeted at the six source quotas below.
+
+Every candidate was refetched through the GitHub API (`gh api`) on
+2026-10-18; each case records its exact `retrievedAt`:
+
+- **Discussions:** GraphQL, with both comment levels paginated.
+- **Issues:** REST issue, comments, timeline and reactions.
+- **PRs:** the issue endpoints plus reviews, review comments and the changed-file
+  count.
+
+Raw responses live only in the git-ignored `.cache/decisions/`.
+`tools/normalise.py` turns each thread into an event skeleton. Every structural
+field comes from the API and is never typed by hand: IDs, actors, `actorId`,
+`actorType`, times, URLs and reply parents.
+
+Annotation was done by Claude subagents, one per batch of threads, following
+[`tools/ANNOTATING.md`](tools/ANNOTATING.md). Each subagent:
+
+- read the digest, screened the thread,
+- wrote one paraphrased summary per event,
+- placed the cutoffs, and
+- filled the decision record.
+
+`tools/assemble.py` merges the summaries into the skeleton.
+`tools/check_ann.py` and `validate.py` checked every annotation. Two
+cross-references that carried the decision were followed one hop and merged
+into their case: a Node.js TSC vote PR and a Home Assistant core PR. Other
+followed links are cited in `sources` only.
+
+Of the 192 candidates:
+
+- **127 were kept** (14 of them seeds).
+- **20 were discarded**: support Q&A, +1 piles, a single-author memo, or a
+  design settled elsewhere.
+- **45 were fetched but not screened**, because their quota was already met
+  when screening stopped. Six are seeds: the very long Go threads #57001,
+  #48429, #49085, #19412 and discussions 58409 and 54763. `candidates.json`
+  records every status and reason. The v2 discussion threads were excluded.
+
+## Holdout check
+
+On 2026-10-19 every candidate, case source, related thread, decision-record
+source and implementation link was checked against the 8 holdout threads: 5 for
+validation from eval-hub partition snapshot
+`dda78480888859ae3eec90846c1263cd28552d05`, and 3 sealed for final tests. None
+matched, and no kept thread or source document links to any of them. Every
+thread URL in that snapshot's conversation corpus was also checked, and none
+matched. The holdout URLs are deliberately not recorded here. Rerun
+`tools/holdout_check.py <urls-file>` after adding cases.
+
+## Licensing approach
+
+GitHub's terms do not clearly permit republishing third-party comments, so
+committed files contain **attributed paraphrases only**:
+
+- Each event's `summary` is a new one- or two-sentence description of the
+  position taken, with the actor's role made clear.
+- Nothing is quoted.
+- Logins are public handles.
+
+`validate.py` compares every summary and every free-text record field with that
+case's own cached source threads. It fails on any run of 8 or more consecutive
+words that also appears in the source. Annotators reworded every hit; the
+committed set has none. The check needs the local raw cache. Without it, the
+check warns and skips.
+
+A few annotators chose not to include linked threads with abusive comments, and
+hidden off-topic comments are summarised only as hidden.
+
+## Counts
+
+Source quotas (target vs kept):
+
+| Quota                                              | Target | Kept |
+| -------------------------------------------------- | -----: | ---: |
+| Product, UI/UX and behaviour (`product-ui`)        |     30 |   35 |
+| Debates inside PR reviews (`pr-debate`)            |     20 |   26 |
+| Formal processes (`formal-process`)                |     15 |   24 |
+| Engineering-team decisions in issues (`eng-issue`) |     15 |   20 |
+| Small teams and solo maintainers (`small-team`)    |     10 |   11 |
+| Open choice (`open-choice`)                        |     10 |   11 |
+
+Annotators reassigned a thread's quota when it fitted better. For example, Home
+Assistant architecture threads settled by an ADR moved to `formal-process`, and
+repos with more than five regular contributors left `small-team`. The overrun
+comes from parallel batches finishing after the quotas filled; no kept case
+was dropped to hit a target.
+
+| Domain                                     | Decisions |
+| ------------------------------------------ | --------: |
+| ui-ux                                      |        32 |
+| product-behaviour                          |        22 |
+| api-design                                 |        22 |
+| architecture                               |        11 |
+| compatibility, language-design, data-model |    8 each |
+| process-governance                         |         6 |
+| tooling, naming                            |    5 each |
+| security                                   |         3 |
+| infra, dependency                          |    1 each |
+
+| Outcome (`how`)     | Decisions |
+| ------------------- | --------: |
+| maintainer-decision |        78 |
+| consensus           |        18 |
+| fcp                 |        12 |
+| default/no-action   |        10 |
+| abandoned           |         7 |
+| unclear             |         4 |
+| vote                |         3 |
+
+117 of the 132 decisions (89%) reached a decision. The rest are open, abandoned
+or postponed.
+
+| Primary shape | Decisions | As secondary shape |
+| ------------- | --------: | -----------------: |
+| choice        |        93 |                 30 |
+| mixed         |        11 |                  — |
+| check         |        11 |                 32 |
+| explore       |        10 |                 33 |
+| tune          |         7 |                 22 |
+
+The median decision involved 18 human participants and took 188 days from
+opening to decision. See [`stats.json`](stats.json) for the gap clusters and
+process shares.
+
+## Known limits
+
+- **One annotator per thread.** Each case was annotated once by a model, with no
+  second annotator and no agreement measure. Shapes, gaps, roles and outcomes
+  are judgements. Read the record, not just the label.
+- **Gap names are free text.** `stats.json` folds them into clusters using the
+  keyword rules in `tools/package.py`. Single-occurrence gaps that match no rule
+  appear as `other: …`.
+- **Sub-questions are over-reported.** Annotators listed sub-questions for
+  nearly every decision (98.5%), so `shareWithSubQuestions` says little. The
+  coupling-gap share (38.6%) is the more useful measure.
+- **Edits.** Summaries describe each comment's final text. `editedAt` is
+  recorded, but no event sets `visibleAt`, so a cutoff may show an edit's
+  content before the edit happened.
+- **The 150-event cap.** Two cases, `go-try-builtin` (822 source events) and
+  `nextjs-app-router-file-conventions` (481), keep the opening, maintainers,
+  bots, process events and the most-reacted comments. Their records are marked
+  `fidelity.truncated`. `participants.count`, `comments` and `reactionsTotal`
+  always count the whole thread.
+- **Missing close events.** GitHub's timeline API omitted the close event on
+  7 issues and PRs, including Go's `try` proposal. Those cases carry a close
+  event built from `closed_at`, with the actor recorded as `unknown`.
+- **What `implementedIn` cites.** Go changes cite `go.dev/cl` Gerrit changes,
+  and Node.js PRs show as closed rather than merged because they land through
+  the commit queue. An open, unmerged PR is never listed as an implementation.
+- **Off-platform evidence.** Meetings, Discord, blog posts and private data are
+  recorded only as other people reported them in the thread. They were not
+  fetched.
+- **REST issue comments are flat.** Their `parentId` is the opening. Discussions
+  and PR review threads keep their true reply structure.
+- **Not a gold set.** As with v2, these are development inputs. A close or merge
+  event is evidence of an outcome, not a verified Chopin settlement.
+
+## Reproducing
+
+```bash
+python3 -I evals/datasets/decisions-v1/tools/fetch.py <urls…>    # cached, idempotent
+python3 -I evals/datasets/decisions-v1/tools/normalise.py
+python3 -I evals/datasets/decisions-v1/tools/assemble.py          # needs .cache/decisions/ann/
+python3 -I evals/datasets/decisions-v1/tools/package.py
+python3 -I evals/datasets/decisions-v1/validate.py
+bun test evals/datasets                                           # structural checks, no cache needed
+```
+
+The annotation files in `.cache/decisions/ann/` are not committed. They contain
+only paraphrases, but their content is already fully represented in `cases/`
+and `decisions.jsonl`.