From b0d3e48152d14bcaa91b94813707c30ab2c39d40 Mon Sep 17 00:00:00 2001 From: Joris Wouter Jonkers Date: Mon, 14 Sep 2026 08:53:09 +0200 Subject: [PATCH 1/2] test: name every enforced behaviour in a ledger, and the test that proves it Adds docs/requirements.md, one row per behaviour this repository enforces today (the ADR, link, manifest and boundary gates, the PR title check, the em-dash ban, the test harness guards and the coverage ratchet), each citing the test file that proves it. scripts/lint-requirements.ts is the meta test that holds the ledger honest: every row parses, every named file exists, is a test file and holds a test, ids are unique, the stated row count matches, and every id cited anywhere in the tracked tree resolves to a row. Records the decision as docs/adr/architecture/0103, next free number after 0102 (reserved on another in-flight branch), and adds its register row. --- docs/adr/README.md | 1 + ...haviour-ledger-names-what-a-test-proves.md | 74 ++++++ docs/requirements.md | 34 +++ scripts/lint-requirements.ts | 206 +++++++++++++++ test/adr-contract.test.ts | 3 + test/boundary-contract.test.ts | 3 + test/emdash.test.ts | 3 + test/harness.test.ts | 6 + test/link-contract.test.ts | 3 + test/manifest-contract.test.ts | 3 + test/pr-title-contract.test.ts | 3 + test/requirements-contract.test.ts | 107 ++++++++ test/requirements-lint-negative.test.ts | 236 ++++++++++++++++++ 13 files changed, 682 insertions(+) create mode 100644 docs/adr/architecture/0103-a-behaviour-ledger-names-what-a-test-proves.md create mode 100644 docs/requirements.md create mode 100644 scripts/lint-requirements.ts create mode 100644 test/requirements-contract.test.ts create mode 100644 test/requirements-lint-negative.test.ts diff --git a/docs/adr/README.md b/docs/adr/README.md index 49357ff..f0020a1 100644 --- a/docs/adr/README.md +++ b/docs/adr/README.md @@ -203,3 +203,4 @@ Decisions about the compiler's own structure, not about the model. Their | [0100](architecture/0100-tests-run-in-process-on-vitest.md) | Tests run in-process on Vitest, and the tooling is TypeScript that Node runs directly | settled | | [0101](architecture/0101-coverage-is-a-ratchet.md) | Coverage is a ratchet: the thresholds sit on what the suite reaches, and only rise | settled | | [0102](architecture/0102-the-gate-grows-with-the-code.md) | A new gate's script and its CI job land in the same pull request, and a test proves the two stay matched | settled | +| [0103](architecture/0103-a-behaviour-ledger-names-what-a-test-proves.md) | A behaviour ledger names every guarantee and the test that proves it, and a meta test holds the two together | settled | diff --git a/docs/adr/architecture/0103-a-behaviour-ledger-names-what-a-test-proves.md b/docs/adr/architecture/0103-a-behaviour-ledger-names-what-a-test-proves.md new file mode 100644 index 0000000..796bde6 --- /dev/null +++ b/docs/adr/architecture/0103-a-behaviour-ledger-names-what-a-test-proves.md @@ -0,0 +1,74 @@ +--- +tier: decision +status: proposed +claim: settled +date: 2026-09-14 +normative: docs/architecture.md#gates +rests-on: ["0001"] +--- + +# A behaviour ledger names every guarantee and the test that proves it, and a meta test holds the two together + +## Rests on + +Every guarantee this repository makes is provable by naming the one test file +that would fail if the guarantee stopped holding, so a ledger of {id, +sentence, test file} triples can be checked by reading the tree rather than by +trusting whoever last touched it. False if: a guarantee exists that no single +test file proves, which would leave a row with nothing honest to cite. +Settled by: `docs/requirements.md` holding one row per gate this repository +enforces today, each resolving to a real, non-empty test file, with none left +over. + +## Why + +The coverage ratchet ([0101](0101-coverage-is-a-ratchet.md)) notices a line +that stops running, but not a test file that is renamed, emptied or deleted +while the code it once proved stays in place and green. Nothing before this +connected a sentence a contributor relies on to the test that makes it true, +so that sentence could go silently unproven and nothing in the suite would +say so. + +The fix is the same shape as the boundary gate and the script-to-workflow +check landing alongside it: state the rule as a comparison over things that +already exist on disk (a markdown table and a directory of test files), +rather than as a convention to remember. A row that no longer resolves, a +file that exists but holds no test, a stated count that drifts from the rows +actually present, or an id cited where no row backs it: each is a comparison +a script can run, not a habit a reviewer can forget, which matters here for +the same reason as [0001](../model/0001-estate-scale-and-ownership.md): one +person reading their own diff later is not a second pair of eyes. + +This ledger is deliberately not `docs/architecture-rules.md` +([issue #29](https://github.com/JorisJonkers-dev/deploy-kit/issues/29), not +yet landed). That one will list the rules a tool enforces, each keyed to its +enforcer; this one lists the behaviours a person depends on, each keyed to +the test that fails when it stops being true. A behaviour can rest on several +rules, and a rule can serve several behaviours, so keeping the ledgers apart +keeps each one answerable to a different question. + +## Alternatives + +| option | cost if taken | why rejected | +|---|---|---| +| No ledger; trust the existing gates and their own tests | Nothing to write or maintain | Exactly the failure mode this decision exists to close: a test file can be renamed, emptied or deleted while the code it covered stays in place, and nothing today would say so | +| A checklist item in the pull request template | Visible to a human reviewer | There is no second reviewer here, and a template item is unchecked by habit long before it is unchecked on purpose | +| Fold behaviours into `docs/architecture-rules.md` once it lands | One document instead of two | A rule and a behaviour answer different questions; merging them would make the rule ledger's per-rule enforcer story and this ledger's per-behaviour test story compete for the same row | + +## Reversibility + +Undo cost today: deleting one document and one gate script. Becomes +irreversible once: never; the ledger describes this repository's own tree and +nothing outside it reads either. + +## Consequences + +- A row must name a real, non-empty test file, so adding a guarantee with no + test to cite is not representable; the guarantee waits until it has one. + Paid by whoever adds a row, once per row. +- Renaming, emptying or deleting a test file a row names fails the suite + instead of merging quietly, and the failure names the row. Paid by whoever + touches that file, in the same pull request. +- The stated row count is one more thing to update by hand when a row is + added or removed; forgetting it fails the same test rather than drifting + unnoticed. Paid once per row added or removed. diff --git a/docs/requirements.md b/docs/requirements.md new file mode 100644 index 0000000..7a8b300 --- /dev/null +++ b/docs/requirements.md @@ -0,0 +1,34 @@ +# Behaviour ledger + +This is not `docs/architecture-rules.md` (issue +[#29](https://github.com/JorisJonkers-dev/deploy-kit/issues/29), not yet +landed). That ledger will list the rules a tool enforces: a dependency-cruiser +check, an ESLint message, a lint's error code, each keyed to its enforcer and +its severity. This one lists the behaviours a person depends on: a sentence a +contributor or a consumer can rely on, keyed to the test that fails the moment +it stops being true. A behaviour can rest on several rules, and one rule can +serve several behaviours, so the two ledgers stay separate on purpose. + +A behaviour can lose its proof without anything saying so: the coverage +ratchet notices a line that stops running, but not a test file that was +renamed, emptied or deleted while the code it covered stayed in place. Every +id below is greppable, and a comment naming one resolves to a row rather than +to nothing. +[`scripts/lint-requirements.ts`](../scripts/lint-requirements.ts) is what +holds this document honest: every row parses; every named file exists, is a +test file and holds at least one test; ids are unique; the count this +document states matches the number of rows it holds; and every id cited +anywhere in the tracked tree resolves to a row here. + +This ledger holds **8** rows. The compiler's behaviours join it as they land. + +| id | a contributor or a consumer can rely on | proved by | +|---|---|---| +| REQ-001 | Every decision record under `docs/adr/` satisfies its frontmatter, register and citation contract | [test/adr-contract.test.ts](../test/adr-contract.test.ts) | +| REQ-002 | Every relative link and heading anchor in tracked Markdown resolves to a real target | [test/link-contract.test.ts](../test/link-contract.test.ts) | +| REQ-003 | Every rendered Kubernetes manifest in the worked examples validates against its pinned schema | [test/manifest-contract.test.ts](../test/manifest-contract.test.ts) | +| REQ-004 | The compiler's layer boundaries and module reachability are enforced on the dependency graph, not on review alone | [test/boundary-contract.test.ts](../test/boundary-contract.test.ts) | +| REQ-005 | A pull request title and its commits use a conventional-commit type release-please reads | [test/pr-title-contract.test.ts](../test/pr-title-contract.test.ts) | +| REQ-006 | No em-dash enters tracked text outside `docs/mde/` and `CHANGELOG.md` | [test/emdash.test.ts](../test/emdash.test.ts) | +| REQ-007 | A test that reaches the network, is committed focused or skipped, sleeps a fixed duration, or asserts nothing never reaches a green build | [test/harness.test.ts](../test/harness.test.ts) | +| REQ-008 | Coverage is a ratchet: no `v8`, `c8` or `istanbul` ignore comment exempts a line from it | [test/harness.test.ts](../test/harness.test.ts) | diff --git a/scripts/lint-requirements.ts b/scripts/lint-requirements.ts new file mode 100644 index 0000000..bdbd5b2 --- /dev/null +++ b/scripts/lint-requirements.ts @@ -0,0 +1,206 @@ +// The behaviour ledger, docs/requirements.md: what keeps it honest. +// +// docs/requirements.md holds one row per behaviour this repository +// guarantees: a greppable id, one sentence a contributor or a consumer can +// rely on, and the test file that proves it. A row can lose its proof +// silently when the test it names is renamed, emptied or deleted while the +// code it covered stays in place; nothing else in the suite would notice. +// This is what notices: every row parses; every named file exists, is a test +// file and holds at least one test; ids are unique; the document's stated row +// count matches what it actually holds; and every id cited anywhere in the +// tracked tree resolves to a row. +// +// This is not docs/architecture-rules.md (issue #29, not yet landed): that +// ledger will list the rules a tool enforces (a lint rule, a dependency- +// cruiser check, an ESLint message), each keyed to its enforcer and severity. +// This one lists the behaviours a person depends on, each keyed to the test +// that fails when it stops being true. A behaviour can rest on several rules, +// and one rule can serve several behaviours, so the two ledgers are +// deliberately not merged. +// +// A library first, like the other gates: tests call lintRequirements() in +// -process, and `node scripts/lint-requirements.ts [root]` is the command. +import { execFileSync } from "node:child_process"; +import { readFileSync } from "node:fs"; +import { join } from "node:path"; +import { isEntrypoint } from "./lib/entrypoint.ts"; +import { processOutput, type GateOutput } from "./lib/output.ts"; + +/** One row of the ledger: an id, its guarantee, and the test that proves it. */ +export interface RequirementRow { + readonly id: string; + readonly sentence: string; + readonly test: string; +} + +/** What one run found: how many rows it read, and every violation. */ +export interface RequirementsLintResult { + readonly rows: number; + readonly errors: readonly string[]; +} + +const REPOSITORY = join(import.meta.dirname, ".."); +const LEDGER = join("docs", "requirements.md"); + +const ID = /^REQ-\d{3}$/; +const CITATION = /\bREQ-\d{3}\b/g; +const STATED_COUNT = /this ledger holds \*\*(\d+)\*\* rows?\b/i; + +// A row's link text is the test path relative to the repository root +// (`test/x.test.ts`); its href is that same path relative to docs/ +// (`../test/x.test.ts`), since the ledger lives in docs/. Both are checked +// against each other, so a row cannot link one file while claiming another. +const ROW = + /^\|\s*(REQ-\d{3})\s*\|\s*(.+?)\s*\|\s*\[([^\]]+)\]\(([^)]+)\)\s*\|$/; + +/** Every line that opens like a ledger row, matched or not. */ +function candidateLines(text: string): string[] { + return text.split("\n").filter((line) => /^\|\s*REQ-\d{3}\s*\|/.test(line)); +} + +/** + * Parse the ledger body into rows. Pure and synchronous, so a malformed row + * is testable against a string with no filesystem involved. A line that opens + * like a row but does not match it is reported and dropped, not guessed at. + */ +export function parseRequirements(text: string): { + readonly rows: readonly RequirementRow[]; + readonly errors: readonly string[]; +} { + const rows: RequirementRow[] = []; + const errors: string[] = []; + for (const line of candidateLines(text)) { + const match = ROW.exec(line); + const id = match?.[1]; + const sentence = match?.[2]; + const linkText = match?.[3]; + const href = match?.[4]; + if ( + id === undefined || + sentence === undefined || + linkText === undefined || + href === undefined + ) { + errors.push(`malformed row: ${line}`); + continue; + } + if (linkText !== href.replace(/^\.\.\//, "")) + errors.push( + `${id}: link text '${linkText}' does not match its target '${href}'`, + ); + rows.push({ id, sentence, test: linkText }); + } + return { rows, errors }; +} + +/** + * Every id cited outside the row that declares it, across the tracked tree, + * that no row carries. Pure over a {rel: content} map, so it is testable + * against a synthetic tree the way emdash.test.ts's offendersIn() is. + */ +export function citationErrors( + files: Readonly>, + ids: ReadonlySet, +): string[] { + const errors: string[] = []; + const reported = new Set(); + for (const rel of Object.keys(files).sort()) { + for (const match of (files[rel] ?? "").matchAll(CITATION)) { + const id = match[0]; + const key = `${rel}\0${id}`; + if (ids.has(id) || reported.has(key)) continue; + reported.add(key); + errors.push(`${rel} cites ${id}, which no row carries`); + } + } + return errors; +} + +/** Every tracked file's text, keyed by its path relative to `root`. */ +function trackedText(root: string): Record { + const files: Record = {}; + const tracked = execFileSync("git", ["ls-files", "-z"], { + cwd: root, + encoding: "utf8", + }).split("\0"); + for (const rel of tracked) { + if (rel === "") continue; + try { + files[rel] = readFileSync(join(root, rel), "utf8"); + } catch { + // A path that cannot be read as text (a symlink to nowhere, a + // directory entry) carries no citation either way. + } + } + return files; +} + +/** Lint the behaviour ledger under `root`. */ +export function lintRequirements(root: string): RequirementsLintResult { + const path = join(root, LEDGER); + let text: string; + try { + text = readFileSync(path, "utf8"); + } catch { + return { rows: 0, errors: [`${LEDGER}: ledger missing`] }; + } + + const { rows, errors: parseErrors } = parseRequirements(text); + const errors = [...parseErrors]; + + const byId = new Map(); + for (const row of rows) { + if (!ID.test(row.id)) errors.push(`${row.id}: id does not match REQ-NNN`); + const seen = byId.get(row.id); + if (seen) + errors.push(`${row.id}: id used twice, also for '${seen.sentence}'`); + else byId.set(row.id, row); + } + + for (const row of rows) { + const testPath = join(root, row.test); + let content: string; + try { + content = readFileSync(testPath, "utf8"); + } catch { + errors.push(`${row.id}: names '${row.test}', which does not exist`); + continue; + } + if (!row.test.endsWith(".test.ts")) + errors.push(`${row.id}: '${row.test}' is not a test file`); + if (!/\b(?:test|it)\s*\(/.test(content)) + errors.push(`${row.id}: '${row.test}' holds no test`); + } + + const stated = STATED_COUNT.exec(text)?.[1]; + if (stated === undefined) + errors.push(`${LEDGER}: does not state how many rows it holds`); + else if (Number(stated) !== rows.length) + errors.push( + `${LEDGER}: states ${stated} rows, holds ${String(rows.length)}`, + ); + + errors.push(...citationErrors(trackedText(root), new Set(byId.keys()))); + + return { rows: rows.length, errors }; +} + +/** Lint the tree named by argv[0], or this repository, and say what was found. */ +export function main( + argv: readonly string[], + output: GateOutput = processOutput, +): number { + const { rows, errors } = lintRequirements(argv[0] ?? REPOSITORY); + if (errors.length > 0) { + output.err( + `requirements lint: ${errors.length} error(s)\n` + + errors.map((e) => ` - ${e}\n`).join(""), + ); + return 1; + } + output.out(`requirements lint: ${rows} rows clean\n`); + return 0; +} + +if (isEntrypoint(import.meta.url, process.argv[1])) + process.exitCode = main(process.argv.slice(2)); diff --git a/test/adr-contract.test.ts b/test/adr-contract.test.ts index 4f4fe56..8340a35 100644 --- a/test/adr-contract.test.ts +++ b/test/adr-contract.test.ts @@ -3,6 +3,9 @@ // part of the test run rather than a thing someone remembers to run. It also // starts the lint the way CI does, as a command, which is what proves the // guard at the bottom of the script still runs it. +// +// REQ-001 (docs/requirements.md): every decision record satisfies its +// frontmatter, register and citation contract. import { execFileSync } from "node:child_process"; import { existsSync, readFileSync, readdirSync } from "node:fs"; import { basename, join } from "node:path"; diff --git a/test/boundary-contract.test.ts b/test/boundary-contract.test.ts index 42cb00b..c23ea5a 100644 --- a/test/boundary-contract.test.ts +++ b/test/boundary-contract.test.ts @@ -4,6 +4,9 @@ // has only ever run against a tree with no violations is untested: nothing // proves it would fail. Each case builds a throwaway src/ tree that crosses // exactly one boundary and asserts the named rule reports it. +// +// REQ-004 (docs/requirements.md): layer boundaries and module reachability +// are enforced on the dependency graph, not on review alone. import { spawnSync } from "node:child_process"; import { mkdirSync, mkdtempSync, symlinkSync, writeFileSync } from "node:fs"; import { dirname, join } from "node:path"; diff --git a/test/emdash.test.ts b/test/emdash.test.ts index c3f04b1..f08aa58 100644 --- a/test/emdash.test.ts +++ b/test/emdash.test.ts @@ -8,6 +8,9 @@ // Two trees are never scanned: `docs/mde/`, which holds third-party papers, // lecture material and coursework kept verbatim, and `CHANGELOG.md`, which // release-please writes. +// +// REQ-006 (docs/requirements.md): no em-dash enters tracked text outside +// docs/mde/ and CHANGELOG.md. import { execFileSync } from "node:child_process"; import { readFileSync } from "node:fs"; import { join } from "node:path"; diff --git a/test/harness.test.ts b/test/harness.test.ts index ed2f2e8..433d610 100644 --- a/test/harness.test.ts +++ b/test/harness.test.ts @@ -1,5 +1,11 @@ // The harness proves its own guards fire. A guard that has only ever run // against well-behaved tests is untested: nothing shows it would catch one. +// +// REQ-007 (docs/requirements.md): a test that reaches the network, is +// committed focused or skipped, sleeps a fixed duration, or asserts nothing +// never reaches a green build. +// REQ-008 (docs/requirements.md): coverage is a ratchet, so no ignore comment +// exempts a line from it (see "coverage ignore comments" below). import { existsSync, readFileSync, readdirSync, writeFileSync } from "node:fs"; import { join } from "node:path"; import { ESLint } from "eslint"; diff --git a/test/link-contract.test.ts b/test/link-contract.test.ts index f624fb4..0b37766 100644 --- a/test/link-contract.test.ts +++ b/test/link-contract.test.ts @@ -3,6 +3,9 @@ // The ADR domain move rewrote about a hundred references outside the decision // set. Nothing checked them, so the next move could break every one of them // with every other gate green. These fixtures prove the check would fail. +// +// REQ-002 (docs/requirements.md): every relative link and heading anchor in +// tracked Markdown resolves to a real target. import { execFileSync, spawnSync } from "node:child_process"; import { mkdirSync, mkdtempSync, writeFileSync } from "node:fs"; import { dirname, join } from "node:path"; diff --git a/test/manifest-contract.test.ts b/test/manifest-contract.test.ts index 9674dcf..a9fd02b 100644 --- a/test/manifest-contract.test.ts +++ b/test/manifest-contract.test.ts @@ -4,6 +4,9 @@ // binary is absent rather than passing quietly, and it fails the build when the // binary fails. Both are what make it a gate rather than a script, and neither // is proven by running it against a clean tree. +// +// REQ-003 (docs/requirements.md): every rendered manifest in the worked +// examples validates against its pinned schema. import { spawnSync } from "node:child_process"; import { chmodSync, mkdirSync, mkdtempSync, writeFileSync } from "node:fs"; import { dirname, join } from "node:path"; diff --git a/test/pr-title-contract.test.ts b/test/pr-title-contract.test.ts index 7ce8436..77a887b 100644 --- a/test/pr-title-contract.test.ts +++ b/test/pr-title-contract.test.ts @@ -3,6 +3,9 @@ // hook is the authority, copied unchanged from the estate; if one side accepts // a type the other does not, contributors get conflicting signals on a laptop // and in CI. +// +// REQ-005 (docs/requirements.md): a pull request title and its commits use a +// conventional-commit type release-please reads. import { execFileSync, spawnSync } from "node:child_process"; import { mkdtempSync, readFileSync } from "node:fs"; import { join } from "node:path"; diff --git a/test/requirements-contract.test.ts b/test/requirements-contract.test.ts new file mode 100644 index 0000000..fd2a496 --- /dev/null +++ b/test/requirements-contract.test.ts @@ -0,0 +1,107 @@ +// The behaviour ledger, executed. docs/requirements.md states the guarantee, +// scripts/lint-requirements.ts enforces that every row still holds, and this +// is what makes that enforcement part of the test run rather than a thing +// someone remembers to run. It also starts the lint the way CI would, as a +// command, which is what proves the guard at the bottom of the script still +// runs it. +import { execFileSync, spawnSync } from "node:child_process"; +import { mkdirSync, mkdtempSync, readFileSync, writeFileSync } from "node:fs"; +import { join } from "node:path"; +import { describe, expect, it, test } from "vitest"; +import { + lintRequirements, + main, + parseRequirements, +} from "../scripts/lint-requirements.ts"; +import { collect } from "./support/collect.ts"; +import { temporary } from "./setup.ts"; + +const REPOSITORY = join(import.meta.dirname, ".."); +const LEDGER = join(REPOSITORY, "docs", "requirements.md"); + +test("the requirements lint, run as a command, passes over the committed ledger", () => { + const out = execFileSync( + process.execPath, + [join(REPOSITORY, "scripts", "lint-requirements.ts")], + { encoding: "utf8" }, + ); + expect(out).toMatch(/^requirements lint: \d+ rows clean$/m); +}); + +test("the ledger is non-empty, and its stated count matches what it holds", () => { + const { rows } = parseRequirements(readFileSync(LEDGER, "utf8")); + expect(rows.length).toBeGreaterThanOrEqual(8); + expect(lintRequirements(REPOSITORY)).toStrictEqual({ + rows: rows.length, + errors: [], + }); +}); + +test("every row's id is unique and shaped REQ-NNN", () => { + const { rows } = parseRequirements(readFileSync(LEDGER, "utf8")); + const ids = rows.map((row) => row.id); + for (const id of ids) expect(id).toMatch(/^REQ-\d{3}$/); + expect(new Set(ids).size).toBe(ids.length); +}); + +test("every row names a real test file that holds at least one test", () => { + const { rows } = parseRequirements(readFileSync(LEDGER, "utf8")); + for (const row of rows) { + const content = readFileSync(join(REPOSITORY, row.test), "utf8"); + expect(row.test, row.id).toMatch(/\.test\.ts$/); + expect(content, row.id).toMatch(/\b(?:test|it)\s*\(/); + } +}); + +describe("the command", () => { + const VALID_TEST = + 'import { expect, it } from "vitest";\n' + + 'it("x", () => {\n expect(1).toBe(1);\n});\n'; + + function fixture(ledgerBody: string): string { + const root = mkdtempSync(join(temporary(), "requirements-command-")); + mkdirSync(join(root, "docs"), { recursive: true }); + mkdirSync(join(root, "test"), { recursive: true }); + writeFileSync(join(root, "docs", "requirements.md"), ledgerBody); + writeFileSync(join(root, "test", "a.test.ts"), VALID_TEST); + execFileSync("git", ["init", "-q"], { cwd: root }); + execFileSync("git", ["add", "-A"], { cwd: root }); + return root; + } + + it("says how many rows are clean, and exits 0", () => { + const output = collect(); + const root = fixture( + "# Behaviour ledger\n\nThis ledger holds **1** rows.\n\n" + + "| id | guarantee | proved by |\n|---|---|---|\n" + + "| REQ-001 | a thing holds | [test/a.test.ts](../test/a.test.ts) |\n", + ); + expect(main([root], output)).toBe(0); + expect(output.text()).toBe("requirements lint: 1 rows clean\n"); + }); + + it("lists every violation under a count, and exits 1", () => { + const output = collect(); + const root = fixture("# Behaviour ledger\n\nno rows here\n"); + expect(main([root], output)).toBe(1); + expect(output.text()).toBe( + "requirements lint: 1 error(s)\n" + + " - docs/requirements.md: does not state how many rows it holds\n", + ); + }); + + it("checks this repository when no tree is named", () => { + expect(main([], collect())).toBe(0); + }); + + it("runs when Node starts the script, which is how CI would run it", () => { + const root = fixture("# Behaviour ledger\n\nno rows here\n"); + const run = spawnSync( + process.execPath, + [join(REPOSITORY, "scripts", "lint-requirements.ts"), root], + { encoding: "utf8" }, + ); + expect(run.status).toBe(1); + expect(run.stderr).toContain("does not state how many rows it holds"); + }); +}); diff --git a/test/requirements-lint-negative.test.ts b/test/requirements-lint-negative.test.ts new file mode 100644 index 0000000..461e61a --- /dev/null +++ b/test/requirements-lint-negative.test.ts @@ -0,0 +1,236 @@ +// Negative fixtures for the behaviour ledger. +// +// A lint that has only ever run against a clean ledger is untested: nothing +// proves it would fail. Each case builds a throwaway tree that violates +// exactly one rule and asserts the lint reports it, naming the row. +import { execFileSync } from "node:child_process"; +import { mkdirSync, mkdtempSync, writeFileSync } from "node:fs"; +import { dirname, join } from "node:path"; +import { describe, expect, it } from "vitest"; +import { + citationErrors, + lintRequirements, + parseRequirements, +} from "../scripts/lint-requirements.ts"; +import { temporary } from "./setup.ts"; + +type Files = Readonly>; + +function write(root: string, files: Files): void { + for (const [rel, content] of Object.entries(files)) { + const target = join(root, rel); + mkdirSync(dirname(target), { recursive: true }); + writeFileSync(target, content); + } +} + +/** A test file that satisfies "exists, is a test file, holds a test". */ +const VALID_TEST = + 'import { expect, it } from "vitest";\n' + + 'it("x", () => {\n expect(1).toBe(1);\n});\n'; + +/** + * A git repository whose ledger and test files are `files`, plus a default + * valid test/a.test.ts so a case can name it without redeclaring it. + */ +function fixture(files: Files): string { + const root = mkdtempSync(join(temporary(), "requirements-lint-")); + write(root, { "test/a.test.ts": VALID_TEST, ...files }); + execFileSync("git", ["init", "-q"], { cwd: root }); + execFileSync("git", ["add", "-A"], { cwd: root }); + return root; +} + +const ledger = (body: string): string => `# Behaviour ledger\n\n${body}\n`; + +// Built from two halves, so this file's own source does not carry the +// citation it tests for: an id no row of the real docs/requirements.md +// carries, which the real ledger's own contract test would otherwise catch +// once this file is tracked. +const BOGUS_ID = ["REQ", "999"].join("-"); + +describe("lintRequirements", () => { + it("passes a clean, minimal ledger, so it is not simply forbidding everything", () => { + const root = fixture({ + "docs/requirements.md": ledger( + "This ledger holds **1** rows.\n\n" + + "| id | guarantee | proved by |\n|---|---|---|\n" + + "| REQ-001 | a thing holds | [test/a.test.ts](../test/a.test.ts) |", + ), + }); + expect(lintRequirements(root)).toStrictEqual({ rows: 1, errors: [] }); + }); + + it("fails, naming the file, when the ledger itself is missing", () => { + const root = fixture({}); + expect(lintRequirements(root).errors).toStrictEqual([ + "docs/requirements.md: ledger missing", + ]); + }); + + it("fails a row that does not parse as a markdown-linked row", () => { + const root = fixture({ + "docs/requirements.md": ledger( + "This ledger holds **1** rows.\n\n" + + "| id | guarantee | proved by |\n|---|---|---|\n" + + "| REQ-001 | a thing holds | test/a.test.ts |", + ), + }); + expect(lintRequirements(root).errors).toContain( + "malformed row: | REQ-001 | a thing holds | test/a.test.ts |", + ); + }); + + it("fails a row whose link text does not match its own target", () => { + const root = fixture({ + "docs/requirements.md": ledger( + "This ledger holds **1** rows.\n\n" + + "| id | guarantee | proved by |\n|---|---|---|\n" + + "| REQ-001 | a thing holds | [test/wrong.test.ts](../test/a.test.ts) |", + ), + }); + expect(lintRequirements(root).errors).toContain( + "REQ-001: link text 'test/wrong.test.ts' does not match its target '../test/a.test.ts'", + ); + }); + + it("fails, naming the row, when the test file it names does not exist", () => { + const root = fixture({ + "docs/requirements.md": ledger( + "This ledger holds **1** rows.\n\n" + + "| id | guarantee | proved by |\n|---|---|---|\n" + + "| REQ-001 | a thing holds | [test/gone.test.ts](../test/gone.test.ts) |", + ), + }); + expect(lintRequirements(root).errors).toStrictEqual([ + "REQ-001: names 'test/gone.test.ts', which does not exist", + ]); + }); + + it("fails, naming the row, when the test file it names holds no test", () => { + const root = fixture({ + "docs/requirements.md": ledger( + "This ledger holds **1** rows.\n\n" + + "| id | guarantee | proved by |\n|---|---|---|\n" + + "| REQ-001 | a thing holds | [test/empty.test.ts](../test/empty.test.ts) |", + ), + "test/empty.test.ts": "", + }); + expect(lintRequirements(root).errors).toStrictEqual([ + "REQ-001: 'test/empty.test.ts' holds no test", + ]); + }); + + it("fails, naming the row, when the file it names is not a test file", () => { + const root = fixture({ + "docs/requirements.md": ledger( + "This ledger holds **1** rows.\n\n" + + "| id | guarantee | proved by |\n|---|---|---|\n" + + "| REQ-001 | a thing holds | [test/notes.txt](../test/notes.txt) |", + ), + "test/notes.txt": "it( is not a test call in a text file\n", + }); + expect(lintRequirements(root).errors).toStrictEqual([ + "REQ-001: 'test/notes.txt' is not a test file", + ]); + }); + + it("fails, naming the file, when the ledger does not state a row count at all", () => { + const root = fixture({ + "docs/requirements.md": + "# Behaviour ledger\n\n" + + "| id | guarantee | proved by |\n|---|---|---|\n" + + "| REQ-001 | a thing holds | [test/a.test.ts](../test/a.test.ts) |\n", + }); + expect(lintRequirements(root).errors).toStrictEqual([ + "docs/requirements.md: does not state how many rows it holds", + ]); + }); + + it("fails two rows sharing one id, naming both sentences", () => { + const root = fixture({ + "docs/requirements.md": ledger( + "This ledger holds **2** rows.\n\n" + + "| id | guarantee | proved by |\n|---|---|---|\n" + + "| REQ-001 | first thing | [test/a.test.ts](../test/a.test.ts) |\n" + + "| REQ-001 | second thing | [test/b.test.ts](../test/b.test.ts) |", + ), + "test/b.test.ts": VALID_TEST, + }); + expect(lintRequirements(root).errors).toContain( + "REQ-001: id used twice, also for 'first thing'", + ); + }); + + it("fails when the stated count disagrees with the rows it holds", () => { + const root = fixture({ + "docs/requirements.md": ledger( + "This ledger holds **2** rows.\n\n" + + "| id | guarantee | proved by |\n|---|---|---|\n" + + "| REQ-001 | a thing holds | [test/a.test.ts](../test/a.test.ts) |", + ), + }); + expect(lintRequirements(root).errors).toStrictEqual([ + "docs/requirements.md: states 2 rows, holds 1", + ]); + }); + + it("fails when an id no row carries is cited elsewhere in the tree", () => { + const root = fixture({ + "docs/requirements.md": ledger( + "This ledger holds **1** rows.\n\n" + + "| id | guarantee | proved by |\n|---|---|---|\n" + + "| REQ-001 | a thing holds | [test/a.test.ts](../test/a.test.ts) |", + ), + "src/note.ts": `// see ${BOGUS_ID} for context\n`, + }); + expect(lintRequirements(root).errors).toStrictEqual([ + `src/note.ts cites ${BOGUS_ID}, which no row carries`, + ]); + }); + + it("ignores an untracked file's citation, the same way the em-dash ban ignores scratch files", () => { + const root = fixture({ + "docs/requirements.md": ledger( + "This ledger holds **1** rows.\n\n" + + "| id | guarantee | proved by |\n|---|---|---|\n" + + "| REQ-001 | a thing holds | [test/a.test.ts](../test/a.test.ts) |", + ), + }); + writeFileSync(join(root, "scratch.md"), `${BOGUS_ID}, never staged\n`); + expect(lintRequirements(root).errors).toStrictEqual([]); + }); +}); + +describe("parseRequirements", () => { + it("returns one row per well-formed line, in document order", () => { + const { rows, errors } = parseRequirements( + "| REQ-002 | second | [test/b.test.ts](../test/b.test.ts) |\n" + + "| REQ-001 | first | [test/a.test.ts](../test/a.test.ts) |", + ); + expect(errors).toStrictEqual([]); + expect(rows.map((row) => row.id)).toStrictEqual(["REQ-002", "REQ-001"]); + }); + + it("ignores prose that merely contains a pipe", () => { + const { rows, errors } = parseRequirements( + "Some table | with a pipe | but no REQ id\n", + ); + expect(rows).toStrictEqual([]); + expect(errors).toStrictEqual([]); + }); +}); + +describe("citationErrors", () => { + it("reports each offending id once per file, not once per occurrence", () => { + expect( + citationErrors( + { + "a.ts": `${BOGUS_ID} and again ${BOGUS_ID}`, + "b.ts": "REQ-001 is fine", + }, + new Set(["REQ-001"]), + ), + ).toStrictEqual([`a.ts cites ${BOGUS_ID}, which no row carries`]); + }); +}); From 4f23b2c0ac600a318f83461c4031b106238b30ac Mon Sep 17 00:00:00 2001 From: Joris Wouter Jonkers Date: Mon, 14 Sep 2026 09:49:05 +0200 Subject: [PATCH 2/2] ci: wire the requirements ledger gate into CI, verify and the register REQ-009 and REQ-010 cover the package-contents gate and the pipeline-wiring test that issue #23 brought; lint:requirements now runs in the Contracts job and in npm run verify; ADR 0103 cites the now-landed ADR 0102; coverage thresholds are raised to what the suite reaches with the new gate covered. --- .github/workflows/ci.yml | 7 ++++++ ...haviour-ledger-names-what-a-test-proves.md | 13 ++++++----- docs/architecture.md | 9 ++++---- docs/requirements.md | 4 +++- package.json | 3 ++- scripts/lint-requirements.ts | 9 ++++---- test/package-contents-contract.test.ts | 3 +++ test/pipeline-wiring.test.ts | 3 +++ test/requirements-lint-negative.test.ts | 15 ++++++++++++ vitest.config.ts | 23 +++++++++---------- 10 files changed, 61 insertions(+), 28 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2dda653..9ad54ac 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -120,6 +120,13 @@ | tar -xz kubeconform KUBECONFORM="${PWD}/kubeconform" npm run lint:manifests + # The behaviour ledger (docs/requirements.md): every row parses, names a + # real test that holds at least one test, its stated count matches what + # it holds, and every REQ-NNN cited in the tracked tree resolves to a + # row. See docs/adr/architecture/0103-a-behaviour-ledger-names-what-a-test-proves.md. + - 'name': 'Requirements ledger' + 'run': 'npm run lint:requirements' + 'tests': 'name': 'Tests' 'runs-on': 'ubuntu-latest' diff --git a/docs/adr/architecture/0103-a-behaviour-ledger-names-what-a-test-proves.md b/docs/adr/architecture/0103-a-behaviour-ledger-names-what-a-test-proves.md index 796bde6..1292c88 100644 --- a/docs/adr/architecture/0103-a-behaviour-ledger-names-what-a-test-proves.md +++ b/docs/adr/architecture/0103-a-behaviour-ledger-names-what-a-test-proves.md @@ -29,12 +29,13 @@ connected a sentence a contributor relies on to the test that makes it true, so that sentence could go silently unproven and nothing in the suite would say so. -The fix is the same shape as the boundary gate and the script-to-workflow -check landing alongside it: state the rule as a comparison over things that -already exist on disk (a markdown table and a directory of test files), -rather than as a convention to remember. A row that no longer resolves, a -file that exists but holds no test, a stated count that drifts from the rows -actually present, or an id cited where no row backs it: each is a comparison +The fix is the same shape as the boundary gate and +[the script-to-workflow check](0102-the-gate-grows-with-the-code.md): state +the rule as a comparison over things that already exist on disk (a markdown +table and a directory of test files), rather than as a convention to +remember. A row that no longer resolves, a file that exists but holds no +test, a stated count that drifts from the rows actually present, or an id +cited where no row backs it: each is a comparison a script can run, not a habit a reviewer can forget, which matters here for the same reason as [0001](../model/0001-estate-scale-and-ownership.md): one person reading their own diff later is not a second pair of eyes. diff --git a/docs/architecture.md b/docs/architecture.md index b3edf42..cffbbd4 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -206,15 +206,16 @@ proves the two never drift apart. | decisions | `npm run lint:adrs` | frontmatter, register integrity, citations, normative anchors per domain | | links | `npm run lint:links` | relative links and heading anchors across every tracked Markdown file | | manifests | `npm run lint:manifests` | every rendered example object against pinned Kubernetes and CRD schemas | +| requirements | `npm run lint:requirements` | a behaviour ledger row that no longer parses, names a missing or empty test, drifts from its stated count, or is cited by an id no row carries | | tests | `npm run test:coverage` | behaviour, plus the coverage ratchet | | package contents | `node scripts/check-package-contents.ts` | `npm pack` shipping a file outside `docs/adr/` and `spec/`, the boundary the package's `files` field states but does not enforce on its own | | actionlint | a pinned `actionlint` binary | invalid workflow syntax, an undefined `${{ }}` expression, a shellcheck finding inside a `run:` step | | secret scan | a pinned `gitleaks` binary | a committed secret matching the default ruleset, or this repository's own allowlist entries | -Decisions, links and manifests share one CI job, `contracts`: all three check a -document against a rule rather than code against a graph. Boundaries runs -alone as `architecture`, because it is the one gate that speaks for -`docs/architecture.md` itself rather than for a document beside it. +Decisions, links, manifests and requirements share one CI job, `contracts`: +all four check a document against a rule rather than code against a graph. +Boundaries runs alone as `architecture`, because it is the one gate that +speaks for `docs/architecture.md` itself rather than for a document beside it. Coverage is a ratchet ([0101](adr/architecture/0101-coverage-is-a-ratchet.md)). The thresholds in `vitest.config.ts` sit on what the suite reaches, over an diff --git a/docs/requirements.md b/docs/requirements.md index 7a8b300..8e8e6f7 100644 --- a/docs/requirements.md +++ b/docs/requirements.md @@ -20,7 +20,7 @@ test file and holds at least one test; ids are unique; the count this document states matches the number of rows it holds; and every id cited anywhere in the tracked tree resolves to a row here. -This ledger holds **8** rows. The compiler's behaviours join it as they land. +This ledger holds **10** rows. The compiler's behaviours join it as they land. | id | a contributor or a consumer can rely on | proved by | |---|---|---| @@ -32,3 +32,5 @@ This ledger holds **8** rows. The compiler's behaviours join it as they land. | REQ-006 | No em-dash enters tracked text outside `docs/mde/` and `CHANGELOG.md` | [test/emdash.test.ts](../test/emdash.test.ts) | | REQ-007 | A test that reaches the network, is committed focused or skipped, sleeps a fixed duration, or asserts nothing never reaches a green build | [test/harness.test.ts](../test/harness.test.ts) | | REQ-008 | Coverage is a ratchet: no `v8`, `c8` or `istanbul` ignore comment exempts a line from it | [test/harness.test.ts](../test/harness.test.ts) | +| REQ-009 | The npm package ships nothing outside `docs/adr/` and `spec/`, checked against what npm would really pack rather than the advisory `files` field | [test/package-contents-contract.test.ts](../test/package-contents-contract.test.ts) | +| REQ-010 | A gate's npm script and the CI job that runs it land in the same pull request, so neither can drift from the other unnoticed | [test/pipeline-wiring.test.ts](../test/pipeline-wiring.test.ts) | diff --git a/package.json b/package.json index 9fc7c24..ac9f5b1 100644 --- a/package.json +++ b/package.json @@ -18,12 +18,13 @@ "lint:links": "node scripts/lint-links.ts", "lint:manifests": "node scripts/lint-manifests.ts", "lint:boundaries": "node scripts/lint-boundaries.ts", + "lint:requirements": "node scripts/lint-requirements.ts", "format": "prettier --write .", "format:check": "prettier --check .", "typecheck": "tsc --noEmit", "test": "vitest run", "test:coverage": "vitest run --coverage", - "verify": "npm run lint && npm run format:check && npm run typecheck && npm run lint:adrs && npm run lint:links && npm run lint:manifests && npm run lint:boundaries && npm run test:coverage" + "verify": "npm run lint && npm run format:check && npm run typecheck && npm run lint:adrs && npm run lint:links && npm run lint:manifests && npm run lint:requirements && npm run lint:boundaries && npm run test:coverage" }, "devDependencies": { "@eslint/js": "10.0.1", diff --git a/scripts/lint-requirements.ts b/scripts/lint-requirements.ts index bdbd5b2..78377a8 100644 --- a/scripts/lint-requirements.ts +++ b/scripts/lint-requirements.ts @@ -42,7 +42,6 @@ export interface RequirementsLintResult { const REPOSITORY = join(import.meta.dirname, ".."); const LEDGER = join("docs", "requirements.md"); -const ID = /^REQ-\d{3}$/; const CITATION = /\bREQ-\d{3}\b/g; const STATED_COUNT = /this ledger holds \*\*(\d+)\*\* rows?\b/i; @@ -104,8 +103,11 @@ export function citationErrors( ): string[] { const errors: string[] = []; const reported = new Set(); - for (const rel of Object.keys(files).sort()) { - for (const match of (files[rel] ?? "").matchAll(CITATION)) { + // Object keys are unique paths, so two entries are never equal: a strict + // less-than is enough to order them, with no equal case to fall through to. + const entries = Object.entries(files).sort(([a], [b]) => (a < b ? -1 : 1)); + for (const [rel, content] of entries) { + for (const match of content.matchAll(CITATION)) { const id = match[0]; const key = `${rel}\0${id}`; if (ids.has(id) || reported.has(key)) continue; @@ -150,7 +152,6 @@ export function lintRequirements(root: string): RequirementsLintResult { const byId = new Map(); for (const row of rows) { - if (!ID.test(row.id)) errors.push(`${row.id}: id does not match REQ-NNN`); const seen = byId.get(row.id); if (seen) errors.push(`${row.id}: id used twice, also for '${seen.sentence}'`); diff --git a/test/package-contents-contract.test.ts b/test/package-contents-contract.test.ts index 4d2bddd..de1be28 100644 --- a/test/package-contents-contract.test.ts +++ b/test/package-contents-contract.test.ts @@ -2,6 +2,9 @@ // package.json is advisory: npm bundles package.json, README and LICENSE // regardless of it, and a stray glob can widen it silently. This proves the // gate catches that against real npm, not only against a fabricated list. +// +// REQ-009 (docs/requirements.md): the npm package ships nothing outside +// docs/adr/ and spec/. import { spawnSync } from "node:child_process"; import { chmodSync, mkdtempSync, mkdirSync, writeFileSync } from "node:fs"; import { join } from "node:path"; diff --git a/test/pipeline-wiring.test.ts b/test/pipeline-wiring.test.ts index db7be00..9a363e2 100644 --- a/test/pipeline-wiring.test.ts +++ b/test/pipeline-wiring.test.ts @@ -5,6 +5,9 @@ // docs/adr/architecture/0102-the-gate-grows-with-the-code.md, a new gate's // script and its CI job land in the same pull request, and this is the test // that makes drift between the two visible instead of silent. +// +// REQ-010 (docs/requirements.md): a gate's npm script and the CI job that +// runs it land together. import { readFileSync, readdirSync } from "node:fs"; import { join } from "node:path"; import { describe, expect, it } from "vitest"; diff --git a/test/requirements-lint-negative.test.ts b/test/requirements-lint-negative.test.ts index 461e61a..e2f563d 100644 --- a/test/requirements-lint-negative.test.ts +++ b/test/requirements-lint-negative.test.ts @@ -233,4 +233,19 @@ describe("citationErrors", () => { ), ).toStrictEqual([`a.ts cites ${BOGUS_ID}, which no row carries`]); }); + + it("walks files in path order regardless of the order they were given in", () => { + expect( + citationErrors( + { + "z.ts": BOGUS_ID, + "a.ts": BOGUS_ID, + }, + new Set(), + ), + ).toStrictEqual([ + `a.ts cites ${BOGUS_ID}, which no row carries`, + `z.ts cites ${BOGUS_ID}, which no row carries`, + ]); + }); }); diff --git a/vitest.config.ts b/vitest.config.ts index 7cd4508..39339f9 100644 --- a/vitest.config.ts +++ b/vitest.config.ts @@ -22,20 +22,19 @@ export default defineConfig({ // A ratchet, per docs/adr/architecture/0101-coverage-is-a-ratchet.md: // set from what the suite reaches, and only ever raised. // - // Measured 2026-09-14, after the package contents gate's own tests - // covered its npm-cannot-run and empty-package branches (and, since - // this gate is small enough that the usual one-line command guard - // would otherwise be a large share of it, its own entrypoint guard - // too), two runs of one tree, identical both times: statements - // 310/319, branches 182/208, functions 53/53, lines 280/289. What is - // left uncovered elsewhere is mostly the one-line command guard at the - // bottom of each other gate and the branches for a tool that cannot be - // started at all. + // Measured 2026-09-14, after the requirements ledger gate's own tests + // covered its citation-ordering branches (and, like the other gates, + // left only its bottom-of-file entrypoint guard uncovered), two runs + // of one tree, identical both times: statements 390/400, branches + // 214/241, functions 62/62, lines 356/366. What is left uncovered + // elsewhere is mostly the one-line command guard at the bottom of each + // other gate and the branches for a tool that cannot be started at + // all. thresholds: { - statements: 97.17, - branches: 87.5, + statements: 97.5, + branches: 88.79, functions: 100, - lines: 96.88, + lines: 97.26, }, }, },