From e7ef695d5b7dbbea80329189cf179827df0c0df4 Mon Sep 17 00:00:00 2001 From: Sander Striker Date: Fri, 30 Oct 2026 10:34:28 +0100 Subject: [PATCH 1/3] ci: check the Markdown the demo reads, on every change A change to docs/ or examples/decisions/ skips make check and CI, but those files are the demo's record. A renamed heading broke 6 of the 14 golden questions and 2 eval tasks, and no check saw it. scripts/corpus-check.sh seeds a scratch record as make demo-ready does. It fails when: - a golden question or an eval task names no entry in the record; - the keywords of a golden question do not find its answer in the top 8; - act 11 does not find its two lists in README.md, or a list is empty. The new CI job, corpus, runs make corpus-check on every change. Each eval task now names its answer in `answers`. Act 11 takes the binary from AMEM. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01Bhj4Ck4y5iwZUsjpgQx8A2 --- .github/workflows/ci.yml | 14 +++++++++ AGENTS.md | 8 ++++- Makefile | 7 ++++- evals/demo.toml | 8 ++++- scripts/corpus-check.sh | 63 ++++++++++++++++++++++++++++++++++++++++ scripts/demo/lib.sh | 2 +- scripts/docs-only.sh | 11 +++---- 7 files changed, 104 insertions(+), 9 deletions(-) create mode 100755 scripts/corpus-check.sh diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 91bc545e..cacf0b28 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -137,6 +137,20 @@ jobs: - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 - run: make wasm-check + # The Markdown the demo reads, checked as the demo reads it (scripts/corpus-check.sh). + # It runs on every change, a docs-only one included: docs/ and examples/decisions/ are + # the demo's record, and a heading renamed there breaks a golden question or an eval. + corpus: + name: corpus + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - uses: dtolnay/rust-toolchain@4cda84d5c5c54efe2404f9d843567869ab1699d4 # 2026-07-16, the commit the ASF allow-list pins + with: + toolchain: stable + - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 + - run: make corpus-check + # Advisories, licences, duplicate versions, sources. Configured in deny.toml. The # cargo-deny action is not on the ASF allow-list, so the tool is installed instead. deny: diff --git a/AGENTS.md b/AGENTS.md index 2402bed2..194d37f3 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -38,7 +38,7 @@ before you override the instruction. counts as Markdown. The hook and CI do not run `make check` for such a change, because nothing builds from those files. `scripts/docs-only.sh` decides which changes are of this type. Its test holds the premise that no crate or tap embeds those files or reads - them. Some scripts read them, and no check runs these scripts: + them. Some scripts read them: - `scripts/seed-corpus.py` seeds the demo record from `docs/` and `examples/decisions/` (`make demo`). `golden/docs.toml` and `evals/demo.toml` score @@ -46,6 +46,12 @@ before you override the instruction. - Act 11 of the demo (`scripts/demo/11-limits.sh`) prints the two lists at the top of `README.md`. + `make corpus-check` (`scripts/corpus-check.sh`) seeds a scratch record from those + files. It fails when a golden question or an eval task names an entry that the record + does not hold, when the keywords of a golden question do not find its answer, or when + act 11 does not find its two lists. CI runs it on every change, a docs-only change + included. It does not run a harness, so it does not show what a model answers. + The demo seeds its record only once. So, after you change a file in `docs/` or `examples/decisions/`, seed a new record with `make demo-ready RESET=1`. Then run `make golden` and act 6 (`make demo-all HARNESS=codex ACTS=6`). Act 6 needs a key diff --git a/Makefile b/Makefile index f29f9967..bd11fbdf 100644 --- a/Makefile +++ b/Makefile @@ -8,7 +8,7 @@ NEXTEST := $(shell cargo nextest --version 2>/dev/null) TEST_CMD := $(if $(NEXTEST),cargo nextest run --workspace --no-fail-fast,cargo test --workspace) .PHONY: demo-ready podman-ready e2e hosted install init check verify fmt fmt-check clippy test guard-test docs-only-test no-image-push-test conformance-test conformance deny machete coverage wasm-check run build wslcsdk-test wslcsdk unsafe-check windows-target \ - local-up local-down local-logs local-key local-memory launch usage golden drift taps-check taps-build wasip2-target reader + local-up local-down local-logs local-key local-memory launch usage golden corpus-check drift taps-check taps-build wasip2-target reader # ---- the local stack: LiteLLM and Postgres in containers, the memory root on disk. # Reads local/.env. See local/README.md. @@ -175,6 +175,11 @@ local-memory: # `make check` is the pre-commit gate: fast, and what every commit must pass. check: wasip2-target fmt-check clippy test guard-test docs-only-test no-image-push-test wslcsdk-test conformance-test unsafe-check taps-check +# The demo's Markdown, checked as the golden suite, the evals and act 11 read it. +corpus-check: + cargo build -p amem + sh scripts/corpus-check.sh target/debug/amem + # `make verify` is what CI runs: the gate plus dependencies, coverage, and the wasm build. verify: check deny machete coverage wasm-check diff --git a/evals/demo.toml b/evals/demo.toml index 5dcc88fd..9bc82b9e 100644 --- a/evals/demo.toml +++ b/evals/demo.toml @@ -1,7 +1,9 @@ # An evaluation set over the demo record: questions whose answers live in the record and # nowhere else, so a harness without memory can only guess. `scripts/eval.py` runs each # task under each condition and grades the answer against `expect`, a case-insensitive -# regular expression that the answer must match. +# regular expression that the answer must match. `answers` names the entries that hold +# the answer, by title or part of one, as in golden/docs.toml; scripts/corpus-check.sh +# fails when the record holds none of them. # # scripts/eval.py evals/demo.toml --root ~/memory-demo2 --harness claude --auth subscription # scripts/eval.py evals/demo.toml --root ~/memory-demo2 --harness codex --auth gateway --model qwen3.8-27b @@ -18,6 +20,7 @@ preamble = "If a memory server is attached to this session, search it and read i [[task]] id = "dr19-decision" +answers = ["DR-19: Does the wiki stay on MoinMoin?: Status"] prompt = "What did DR-19 decide about the wiki staying on MoinMoin? Answer in one or two sentences from what you know; do not browse the web." # "no decision" alone also matches a refusal ("no decision entries recorded"), so only # the record's own answer counts: proposed, not yet taken, no final decision @@ -25,11 +28,13 @@ expect = "(not (yet )?(been )?taken|no final decision|(hasn't|has not) (yet )?(m [[task]] id = "dr19-spike" +answers = ["DR-19: Does the wiki stay on MoinMoin?: What settles it"] prompt = "Which spike in the work plan would settle DR-19, the question of whether the wiki stays on MoinMoin? Name it. Do not browse the web." expect = "S2" [[task]] id = "asf-pressures" +answers = ["Seeding a Foundation: 10. What the Foundation taught the design"] prompt = "In the 'Seeding a Foundation' notes, how many pressures does the part 'What the Foundation taught the design' list? Give the number." # the part cites "paper §6" and another part is "6. Trust and review", so a six or 6 # counts unless it is a section number, a link anchor, or the end of a range or a list of @@ -39,6 +44,7 @@ expect = "(?/dev/null +python3 scripts/seed-corpus.py --root "$root" --memory "$amem" examples/decisions docs | tail -n 1 + +"$amem" --root "$root" golden golden/docs.toml --json >"$work/golden.json" +python3 - evals/demo.toml "$work/evals.toml" <<'PY' +import json, sys, tomllib +tasks = tomllib.load(open(sys.argv[1], "rb"))["task"] +bad = [t["id"] for t in tasks if not t.get("answers")] +if bad: + sys.exit(f"evals/demo.toml: tasks without `answers`: {', '.join(bad)}") +with open(sys.argv[2], "w") as out: + out.write('domain = "dom-demo"\n') + for t in tasks: + out.write(f"\n[[question]]\nask = {json.dumps(t['id'])}\nanswers = {json.dumps(t['answers'])}\n") +PY +"$amem" --root "$root" golden "$work/evals.toml" --json >"$work/evals.json" +python3 - "$work/golden.json" "$work/evals.json" <<'PY' +import json, sys +golden, evals = (json.load(open(p)) for p in sys.argv[1:3]) +fail = [f"golden/docs.toml: no entry is titled as the answer to: {q}" for q in golden["unresolved"]] +fail += [ + f"golden/docs.toml: its keywords do not find the answer in the top {golden['k']}: {r['ask']}" + for r in golden["rows"] + if r["strategy"] == "keywords" and r["rank"] is None +] +fail += [f"evals/demo.toml: no entry is titled as the answer to task {t}" for t in evals["unresolved"]] +for s in golden["scores"]: + print(f" golden {s['strategy']:<9} {s['hits']} of {s['asked']}, MRR {s['mrr']:.2f}") +print(f" evals: {evals['questions'] - len(evals['unresolved'])} of {evals['questions']} tasks name an entry") +if fail: + sys.exit("\n".join(fail)) +PY + +AMEM=$amem sh scripts/demo/11-limits.sh "$root" >"$work/act11.txt" +python3 - "$work/act11.txt" <<'PY' +import re, sys +text = open(sys.argv[1]).read() +parts = re.split(r"^11\.\d ", text, flags=re.M)[1:] +counts = [len(re.findall(r"^ - ", p, re.M)) for p in parts] +print(f" act 11: {' and '.join(map(str, counts))} items") +if len(counts) != 2 or 0 in counts: + sys.exit(f"act 11: README.md does not give it two lists with items:\n{text}") +PY diff --git a/scripts/demo/lib.sh b/scripts/demo/lib.sh index 2c719640..35f0290f 100644 --- a/scripts/demo/lib.sh +++ b/scripts/demo/lib.sh @@ -4,7 +4,7 @@ set -eu root=${1:?demo root} here=$(cd "$(dirname "$0")/../.." && pwd) -memory="$here/target/release/amem" +memory=${AMEM:-$here/target/release/amem} [ -x "$memory" ] || { echo "build first: make build" >&2; exit 2; } [ -f "$root/grants.toml" ] || { echo "no record at $root: run the first act, make demo" >&2; exit 2; } a() { "$memory" --root "$root" "$@"; } diff --git a/scripts/docs-only.sh b/scripts/docs-only.sh index aa03d111..f521f284 100755 --- a/scripts/docs-only.sh +++ b/scripts/docs-only.sh @@ -5,11 +5,12 @@ # `crates/` and `taps/` a `.md` file counts as code, since a fixture or a crate's README is # what a test or a doc attribute would read; an `AGENTS.md` there is instructions, not # input. No crate or tap embeds a Markdown file from anywhere else or reads one -# (scripts/docs-only.test.sh holds that). Some scripts do, and no check runs them: -# scripts/seed-corpus.py seeds the demo record from docs/ and examples/decisions/, which -# golden/docs.toml and evals/demo.toml score against, and act 11 of the demo prints the -# lists at the top of README.md. An empty list is not docs-only, so a caller that failed -# to list its changes runs everything. +# (scripts/docs-only.test.sh holds that). Some scripts do: scripts/seed-corpus.py seeds +# the demo record from docs/ and examples/decisions/, which golden/docs.toml and +# evals/demo.toml score against, and act 11 of the demo prints the lists at the top of +# README.md. scripts/corpus-check.sh checks those readers, and CI runs it on every +# change, this kind included. An empty list is not docs-only, so a caller that failed to +# list its changes runs everything. # # git diff --cached --name-only --no-renames | scripts/docs-only.sh # From 759b62154ce2aebc90915850d37452f252f21ee4 Mon Sep 17 00:00:00 2001 From: Sander Striker Date: Sat, 31 Oct 2026 02:13:48 +0100 Subject: [PATCH 2/3] fix: answer the adversarial pass on the corpus check The check now scores act 7's recall suite, which moves to golden/platform.toml, with the answer "Trust and review" in place of the old heading. golden/acts.toml names the DR-19 and DR-22 entries that acts 4, 7 and 10 ask about. The text of each eval task's entries must match its expect; for asf-pressures, the number of table rows must. Act 11 must print every item of the two README lists. The seed writes to a file, so a crash fails the check. The eval answers name parts by title, not by number. The check seeds from DEMO_FROM, keeps non-ASCII text as it is, and removes its directory on a signal. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01Bhj4Ck4y5iwZUsjpgQx8A2 --- .github/workflows/ci.yml | 2 +- AGENTS.md | 10 ++-- Makefile | 4 +- evals/demo.toml | 7 ++- golden/acts.toml | 13 +++++ golden/platform.toml | 16 ++++++ scripts/corpus-check.sh | 103 ++++++++++++++++++++++++++++--------- scripts/demo/7-platform.sh | 20 +------ scripts/docs-only.sh | 2 +- 9 files changed, 124 insertions(+), 53 deletions(-) create mode 100644 golden/acts.toml create mode 100644 golden/platform.toml diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index cacf0b28..f186dca0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -14,7 +14,7 @@ env: jobs: # Whether this push or pull request changes only Markdown outside the code, which cannot - # change what builds (scripts/docs-only.sh says which files those are). The jobs below skip on a yes; a job skipped by its + # change what builds (scripts/docs-only.sh says which files those are). The jobs below, all but `corpus`, skip on a yes; a job skipped by its # condition reports success, so a required check does not sit waiting for it. Anything # the diff cannot answer, a first push or a rewritten base, runs everything. changes: diff --git a/AGENTS.md b/AGENTS.md index 194d37f3..7e404ed0 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -47,10 +47,12 @@ before you override the instruction. `README.md`. `make corpus-check` (`scripts/corpus-check.sh`) seeds a scratch record from those - files. It fails when a golden question or an eval task names an entry that the record - does not hold, when the keywords of a golden question do not find its answer, or when - act 11 does not find its two lists. CI runs it on every change, a docs-only change - included. It does not run a harness, so it does not show what a model answers. + files. It fails when a golden question (`golden/docs.toml`, `golden/platform.toml`, + `golden/acts.toml`) or an eval task names an entry that the record does not hold, when + the keywords of a golden question do not find its answer, when the text of an eval + task's entry no longer matches its `expect`, or when act 11 does not print every item + of its two lists. CI runs it on every change, a docs-only change included. It does not + run a harness, so it does not show what a model answers. The demo seeds its record only once. So, after you change a file in `docs/` or `examples/decisions/`, seed a new record with `make demo-ready RESET=1`. Then run diff --git a/Makefile b/Makefile index bd11fbdf..faf2789f 100644 --- a/Makefile +++ b/Makefile @@ -175,10 +175,10 @@ local-memory: # `make check` is the pre-commit gate: fast, and what every commit must pass. check: wasip2-target fmt-check clippy test guard-test docs-only-test no-image-push-test wslcsdk-test conformance-test unsafe-check taps-check -# The demo's Markdown, checked as the golden suite, the evals and act 11 read it. +# The demo's Markdown, checked as the golden suites, the evals and the acts read it. corpus-check: cargo build -p amem - sh scripts/corpus-check.sh target/debug/amem + sh scripts/corpus-check.sh target/debug/amem $(DEMO_FROM) # `make verify` is what CI runs: the gate plus dependencies, coverage, and the wasm build. verify: check deny machete coverage wasm-check diff --git a/evals/demo.toml b/evals/demo.toml index 9bc82b9e..a4cad6c4 100644 --- a/evals/demo.toml +++ b/evals/demo.toml @@ -34,7 +34,10 @@ expect = "S2" [[task]] id = "asf-pressures" -answers = ["Seeding a Foundation: 10. What the Foundation taught the design"] +answers = ["What the Foundation taught the design"] +# the number is not in the text: scripts/corpus-check.sh counts the lines of the answer +# entry that match `counts`, and checks `expect` against that count +counts = "^\\| \\*\\*" prompt = "In the 'Seeding a Foundation' notes, how many pressures does the part 'What the Foundation taught the design' list? Give the number." # the part cites "paper §6" and another part is "6. Trust and review", so a six or 6 # counts unless it is a section number, a link anchor, or the end of a range or a list of @@ -44,7 +47,7 @@ expect = "(?/dev/null -python3 scripts/seed-corpus.py --root "$root" --memory "$amem" examples/decisions docs | tail -n 1 +python3 scripts/seed-corpus.py --root "$root" --memory "$amem" "$@" >"$work/seed.txt" +tail -n 1 "$work/seed.txt" -"$amem" --root "$root" golden golden/docs.toml --json >"$work/golden.json" +for suite in docs platform acts; do + "$amem" --root "$root" golden "golden/$suite.toml" --json >"$work/$suite.json" +done python3 - evals/demo.toml "$work/evals.toml" <<'PY' import json, sys, tomllib tasks = tomllib.load(open(sys.argv[1], "rb"))["task"] @@ -31,33 +43,74 @@ if bad: with open(sys.argv[2], "w") as out: out.write('domain = "dom-demo"\n') for t in tasks: - out.write(f"\n[[question]]\nask = {json.dumps(t['id'])}\nanswers = {json.dumps(t['answers'])}\n") + out.write(f"\n[[question]]\nask = {json.dumps(t['id'], ensure_ascii=False)}\n") + out.write(f"answers = {json.dumps(t['answers'], ensure_ascii=False)}\n") PY "$amem" --root "$root" golden "$work/evals.toml" --json >"$work/evals.json" -python3 - "$work/golden.json" "$work/evals.json" <<'PY' -import json, sys -golden, evals = (json.load(open(p)) for p in sys.argv[1:3]) -fail = [f"golden/docs.toml: no entry is titled as the answer to: {q}" for q in golden["unresolved"]] -fail += [ - f"golden/docs.toml: its keywords do not find the answer in the top {golden['k']}: {r['ask']}" - for r in golden["rows"] - if r["strategy"] == "keywords" and r["rank"] is None -] -fail += [f"evals/demo.toml: no entry is titled as the answer to task {t}" for t in evals["unresolved"]] -for s in golden["scores"]: - print(f" golden {s['strategy']:<9} {s['hits']} of {s['asked']}, MRR {s['mrr']:.2f}") -print(f" evals: {evals['questions'] - len(evals['unresolved'])} of {evals['questions']} tasks name an entry") +python3 - "$work" "$root/record/dom-demo" evals/demo.toml <<'PY' +import json, pathlib, re, sys, tomllib +work, record, evals_toml = pathlib.Path(sys.argv[1]), pathlib.Path(sys.argv[2]), sys.argv[3] +fail = [] +for suite in ("docs", "platform", "acts"): + g = json.loads((work / f"{suite}.json").read_text()) + fail += [f"golden/{suite}.toml: no entry is titled as the answer to: {q}" for q in g["unresolved"]] + if suite != "acts": + fail += [ + f"golden/{suite}.toml: its keywords do not find the answer in the top {g['k']}: {r['ask']}" + for r in g["rows"] + if r["strategy"] == "keywords" and r["rank"] is None + ] + for s in g["scores"]: + print(f" golden/{suite}.toml {s['strategy']:<9} {s['hits']} of {s['asked']}, MRR {s['mrr']:.2f}") + else: + print(f" golden/acts.toml: {g['questions'] - len(g['unresolved'])} of {g['questions']} questions name an entry") +e = json.loads((work / "evals.json").read_text()) +fail += [f"evals/demo.toml: no entry is titled as the answer to task {t}" for t in e["unresolved"]] + +# the entries as golden matches them: an answer is a part of the title, in any case +entries = [] +for p in record.glob("*.md"): + _, front, body = p.read_text().split("---\n", 2) + title = next((l[7:].strip() for l in front.splitlines() if l.startswith("title: ")), "") + if title[:1] == "'" and title[-1:] == "'": + title = title[1:-1].replace("''", "'") + elif title[:1] == '"' and title[-1:] == '"': + title = json.loads(title) + entries.append((title.lower(), body)) +matched = 0 +for t in tomllib.load(open(evals_toml, "rb"))["task"]: + bodies = [b for title, b in entries if any(a.strip().lower() in title for a in t["answers"])] + if "counts" in t: + bodies = [str(len(re.findall(t["counts"], b, re.M))) for b in bodies] + if bodies and not any(re.search(t["expect"], b, re.I | re.S) for b in bodies): + fail.append(f"evals/demo.toml: task {t['id']}: its `expect` matches no text of the entries its `answers` name") + elif bodies: + matched += 1 +print(f" evals: {e['questions'] - len(e['unresolved'])} of {e['questions']} tasks name an entry, {matched} of them hold the answer") if fail: sys.exit("\n".join(fail)) PY AMEM=$amem sh scripts/demo/11-limits.sh "$root" >"$work/act11.txt" -python3 - "$work/act11.txt" <<'PY' +python3 - "$work/act11.txt" README.md <<'PY' import re, sys -text = open(sys.argv[1]).read() +text, readme = open(sys.argv[1]).read(), open(sys.argv[2]).read() parts = re.split(r"^11\.\d ", text, flags=re.M)[1:] counts = [len(re.findall(r"^ - ", p, re.M)) for p in parts] -print(f" act 11: {' and '.join(map(str, counts))} items") -if len(counts) != 2 or 0 in counts: - sys.exit(f"act 11: README.md does not give it two lists with items:\n{text}") +heads = ("What the paper asks for that is not built", "What is built but works only at demo scale") +# the items of the first list under each heading: it ends at the first line that is not +# an item, a line of one, or blank +wanted = [] +for head in heads: + part = readme.split("### " + head, 1)[1].split("\n#", 1)[0] if "### " + head in readme else "" + items, started = 0, False + for line in part.splitlines(): + if line.startswith("- "): + items, started = items + 1, True + elif started and line and not line.startswith(" "): + break + wanted.append(items) +print(f" act 11: {' and '.join(map(str, counts))} items, of {' and '.join(map(str, wanted))} in README.md") +if len(counts) != 2 or 0 in counts or counts != wanted: + sys.exit(f"act 11: it does not print every item of the two lists in README.md:\n{text}") PY diff --git a/scripts/demo/7-platform.sh b/scripts/demo/7-platform.sh index f725242c..211c2cb9 100644 --- a/scripts/demo/7-platform.sh +++ b/scripts/demo/7-platform.sh @@ -24,26 +24,10 @@ print(json.dumps({"title": f"How a {h} worker answers a question", "domain": "do "body": f"```platform agent {h}\nSearch the record for the decision your task names and read it before you answer. End your handoff with this line, exactly: Checked against the platform definition.\n```\n"})) PY )" | j "d['id']") -suite=$(agent agent:architect write "$(python3 - "$WORKER_HARNESS" "$WORKER_MODEL" <<'PY' +suite=$(agent agent:architect write "$(python3 - "$WORKER_HARNESS" "$WORKER_MODEL" "$here/golden/platform.toml" <<'PY' import json, sys h, model = sys.argv[1], sys.argv[2] -suite = '''domain = "dom-demo" - -[[question]] -ask = "What did DR-19 decide about the wiki staying on MoinMoin?" -answers = ["DR-19"] -keywords = "DR-19 MoinMoin wiki" - -[[question]] -ask = "Why is the domain the partition key everywhere?" -answers = ["the domain is the partition key"] -keywords = "domain partition key" - -[[question]] -ask = "Who is the promotion gate at a foundation?" -answers = ["Trust, review"] -keywords = "committers promotion gate" -''' +suite = open(sys.argv[3]).read() print(json.dumps({"title": "The recall suite, and the worker's model", "domain": "dom-platform", "entry_type": "method", "body": f"```platform eval recall\n{suite}```\n\n```platform harness {h}\n{{\"model\": \"{model}\"}}\n```\n"})) diff --git a/scripts/docs-only.sh b/scripts/docs-only.sh index f521f284..cd97d955 100755 --- a/scripts/docs-only.sh +++ b/scripts/docs-only.sh @@ -6,7 +6,7 @@ # what a test or a doc attribute would read; an `AGENTS.md` there is instructions, not # input. No crate or tap embeds a Markdown file from anywhere else or reads one # (scripts/docs-only.test.sh holds that). Some scripts do: scripts/seed-corpus.py seeds -# the demo record from docs/ and examples/decisions/, which golden/docs.toml and +# the demo record from docs/ and examples/decisions/, which the suites in golden/ and # evals/demo.toml score against, and act 11 of the demo prints the lists at the top of # README.md. scripts/corpus-check.sh checks those readers, and CI runs it on every # change, this kind included. An empty list is not docs-only, so a caller that failed to From 1436c8eecfea8f9a89311eff611fca41b5697478 Mon Sep 17 00:00:00 2001 From: Sander Striker Date: Sat, 31 Oct 2026 17:45:36 +0100 Subject: [PATCH 3/3] fix: count every data row, and fail the corpus check on a failed write Copilot found two gaps. The asf-pressures pattern counted only rows with bold text, so a seventh row without bold was not counted. Now it counts each row that follows another row and is not the separator. The seed went on after a failed write and exited 0, so the check could pass on a partial record. Now seed-corpus.py has --strict, and the check uses it. With one failed write, the check exited 0 before this change and exits 1 now. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01Bhj4Ck4y5iwZUsjpgQx8A2 --- evals/demo.toml | 5 +++-- scripts/corpus-check.sh | 2 +- scripts/seed-corpus.py | 12 +++++++++--- 3 files changed, 13 insertions(+), 6 deletions(-) diff --git a/evals/demo.toml b/evals/demo.toml index a4cad6c4..0078d253 100644 --- a/evals/demo.toml +++ b/evals/demo.toml @@ -36,8 +36,9 @@ expect = "S2" id = "asf-pressures" answers = ["What the Foundation taught the design"] # the number is not in the text: scripts/corpus-check.sh counts the lines of the answer -# entry that match `counts`, and checks `expect` against that count -counts = "^\\| \\*\\*" +# entry that match `counts`, and checks `expect` against that count. A match is a table +# row that follows another row and is not the separator: the data rows, bold or not +counts = "(?<=\\|\n)\\|(?![\\s:|-]*$)" prompt = "In the 'Seeding a Foundation' notes, how many pressures does the part 'What the Foundation taught the design' list? Give the number." # the part cites "paper §6" and another part is "6. Trust and review", so a six or 6 # counts unless it is a section number, a link anchor, or the end of a range or a list of diff --git a/scripts/corpus-check.sh b/scripts/corpus-check.sh index 12273b3f..4bff995b 100755 --- a/scripts/corpus-check.sh +++ b/scripts/corpus-check.sh @@ -28,7 +28,7 @@ root=$work/record cd "$here" "$amem" --root "$root" init --owner human:demo:check --domain dom-demo >/dev/null -python3 scripts/seed-corpus.py --root "$root" --memory "$amem" "$@" >"$work/seed.txt" +python3 scripts/seed-corpus.py --strict --root "$root" --memory "$amem" "$@" >"$work/seed.txt" tail -n 1 "$work/seed.txt" for suite in docs platform acts; do diff --git a/scripts/seed-corpus.py b/scripts/seed-corpus.py index 89bf92c2..b934a710 100755 --- a/scripts/seed-corpus.py +++ b/scripts/seed-corpus.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 """Seed a memory record from Markdown, one entry per section, for a realistic demo corpus. -Usage: seed-corpus.py --root ROOT [--kind KIND] [--min-words N] [--memory AMEM] PATH... +Usage: seed-corpus.py --root ROOT [--kind KIND] [--min-words N] [--memory AMEM] [--strict] PATH... Each .md file under PATH is split on second-level headings; each section becomes an entry titled ":
". Files under a directory named decisions become @@ -35,6 +35,9 @@ def main(): ap = argparse.ArgumentParser() ap.add_argument("--root", required=True); ap.add_argument("--kind", default="fact") ap.add_argument("--min-words", type=int, default=40); ap.add_argument("--memory", default="amem") + # a write that fails is reported and skipped; under --strict it also fails the run, + # for a check that must not pass on a partial record + ap.add_argument("--strict", action="store_true") ap.add_argument("paths", nargs="+") a = ap.parse_args() files = [] @@ -43,7 +46,7 @@ def main(): for d, _, fs in os.walk(p): files += [os.path.join(d, f) for f in sorted(fs) if f.endswith(".md") and f.upper() not in SKIP] elif p.endswith(".md"): files.append(p) - written = 0 + written = failed = 0 for f in files: kind = "decision" if "/decisions/" in f or f.startswith("decisions/") else a.kind title, secs = sections(open(f, encoding="utf-8").read()) @@ -60,9 +63,12 @@ def main(): r = subprocess.run([a.memory, "--root", a.root, "write", f"--kind={kind}", f"--title={t[:140]}", f"--body={body[:12000]}"], capture_output=True, text=True) if r.returncode != 0: - print(f"skip {f} [{head}]: {r.stderr.strip()[:120]}", file=sys.stderr); continue + print(f"skip {f} [{head}]: {r.stderr.strip()[:120]}", file=sys.stderr) + failed += 1; continue written += 1 print(f"wrote {written} entries from {len(files)} files") + if failed and a.strict: + sys.exit(f"{failed} writes failed") if __name__ == "__main__": main()