ci: check the Markdown the demo reads, on every change

.github/workflows/ci.yml

@@ -14,7 +14,7 @@ env:
 
 jobs:
   # Whether this push or pull request changes only Markdown outside the code, which cannot
-  # change what builds (scripts/docs-only.sh says which files those are). The jobs below skip on a yes; a job skipped by its
+  # change what builds (scripts/docs-only.sh says which files those are). The jobs below, all but `corpus`, skip on a yes; a job skipped by its
   # condition reports success, so a required check does not sit waiting for it. Anything
   # the diff cannot answer, a first push or a rewritten base, runs everything.
   changes:
@@ -137,6 +137,20 @@ jobs:
       - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2
       - run: make wasm-check
 
+  # The Markdown the demo reads, checked as the demo reads it (scripts/corpus-check.sh).
+  # It runs on every change, a docs-only one included: docs/ and examples/decisions/ are
+  # the demo's record, and a heading renamed there breaks a golden question or an eval.
+  corpus:
+    name: corpus
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+      - uses: dtolnay/rust-toolchain@4cda84d5c5c54efe2404f9d843567869ab1699d4 # 2026-07-16, the commit the ASF allow-list pins
+        with:
+          toolchain: stable
+      - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2
+      - run: make corpus-check
+
   # The rust-version in Cargo.toml, built with that release of Rust (`make msrv-check`), so
   # the declared minimum cannot fall behind what the locked dependencies need.
   # RUSTUP_TOOLCHAIN outranks rust-toolchain.toml, which would otherwise fetch stable and

AGENTS.md

@@ -38,14 +38,22 @@ before you override the instruction.
   counts as Markdown. The hook and CI do not run `make check` for such a change, because
   nothing builds from those files. `scripts/docs-only.sh` decides which changes are of
   this type. Its test holds the premise that no crate or tap embeds those files or reads
-  them. Some scripts read them, and no check runs these scripts:
+  them. Some scripts read them:
 
   - `scripts/seed-corpus.py` seeds the demo record from `docs/` and
     `examples/decisions/` (`make demo`). `golden/docs.toml` and `evals/demo.toml` score
     answers against that text.
   - Act 11 of the demo (`scripts/demo/11-limits.sh`) prints the two lists at the top of
     `README.md`.
 
+  `make corpus-check` (`scripts/corpus-check.sh`) seeds a scratch record from those
+  files. It fails when a golden question (`golden/docs.toml`, `golden/platform.toml`,
+  `golden/acts.toml`) or an eval task names an entry that the record does not hold, when
+  the keywords of a golden question do not find its answer, when the text of an eval
+  task's entry no longer matches its `expect`, or when act 11 does not print every item
+  of its two lists. CI runs it on every change, a docs-only change included. It does not
+  run a harness, so it does not show what a model answers.
+
   The demo seeds its record only once. So, after you change a file in `docs/` or
   `examples/decisions/`, seed a new record with `make demo-ready RESET=1`. Then run
   `make golden` and act 6 (`make demo-all HARNESS=codex ACTS=6`). Act 6 needs a key

Makefile

@@ -8,7 +8,7 @@ NEXTEST := $(shell cargo nextest --version 2>/dev/null)
 TEST_CMD := $(if $(NEXTEST),cargo nextest run --workspace --no-fail-fast,cargo test --workspace)
 
 .PHONY: demo-ready podman-ready e2e hosted install init check verify fmt fmt-check clippy test guard-test docs-only-test no-image-push-test conformance-test conformance deny machete coverage wasm-check msrv-check run build wslcsdk-test wslcsdk unsafe-check windows-target \
-	local-up local-down local-logs local-key local-memory launch usage golden drift taps-check taps-build wasip2-target reader
+	local-up local-down local-logs local-key local-memory launch usage golden corpus-check drift taps-check taps-build wasip2-target reader
 
 # ---- the local stack: LiteLLM and Postgres in containers, the memory root on disk.
 # Reads local/.env. See local/README.md.
@@ -175,6 +175,11 @@ local-memory:
 # `make check` is the pre-commit gate: fast, and what every commit must pass.
 check: wasip2-target fmt-check clippy test guard-test docs-only-test no-image-push-test wslcsdk-test conformance-test unsafe-check taps-check
 
+# The demo's Markdown, checked as the golden suites, the evals and the acts read it.
+corpus-check:
+	cargo build -p amem
+	sh scripts/corpus-check.sh target/debug/amem $(DEMO_FROM)
+
 # `make verify` is what CI runs: the gate plus dependencies, coverage, and the wasm build.
 verify: check deny machete coverage wasm-check
 

evals/demo.toml

@@ -1,7 +1,9 @@
 # An evaluation set over the demo record: questions whose answers live in the record and
 # nowhere else, so a harness without memory can only guess. `scripts/eval.py` runs each
 # task under each condition and grades the answer against `expect`, a case-insensitive
-# regular expression that the answer must match.
+# regular expression that the answer must match. `answers` names the entries that hold
+# the answer, by title or part of one, as in golden/docs.toml; scripts/corpus-check.sh
+# fails when the record holds none of them.
 #
 #   scripts/eval.py evals/demo.toml --root ~/memory-demo2 --harness claude --auth subscription
 #   scripts/eval.py evals/demo.toml --root ~/memory-demo2 --harness codex --auth gateway --model qwen3.8-27b
@@ -18,18 +20,25 @@ preamble = "If a memory server is attached to this session, search it and read i
 
 [[task]]
 id = "dr19-decision"
+answers = ["DR-19: Does the wiki stay on MoinMoin?: Status"]
 prompt = "What did DR-19 decide about the wiki staying on MoinMoin? Answer in one or two sentences from what you know; do not browse the web."
 # "no decision" alone also matches a refusal ("no decision entries recorded"), so only
 # the record's own answer counts: proposed, not yet taken, no final decision
 expect = "(not (yet )?(been )?taken|no final decision|(hasn't|has not) (yet )?(made|taken|reached) a (final )?decision|not (yet )?decided|still open|stays on MoinMoin|proposed)"
 
 [[task]]
 id = "dr19-spike"
+answers = ["DR-19: Does the wiki stay on MoinMoin?: What settles it"]
 prompt = "Which spike in the work plan would settle DR-19, the question of whether the wiki stays on MoinMoin? Name it. Do not browse the web."
 expect = "S2"
 
 [[task]]
 id = "asf-pressures"
+answers = ["What the Foundation taught the design"]
+# the number is not in the text: scripts/corpus-check.sh counts the lines of the answer
+# entry that match `counts`, and checks `expect` against that count. A match is a table
+# row that follows another row and is not the separator: the data rows, bold or not
+counts = "(?<=\\|\n)\\|(?![\\s:|-]*$)"
 prompt = "In the 'Seeding a Foundation' notes, how many pressures does the part 'What the Foundation taught the design' list? Give the number."
 # the part cites "paper §6" and another part is "6. Trust and review", so a six or 6
 # counts unless it is a section number, a link anchor, or the end of a range or a list of
@@ -39,6 +48,7 @@ expect = "(?<![§#–-])(?<!§ )(?<!section )(?<!part )\\b(six|6)\\b(?![–-])(?
 
 [[task]]
 id = "committers-gate"
+answers = ["Trust and review"]
 prompt = "In the 'Seeding a Foundation' notes on trust and review, who is the promotion gate? One sentence."
 # the gate is named people that domain policy authorises, under PMC oversight; the case
 # study gives committers sampled review, so "committers" is no longer the answer, and

golden/acts.toml

@@ -0,0 +1,13 @@
+# What the acts ask of the seeded record, apart from the golden suites they score: act 7
+# files the DR-22 question as a task, act 4 asks a session to summarise DR-19 and DR-22,
+# and act 10 searches for DR-19. scripts/corpus-check.sh checks that each names an entry
+# the record holds.
+domain = "dom-demo"
+
+[[question]]
+ask = "What did DR-22 decide about the format memory entries travel in?"
+answers = ["DR-22"]
+
+[[question]]
+ask = "DR-19 MoinMoin"
+answers = ["DR-19"]

golden/platform.toml

@@ -0,0 +1,16 @@
+domain = "dom-demo"
+
+[[question]]
+ask = "What did DR-19 decide about the wiki staying on MoinMoin?"
+answers = ["DR-19"]
+keywords = "DR-19 MoinMoin wiki"
+
+[[question]]
+ask = "Why is the domain the partition key everywhere?"
+answers = ["the domain is the partition key"]
+keywords = "domain partition key"
+
+[[question]]
+ask = "Who is the promotion gate at a foundation?"
+answers = ["Trust and review"]
+keywords = "committers promotion gate"

scripts/corpus-check.sh

@@ -0,0 +1,116 @@
+#!/bin/sh
+# The Markdown the demo reads, checked as the demo reads it. scripts/docs-only.sh lets a
+# change to docs/ or examples/decisions/ skip the build and the tests, but those files are
+# the demo's record: this seeds a scratch record from them as `make demo-ready` does, and
+# fails when what reads that record would break.
+#
+#   - every question in the golden suites the demo scores (golden/docs.toml, and
+#     golden/platform.toml, which act 7 files) names an entry the record holds, and its
+#     keywords find that entry in the top k;
+#   - every question an act puts to a harness (golden/acts.toml) names an entry the
+#     record holds;
+#   - every task in evals/demo.toml names, in `answers`, an entry the record holds, and
+#     its `expect` matches the text of one of them, or, for a task with `counts`, the
+#     number of lines of one that match `counts`;
+#   - act 11 finds both of its lists at the top of README.md, and prints every item of
+#     each.
+#
+#   scripts/corpus-check.sh target/debug/amem [directories to seed from]
+set -eu
+amem=$(cd "$(dirname "$1")" && pwd)/$(basename "$1")
+shift
+[ $# -gt 0 ] || set -- examples/decisions docs
+here=$(cd "$(dirname "$0")/.." && pwd)
+work=$(mktemp -d)
+trap 'rm -rf "$work"' EXIT
+trap 'exit 1' INT TERM HUP
+root=$work/record
+cd "$here"
+
+"$amem" --root "$root" init --owner human:demo:check --domain dom-demo >/dev/null
+python3 scripts/seed-corpus.py --strict --root "$root" --memory "$amem" "$@" >"$work/seed.txt"
+tail -n 1 "$work/seed.txt"
+
+for suite in docs platform acts; do
+  "$amem" --root "$root" golden "golden/$suite.toml" --json >"$work/$suite.json"
+done
+python3 - evals/demo.toml "$work/evals.toml" <<'PY'
+import json, sys, tomllib
+tasks = tomllib.load(open(sys.argv[1], "rb"))["task"]
+bad = [t["id"] for t in tasks if not t.get("answers")]
+if bad:
+    sys.exit(f"evals/demo.toml: tasks without `answers`: {', '.join(bad)}")
+with open(sys.argv[2], "w") as out:
+    out.write('domain = "dom-demo"\n')
+    for t in tasks:
+        out.write(f"\n[[question]]\nask = {json.dumps(t['id'], ensure_ascii=False)}\n")
+        out.write(f"answers = {json.dumps(t['answers'], ensure_ascii=False)}\n")
+PY
+"$amem" --root "$root" golden "$work/evals.toml" --json >"$work/evals.json"
+python3 - "$work" "$root/record/dom-demo" evals/demo.toml <<'PY'
+import json, pathlib, re, sys, tomllib
+work, record, evals_toml = pathlib.Path(sys.argv[1]), pathlib.Path(sys.argv[2]), sys.argv[3]
+fail = []
+for suite in ("docs", "platform", "acts"):
+    g = json.loads((work / f"{suite}.json").read_text())
+    fail += [f"golden/{suite}.toml: no entry is titled as the answer to: {q}" for q in g["unresolved"]]
+    if suite != "acts":
+        fail += [
+            f"golden/{suite}.toml: its keywords do not find the answer in the top {g['k']}: {r['ask']}"
+            for r in g["rows"]
+            if r["strategy"] == "keywords" and r["rank"] is None
+        ]
+        for s in g["scores"]:
+            print(f"  golden/{suite}.toml {s['strategy']:<9} {s['hits']} of {s['asked']}, MRR {s['mrr']:.2f}")
+    else:
+        print(f"  golden/acts.toml: {g['questions'] - len(g['unresolved'])} of {g['questions']} questions name an entry")
+e = json.loads((work / "evals.json").read_text())
+fail += [f"evals/demo.toml: no entry is titled as the answer to task {t}" for t in e["unresolved"]]
+
+# the entries as golden matches them: an answer is a part of the title, in any case
+entries = []
+for p in record.glob("*.md"):
+    _, front, body = p.read_text().split("---\n", 2)
+    title = next((l[7:].strip() for l in front.splitlines() if l.startswith("title: ")), "")
+    if title[:1] == "'" and title[-1:] == "'":
+        title = title[1:-1].replace("''", "'")
+    elif title[:1] == '"' and title[-1:] == '"':
+        title = json.loads(title)
+    entries.append((title.lower(), body))
+matched = 0
+for t in tomllib.load(open(evals_toml, "rb"))["task"]:
+    bodies = [b for title, b in entries if any(a.strip().lower() in title for a in t["answers"])]
+    if "counts" in t:
+        bodies = [str(len(re.findall(t["counts"], b, re.M))) for b in bodies]
+    if bodies and not any(re.search(t["expect"], b, re.I | re.S) for b in bodies):
+        fail.append(f"evals/demo.toml: task {t['id']}: its `expect` matches no text of the entries its `answers` name")
+    elif bodies:
+        matched += 1
+print(f"  evals: {e['questions'] - len(e['unresolved'])} of {e['questions']} tasks name an entry, {matched} of them hold the answer")
+if fail:
+    sys.exit("\n".join(fail))
+PY
+
+AMEM=$amem sh scripts/demo/11-limits.sh "$root" >"$work/act11.txt"
+python3 - "$work/act11.txt" README.md <<'PY'
+import re, sys
+text, readme = open(sys.argv[1]).read(), open(sys.argv[2]).read()
+parts = re.split(r"^11\.\d  ", text, flags=re.M)[1:]
+counts = [len(re.findall(r"^  - ", p, re.M)) for p in parts]
+heads = ("What the paper asks for that is not built", "What is built but works only at demo scale")
+# the items of the first list under each heading: it ends at the first line that is not
+# an item, a line of one, or blank
+wanted = []
+for head in heads:
+    part = readme.split("### " + head, 1)[1].split("\n#", 1)[0] if "### " + head in readme else ""
+    items, started = 0, False
+    for line in part.splitlines():
+        if line.startswith("- "):
+            items, started = items + 1, True
+        elif started and line and not line.startswith(" "):
+            break
+    wanted.append(items)
+print(f"  act 11: {' and '.join(map(str, counts))} items, of {' and '.join(map(str, wanted))} in README.md")
+if len(counts) != 2 or 0 in counts or counts != wanted:
+    sys.exit(f"act 11: it does not print every item of the two lists in README.md:\n{text}")
+PY

scripts/demo/7-platform.sh

@@ -24,26 +24,10 @@ print(json.dumps({"title": f"How a {h} worker answers a question", "domain": "do
                   "body": f"```platform agent {h}\nSearch the record for the decision your task names and read it before you answer. End your handoff with this line, exactly: Checked against the platform definition.\n```\n"}))
 PY
 )" | j "d['id']")
-suite=$(agent agent:architect write "$(python3 - "$WORKER_HARNESS" "$WORKER_MODEL" <<'PY'
+suite=$(agent agent:architect write "$(python3 - "$WORKER_HARNESS" "$WORKER_MODEL" "$here/golden/platform.toml" <<'PY'
 import json, sys
 h, model = sys.argv[1], sys.argv[2]
-suite = '''domain = "dom-demo"
-
-[[question]]
-ask = "What did DR-19 decide about the wiki staying on MoinMoin?"
-answers = ["DR-19"]
-keywords = "DR-19 MoinMoin wiki"
-
-[[question]]
-ask = "Why is the domain the partition key everywhere?"
-answers = ["the domain is the partition key"]
-keywords = "domain partition key"
-
-[[question]]
-ask = "Who is the promotion gate at a foundation?"
-answers = ["Trust, review"]
-keywords = "committers promotion gate"
-'''
+suite = open(sys.argv[3]).read()
 print(json.dumps({"title": "The recall suite, and the worker's model", "domain": "dom-platform",
                   "entry_type": "method",
                   "body": f"```platform eval recall\n{suite}```\n\n```platform harness {h}\n{{\"model\": \"{model}\"}}\n```\n"}))

scripts/demo/lib.sh

@@ -4,7 +4,7 @@
 set -eu
 root=${1:?demo root}
 here=$(cd "$(dirname "$0")/../.." && pwd)
-memory="$here/target/release/amem"
+memory=${AMEM:-$here/target/release/amem}
 [ -x "$memory" ] || { echo "build first: make build" >&2; exit 2; }
 [ -f "$root/grants.toml" ] || { echo "no record at $root: run the first act, make demo" >&2; exit 2; }
 a() { "$memory" --root "$root" "$@"; }

scripts/docs-only.sh

@@ -5,11 +5,12 @@
 # `crates/` and `taps/` a `.md` file counts as code, since a fixture or a crate's README is
 # what a test or a doc attribute would read; an `AGENTS.md` there is instructions, not
 # input. No crate or tap embeds a Markdown file from anywhere else or reads one
-# (scripts/docs-only.test.sh holds that). Some scripts do, and no check runs them:
-# scripts/seed-corpus.py seeds the demo record from docs/ and examples/decisions/, which
-# golden/docs.toml and evals/demo.toml score against, and act 11 of the demo prints the
-# lists at the top of README.md. An empty list is not docs-only, so a caller that failed
-# to list its changes runs everything.
+# (scripts/docs-only.test.sh holds that). Some scripts do: scripts/seed-corpus.py seeds
+# the demo record from docs/ and examples/decisions/, which the suites in golden/ and
+# evals/demo.toml score against, and act 11 of the demo prints the lists at the top of
+# README.md. scripts/corpus-check.sh checks those readers, and CI runs it on every
+# change, this kind included. An empty list is not docs-only, so a caller that failed to
+# list its changes runs everything.
 #
 #   git diff --cached --name-only --no-renames | scripts/docs-only.sh
 #

scripts/seed-corpus.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python3
 """Seed a memory record from Markdown, one entry per section, for a realistic demo corpus.
 
-Usage: seed-corpus.py --root ROOT [--kind KIND] [--min-words N] [--memory AMEM] PATH...
+Usage: seed-corpus.py --root ROOT [--kind KIND] [--min-words N] [--memory AMEM] [--strict] PATH...
 
 Each .md file under PATH is split on second-level headings; each section becomes an entry
 titled "<file title>: <section heading>". Files under a directory named decisions become
@@ -35,6 +35,9 @@ def main():
     ap = argparse.ArgumentParser()
     ap.add_argument("--root", required=True); ap.add_argument("--kind", default="fact")
     ap.add_argument("--min-words", type=int, default=40); ap.add_argument("--memory", default="amem")
+    # a write that fails is reported and skipped; under --strict it also fails the run,
+    # for a check that must not pass on a partial record
+    ap.add_argument("--strict", action="store_true")
     ap.add_argument("paths", nargs="+")
     a = ap.parse_args()
     files = []
@@ -43,7 +46,7 @@ def main():
             for d, _, fs in os.walk(p):
                 files += [os.path.join(d, f) for f in sorted(fs) if f.endswith(".md") and f.upper() not in SKIP]
         elif p.endswith(".md"): files.append(p)
-    written = 0
+    written = failed = 0
     for f in files:
         kind = "decision" if "/decisions/" in f or f.startswith("decisions/") else a.kind
         title, secs = sections(open(f, encoding="utf-8").read())
@@ -60,9 +63,12 @@ def main():
             r = subprocess.run([a.memory, "--root", a.root, "write", f"--kind={kind}", f"--title={t[:140]}",
                                 f"--body={body[:12000]}"], capture_output=True, text=True)
             if r.returncode != 0:
-                print(f"skip {f} [{head}]: {r.stderr.strip()[:120]}", file=sys.stderr); continue
+                print(f"skip {f} [{head}]: {r.stderr.strip()[:120]}", file=sys.stderr)
+                failed += 1; continue
             written += 1
     print(f"wrote {written} entries from {len(files)} files")
+    if failed and a.strict:
+        sys.exit(f"{failed} writes failed")
 
 if __name__ == "__main__":
     main()