diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9ad49dc8..cfc62b2d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -14,7 +14,7 @@ env: jobs: # Whether this push or pull request changes only Markdown outside the code, which cannot - # change what builds (scripts/docs-only.sh says which files those are). The jobs below skip on a yes; a job skipped by its + # change what builds (scripts/docs-only.sh says which files those are). The jobs below, all but `corpus`, skip on a yes; a job skipped by its # condition reports success, so a required check does not sit waiting for it. Anything # the diff cannot answer, a first push or a rewritten base, runs everything. changes: @@ -137,6 +137,20 @@ jobs: - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 - run: make wasm-check + # The Markdown the demo reads, checked as the demo reads it (scripts/corpus-check.sh). + # It runs on every change, a docs-only one included: docs/ and examples/decisions/ are + # the demo's record, and a heading renamed there breaks a golden question or an eval. + corpus: + name: corpus + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - uses: dtolnay/rust-toolchain@4cda84d5c5c54efe2404f9d843567869ab1699d4 # 2026-07-16, the commit the ASF allow-list pins + with: + toolchain: stable + - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 + - run: make corpus-check + # The rust-version in Cargo.toml, built with that release of Rust (`make msrv-check`), so # the declared minimum cannot fall behind what the locked dependencies need. # RUSTUP_TOOLCHAIN outranks rust-toolchain.toml, which would otherwise fetch stable and diff --git a/AGENTS.md b/AGENTS.md index 2402bed2..7e404ed0 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -38,7 +38,7 @@ before you override the instruction. counts as Markdown. The hook and CI do not run `make check` for such a change, because nothing builds from those files. `scripts/docs-only.sh` decides which changes are of this type. Its test holds the premise that no crate or tap embeds those files or reads - them. Some scripts read them, and no check runs these scripts: + them. Some scripts read them: - `scripts/seed-corpus.py` seeds the demo record from `docs/` and `examples/decisions/` (`make demo`). `golden/docs.toml` and `evals/demo.toml` score @@ -46,6 +46,14 @@ before you override the instruction. - Act 11 of the demo (`scripts/demo/11-limits.sh`) prints the two lists at the top of `README.md`. + `make corpus-check` (`scripts/corpus-check.sh`) seeds a scratch record from those + files. It fails when a golden question (`golden/docs.toml`, `golden/platform.toml`, + `golden/acts.toml`) or an eval task names an entry that the record does not hold, when + the keywords of a golden question do not find its answer, when the text of an eval + task's entry no longer matches its `expect`, or when act 11 does not print every item + of its two lists. CI runs it on every change, a docs-only change included. It does not + run a harness, so it does not show what a model answers. + The demo seeds its record only once. So, after you change a file in `docs/` or `examples/decisions/`, seed a new record with `make demo-ready RESET=1`. Then run `make golden` and act 6 (`make demo-all HARNESS=codex ACTS=6`). Act 6 needs a key diff --git a/Makefile b/Makefile index f35bd957..3a9f2366 100644 --- a/Makefile +++ b/Makefile @@ -8,7 +8,7 @@ NEXTEST := $(shell cargo nextest --version 2>/dev/null) TEST_CMD := $(if $(NEXTEST),cargo nextest run --workspace --no-fail-fast,cargo test --workspace) .PHONY: demo-ready podman-ready e2e hosted install init check verify fmt fmt-check clippy test guard-test docs-only-test no-image-push-test conformance-test conformance deny machete coverage wasm-check msrv-check run build wslcsdk-test wslcsdk unsafe-check windows-target \ - local-up local-down local-logs local-key local-memory launch usage golden drift taps-check taps-build wasip2-target reader + local-up local-down local-logs local-key local-memory launch usage golden corpus-check drift taps-check taps-build wasip2-target reader # ---- the local stack: LiteLLM and Postgres in containers, the memory root on disk. # Reads local/.env. See local/README.md. @@ -175,6 +175,11 @@ local-memory: # `make check` is the pre-commit gate: fast, and what every commit must pass. check: wasip2-target fmt-check clippy test guard-test docs-only-test no-image-push-test wslcsdk-test conformance-test unsafe-check taps-check +# The demo's Markdown, checked as the golden suites, the evals and the acts read it. +corpus-check: + cargo build -p amem + sh scripts/corpus-check.sh target/debug/amem $(DEMO_FROM) + # `make verify` is what CI runs: the gate plus dependencies, coverage, and the wasm build. verify: check deny machete coverage wasm-check diff --git a/evals/demo.toml b/evals/demo.toml index 5dcc88fd..0078d253 100644 --- a/evals/demo.toml +++ b/evals/demo.toml @@ -1,7 +1,9 @@ # An evaluation set over the demo record: questions whose answers live in the record and # nowhere else, so a harness without memory can only guess. `scripts/eval.py` runs each # task under each condition and grades the answer against `expect`, a case-insensitive -# regular expression that the answer must match. +# regular expression that the answer must match. `answers` names the entries that hold +# the answer, by title or part of one, as in golden/docs.toml; scripts/corpus-check.sh +# fails when the record holds none of them. # # scripts/eval.py evals/demo.toml --root ~/memory-demo2 --harness claude --auth subscription # scripts/eval.py evals/demo.toml --root ~/memory-demo2 --harness codex --auth gateway --model qwen3.8-27b @@ -18,6 +20,7 @@ preamble = "If a memory server is attached to this session, search it and read i [[task]] id = "dr19-decision" +answers = ["DR-19: Does the wiki stay on MoinMoin?: Status"] prompt = "What did DR-19 decide about the wiki staying on MoinMoin? Answer in one or two sentences from what you know; do not browse the web." # "no decision" alone also matches a refusal ("no decision entries recorded"), so only # the record's own answer counts: proposed, not yet taken, no final decision @@ -25,11 +28,17 @@ expect = "(not (yet )?(been )?taken|no final decision|(hasn't|has not) (yet )?(m [[task]] id = "dr19-spike" +answers = ["DR-19: Does the wiki stay on MoinMoin?: What settles it"] prompt = "Which spike in the work plan would settle DR-19, the question of whether the wiki stays on MoinMoin? Name it. Do not browse the web." expect = "S2" [[task]] id = "asf-pressures" +answers = ["What the Foundation taught the design"] +# the number is not in the text: scripts/corpus-check.sh counts the lines of the answer +# entry that match `counts`, and checks `expect` against that count. A match is a table +# row that follows another row and is not the separator: the data rows, bold or not +counts = "(?<=\\|\n)\\|(?![\\s:|-]*$)" prompt = "In the 'Seeding a Foundation' notes, how many pressures does the part 'What the Foundation taught the design' list? Give the number." # the part cites "paper §6" and another part is "6. Trust and review", so a six or 6 # counts unless it is a section number, a link anchor, or the end of a range or a list of @@ -39,6 +48,7 @@ expect = "(?/dev/null +python3 scripts/seed-corpus.py --strict --root "$root" --memory "$amem" "$@" >"$work/seed.txt" +tail -n 1 "$work/seed.txt" + +for suite in docs platform acts; do + "$amem" --root "$root" golden "golden/$suite.toml" --json >"$work/$suite.json" +done +python3 - evals/demo.toml "$work/evals.toml" <<'PY' +import json, sys, tomllib +tasks = tomllib.load(open(sys.argv[1], "rb"))["task"] +bad = [t["id"] for t in tasks if not t.get("answers")] +if bad: + sys.exit(f"evals/demo.toml: tasks without `answers`: {', '.join(bad)}") +with open(sys.argv[2], "w") as out: + out.write('domain = "dom-demo"\n') + for t in tasks: + out.write(f"\n[[question]]\nask = {json.dumps(t['id'], ensure_ascii=False)}\n") + out.write(f"answers = {json.dumps(t['answers'], ensure_ascii=False)}\n") +PY +"$amem" --root "$root" golden "$work/evals.toml" --json >"$work/evals.json" +python3 - "$work" "$root/record/dom-demo" evals/demo.toml <<'PY' +import json, pathlib, re, sys, tomllib +work, record, evals_toml = pathlib.Path(sys.argv[1]), pathlib.Path(sys.argv[2]), sys.argv[3] +fail = [] +for suite in ("docs", "platform", "acts"): + g = json.loads((work / f"{suite}.json").read_text()) + fail += [f"golden/{suite}.toml: no entry is titled as the answer to: {q}" for q in g["unresolved"]] + if suite != "acts": + fail += [ + f"golden/{suite}.toml: its keywords do not find the answer in the top {g['k']}: {r['ask']}" + for r in g["rows"] + if r["strategy"] == "keywords" and r["rank"] is None + ] + for s in g["scores"]: + print(f" golden/{suite}.toml {s['strategy']:<9} {s['hits']} of {s['asked']}, MRR {s['mrr']:.2f}") + else: + print(f" golden/acts.toml: {g['questions'] - len(g['unresolved'])} of {g['questions']} questions name an entry") +e = json.loads((work / "evals.json").read_text()) +fail += [f"evals/demo.toml: no entry is titled as the answer to task {t}" for t in e["unresolved"]] + +# the entries as golden matches them: an answer is a part of the title, in any case +entries = [] +for p in record.glob("*.md"): + _, front, body = p.read_text().split("---\n", 2) + title = next((l[7:].strip() for l in front.splitlines() if l.startswith("title: ")), "") + if title[:1] == "'" and title[-1:] == "'": + title = title[1:-1].replace("''", "'") + elif title[:1] == '"' and title[-1:] == '"': + title = json.loads(title) + entries.append((title.lower(), body)) +matched = 0 +for t in tomllib.load(open(evals_toml, "rb"))["task"]: + bodies = [b for title, b in entries if any(a.strip().lower() in title for a in t["answers"])] + if "counts" in t: + bodies = [str(len(re.findall(t["counts"], b, re.M))) for b in bodies] + if bodies and not any(re.search(t["expect"], b, re.I | re.S) for b in bodies): + fail.append(f"evals/demo.toml: task {t['id']}: its `expect` matches no text of the entries its `answers` name") + elif bodies: + matched += 1 +print(f" evals: {e['questions'] - len(e['unresolved'])} of {e['questions']} tasks name an entry, {matched} of them hold the answer") +if fail: + sys.exit("\n".join(fail)) +PY + +AMEM=$amem sh scripts/demo/11-limits.sh "$root" >"$work/act11.txt" +python3 - "$work/act11.txt" README.md <<'PY' +import re, sys +text, readme = open(sys.argv[1]).read(), open(sys.argv[2]).read() +parts = re.split(r"^11\.\d ", text, flags=re.M)[1:] +counts = [len(re.findall(r"^ - ", p, re.M)) for p in parts] +heads = ("What the paper asks for that is not built", "What is built but works only at demo scale") +# the items of the first list under each heading: it ends at the first line that is not +# an item, a line of one, or blank +wanted = [] +for head in heads: + part = readme.split("### " + head, 1)[1].split("\n#", 1)[0] if "### " + head in readme else "" + items, started = 0, False + for line in part.splitlines(): + if line.startswith("- "): + items, started = items + 1, True + elif started and line and not line.startswith(" "): + break + wanted.append(items) +print(f" act 11: {' and '.join(map(str, counts))} items, of {' and '.join(map(str, wanted))} in README.md") +if len(counts) != 2 or 0 in counts or counts != wanted: + sys.exit(f"act 11: it does not print every item of the two lists in README.md:\n{text}") +PY diff --git a/scripts/demo/7-platform.sh b/scripts/demo/7-platform.sh index f725242c..211c2cb9 100644 --- a/scripts/demo/7-platform.sh +++ b/scripts/demo/7-platform.sh @@ -24,26 +24,10 @@ print(json.dumps({"title": f"How a {h} worker answers a question", "domain": "do "body": f"```platform agent {h}\nSearch the record for the decision your task names and read it before you answer. End your handoff with this line, exactly: Checked against the platform definition.\n```\n"})) PY )" | j "d['id']") -suite=$(agent agent:architect write "$(python3 - "$WORKER_HARNESS" "$WORKER_MODEL" <<'PY' +suite=$(agent agent:architect write "$(python3 - "$WORKER_HARNESS" "$WORKER_MODEL" "$here/golden/platform.toml" <<'PY' import json, sys h, model = sys.argv[1], sys.argv[2] -suite = '''domain = "dom-demo" - -[[question]] -ask = "What did DR-19 decide about the wiki staying on MoinMoin?" -answers = ["DR-19"] -keywords = "DR-19 MoinMoin wiki" - -[[question]] -ask = "Why is the domain the partition key everywhere?" -answers = ["the domain is the partition key"] -keywords = "domain partition key" - -[[question]] -ask = "Who is the promotion gate at a foundation?" -answers = ["Trust, review"] -keywords = "committers promotion gate" -''' +suite = open(sys.argv[3]).read() print(json.dumps({"title": "The recall suite, and the worker's model", "domain": "dom-platform", "entry_type": "method", "body": f"```platform eval recall\n{suite}```\n\n```platform harness {h}\n{{\"model\": \"{model}\"}}\n```\n"})) diff --git a/scripts/demo/lib.sh b/scripts/demo/lib.sh index 2c719640..35f0290f 100644 --- a/scripts/demo/lib.sh +++ b/scripts/demo/lib.sh @@ -4,7 +4,7 @@ set -eu root=${1:?demo root} here=$(cd "$(dirname "$0")/../.." && pwd) -memory="$here/target/release/amem" +memory=${AMEM:-$here/target/release/amem} [ -x "$memory" ] || { echo "build first: make build" >&2; exit 2; } [ -f "$root/grants.toml" ] || { echo "no record at $root: run the first act, make demo" >&2; exit 2; } a() { "$memory" --root "$root" "$@"; } diff --git a/scripts/docs-only.sh b/scripts/docs-only.sh index aa03d111..cd97d955 100755 --- a/scripts/docs-only.sh +++ b/scripts/docs-only.sh @@ -5,11 +5,12 @@ # `crates/` and `taps/` a `.md` file counts as code, since a fixture or a crate's README is # what a test or a doc attribute would read; an `AGENTS.md` there is instructions, not # input. No crate or tap embeds a Markdown file from anywhere else or reads one -# (scripts/docs-only.test.sh holds that). Some scripts do, and no check runs them: -# scripts/seed-corpus.py seeds the demo record from docs/ and examples/decisions/, which -# golden/docs.toml and evals/demo.toml score against, and act 11 of the demo prints the -# lists at the top of README.md. An empty list is not docs-only, so a caller that failed -# to list its changes runs everything. +# (scripts/docs-only.test.sh holds that). Some scripts do: scripts/seed-corpus.py seeds +# the demo record from docs/ and examples/decisions/, which the suites in golden/ and +# evals/demo.toml score against, and act 11 of the demo prints the lists at the top of +# README.md. scripts/corpus-check.sh checks those readers, and CI runs it on every +# change, this kind included. An empty list is not docs-only, so a caller that failed to +# list its changes runs everything. # # git diff --cached --name-only --no-renames | scripts/docs-only.sh # diff --git a/scripts/seed-corpus.py b/scripts/seed-corpus.py index 89bf92c2..b934a710 100755 --- a/scripts/seed-corpus.py +++ b/scripts/seed-corpus.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 """Seed a memory record from Markdown, one entry per section, for a realistic demo corpus. -Usage: seed-corpus.py --root ROOT [--kind KIND] [--min-words N] [--memory AMEM] PATH... +Usage: seed-corpus.py --root ROOT [--kind KIND] [--min-words N] [--memory AMEM] [--strict] PATH... Each .md file under PATH is split on second-level headings; each section becomes an entry titled ":
". Files under a directory named decisions become @@ -35,6 +35,9 @@ def main(): ap = argparse.ArgumentParser() ap.add_argument("--root", required=True); ap.add_argument("--kind", default="fact") ap.add_argument("--min-words", type=int, default=40); ap.add_argument("--memory", default="amem") + # a write that fails is reported and skipped; under --strict it also fails the run, + # for a check that must not pass on a partial record + ap.add_argument("--strict", action="store_true") ap.add_argument("paths", nargs="+") a = ap.parse_args() files = [] @@ -43,7 +46,7 @@ def main(): for d, _, fs in os.walk(p): files += [os.path.join(d, f) for f in sorted(fs) if f.endswith(".md") and f.upper() not in SKIP] elif p.endswith(".md"): files.append(p) - written = 0 + written = failed = 0 for f in files: kind = "decision" if "/decisions/" in f or f.startswith("decisions/") else a.kind title, secs = sections(open(f, encoding="utf-8").read()) @@ -60,9 +63,12 @@ def main(): r = subprocess.run([a.memory, "--root", a.root, "write", f"--kind={kind}", f"--title={t[:140]}", f"--body={body[:12000]}"], capture_output=True, text=True) if r.returncode != 0: - print(f"skip {f} [{head}]: {r.stderr.strip()[:120]}", file=sys.stderr); continue + print(f"skip {f} [{head}]: {r.stderr.strip()[:120]}", file=sys.stderr) + failed += 1; continue written += 1 print(f"wrote {written} entries from {len(files)} files") + if failed and a.strict: + sys.exit(f"{failed} writes failed") if __name__ == "__main__": main()