Check curve, ablation and baseline completion in code

Makefile

@@ -15,6 +15,16 @@ UV_ENV := $(if $(ENV_FILE),--env-file $(ENV_FILE),)
 # checkpoint, and can evict files mid-training. See README "iCloud".
 NOSYNC_LINKS := .venv checkpoints
 
+# No default target. On 2026-09-23 a zsh loop ran `make $t` with
+# t="curve MODEL=bert"; make got one argument, read it as a variable
+# assignment, ran the first target (setup) and exited 0, and two curves
+# were reported done with no point run. A bare `make` (or only variable
+# assignments) now fails instead of silently running setup.
+# tests/test_makefile.py pins this.
+ifeq ($(strip $(MAKECMDGOALS)),)
+$(error no target given; a quoted "curve MODEL=bert" is one variable assignment, not a target)
+endif
+
 setup:
 	@for name in $(NOSYNC_LINKS); do \
 		if [ -L "$$name" ]; then continue; fi; \
@@ -73,6 +83,13 @@ ac2:
 # never edit configs/curve.yaml. Curves and the ablation resume like ac2 and
 # keep no weights, only logits. `curve` runs the cheap baselines first.
 # `make baselines` runs them alone.
+# Completion: each command's last line is `completed N/N ...` only after its
+# index passed the checks in src/tinyrouter/completeness.py. `make curve`
+# prints `completed 36/36 baseline points` before the curve starts, so a
+# curve that fails later still has one completed line in its output. To
+# decide a curve is done, match the whole line
+# `completed 18/18 encoder points (<model>)` (the ablation:
+# `completed 3/3 ablation points`), never just `completed`.
 pilot-lr:
 	uv run $(UV_ENV) python -m tinyrouter.pilots lr
 

README.md

@@ -40,6 +40,10 @@ Other targets:
 
 Run every target from the repository root: `checkpoint_root` and `results_root` in the configs are relative paths.
 
+There is no default target: a bare `make`, or a quoted `make "curve MODEL=bert"` (one argument: GNU make 3.81, the macOS default, reads it as a variable assignment and used to run `make setup` and exit 0; make 4.x reads it as an unknown target), stops with an error.
+
+**How to tell a curve finished.** Not from the exit code alone. `make baselines`, `make curve` and `make oos-ablation` end with one line that is printed only after the index was read back from disk and checked: exactly the expected points, each once, and every archive's SHA-256 equal on disk, in the manifest and in the index. Match the whole line: `completed 36/36 baseline points`, `completed 18/18 encoder points (bert)` (or `(modernbert)`), `completed 3/3 ablation points`. `make curve` runs the baselines first and prints their `completed 36/36 baseline points` before any encoder point, so a curve that fails afterwards still has a line starting with `completed` in its output; grepping for `completed` alone is not enough.
+
 **Tuning after an AC2 FAIL uses validation only.** `results/ac2.json` and the `make ac2` printout list each seed's validation in-scope accuracy, 8-way accuracy and OOS recall for that purpose; the test numbers are the verdict and are not looked at while choosing hyperparameters.
 
 Put `HF_TOKEN` and `ANTHROPIC_API_KEY` in a `.env` (copy `.env.example`) if you need them; the Makefile passes it to `uv run`. Nothing in the tests or the smoke run calls a paid API.

src/tinyrouter/baselines.py

@@ -39,13 +39,19 @@
   (docs/PLAN.md section 4.1). The factor was chosen so that the
   temperature fit (T in [0.05, 100]) has an interior optimum: on plain
   cosines it hit the lower bound at every k >= 5.
+
+``run_all`` checks its own completion (completeness.py): the plan must be
+exactly 2 baselines x 6 values of k x 3 seeds, each once, before anything
+is fit; the index is read back and must hold the same 36 points with
+every archive's SHA-256 equal on disk, in the manifest and in the index
+before it is moved into place and ``completed 36/36 baseline points`` is
+printed.
 """
 
 from __future__ import annotations
 
 import argparse
 import json
-import os
 from collections.abc import Callable
 from pathlib import Path
 
@@ -61,6 +67,7 @@
     utc_now,
 )
 from tinyrouter.calibrate import SplitLogits
+from tinyrouter.completeness import BASELINE_KEY, BASELINE_POINTS, check_points, publish_index
 from tinyrouter.data import DATASET_REVISION, Split, load_split
 from tinyrouter.evaluate import RunPaths, score
 from tinyrouter.labels import load_label_space
@@ -221,25 +228,34 @@ def environment() -> dict[str, object]:
     }
 
 
+def planned_points() -> list[tuple[str, int, int]]:
+    """(baseline, k, seed) in run order: every baseline of one (k, seed) shares its sample."""
+    return [(name, k, seed) for k in CURVE_KS for seed in SEEDS for name in BASELINES]
+
+
 def run_all(
     results_root: Path,
     sample_fn: SampleFn = curve_sample,
     eval_fn: EvalFn = load_split,  # type: ignore[assignment]
     log: Log = print,
 ) -> dict[str, object]:
-    """Every baseline at every (k, seed); writes results/curves/baselines.json."""
+    """Every baseline at every (k, seed); writes and verifies results/curves/baselines.json."""
+    plan = planned_points()
+    check_points(plan, BASELINE_POINTS, "baselines plan")
     evals = {name: eval_fn(name) for name in ("validation", "test")}
     entries = []
-    for k in CURVE_KS:
-        for seed in SEEDS:
-            train = sample_fn(k, seed)
-            for name in BASELINES:
-                record = run_baseline(name, k, seed, train, evals, results_root)
-                entries.append(index_entry(name, k, seed, record))
+    samples: dict[tuple[int, int], Split] = {}
+    for name, k, seed in plan:
+        if (k, seed) not in samples:
+            samples[(k, seed)] = sample_fn(k, seed)
+        record = run_baseline(name, k, seed, samples[(k, seed)], evals, results_root)
+        entries.append(index_entry(name, k, seed, record))
+        if name == BASELINES[-1]:
             log(f"baselines k={k} seed={seed}: done")
     body = {"baselines": {n: definition(n) for n in BASELINES}, "points": entries}
     out = results_root / "curves" / "baselines.json"
-    write_index(out, body)
+    count = publish_index(out, body, BASELINE_KEY, BASELINE_POINTS, results_root, "baselines")
+    log(f"completed {count}/{len(BASELINE_POINTS)} baseline points")
     return body
 
 
@@ -259,19 +275,11 @@ def index_entry(name: str, k: int, seed: int, record: dict[str, object]) -> dict
     }
 
 
-def write_index(path: Path, body: dict[str, object]) -> None:
-    path.parent.mkdir(parents=True, exist_ok=True)
-    tmp = path.with_name(path.name + ".tmp")
-    tmp.write_text(json.dumps(body, indent=2) + "\n", encoding="utf-8")
-    os.replace(tmp, path)
-
-
 def main(argv: list[str] | None = None) -> None:
     parser = argparse.ArgumentParser(description=__doc__)
     parser.add_argument("--results-root", default="results")
     args = parser.parse_args(argv)
-    body = run_all(Path(args.results_root))
-    print(f"wrote {len(body['points'])} baseline runs")  # type: ignore[arg-type]
+    run_all(Path(args.results_root))
 
 
 if __name__ == "__main__":

src/tinyrouter/completeness.py

@@ -0,0 +1,130 @@
+"""Completion checks for the curve, ablation and baseline indexes.
+
+A command that exits 0 is not evidence that it ran anything: a shell
+loop once handed ``make`` a single argument ``curve MODEL=bert``, make
+read it as a variable assignment, ran its default target and exited 0,
+and two curves were reported done with no point run. So each index
+command checks its own work twice and says so in one line only when
+both checks pass:
+
+1. before running, the planned points are exactly the expected set,
+   each once (``check_points``);
+2. after running, the index read back from disk has exactly the expected
+   set, each once, and every point's archive has the same SHA-256 on
+   disk, in the manifest and in the index, and no two points share an
+   archive (``publish_index``). The index is written to a temporary file
+   and that file, not the in-memory body, is what gets checked, so a
+   short or failed write is caught too; it is moved into place only after
+   this passes.
+
+The expected sets are written out here as literals, not derived from
+``sampling.CURVE_KS`` or ``protocol.SEEDS``: a check that reads the same
+constant as the loop it guards would agree with a wrong loop.
+"""
+
+from __future__ import annotations
+
+import json
+import os
+from collections import Counter
+from collections.abc import Iterable, Sequence
+from pathlib import Path
+
+from tinyrouter.archive import MANIFEST_NAME, read_manifest
+from tinyrouter.data import sha256_of
+from tinyrouter.evaluate import RunPaths
+
+EXPECTED_KS = (1, 5, 10, 25, 50, 100)
+EXPECTED_SEEDS = (42, 43, 44)
+EXPECTED_BASELINES = ("majority", "tfidf-centroid")
+ABLATION_K = 100
+ABLATION_OOS_ROWS = 0
+
+Key = tuple[object, ...]
+# Index fields that identify a point; ``oos_train`` is None on a curve and 0 on the ablation.
+ENCODER_KEY = ("k", "seed", "oos_train")
+BASELINE_KEY = ("baseline", "k", "seed")
+CURVE_POINTS: frozenset[Key] = frozenset((k, s, None) for k in EXPECTED_KS for s in EXPECTED_SEEDS)
+ABLATION_POINTS: frozenset[Key] = frozenset(
+    (ABLATION_K, s, ABLATION_OOS_ROWS) for s in EXPECTED_SEEDS
+)
+BASELINE_POINTS: frozenset[Key] = frozenset(
+    (b, k, s) for b in EXPECTED_BASELINES for k in EXPECTED_KS for s in EXPECTED_SEEDS
+)
+
+
+class IncompleteError(RuntimeError):
+    """A planned or written set of points is not exactly the expected one."""
+
+
+def check_points(keys: Iterable[Key], expected: frozenset[Key], what: str) -> None:
+    """``keys`` must be ``expected`` with every key exactly once."""
+    counts = Counter(keys)
+    duplicated = sorted((k for k, n in counts.items() if n > 1), key=repr)
+    missing = sorted(expected - set(counts), key=repr)
+    unexpected = sorted(set(counts) - expected, key=repr)
+    if duplicated or missing or unexpected:
+        raise IncompleteError(
+            f"{what}: expected {len(expected)} unique points, got {sum(counts.values())} "
+            f"({len(counts)} unique); missing {missing}, duplicated {duplicated}, "
+            f"unexpected {unexpected}"
+        )
+
+
+def check_archives(points: Sequence[dict], results_root: Path, what: str) -> None:
+    """Each point has its own archive, whose SHA-256 agrees on disk, in manifest and index."""
+    manifest = read_manifest(results_root / MANIFEST_NAME)
+    owner: dict[object, object] = {}
+    for point in points:
+        paths = RunPaths.named(results_root, str(point["run_name"]))
+        name = paths.logits.name
+        for held in (point.get("logits_file"), point.get("logits_sha256")):
+            if held in owner:
+                raise IncompleteError(
+                    f"{what}: {point['run_name']} and {owner[held]} share one logits archive"
+                )
+            owner[held] = point["run_name"]
+        if point.get("logits_file") != name:
+            raise IncompleteError(
+                f"{what}: {point['run_name']} lists archive {point.get('logits_file')}"
+            )
+        if not paths.logits.exists():
+            raise IncompleteError(f"{what}: {paths.logits} is missing")
+        listed = manifest.get(name, {}).get("sha256")
+        on_disk = sha256_of(paths.logits)
+        if not point.get("logits_sha256") == listed == on_disk:
+            raise IncompleteError(
+                f"{what}: {name} SHA-256 differs: index {point.get('logits_sha256')}, "
+                f"manifest {listed}, file {on_disk}"
+            )
+
+
+def verify_index(
+    path: Path, key: Sequence[str], expected: frozenset[Key], results_root: Path, what: str
+) -> int:
+    """Read the index at ``path`` and run both checks on it; returns the number of points."""
+    points = json.loads(path.read_text(encoding="utf-8"))["points"]
+    check_points((tuple(p[f] for f in key) for p in points), expected, what)
+    check_archives(points, results_root, what)
+    return len(points)
+
+
+def publish_index(
+    out: Path,
+    body: dict[str, object],
+    key: Sequence[str],
+    expected: frozenset[Key],
+    results_root: Path,
+    what: str,
+) -> int:
+    """Write ``body`` to ``out`` only if, read back from disk, it passes ``verify_index``."""
+    out.parent.mkdir(parents=True, exist_ok=True)
+    tmp = out.with_name(out.name + ".tmp")
+    tmp.write_text(json.dumps(body, indent=2) + "\n", encoding="utf-8")
+    try:
+        count = verify_index(tmp, key, expected, results_root, what)
+    except Exception:
+        tmp.unlink(missing_ok=True)
+        raise
+    os.replace(tmp, out)
+    return count

src/tinyrouter/curves.py

@@ -22,6 +22,16 @@
 ``min_train_steps``), the steps actually run, and ``reused_from`` when the
 point is an AC2 run.
 
+Completion is checked, not inferred from the exit code (completeness.py).
+Before any point runs, the configs must be exactly 6 values of k x 3
+seeds (the ablation: k=100 with 0 OOS rows x 3 seeds) of the named
+model, each once. After, the index is written to a temporary file, read
+back and checked again: the same points, each once, and every archive's
+SHA-256 equal on disk, in the manifest and in the index. Only then is it
+moved into place and ``completed 18/18 encoder points (bert)`` (or
+``completed 3/3 ablation points``) printed; any failure raises and the
+command exits non-zero.
+
 Every point's training sample is checked against ``curve_sample(k, seed)``
 recomputed now (``sample_fingerprint``). The encoder's resume check
 compares configs only, so a numpy release that changed the sampler's
@@ -34,16 +44,28 @@
 from __future__ import annotations
 
 import argparse
-import json
-import os
 from collections.abc import Callable
 from functools import cache
 from pathlib import Path
 
 from tinyrouter.baselines import run_all as run_baselines
+from tinyrouter.completeness import (
+    ABLATION_POINTS,
+    CURVE_POINTS,
+    ENCODER_KEY,
+    Key,
+    check_points,
+    publish_index,
+)
 from tinyrouter.config import RunConfig
 from tinyrouter.evaluate import RunPaths
-from tinyrouter.protocol import MODELS, CurveProtocol, ac2_donor, load_protocol
+from tinyrouter.protocol import (
+    ABLATION_MODEL,
+    MODELS,
+    CurveProtocol,
+    ac2_donor,
+    load_protocol,
+)
 from tinyrouter.runs import (
     EvaluateFn,
     Log,
@@ -60,6 +82,7 @@
 
 FingerprintFn = Callable[[RunConfig], str]
 BACKFILLED_SAMPLE = "computed at index time from curve_sample(k, seed); the run predates the field"
+ABLATION_NAME = "oos-ablation"
 
 
 @cache
@@ -154,6 +177,31 @@ def run_point(
     return index_entry(config, record, None, fingerprint_fn)
 
 
+def planned_shape(name: str, protocol: CurveProtocol) -> tuple[frozenset[Key], str]:
+    """The (k, seed, oos_train) points and the model the index called ``name`` must hold."""
+    if name == ABLATION_NAME:
+        return ABLATION_POINTS, protocol.base(ABLATION_MODEL).model_name
+    if name in MODELS:
+        return CURVE_POINTS, protocol.base(name).model_name
+    raise CurveError(f"unknown curve '{name}'; expected one of {(*MODELS, ABLATION_NAME)}")
+
+
+def check_plan(name: str, configs: list[RunConfig], protocol: CurveProtocol) -> frozenset[Key]:
+    """Refuse to start unless ``configs`` are the expected points of one model, each once."""
+    expected, model = planned_shape(name, protocol)
+    others = sorted({c.model_name for c in configs} - {model})
+    if others:
+        raise CurveError(f"{name}: configs for {others}; this index holds only {model}")
+    check_points(((c.k_shot, c.seed, c.oos_train) for c in configs), expected, f"{name} plan")
+    return expected
+
+
+def completion_message(name: str, count: int, expected: int) -> str:
+    if name == ABLATION_NAME:
+        return f"completed {count}/{expected} ablation points"
+    return f"completed {count}/{expected} encoder points ({name})"
+
+
 def run_curve(
     name: str,
     configs: list[RunConfig],
@@ -163,7 +211,8 @@ def run_curve(
     log: Log = print,
     fingerprint_fn: FingerprintFn | None = None,
 ) -> dict[str, object]:
-    """Run or resume every config, then write results/curves/<name>.json."""
+    """Run or resume every config, then write and verify results/curves/<name>.json."""
+    expected = check_plan(name, configs, protocol)
     entries = [run_point(c, protocol, train_fn, evaluate_fn, log, fingerprint_fn) for c in configs]
     shas = [e["logits_sha256"] for e in entries]
     if len(set(shas)) != len(shas):
@@ -177,12 +226,11 @@ def run_curve(
         "min_train_steps": first.min_train_steps,
         "points": entries,
     }
-    out = Path(first.results_root) / "curves" / f"{name}.json"
-    out.parent.mkdir(parents=True, exist_ok=True)
-    tmp = out.with_name(out.name + ".tmp")
-    tmp.write_text(json.dumps(body, indent=2) + "\n", encoding="utf-8")
-    os.replace(tmp, out)
-    log(f"wrote {out} ({len(entries)} points)")
+    root = Path(first.results_root)
+    out = root / "curves" / f"{name}.json"
+    count = publish_index(out, body, ENCODER_KEY, expected, root, name)
+    log(f"wrote {out} ({count} points)")
+    log(completion_message(name, count, len(expected)))
     return body
 
 
@@ -197,7 +245,7 @@ def main(argv: list[str] | None = None) -> None:
     # Build every config first: a protocol still missing a pilot value stops here,
     # before anything is computed or written.
     if args.ablation:
-        run_curve("oos-ablation", protocol.ablation_configs(), protocol)
+        run_curve(ABLATION_NAME, protocol.ablation_configs(), protocol)
         return
     configs = protocol.curve_configs(args.model)
     run_baselines(Path(configs[0].results_root))