diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index fbf7e8d..1402977 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -18,6 +18,8 @@ jobs: run: test -z "$(gofmt -l .)" || { gofmt -l .; exit 1; } - run: go vet ./... - run: go test -race ./... + - name: device binding conformance kit (reference fake binding, B1 and B2) + run: python3 conformance/binding/conformance.py --level B2 && python3 conformance/binding/conformance.py --level B1 - name: chunker benchmark (reported, not gated yet) run: go test -run xxx -bench . -benchtime 3x ./pkg/chunker - name: build linux/amd64 CLI diff --git a/cmd/onggi/store.go b/cmd/onggi/store.go index 21ba66c..83204da 100644 --- a/cmd/onggi/store.go +++ b/cmd/onggi/store.go @@ -375,7 +375,7 @@ func diff(args []string, stdout, stderr io.Writer) int { fs.SetOutput(stderr) sf := addStoreFlags(fs) asJSON := fs.Bool("json", false, "print the diff as JSON") - ranges := fs.Bool("ranges", false, "list the changed byte ranges of each file (with --json: [offset, length] pairs)") + ranges := fs.Bool("ranges", false, "list the changed byte ranges of each file; with --json also the range plan of each file, schema onggi.switchplan.v1 (ADR-0010)") if err := fs.Parse(args); err != nil || fs.NArg() != 2 { fmt.Fprintln(stderr, "usage: onggi diff --store S [--json] [--ranges] ") return 2 diff --git a/conformance/binding/README.md b/conformance/binding/README.md new file mode 100644 index 0000000..2d9a811 --- /dev/null +++ b/conformance/binding/README.md @@ -0,0 +1,47 @@ +# Device binding conformance kit + +Checks a device binding against the interface of [ADR-0010](../../docs/adr/0010-device-binding-contract.md). +Python 3.6+, standard library only, Apache-2.0. + +```bash +python3 conformance.py # the reference fake binding, level B2 +python3 conformance.py --level B1 # the fake binding acting as B1 (plan always reloads) +python3 conformance.py --level B2 -- ./my-binding --device 0 # your binding +``` + +Your binding is run as `COMMAND [args]` with JSON on stdin and stdout and a non-zero exit on error: + +| operation | stdin | stdout | +|---|---|---| +| `describe ARTIFACT_DIR` | — | `onggi.profile.v1` | +| `resident` | — | `onggi.resident.v1` | +| `plan` | `onggi.switchplan.v1` | `onggi.updateplan.v1` | +| `apply --target DIR` | `onggi.updateplan.v1` | `onggi.verification.v1` | +| `rollback` | — | `onggi.verification.v1` | +| `conformance-load ARTIFACT_DIR MANIFEST` | — | — (kit hook: load the base artifact the normal way and remember MANIFEST as its identity) | + +Schemas and examples: [`docs/schemas/`](../../docs/schemas/). + +## What is in here + +- `golden/*.switchplan.json`: what `onggi diff --ranges --json` prints for four synthetic switches of a 2 MiB + `model.bin` and a `config.json`: `identical`, `inplace` (40 KiB overwritten, config changed), `shifted` (17,952 B + inserted in front, as a larger TensorRT-LLM batch does) and `rearranged` (the same bytes in 32 KiB blocks in another + order). `go test ./pkg/store -run GoldenSwitchPlans` fails when onggi's output drifts from them (`-update` + rewrites them). +- `golden_data.py`: generates the bytes of those artifacts (SHA-256 in counter mode, the same in Go). +- `fake_binding.py`: the reference binding on a simulated device (a directory of 4 KiB rows). It maps changed runs to + rows, answers `reload` when the base is not resident or more than 64 runs or 25 % of the bytes change, stages and + journals before writing, reads back to verify, and rolls back from the journal. +- `conformance.py`: the tests, per golden switch: + 1. `describe` returns `onggi.profile.v1`; after `conformance-load`, `resident` names the base. + 2. `plan` is pure: the same answer twice and the resident state unchanged. + 3. The update plan names the switch plan's base and target; at B1 it is always `reload`; at B2 the `identical` and + `inplace` switches are applied as units, and units cover every byte that is not `same`. + 4. A plan for a base that is not resident is refused (`reload` or an error). + 5. B2: `apply` returns digests equal to the target's bytes (all target files for a reload), the target is then + resident, `rollback` returns digests equal to the base's bytes, the base is resident again, and a second + rollback fails. + +A real binding computes the digests from what it reads back from the device, mapped to artifact byte order. The kit +does not check timing or draining; those belong to the runtime's own tests. diff --git a/conformance/binding/conformance.py b/conformance/binding/conformance.py new file mode 100644 index 0000000..8c9f01c --- /dev/null +++ b/conformance/binding/conformance.py @@ -0,0 +1,143 @@ +#!/usr/bin/env python3 +# Copyright 2026 The onggi authors. SPDX-License-Identifier: Apache-2.0 +"""Conformance tests for an onggi device binding (ADR-0010). Python 3.6+, standard library only. + + python3 conformance.py [--level B1|B2] [-- BINDING COMMAND...] + +Without a command it tests the reference fake binding. The binding command is run as +`COMMAND [args]` with JSON on stdin and stdout; see README.md. +""" +import hashlib +import json +import os +import subprocess +import sys +import tempfile + +HERE = os.path.dirname(os.path.abspath(__file__)) +sys.path.insert(0, HERE) +import golden_data # noqa: E402 + +FAILS = [] + + +def check(cond, what): + print(("ok " if cond else "FAIL ") + what) + if not cond: + FAILS.append(what) + return cond + + +def run(cmd, op, *args, stdin=None, expect_ok=True): + p = subprocess.run(cmd + [op] + list(args), input=None if stdin is None else json.dumps(stdin).encode(), + stdout=subprocess.PIPE, stderr=subprocess.PIPE) + if expect_ok and p.returncode != 0: + raise RuntimeError("%s %s: exit %d: %s" % (op, " ".join(args), p.returncode, p.stderr.decode()[-500:])) + try: + return p.returncode, json.loads(p.stdout.decode()) if p.stdout.strip() else None + except ValueError: + raise RuntimeError("%s: stdout is not JSON: %r" % (op, p.stdout[:200])) + + +def sha(b): + return hashlib.sha256(b).hexdigest() + + +def covers(update, switch): + """Every byte that is not `same` in the switch plan lies in a unit of the same file.""" + units = {} + for u in update["units"]: + units.setdefault(u["file"], []).append((u["offset"], u["offset"] + u["length"])) + for f in switch["files"]: + for r in f["ranges"]: + if r["kind"] == "same": + continue + pos, end = r["offset"], r["offset"] + r["length"] + for a, b in sorted(units.get(f["path"], [])): + if a <= pos < b: + pos = b + if pos < end: + return False + return True + + +def verify_units(ver, files, what): + ok = ver.get("schema") == "onggi.verification.v1" and ver.get("ok") is True + for u in ver.get("units", []): + want = files.get(u["file"], b"")[u["offset"]:u["offset"] + u["length"]] + ok = ok and len(want) == u["length"] and u["sha256"] == sha(want) + return check(ok, what) + + +def main(argv): + level, cmd = "B2", None + if "--" in argv: + cmd = argv[argv.index("--") + 1:] + argv = argv[:argv.index("--")] + if "--level" in argv: + level = argv[argv.index("--level") + 1] + work = tempfile.mkdtemp(prefix="onggi-binding-") + if not cmd: + os.environ["ONGGI_FAKE_DEVICE"] = os.path.join(work, "device") + if level == "B1": + os.environ["ONGGI_FAKE_LEVEL"] = "B1" + cmd = [sys.executable, os.path.join(HERE, "fake_binding.py")] + for name in golden_data.CASES: + base, target = golden_data.case(name) + bdir, tdir = os.path.join(work, name, "base"), os.path.join(work, name, "target") + golden_data.write(bdir, base) + golden_data.write(tdir, target) + with open(os.path.join(HERE, "golden", name + ".switchplan.json")) as f: + sp = json.load(f) + print("== %s (%s)" % (name, level)) + + _, prof = run(cmd, "describe", tdir) + check(prof.get("schema") == "onggi.profile.v1", "describe returns onggi.profile.v1") + + run(cmd, "conformance-load", bdir, sp["base_manifest"]) + _, res = run(cmd, "resident") + check(res.get("schema") == "onggi.resident.v1" and res.get("manifest") == sp["base_manifest"], + "resident names the loaded base") + + _, up1 = run(cmd, "plan", stdin=sp) + _, up2 = run(cmd, "plan", stdin=sp) + _, res2 = run(cmd, "resident") + check(up1 == up2 and res2 == res, "plan is pure: same answer twice, resident state unchanged") + check(up1.get("schema") == "onggi.updateplan.v1" and up1.get("mode") in ("units", "reload"), + "plan returns onggi.updateplan.v1 with mode units or reload") + check(up1["base_manifest"] == sp["base_manifest"] and up1["manifest"] == sp["manifest"], + "update plan names the switch plan's base and target") + if level == "B1": + check(up1["mode"] == "reload", "B1: plan always answers reload") + elif name in ("identical", "inplace"): + check(up1["mode"] == "units", "B2: a switch that keeps 97 % in place is applied as units") + if up1["mode"] == "units": + check(covers(up1, sp), "units cover every byte that is not at the same offset") + + wrong = dict(sp, base_manifest="sha256:" + "0" * 64) + rc, upw = run(cmd, "plan", stdin=wrong, expect_ok=False) + check(rc != 0 or (upw or {}).get("mode") == "reload", "plan against a base that is not resident refuses (reload)") + + if level == "B1": + continue + _, ver = run(cmd, "apply", "--target", tdir, stdin=up1) + verify_units(ver, target, "apply verifies: unit digests equal the target bytes") + if up1["mode"] == "reload": + check(sorted(u["file"] for u in ver["units"]) == sorted(target), "reload writes every target file") + _, res3 = run(cmd, "resident") + check(res3.get("manifest") == sp["manifest"], "after apply the target is resident") + + _, rb = run(cmd, "rollback") + check(rb.get("op") == "rollback", "rollback returns a verification") + verify_units(rb, base, "rollback restores: unit digests equal the base bytes") + _, res4 = run(cmd, "resident") + check(res4.get("manifest") == sp["base_manifest"], "after rollback the base is resident") + rc, _ = run(cmd, "rollback", expect_ok=False) + check(rc != 0, "a second rollback has nothing to undo and fails") + + print("\n%d failed" % len(FAILS)) + return 1 if FAILS else 0 + + +if __name__ == "__main__": + sys.exit(main(sys.argv[1:])) diff --git a/conformance/binding/fake_binding.py b/conformance/binding/fake_binding.py new file mode 100644 index 0000000..477b2aa --- /dev/null +++ b/conformance/binding/fake_binding.py @@ -0,0 +1,252 @@ +#!/usr/bin/env python3 +# Copyright 2026 The onggi authors. SPDX-License-Identifier: Apache-2.0 +"""Reference device binding (ADR-0010) on a simulated device: a directory that holds the resident +artifact's files, updated in rows of 4 KiB. It shows the contract, not a runtime; Python 3.6+, +standard library only. + + fake_binding.py describe ARTIFACT_DIR -> onggi.profile.v1 + fake_binding.py resident -> onggi.resident.v1 + fake_binding.py plan < switchplan.json -> onggi.updateplan.v1 (no device side effects) + fake_binding.py apply --target DIR < updateplan.json -> onggi.verification.v1 + fake_binding.py rollback -> onggi.verification.v1 + fake_binding.py conformance-load ARTIFACT_DIR MANIFEST (kit hook: load a base the normal way) + +The device directory is $ONGGI_FAKE_DEVICE (default ./.fake-device). ONGGI_FAKE_LEVEL=B1 makes plan +always answer reload, as a B1 binding does. +""" +import hashlib +import json +import os +import shutil +import sys + +ROW = 4096 # the device unit: a "row" +MAX_SHARE = 0.25 # more of the resident bytes to rewrite than this: reload instead +MAX_RUNS = 64 # more changed runs than this: reload instead + +DEV = os.environ.get("ONGGI_FAKE_DEVICE", ".fake-device") +MEM, STATE, JOURNAL = os.path.join(DEV, "mem"), os.path.join(DEV, "state.json"), os.path.join(DEV, "journal") + + +def fail(msg, code=2): + sys.stderr.write("fake-binding: %s\n" % msg) + sys.exit(code) + + +def out(obj): + json.dump(obj, sys.stdout, indent=2, sort_keys=True) + sys.stdout.write("\n") + + +def state(): + try: + with open(STATE) as f: + return json.load(f) + except (OSError, ValueError): + return {"manifest": None} + + +def save_state(s): + tmp = STATE + ".tmp" + with open(tmp, "w") as f: + json.dump(s, f) + os.replace(tmp, STATE) + + +def files_of(root): + res = {} + for dp, _, fs in os.walk(root): + for f in fs: + p = os.path.join(dp, f) + res[os.path.relpath(p, root).replace(os.sep, "/")] = p + return res + + +def read(path, off, n): + with open(path, "rb") as f: + f.seek(off) + return f.read(n) + + +def sha(b): + return hashlib.sha256(b).hexdigest() + + +def describe(artifact): + """The variant coordinate from documented plain metadata only (ADR-0009): here config.json.""" + prof = {"schema": "onggi.profile.v1", "device": "fake", "runtime": "onggi-fake-binding", "format": "bytes", + "layout": "rows-4KiB"} + try: + with open(os.path.join(artifact, "config.json")) as f: + prof["batch"] = int(json.load(f).get("batch", 0)) + except (OSError, ValueError): + pass + out(prof) + + +def resident(): + m = state()["manifest"] + out({"schema": "onggi.resident.v1", "manifest": m, "state": "ready" if m else "empty"}) + + +def reload_plan(sp, reason): + return {"schema": "onggi.updateplan.v1", "base_manifest": sp.get("base_manifest"), "manifest": sp.get("manifest"), + "mode": "reload", "reason": reason, "units": [], "bytes": sum(f["size"] for f in sp.get("files", []))} + + +def plan(sp): + """Pure: reads the resident state, writes nothing.""" + if sp.get("schema") != "onggi.switchplan.v1": + fail("unknown switch plan schema %r" % sp.get("schema")) + if state()["manifest"] != sp.get("base_manifest"): + return reload_plan(sp, "base not resident") + if os.environ.get("ONGGI_FAKE_LEVEL") == "B1": + return reload_plan(sp, "B1 binding: always reload") + resident_files = files_of(MEM) + units, runs, total = [], 0, 0 + for f in sp["files"]: + if "ranges" not in f: + return reload_plan(sp, "switch plan without ranges (onggi diff --ranges --json)") + base = resident_files.get(f.get("base") or "") + if f.get("base") != f["path"] or base is None or os.path.getsize(base) != f["size"]: + return reload_plan(sp, "%s: not the same file and size on the device" % f["path"]) + total += f["size"] + rows = set() + for r in f["ranges"]: + if r["kind"] == "same": + continue + runs += 1 + rows.update(range(r["offset"] // ROW, (r["offset"] + r["length"] - 1) // ROW + 1)) + for row in sorted(rows): # merge adjacent rows into units + off = row * ROW + n = min(ROW, f["size"] - off) + if units and units[-1]["file"] == f["path"] and units[-1]["offset"] + units[-1]["length"] == off: + units[-1]["length"] += n + else: + units.append({"file": f["path"], "offset": off, "length": n, "source": "target"}) + nbytes = sum(u["length"] for u in units) + if runs > MAX_RUNS or nbytes > MAX_SHARE * max(total, 1): + return reload_plan(sp, "fragmented: %d changed runs, %d of %d bytes" % (runs, nbytes, total)) + for u in units: + u["unit"] = "rows:%d-%d" % (u["offset"] // ROW, (u["offset"] + u["length"] - 1) // ROW) + return {"schema": "onggi.updateplan.v1", "base_manifest": sp["base_manifest"], "manifest": sp["manifest"], + "mode": "units", "units": units, "bytes": nbytes} + + +def verification(op, mode, manifest, units, written): + return {"schema": "onggi.verification.v1", "op": op, "mode": mode, "manifest": manifest, "ok": True, + "units": units, "bytes_written": written} + + +def apply(up, target): + if up.get("schema") != "onggi.updateplan.v1": + fail("unknown update plan schema %r" % up.get("schema")) + st = state() + if up["mode"] == "units" and st["manifest"] != up["base_manifest"]: + fail("base not resident; nothing changed") + tfiles = files_of(target) + # stage: read and hash everything to write before touching the device + if up["mode"] == "reload": + staged = [{"file": p, "offset": 0, "data": read(a, 0, os.path.getsize(a))} for p, a in sorted(tfiles.items())] + else: + staged = [] + for u in up["units"]: + if u["file"] not in tfiles: + fail("%s: not in the target; nothing changed" % u["file"]) + d = read(tfiles[u["file"]], u["offset"], u["length"]) + if len(d) != u["length"]: + fail("%s: target shorter than the unit; nothing changed" % u["file"]) + staged.append({"file": u["file"], "offset": u["offset"], "data": d}) + # journal: what rollback needs (here: the whole resident state for a reload, the old rows otherwise) + shutil.rmtree(JOURNAL, ignore_errors=True) + os.makedirs(JOURNAL) + if up["mode"] == "reload": + if os.path.isdir(MEM): + shutil.copytree(MEM, os.path.join(JOURNAL, "mem")) + else: + for i, s in enumerate(staged): + with open(os.path.join(JOURNAL, "%d" % i), "wb") as f: + f.write(read(os.path.join(MEM, s["file"]), s["offset"], len(s["data"]))) + with open(os.path.join(JOURNAL, "journal.json"), "w") as f: + json.dump({"manifest": st["manifest"], "mode": up["mode"], + "units": [{"file": s["file"], "offset": s["offset"], "length": len(s["data"])} for s in staged]}, f) + # apply (a real binding does this at a drained boundary), then read back and verify + if up["mode"] == "reload": + shutil.rmtree(MEM, ignore_errors=True) + written, units = 0, [] + for s in staged: + p = os.path.join(MEM, s["file"]) + os.makedirs(os.path.dirname(p), exist_ok=True) + with open(p, "r+b" if os.path.exists(p) else "wb") as f: + f.seek(s["offset"]) + f.write(s["data"]) + written += len(s["data"]) + got = read(p, s["offset"], len(s["data"])) + if got != s["data"]: + rollback() + fail("read-back mismatch in %s; rolled back" % s["file"], 3) + units.append({"file": s["file"], "offset": s["offset"], "length": len(got), "sha256": sha(got)}) + save_state({"manifest": up["manifest"]}) + out(verification("apply", up["mode"], up["manifest"], units, written)) + + +def rollback(emit=False): + try: + with open(os.path.join(JOURNAL, "journal.json")) as f: + j = json.load(f) + except (OSError, ValueError): + fail("nothing to roll back") + units, written = [], 0 + if j["mode"] == "reload": + shutil.rmtree(MEM, ignore_errors=True) + if os.path.isdir(os.path.join(JOURNAL, "mem")): + shutil.copytree(os.path.join(JOURNAL, "mem"), MEM) + for p, a in sorted(files_of(MEM).items()): + d = read(a, 0, os.path.getsize(a)) + written += len(d) + units.append({"file": p, "offset": 0, "length": len(d), "sha256": sha(d)}) + else: + for i, u in enumerate(j["units"]): + with open(os.path.join(JOURNAL, "%d" % i), "rb") as f: + old = f.read() + p = os.path.join(MEM, u["file"]) + with open(p, "r+b") as f: + f.seek(u["offset"]) + f.write(old) + written += len(old) + units.append({"file": u["file"], "offset": u["offset"], "length": len(old), + "sha256": sha(read(p, u["offset"], len(old)))}) + save_state({"manifest": j["manifest"]}) + shutil.rmtree(JOURNAL, ignore_errors=True) + if emit: + out(verification("rollback", j["mode"], j["manifest"], units, written)) + + +def load(artifact, manifest): + shutil.rmtree(DEV, ignore_errors=True) + shutil.copytree(artifact, MEM) + save_state({"manifest": manifest}) + + +def main(argv): + if not argv: + fail(__doc__, 64) + cmd, args = argv[0], argv[1:] + if cmd == "describe" and len(args) == 1: + describe(args[0]) + elif cmd == "resident" and not args: + resident() + elif cmd == "plan" and not args: + out(plan(json.load(sys.stdin))) + elif cmd == "apply" and len(args) == 2 and args[0] == "--target": + apply(json.load(sys.stdin), args[1]) + elif cmd == "rollback" and not args: + rollback(emit=True) + elif cmd == "conformance-load" and len(args) == 2: + load(args[0], args[1]) + else: + fail("usage: see the module docstring", 64) + + +if __name__ == "__main__": + main(sys.argv[1:]) diff --git a/conformance/binding/golden/identical.switchplan.json b/conformance/binding/golden/identical.switchplan.json new file mode 100644 index 0000000..d27fa0e --- /dev/null +++ b/conformance/binding/golden/identical.switchplan.json @@ -0,0 +1,51 @@ +{ + "schema": "onggi.switchplan.v1", + "base_manifest": "sha256:a054eea1a32af6a08f2c1452669b6012ac2cecdba7577b9edf233e438573f3fb", + "manifest": "sha256:bf3ea2ef3ee936e96db31c083505cee4047c6b6b9d13f662814e506461bcc0bc", + "files": [ + { + "path": "config.json", + "base": "config.json", + "size": 13, + "same": 13, + "moved": 0, + "new": 0, + "same_blocks": 13, + "changed_runs": 0, + "largest_run": 0, + "ranges": [ + { + "offset": 0, + "length": 13, + "kind": "same" + } + ] + }, + { + "path": "model.bin", + "base": "model.bin", + "size": 2097152, + "same": 2097152, + "moved": 0, + "new": 0, + "same_blocks": 2097152, + "changed_runs": 0, + "largest_run": 0, + "ranges": [ + { + "offset": 0, + "length": 2097152, + "kind": "same" + } + ] + } + ], + "size": 2097165, + "same": 2097165, + "moved": 0, + "new": 0, + "new_stored": 0, + "same_blocks": 2097165, + "changed_runs": 0, + "largest_run": 0 +} diff --git a/conformance/binding/golden/inplace.switchplan.json b/conformance/binding/golden/inplace.switchplan.json new file mode 100644 index 0000000..31f4341 --- /dev/null +++ b/conformance/binding/golden/inplace.switchplan.json @@ -0,0 +1,73 @@ +{ + "schema": "onggi.switchplan.v1", + "base_manifest": "sha256:7c6928f0385a257b5ceb9d13ea205a5ab0643a4fcec6b2a421ef5756aea93280", + "manifest": "sha256:cb4a0497f3a7c591b276b4526d0e305718fff0e98f35f3104185a6fdb0117a03", + "files": [ + { + "path": "config.json", + "base": "config.json", + "size": 13, + "same": 0, + "moved": 0, + "new": 13, + "same_blocks": 0, + "changed_runs": 1, + "largest_run": 13, + "changed": [ + [ + 0, + 13 + ] + ], + "ranges": [ + { + "offset": 0, + "length": 13, + "kind": "new" + } + ] + }, + { + "path": "model.bin", + "base": "model.bin", + "size": 2097152, + "same": 2031875, + "moved": 0, + "new": 65277, + "same_blocks": 2027520, + "changed_runs": 1, + "largest_run": 65277, + "changed": [ + [ + 1046344, + 65277 + ] + ], + "ranges": [ + { + "offset": 0, + "length": 1046344, + "kind": "same" + }, + { + "offset": 1046344, + "length": 65277, + "kind": "new" + }, + { + "offset": 1111621, + "length": 985531, + "kind": "same" + } + ] + } + ], + "size": 2097165, + "same": 2031875, + "moved": 0, + "new": 65290, + "new_stored": 65290, + "same_blocks": 2027520, + "changed_runs": 2, + "largest_run": 65277 +} diff --git a/conformance/binding/golden/rearranged.switchplan.json b/conformance/binding/golden/rearranged.switchplan.json new file mode 100644 index 0000000..2d764f5 --- /dev/null +++ b/conformance/binding/golden/rearranged.switchplan.json @@ -0,0 +1,805 @@ +{ + "schema": "onggi.switchplan.v1", + "base_manifest": "sha256:a11ae743b61036855dc258adeb9ccdc3773affeb97db265e021124894f2c9a14", + "manifest": "sha256:8442a4dc9e4ccb1b606c9ec3a9fde1b57cb02bec0fd06f71f4b6805767c28e2a", + "files": [ + { + "path": "config.json", + "base": "config.json", + "size": 13, + "same": 13, + "moved": 0, + "new": 0, + "same_blocks": 13, + "changed_runs": 0, + "largest_run": 0, + "ranges": [ + { + "offset": 0, + "length": 13, + "kind": "same" + } + ] + }, + { + "path": "model.bin", + "base": "model.bin", + "size": 2097152, + "same": 49926, + "moved": 1438770, + "new": 608456, + "same_blocks": 40960, + "changed_runs": 3, + "largest_run": 1257615, + "changed": [ + [ + 0, + 689135 + ], + [ + 718479, + 1257615 + ], + [ + 1996676, + 100476 + ] + ], + "ranges": [ + { + "offset": 0, + "length": 7287, + "kind": "new" + }, + { + "offset": 7287, + "length": 22990, + "kind": "elsewhere", + "source_file": "model.bin", + "source_offset": 1088631 + }, + { + "offset": 30277, + "length": 5515, + "kind": "new" + }, + { + "offset": 35792, + "length": 24471, + "kind": "elsewhere", + "source_file": "model.bin", + "source_offset": 986064 + }, + { + "offset": 60263, + "length": 10010, + "kind": "new" + }, + { + "offset": 70273, + "length": 17749, + "kind": "elsewhere", + "source_file": "model.bin", + "source_offset": 1282689 + }, + { + "offset": 88022, + "length": 23316, + "kind": "new" + }, + { + "offset": 111338, + "length": 15885, + "kind": "elsewhere", + "source_file": "model.bin", + "source_offset": 865002 + }, + { + "offset": 127223, + "length": 5209, + "kind": "new" + }, + { + "offset": 132432, + "length": 25382, + "kind": "elsewhere", + "source_file": "model.bin", + "source_offset": 1246544 + }, + { + "offset": 157814, + "length": 7399, + "kind": "new" + }, + { + "offset": 165213, + "length": 26385, + "kind": "elsewhere", + "source_file": "model.bin", + "source_offset": 492893 + }, + { + "offset": 191598, + "length": 9579, + "kind": "new" + }, + { + "offset": 201177, + "length": 25849, + "kind": "elsewhere", + "source_file": "model.bin", + "source_offset": 1839577 + }, + { + "offset": 227026, + "length": 5528, + "kind": "new" + }, + { + "offset": 232554, + "length": 27788, + "kind": "elsewhere", + "source_file": "model.bin", + "source_offset": 363626 + }, + { + "offset": 260342, + "length": 4594, + "kind": "new" + }, + { + "offset": 264936, + "length": 26051, + "kind": "elsewhere", + "source_file": "model.bin", + "source_offset": 330472 + }, + { + "offset": 290987, + "length": 14947, + "kind": "new" + }, + { + "offset": 305934, + "length": 15019, + "kind": "elsewhere", + "source_file": "model.bin", + "source_offset": 2075406 + }, + { + "offset": 320953, + "length": 8440, + "kind": "new" + }, + { + "offset": 329393, + "length": 30791, + "kind": "elsewhere", + "source_file": "model.bin", + "source_offset": 394929 + }, + { + "offset": 360184, + "length": 11765, + "kind": "new" + }, + { + "offset": 371949, + "length": 19433, + "kind": "elsewhere", + "source_file": "model.bin", + "source_offset": 1158381 + }, + { + "offset": 391382, + "length": 13539, + "kind": "new" + }, + { + "offset": 404921, + "length": 17549, + "kind": "elsewhere", + "source_file": "model.bin", + "source_offset": 1420729 + }, + { + "offset": 422470, + "length": 18415, + "kind": "new" + }, + { + "offset": 440885, + "length": 14682, + "kind": "elsewhere", + "source_file": "model.bin", + "source_offset": 211509 + }, + { + "offset": 455567, + "length": 9048, + "kind": "new" + }, + { + "offset": 464615, + "length": 22049, + "kind": "elsewhere", + "source_file": "model.bin", + "source_offset": 1316583 + }, + { + "offset": 486664, + "length": 6594, + "kind": "new" + }, + { + "offset": 493258, + "length": 25811, + "kind": "elsewhere", + "source_file": "model.bin", + "source_offset": 1443530 + }, + { + "offset": 519069, + "length": 5843, + "kind": "new" + }, + { + "offset": 524912, + "length": 31289, + "kind": "elsewhere", + "source_file": "model.bin", + "source_offset": 918128 + }, + { + "offset": 556201, + "length": 23368, + "kind": "new" + }, + { + "offset": 579569, + "length": 7367, + "kind": "elsewhere", + "source_file": "model.bin", + "source_offset": 1234929 + }, + { + "offset": 586936, + "length": 5219, + "kind": "new" + }, + { + "offset": 592155, + "length": 25988, + "kind": "elsewhere", + "source_file": "model.bin", + "source_offset": 1345819 + }, + { + "offset": 618143, + "length": 8870, + "kind": "new" + }, + { + "offset": 627013, + "length": 24791, + "kind": "elsewhere", + "source_file": "model.bin", + "source_offset": 1773893 + }, + { + "offset": 651804, + "length": 6670, + "kind": "new" + }, + { + "offset": 658474, + "length": 26363, + "kind": "elsewhere", + "source_file": "model.bin", + "source_offset": 1641514 + }, + { + "offset": 684837, + "length": 4298, + "kind": "new" + }, + { + "offset": 689135, + "length": 29344, + "kind": "same" + }, + { + "offset": 718479, + "length": 17031, + "kind": "new" + }, + { + "offset": 735510, + "length": 15412, + "kind": "elsewhere", + "source_file": "model.bin", + "source_offset": 637206 + }, + { + "offset": 750922, + "length": 12653, + "kind": "new" + }, + { + "offset": 763575, + "length": 19737, + "kind": "elsewhere", + "source_file": "model.bin", + "source_offset": 1517239 + }, + { + "offset": 783312, + "length": 6781, + "kind": "new" + }, + { + "offset": 790093, + "length": 26302, + "kind": "elsewhere", + "source_file": "model.bin", + "source_offset": 1674829 + }, + { + "offset": 816395, + "length": 5100, + "kind": "new" + }, + { + "offset": 821495, + "length": 26950, + "kind": "elsewhere", + "source_file": "model.bin", + "source_offset": 35063 + }, + { + "offset": 848445, + "length": 6697, + "kind": "new" + }, + { + "offset": 855142, + "length": 27159, + "kind": "elsewhere", + "source_file": "model.bin", + "source_offset": 298086 + }, + { + "offset": 882301, + "length": 22156, + "kind": "new" + }, + { + "offset": 904457, + "length": 9492, + "kind": "elsewhere", + "source_file": "model.bin", + "source_offset": 1494281 + }, + { + "offset": 913949, + "length": 8793, + "kind": "new" + }, + { + "offset": 922742, + "length": 23482, + "kind": "elsewhere", + "source_file": "model.bin", + "source_offset": 1184886 + }, + { + "offset": 946224, + "length": 12968, + "kind": "new" + }, + { + "offset": 959192, + "length": 21098, + "kind": "elsewhere", + "source_file": "model.bin", + "source_offset": 1909464 + }, + { + "offset": 980290, + "length": 5191, + "kind": "new" + }, + { + "offset": 985481, + "length": 28095, + "kind": "elsewhere", + "source_file": "model.bin", + "source_offset": 1018249 + }, + { + "offset": 1013576, + "length": 5078, + "kind": "new" + }, + { + "offset": 1018654, + "length": 26764, + "kind": "elsewhere", + "source_file": "model.bin", + "source_offset": 1870622 + }, + { + "offset": 1045418, + "length": 4826, + "kind": "new" + }, + { + "offset": 1050244, + "length": 60588, + "kind": "elsewhere", + "source_file": "model.bin", + "source_offset": 1705604 + }, + { + "offset": 1110832, + "length": 3754, + "kind": "new" + }, + { + "offset": 1114586, + "length": 28714, + "kind": "elsewhere", + "source_file": "model.bin", + "source_offset": 1376730 + }, + { + "offset": 1143300, + "length": 8874, + "kind": "new" + }, + { + "offset": 1152174, + "length": 26930, + "kind": "elsewhere", + "source_file": "model.bin", + "source_offset": 1610926 + }, + { + "offset": 1179104, + "length": 27685, + "kind": "new" + }, + { + "offset": 1206789, + "length": 3585, + "kind": "elsewhere", + "source_file": "model.bin", + "source_offset": 158213 + }, + { + "offset": 1210374, + "length": 14247, + "kind": "new" + }, + { + "offset": 1224621, + "length": 16821, + "kind": "elsewhere", + "source_file": "model.bin", + "source_offset": 896941 + }, + { + "offset": 1241442, + "length": 6229, + "kind": "new" + }, + { + "offset": 1247671, + "length": 22083, + "kind": "elsewhere", + "source_file": "model.bin", + "source_offset": 1051063 + }, + { + "offset": 1269754, + "length": 10238, + "kind": "new" + }, + { + "offset": 1279992, + "length": 27342, + "kind": "elsewhere", + "source_file": "model.bin", + "source_offset": 591864 + }, + { + "offset": 1307334, + "length": 9950, + "kind": "new" + }, + { + "offset": 1317284, + "length": 25152, + "kind": "elsewhere", + "source_file": "model.bin", + "source_offset": 825764 + }, + { + "offset": 1342436, + "length": 2962, + "kind": "new" + }, + { + "offset": 1345398, + "length": 27741, + "kind": "elsewhere", + "source_file": "model.bin", + "source_offset": 427894 + }, + { + "offset": 1373139, + "length": 15298, + "kind": "new" + }, + { + "offset": 1388437, + "length": 17316, + "kind": "elsewhere", + "source_file": "model.bin", + "source_offset": 77717 + }, + { + "offset": 1405753, + "length": 20362, + "kind": "new" + }, + { + "offset": 1426115, + "length": 11060, + "kind": "elsewhere", + "source_file": "model.bin", + "source_offset": 770755 + }, + { + "offset": 1437175, + "length": 10904, + "kind": "new" + }, + { + "offset": 1448079, + "length": 26134, + "kind": "elsewhere", + "source_file": "model.bin", + "source_offset": 727183 + }, + { + "offset": 1474213, + "length": 4538, + "kind": "new" + }, + { + "offset": 1478751, + "length": 25776, + "kind": "elsewhere", + "source_file": "model.bin", + "source_offset": 4191 + }, + { + "offset": 1504527, + "length": 3094, + "kind": "new" + }, + { + "offset": 1507621, + "length": 27073, + "kind": "elsewhere", + "source_file": "model.bin", + "source_offset": 229669 + }, + { + "offset": 1534694, + "length": 8676, + "kind": "new" + }, + { + "offset": 1543370, + "length": 28855, + "kind": "elsewhere", + "source_file": "model.bin", + "source_offset": 2034890 + }, + { + "offset": 1572225, + "length": 17061, + "kind": "new" + }, + { + "offset": 1589286, + "length": 15134, + "kind": "elsewhere", + "source_file": "model.bin", + "source_offset": 180262 + }, + { + "offset": 1604420, + "length": 13442, + "kind": "new" + }, + { + "offset": 1617862, + "length": 18250, + "kind": "elsewhere", + "source_file": "model.bin", + "source_offset": 1945542 + }, + { + "offset": 1636112, + "length": 9040, + "kind": "new" + }, + { + "offset": 1645152, + "length": 21706, + "kind": "elsewhere", + "source_file": "model.bin", + "source_offset": 105056 + }, + { + "offset": 1666858, + "length": 4529, + "kind": "new" + }, + { + "offset": 1671387, + "length": 27129, + "kind": "elsewhere", + "source_file": "model.bin", + "source_offset": 557275 + }, + { + "offset": 1698516, + "length": 12995, + "kind": "new" + }, + { + "offset": 1711511, + "length": 22271, + "kind": "elsewhere", + "source_file": "model.bin", + "source_offset": 269719 + }, + { + "offset": 1733782, + "length": 24371, + "kind": "new" + }, + { + "offset": 1758153, + "length": 10063, + "kind": "elsewhere", + "source_file": "model.bin", + "source_offset": 545737 + }, + { + "offset": 1768216, + "length": 2609, + "kind": "new" + }, + { + "offset": 1770825, + "length": 27809, + "kind": "elsewhere", + "source_file": "model.bin", + "source_offset": 1803593 + }, + { + "offset": 1798634, + "length": 6022, + "kind": "new" + }, + { + "offset": 1804656, + "length": 28243, + "kind": "elsewhere", + "source_file": "model.bin", + "source_offset": 1116528 + }, + { + "offset": 1832899, + "length": 5219, + "kind": "new" + }, + { + "offset": 1838118, + "length": 25984, + "kind": "elsewhere", + "source_file": "model.bin", + "source_offset": 658470 + }, + { + "offset": 1864102, + "length": 4103, + "kind": "new" + }, + { + "offset": 1868205, + "length": 31025, + "kind": "elsewhere", + "source_file": "model.bin", + "source_offset": 1999277 + }, + { + "offset": 1899230, + "length": 4601, + "kind": "new" + }, + { + "offset": 1903831, + "length": 24971, + "kind": "elsewhere", + "source_file": "model.bin", + "source_offset": 462039 + }, + { + "offset": 1928802, + "length": 5374, + "kind": "new" + }, + { + "offset": 1934176, + "length": 26040, + "kind": "elsewhere", + "source_file": "model.bin", + "source_offset": 787296 + }, + { + "offset": 1960216, + "length": 15878, + "kind": "new" + }, + { + "offset": 1976094, + "length": 20582, + "kind": "same" + }, + { + "offset": 1996676, + "length": 5382, + "kind": "new" + }, + { + "offset": 2002058, + "length": 26930, + "kind": "elsewhere", + "source_file": "model.bin", + "source_offset": 953482 + }, + { + "offset": 2028988, + "length": 4085, + "kind": "new" + }, + { + "offset": 2033073, + "length": 59872, + "kind": "elsewhere", + "source_file": "model.bin", + "source_offset": 1541553 + }, + { + "offset": 2092945, + "length": 4207, + "kind": "new" + } + ] + } + ], + "size": 2097165, + "same": 49939, + "moved": 1438770, + "new": 608456, + "new_stored": 608456, + "same_blocks": 40973, + "changed_runs": 3, + "largest_run": 1257615 +} diff --git a/conformance/binding/golden/shifted.switchplan.json b/conformance/binding/golden/shifted.switchplan.json new file mode 100644 index 0000000..8406020 --- /dev/null +++ b/conformance/binding/golden/shifted.switchplan.json @@ -0,0 +1,64 @@ +{ + "schema": "onggi.switchplan.v1", + "base_manifest": "sha256:4c45361a32a82eba30135c1b12e30300fb794044562344277d56b7d21be1b9f0", + "manifest": "sha256:057328b9fbbb63443c9cbc6abbc3265aff5229d51d396534f290cad78c8a059b", + "files": [ + { + "path": "config.json", + "base": "config.json", + "size": 13, + "same": 13, + "moved": 0, + "new": 0, + "same_blocks": 13, + "changed_runs": 0, + "largest_run": 0, + "ranges": [ + { + "offset": 0, + "length": 13, + "kind": "same" + } + ] + }, + { + "path": "model.bin", + "base": "model.bin", + "size": 2115104, + "same": 0, + "moved": 2092961, + "new": 22143, + "same_blocks": 0, + "changed_runs": 1, + "largest_run": 2115104, + "changed": [ + [ + 0, + 2115104 + ] + ], + "ranges": [ + { + "offset": 0, + "length": 22143, + "kind": "new" + }, + { + "offset": 22143, + "length": 2092961, + "kind": "elsewhere", + "source_file": "model.bin", + "source_offset": 4191 + } + ] + } + ], + "size": 2115117, + "same": 13, + "moved": 2092961, + "new": 22143, + "new_stored": 22143, + "same_blocks": 13, + "changed_runs": 1, + "largest_run": 2115104 +} diff --git a/conformance/binding/golden_data.py b/conformance/binding/golden_data.py new file mode 100644 index 0000000..32b8ea2 --- /dev/null +++ b/conformance/binding/golden_data.py @@ -0,0 +1,60 @@ +# Copyright 2026 The onggi authors. SPDX-License-Identifier: Apache-2.0 +"""Synthetic artifacts of the golden switch plans (same bytes as pkg/store/conformance_test.go). + + python3 golden_data.py OUT_DIR writes OUT_DIR//{base,target}/ +""" +import hashlib +import os +import sys + +CASES = ("identical", "inplace", "shifted", "rearranged") +SIZE = 2 << 20 +BLOCK = 32 << 10 + + +def data(seed, n): + out = bytearray() + i = 0 + while len(out) < n: + out += hashlib.sha256(("onggi-conformance:%d:%d" % (seed, i)).encode()).digest() + i += 1 + return bytes(out[:n]) + + +def config(batch): + return ('{"batch": %d}\n' % batch).encode() + + +def case(name): + """Returns (base files, target files) as {path: bytes}.""" + base = data(1, SIZE) + b = {"model.bin": base, "config.json": config(2)} + if name == "identical": + t = dict(b) + elif name == "inplace": + m = bytearray(base) + m[1 << 20:(1 << 20) + (40 << 10)] = data(2, 40 << 10) + t = {"model.bin": bytes(m), "config.json": config(8)} + elif name == "shifted": + t = {"model.bin": data(3, 17952) + base, "config.json": config(2)} + elif name == "rearranged": + idx = sorted(range(SIZE // BLOCK), key=lambda i: hashlib.sha256(("perm:%d" % i).encode()).digest()) + t = {"model.bin": b"".join(base[i * BLOCK:(i + 1) * BLOCK] for i in idx), "config.json": config(2)} + else: + raise KeyError(name) + return b, t + + +def write(root, files): + for path, content in files.items(): + p = os.path.join(root, path) + os.makedirs(os.path.dirname(p), exist_ok=True) + with open(p, "wb") as f: + f.write(content) + + +if __name__ == "__main__": + for name in CASES: + base, target = case(name) + write(os.path.join(sys.argv[1], name, "base"), base) + write(os.path.join(sys.argv[1], name, "target"), target) diff --git a/docs/adr/0010-device-binding-contract.md b/docs/adr/0010-device-binding-contract.md new file mode 100644 index 0000000..54012a6 --- /dev/null +++ b/docs/adr/0010-device-binding-contract.md @@ -0,0 +1,105 @@ +# ADR-0010: Device binding interface + +Status: proposed · 2026-10-30 (#75) + +## Context +onggi stores, ships and switches compiled model variants as bytes. Only two parts of the system depend on the +accelerator vendor: **which variant an artifact is** (its coordinate: toolchain, device, topology, shape), which the +publisher knows, and **where an artifact's bytes live in device memory**, which only the runtime knows. Everything +between them (chunking, dedup, transfer, the three-way diff, in-place patching of files on a node) works without +reading any vendor format (ADR-0009). + +A switch between variants changes few bytes (docs/evaluation.md: decode B2 → B8 keeps 98.6 % at the same offset; RNGD +pfbatch → preset 99.77 %), and `onggi diff --ranges --json` already lists them (#46). Runtime experiments (the rngd-pd +resident delta) consumed that output, but it had no schema or version, did not say which runs the base holds elsewhere +or where, and nothing stated what a runtime may assume or must guarantee. For onggi to work with any accelerator, the +runtime side needs one public, versioned interface that a vendor or runtime team can implement alone. + +## Decision + +### 1. A binding implements five operations +A **device binding** is a program or library on the serving side that implements the operations below. They are +defined as JSON in, JSON out, so a binding can be a separate process in any language (the reference is an executable: +`BINDING [args]`, input on stdin, output on stdout, non-zero exit on error with the device unchanged). No +Go type crosses the boundary. + +| operation | input | output | side effects | +|---|---|---|---| +| `describe(artifact)` | an artifact directory | the coordinate, `onggi.profile.v1` (#71/#72) | none; runs on the publisher or compiler side, reads only documented metadata | +| `resident()` | — | `onggi.resident.v1`: manifest digest of the artifact resident on the device, and its state | none | +| `plan(switch_plan)` | `onggi.switchplan.v1` | `onggi.updateplan.v1`: device units to rewrite (`mode: units`) or `mode: reload` with a reason | **none** (pure); may always answer reload | +| `apply(update_plan, target)` | `onggi.updateplan.v1` and the verified target artifact directory | `onggi.verification.v1`: per unit, SHA-256 of what the device holds, in artifact byte space | stages, applies at a drained boundary, verifies | +| `rollback()` | — | `onggi.verification.v1` of the restored units | restores the state before the last apply | + +### 2. Schemas +Checked in under [`docs/schemas/`](../schemas/) as JSON Schema (2020-12), with examples: + +- [`switchplan.v1.schema.json`](../schemas/switchplan.v1.schema.json): exactly what `onggi diff --ranges --json BASE + TARGET` prints. `schema: "onggi.switchplan.v1"`, `base_manifest`, `manifest`, and per file `ranges[]` = + `{offset, length, kind: same|elsewhere|new, source_file, source_offset}` covering the file in order without gaps + (`source_*` only for `elsewhere`). Every field of the earlier output (`same`, `moved`, `new`, `new_stored`, + `same_blocks`, `changed_runs`, `largest_run`, per-file `changed` `[offset, length]` pairs) keeps its meaning, so + existing consumers keep working. +- [`updateplan.v1.schema.json`](../schemas/updateplan.v1.schema.json): `mode`, `reason`, `bytes`, `units[]` = + `{file, offset, length, unit, source}`, in artifact byte space, rounded out to whole device units (rows, tensors, + pages: the binding's choice; `unit` is its own opaque name). +- [`verification.v1.schema.json`](../schemas/verification.v1.schema.json): `op`, `mode`, `manifest` now resident, + `ok`, `bytes_written`, `units[]` = `{file, offset, length, sha256}`. +- [`resident.v1.schema.json`](../schemas/resident.v1.schema.json): `manifest` (or null), `state` + (`empty`/`ready`/`staged`). +- `onggi.profile.v1` is the publisher-declared coordinate of #71/#72 (`store.Profile`); this ADR uses it unchanged. + +Within v1, fields may only be added; consumers ignore fields they do not know. An incompatible change is a v2, and a +binding rejects a schema it does not know. + +### 3. What onggi guarantees +- The switch plan comes from the two artifacts' content-addressed chunk lists: a `same` or `elsewhere` run is + byte-identical to its source in the base artifact (SHA-256 of the chunks). It is exact at chunk granularity: bytes + that chunk differently are reported as `elsewhere` or `new` even if some equal the base. +- The target's bytes are available as verified files (an onggi CSI volume or `onggi pull`). +- onggi never parses a vendor format or device layout, and never calls a vendor API. + +### 4. What a binding must do +- **Plan purely.** `plan` reads the switch plan and the binding's own knowledge of the resident state and changes + nothing. Given the same inputs it gives the same answer. +- **Map or refuse.** Every byte that is not `same` lies in some unit of the update plan, or the plan says `reload`. A + binding answers `reload` when the resident artifact is not `base_manifest`, when a run cannot be mapped, or when the + plan is too fragmented to be worth it (its own threshold). It never leaves a byte unmapped. +- **Read verified bytes.** `apply` reads target bytes from the verified target directory, not from the plan. +- **Stage, drain, verify.** Read and stage all units first, apply at a drained boundary (no request in flight on the + affected model), read back and return a digest per unit. `ok: true` only if every unit verified. +- **Roll back.** Keep what restores the base state of every written unit until the caller accepts the verification; + on any failure restore the base or unload the model. A failed update never leaves a mix of base and target serving. +- **Use public APIs only.** A binding may use only the vendor's public runtime APIs and documented metadata. It does + not decode undocumented formats; onggi keeps its side of ADR-0009 by passing byte ranges only. +- **Report** the path taken (`units` or `reload`) and the bytes written, so each path's cost is measured. + +### 5. Conformance levels + +| level | binding | what onggi does | +|---|---|---| +| **B0** | none: the runtime reads files | store less (chunk dedup across variants and checkpoints), node delta (only missing chunks cross the network; in-place patching of a base copy on the node), CSI volumes. A switch restarts or reloads the engine from files. | +| **B1** | `describe`, `resident`, and `plan` that always answers `reload` | B0, plus selection of the compatible artifact by coordinate, and choosing the resident artifact as the base of the node switch (the files the runtime will reload are patched, not refetched). Graphs or tensors the runtime keeps across a reload stay its business. | +| **B2** | all five | B1, plus resident updates: only the device units an update plan names are rewritten, verified and roll-backable; `reload` remains the fallback for fragmented switches. | + +A runtime starts at B0 and adds a binding when it wants device-level switches; each level is useful on its own. + +### 6. Conformance kit +[`conformance/binding/`](../../conformance/binding/) holds what a vendor runs against its binding: golden switch plans +for four small synthetic switches (identical, in place, shifted by 17,952 B, rearranged in 32 KiB blocks) that a Go +test keeps equal to what `onggi diff --ranges --json` produces; a generator for their bytes; a reference fake binding +(Python, standard library, Apache-2.0) on a simulated device of 4 KiB rows; and `conformance.py`, which checks plan +purity, the refusal path (a base that is not resident), coverage of every changed byte, apply with digests equal to +the target, the resident identity after apply, rollback to digests equal to the base, and that B1 always reloads. +Bindings live with their runtimes (for example rngd-pd), not in onggi. + +## Consequences +- A new accelerator joins by implementing a binding against public schemas and passing the kit; onggi needs no + change and learns no format. +- Rearranged switches (decode → prefill: 0 % at the same offset) produce many `elsewhere` runs; bindings will usually + answer `reload`, which is the intended outcome. The switch plan grows with the number of runs and is printed only + with `--ranges`. +- File pairing in the switch plan is a heuristic (same path, else the closest size, preferring the same file kind by + name); a binding that needs other pairs compares the files with the same rules. +- onggi does not yet invoke bindings itself; a node agent that calls `resident` to choose the base and `plan`/`apply` + after a node switch is follow-up work. diff --git a/docs/adr/README.md b/docs/adr/README.md index d9b792c..9bff14b 100644 --- a/docs/adr/README.md +++ b/docs/adr/README.md @@ -14,3 +14,4 @@ A decision changes only through a new ADR that supersedes the old one. | [0007](0007-refs-in-the-kubernetes-api.md) | Only refs live in the Kubernetes API; manifests are content-addressed objects | proposed | | [0008](0008-pluggable-storage-backends.md) | Pluggable storage backends behind one interface; durability delegated to the backend | proposed | | [0009](0009-clean-room-and-vendor-formats.md) | Vendor artifacts are opaque; clean-room code; published algorithms implemented as published | accepted | +| [0010](0010-device-binding-contract.md) | Device binding interface: five JSON operations, versioned schemas (`onggi.switchplan.v1` …), levels B0–B2, conformance kit | proposed | diff --git a/docs/schemas/README.md b/docs/schemas/README.md new file mode 100644 index 0000000..9c7eb85 --- /dev/null +++ b/docs/schemas/README.md @@ -0,0 +1,23 @@ +# Schemas of the device binding interface + +JSON Schema (draft 2020-12) of the documents exchanged between onggi and a device binding +([ADR-0010](../adr/0010-device-binding-contract.md)). + +| schema | file | produced by | example | +|---|---|---|---| +| `onggi.switchplan.v1` | [switchplan.v1.schema.json](switchplan.v1.schema.json) | `onggi diff --ranges --json BASE TARGET` | [examples/switchplan.json](examples/switchplan.json) | +| `onggi.updateplan.v1` | [updateplan.v1.schema.json](updateplan.v1.schema.json) | binding `plan` | [examples/updateplan.json](examples/updateplan.json), [examples/updateplan-reload.json](examples/updateplan-reload.json) | +| `onggi.verification.v1` | [verification.v1.schema.json](verification.v1.schema.json) | binding `apply`, `rollback` | [examples/verification.json](examples/verification.json) | +| `onggi.resident.v1` | [resident.v1.schema.json](resident.v1.schema.json) | binding `resident` | [examples/resident.json](examples/resident.json) | +| `onggi.profile.v1` | `store.Profile` (#71/#72) | publisher, binding `describe` | [examples/profile.json](examples/profile.json) | + +The examples come from the `inplace` and `rearranged` golden switches of +[`conformance/binding`](../../conformance/binding/) and the reference fake binding. To validate them (needs the +`jsonschema` package, not used by onggi itself): + +```bash +python3 -c 'import json,glob,jsonschema; from referencing import Registry, Resource +s={json.load(open(p))["$id"]:json.load(open(p)) for p in glob.glob("docs/schemas/*.schema.json")} +r=Registry().with_resources([(k,Resource.from_contents(v)) for k,v in s.items()]); t={v["title"]:v for v in s.values()} +[jsonschema.Draft202012Validator(t[d["schema"]],registry=r).validate(d) for p in glob.glob("docs/schemas/examples/*.json")+glob.glob("conformance/binding/golden/*.json") for d in [json.load(open(p))] if d["schema"] in t]' +``` diff --git a/docs/schemas/examples/profile.json b/docs/schemas/examples/profile.json new file mode 100644 index 0000000..05f0f48 --- /dev/null +++ b/docs/schemas/examples/profile.json @@ -0,0 +1,8 @@ +{ + "batch": 8, + "device": "fake", + "format": "bytes", + "layout": "rows-4KiB", + "runtime": "onggi-fake-binding", + "schema": "onggi.profile.v1" +} diff --git a/docs/schemas/examples/resident.json b/docs/schemas/examples/resident.json new file mode 100644 index 0000000..0ba0e47 --- /dev/null +++ b/docs/schemas/examples/resident.json @@ -0,0 +1,5 @@ +{ + "manifest": "sha256:7c6928f0385a257b5ceb9d13ea205a5ab0643a4fcec6b2a421ef5756aea93280", + "schema": "onggi.resident.v1", + "state": "ready" +} diff --git a/docs/schemas/examples/switchplan.json b/docs/schemas/examples/switchplan.json new file mode 100644 index 0000000..31f4341 --- /dev/null +++ b/docs/schemas/examples/switchplan.json @@ -0,0 +1,73 @@ +{ + "schema": "onggi.switchplan.v1", + "base_manifest": "sha256:7c6928f0385a257b5ceb9d13ea205a5ab0643a4fcec6b2a421ef5756aea93280", + "manifest": "sha256:cb4a0497f3a7c591b276b4526d0e305718fff0e98f35f3104185a6fdb0117a03", + "files": [ + { + "path": "config.json", + "base": "config.json", + "size": 13, + "same": 0, + "moved": 0, + "new": 13, + "same_blocks": 0, + "changed_runs": 1, + "largest_run": 13, + "changed": [ + [ + 0, + 13 + ] + ], + "ranges": [ + { + "offset": 0, + "length": 13, + "kind": "new" + } + ] + }, + { + "path": "model.bin", + "base": "model.bin", + "size": 2097152, + "same": 2031875, + "moved": 0, + "new": 65277, + "same_blocks": 2027520, + "changed_runs": 1, + "largest_run": 65277, + "changed": [ + [ + 1046344, + 65277 + ] + ], + "ranges": [ + { + "offset": 0, + "length": 1046344, + "kind": "same" + }, + { + "offset": 1046344, + "length": 65277, + "kind": "new" + }, + { + "offset": 1111621, + "length": 985531, + "kind": "same" + } + ] + } + ], + "size": 2097165, + "same": 2031875, + "moved": 0, + "new": 65290, + "new_stored": 65290, + "same_blocks": 2027520, + "changed_runs": 2, + "largest_run": 65277 +} diff --git a/docs/schemas/examples/updateplan-reload.json b/docs/schemas/examples/updateplan-reload.json new file mode 100644 index 0000000..b3d611a --- /dev/null +++ b/docs/schemas/examples/updateplan-reload.json @@ -0,0 +1,9 @@ +{ + "base_manifest": "sha256:a11ae743b61036855dc258adeb9ccdc3773affeb97db265e021124894f2c9a14", + "bytes": 2097165, + "manifest": "sha256:8442a4dc9e4ccb1b606c9ec3a9fde1b57cb02bec0fd06f71f4b6805767c28e2a", + "mode": "reload", + "reason": "fragmented: 123 changed runs, 2056192 of 2097165 bytes", + "schema": "onggi.updateplan.v1", + "units": [] +} diff --git a/docs/schemas/examples/updateplan.json b/docs/schemas/examples/updateplan.json new file mode 100644 index 0000000..1c138ac --- /dev/null +++ b/docs/schemas/examples/updateplan.json @@ -0,0 +1,23 @@ +{ + "base_manifest": "sha256:7c6928f0385a257b5ceb9d13ea205a5ab0643a4fcec6b2a421ef5756aea93280", + "bytes": 69645, + "manifest": "sha256:cb4a0497f3a7c591b276b4526d0e305718fff0e98f35f3104185a6fdb0117a03", + "mode": "units", + "schema": "onggi.updateplan.v1", + "units": [ + { + "file": "config.json", + "length": 13, + "offset": 0, + "source": "target", + "unit": "rows:0-0" + }, + { + "file": "model.bin", + "length": 69632, + "offset": 1044480, + "source": "target", + "unit": "rows:255-271" + } + ] +} diff --git a/docs/schemas/examples/verification.json b/docs/schemas/examples/verification.json new file mode 100644 index 0000000..a7fb3d6 --- /dev/null +++ b/docs/schemas/examples/verification.json @@ -0,0 +1,22 @@ +{ + "bytes_written": 69645, + "manifest": "sha256:cb4a0497f3a7c591b276b4526d0e305718fff0e98f35f3104185a6fdb0117a03", + "mode": "units", + "ok": true, + "op": "apply", + "schema": "onggi.verification.v1", + "units": [ + { + "file": "config.json", + "length": 13, + "offset": 0, + "sha256": "d2c3ed81e63d9795d2471722d6c381bdfdadad21c00c926a044c40107187cc68" + }, + { + "file": "model.bin", + "length": 69632, + "offset": 1044480, + "sha256": "9b865032bc8b647286722b4495a85cb32cc877135e3b7e0651993f478d51f68e" + } + ] +} diff --git a/docs/schemas/resident.v1.schema.json b/docs/schemas/resident.v1.schema.json new file mode 100644 index 0000000..e8401e8 --- /dev/null +++ b/docs/schemas/resident.v1.schema.json @@ -0,0 +1,13 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://github.com/ai-computing/onggi/docs/schemas/resident.v1.schema.json", + "title": "onggi.resident.v1", + "description": "Which artifact is resident on the device (ADR-0010 resident), so a node can choose the base of a switch.", + "type": "object", + "required": ["schema", "manifest", "state"], + "properties": { + "schema": {"const": "onggi.resident.v1"}, + "manifest": {"oneOf": [{"$ref": "switchplan.v1.schema.json#/$defs/digest"}, {"type": "null"}]}, + "state": {"enum": ["empty", "ready", "staged"], "description": "staged: an apply has not finished or not been verified"} + } +} diff --git a/docs/schemas/switchplan.v1.schema.json b/docs/schemas/switchplan.v1.schema.json new file mode 100644 index 0000000..75c5623 --- /dev/null +++ b/docs/schemas/switchplan.v1.schema.json @@ -0,0 +1,63 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://github.com/ai-computing/onggi/docs/schemas/switchplan.v1.schema.json", + "title": "onggi.switchplan.v1", + "description": "What turning a resident base artifact into a target takes, per byte range (ADR-0010). Printed by `onggi diff --ranges --json BASE TARGET`. Fields may be added within v1; consumers ignore unknown fields.", + "type": "object", + "required": ["schema", "base_manifest", "manifest", "files", "size", "same", "moved", "new"], + "properties": { + "schema": {"const": "onggi.switchplan.v1"}, + "base_manifest": {"$ref": "#/$defs/digest", "description": "the artifact the plan applies to"}, + "manifest": {"$ref": "#/$defs/digest", "description": "the artifact the plan produces"}, + "size": {"type": "integer", "minimum": 0}, + "same": {"type": "integer", "minimum": 0, "description": "bytes at the same offset of the paired base file"}, + "moved": {"type": "integer", "minimum": 0, "description": "bytes the base holds elsewhere"}, + "new": {"type": "integer", "minimum": 0, "description": "bytes the base does not hold"}, + "new_stored": {"type": "integer", "minimum": 0, "description": "stored (compressed) bytes of the distinct new chunks"}, + "same_blocks": {"type": "integer", "minimum": 0}, + "changed_runs": {"type": "integer", "minimum": 0}, + "largest_run": {"type": "integer", "minimum": 0}, + "files": { + "type": "array", + "items": { + "type": "object", + "required": ["path", "size", "same", "moved", "new"], + "properties": { + "path": {"type": "string", "description": "target file, slash-separated, relative to the artifact root"}, + "base": {"type": "string", "description": "paired base file; absent when there is none"}, + "size": {"type": "integer", "minimum": 0}, + "same": {"type": "integer", "minimum": 0}, + "moved": {"type": "integer", "minimum": 0}, + "new": {"type": "integer", "minimum": 0}, + "same_blocks": {"type": "integer", "minimum": 0}, + "changed_runs": {"type": "integer", "minimum": 0}, + "largest_run": {"type": "integer", "minimum": 0}, + "changed": { + "description": "runs that are not same, as [offset, length] (output before v1; kept)", + "type": "array", + "items": {"type": "array", "items": {"type": "integer"}, "minItems": 2, "maxItems": 2} + }, + "ranges": { + "description": "with --ranges: runs that cover the file from 0 to size, in order, without gaps", + "type": "array", + "items": { + "type": "object", + "required": ["offset", "length", "kind"], + "properties": { + "offset": {"type": "integer", "minimum": 0}, + "length": {"type": "integer", "minimum": 1}, + "kind": {"enum": ["same", "elsewhere", "new"]}, + "source_file": {"type": "string", "description": "elsewhere only: a base file holding the same bytes"}, + "source_offset": {"type": "integer", "minimum": 0, "description": "elsewhere only"} + }, + "if": {"properties": {"kind": {"const": "elsewhere"}}}, + "then": {"required": ["source_file", "source_offset"]}, + "else": {"not": {"anyOf": [{"required": ["source_file"]}, {"required": ["source_offset"]}]}} + } + } + } + } + } + }, + "$defs": {"digest": {"type": "string", "pattern": "^sha256:[0-9a-f]{64}$"}} +} diff --git a/docs/schemas/updateplan.v1.schema.json b/docs/schemas/updateplan.v1.schema.json new file mode 100644 index 0000000..267efd7 --- /dev/null +++ b/docs/schemas/updateplan.v1.schema.json @@ -0,0 +1,35 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://github.com/ai-computing/onggi/docs/schemas/updateplan.v1.schema.json", + "title": "onggi.updateplan.v1", + "description": "A binding's answer to a switch plan (ADR-0010 plan): the device units to rewrite, or a full reload. Produced without device side effects.", + "type": "object", + "required": ["schema", "base_manifest", "manifest", "mode", "units"], + "properties": { + "schema": {"const": "onggi.updateplan.v1"}, + "base_manifest": {"$ref": "switchplan.v1.schema.json#/$defs/digest"}, + "manifest": {"$ref": "switchplan.v1.schema.json#/$defs/digest"}, + "mode": {"enum": ["units", "reload"], "description": "units: rewrite only the listed units; reload: load the target whole"}, + "reason": {"type": "string", "description": "why a reload was chosen (fragmented, base not resident, unmappable, ...)"}, + "bytes": {"type": "integer", "minimum": 0, "description": "bytes the update writes to the device (an estimate for reload)"}, + "units": { + "type": "array", + "description": "mode units: device units (rows, tensors, pages) in artifact byte space; together they cover every byte of the switch plan that is not same. Empty for reload.", + "items": { + "type": "object", + "required": ["file", "offset", "length"], + "properties": { + "file": {"type": "string"}, + "offset": {"type": "integer", "minimum": 0}, + "length": {"type": "integer", "minimum": 1}, + "unit": {"type": "string", "description": "the name the binding gives the device unit (opaque to onggi)"}, + "source": {"enum": ["target", "base"], "description": "where the bytes come from: the verified target files, or a device-side copy from the base"}, + "source_file": {"type": "string"}, + "source_offset": {"type": "integer", "minimum": 0} + } + } + } + }, + "if": {"properties": {"mode": {"const": "reload"}}}, + "then": {"properties": {"units": {"maxItems": 0}}} +} diff --git a/docs/schemas/verification.v1.schema.json b/docs/schemas/verification.v1.schema.json new file mode 100644 index 0000000..bee17da --- /dev/null +++ b/docs/schemas/verification.v1.schema.json @@ -0,0 +1,29 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://github.com/ai-computing/onggi/docs/schemas/verification.v1.schema.json", + "title": "onggi.verification.v1", + "description": "Result of apply or rollback (ADR-0010): what the device holds now, with a digest per written unit in artifact byte space, so the caller can check it against the artifact.", + "type": "object", + "required": ["schema", "op", "mode", "manifest", "ok", "units"], + "properties": { + "schema": {"const": "onggi.verification.v1"}, + "op": {"enum": ["apply", "rollback"]}, + "mode": {"enum": ["units", "reload"]}, + "manifest": {"$ref": "switchplan.v1.schema.json#/$defs/digest", "description": "the artifact resident after the operation"}, + "ok": {"type": "boolean", "description": "true only when every unit verified; when false the binding has restored the base or unloaded the model"}, + "bytes_written": {"type": "integer", "minimum": 0}, + "units": { + "type": "array", + "items": { + "type": "object", + "required": ["file", "offset", "length", "sha256"], + "properties": { + "file": {"type": "string"}, + "offset": {"type": "integer", "minimum": 0}, + "length": {"type": "integer", "minimum": 0}, + "sha256": {"type": "string", "pattern": "^[0-9a-f]{64}$", "description": "SHA-256 of the unit as read back from the device, in artifact byte order"} + } + } + } + } +} diff --git a/pkg/store/conformance_test.go b/pkg/store/conformance_test.go new file mode 100644 index 0000000..059a386 --- /dev/null +++ b/pkg/store/conformance_test.go @@ -0,0 +1,94 @@ +package store + +import ( + "bytes" + "context" + "crypto/sha256" + "encoding/json" + "flag" + "fmt" + "os" + "path/filepath" + "sort" + "testing" +) + +var updateGolden = flag.Bool("update", false, "rewrite the golden switch plans of conformance/binding") + +// goldenData is the generator of conformance/binding/golden_data.py: SHA-256 in counter mode, so +// Go and Python produce the same bytes. +func goldenData(seed, n int) []byte { + var out []byte + for i := 0; len(out) < n; i++ { + h := sha256.Sum256([]byte(fmt.Sprintf("onggi-conformance:%d:%d", seed, i))) + out = append(out, h[:]...) + } + return out[:n] +} + +// goldenCases mirrors golden_data.py: base and target files of each synthetic switch. +func goldenCases() map[string][2]map[string][]byte { + const n = 2 << 20 + base := goldenData(1, n) + cfg := func(b int) []byte { return []byte(fmt.Sprintf("{\"batch\": %d}\n", b)) } + inplace := append([]byte(nil), base...) + copy(inplace[1<<20:], goldenData(2, 40<<10)) + const blk = 32 << 10 + idx := make([]int, n/blk) + keys := make([][32]byte, len(idx)) + for i := range idx { + idx[i], keys[i] = i, sha256.Sum256([]byte(fmt.Sprintf("perm:%d", i))) + } + sort.Slice(idx, func(a, b int) bool { return bytes.Compare(keys[idx[a]][:], keys[idx[b]][:]) < 0 }) + var rearranged []byte + for _, i := range idx { + rearranged = append(rearranged, base[i*blk:(i+1)*blk]...) + } + b := map[string][]byte{"model.bin": base, "config.json": cfg(2)} + return map[string][2]map[string][]byte{ + "identical": {b, {"model.bin": base, "config.json": cfg(2)}}, + "inplace": {b, {"model.bin": inplace, "config.json": cfg(8)}}, + "shifted": {b, {"model.bin": append(goldenData(3, 17952), base...), "config.json": cfg(2)}}, + "rearranged": {b, {"model.bin": rearranged, "config.json": cfg(2)}}, + } +} + +// The golden switch plans that the binding conformance kit replays are what onggi diff --ranges +// --json produces for the synthetic artifacts (ADR-0010). Manifest digests carry a creation time, +// so they are compared only for presence. +func TestGoldenSwitchPlans(t *testing.T) { + ctx := context.Background() + for name, c := range goldenCases() { + s := openTwoLevel(t, t.TempDir()) + bm, nm := pushDir(t, s, c[0]), pushDir(t, s, c[1]) + st, err := s.Diff(ctx, bm, nm, DiffOptions{Ranges: true}) + if err != nil { + t.Fatal(err) + } + path := filepath.Join("..", "..", "conformance", "binding", "golden", name+".switchplan.json") + if *updateGolden { + data, _ := json.MarshalIndent(st, "", " ") + if err := os.WriteFile(path, append(data, '\n'), 0o644); err != nil { + t.Fatal(err) + } + continue + } + data, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + var want DiffStats + if err := json.Unmarshal(data, &want); err != nil { + t.Fatal(err) + } + if want.BaseManifest == "" || want.Manifest == "" { + t.Fatalf("%s: golden plan without manifest digests", name) + } + want.BaseManifest, want.Manifest = st.BaseManifest, st.Manifest + got, _ := json.Marshal(st) + exp, _ := json.Marshal(want) + if !bytes.Equal(got, exp) { + t.Errorf("%s: onggi diff no longer produces the golden switch plan; if intended, run go test ./pkg/store -run GoldenSwitchPlans -update\n got %s\nwant %s", name, got, exp) + } + } +} diff --git a/pkg/store/diff.go b/pkg/store/diff.go index 809ef30..fdf3efb 100644 --- a/pkg/store/diff.go +++ b/pkg/store/diff.go @@ -93,6 +93,30 @@ type FileDiff struct { // every other byte is at the same offset in Base. A runtime that holds Base can update only // these ranges. Changed [][2]int64 `json:"changed,omitempty"` + // Ranges is the range plan of the file when DiffOptions.Ranges is set (ADR-0010): runs that + // cover the file in order, each the same, elsewhere in the base, or new. + Ranges []Range `json:"ranges,omitempty"` +} + +// SwitchPlanSchema names the format of DiffStats as JSON (ADR-0010). The switch plan +// is the only interface between onggi and a runtime's device binding. +const SwitchPlanSchema = "onggi.switchplan.v1" + +// Range kinds (ADR-0010). +const ( + RangeSame = "same" // at the same offset in the paired base file + RangeElsewhere = "elsewhere" // in the base at SourceFile, SourceOffset + RangeNew = "new" // not in the base +) + +// Range is one run of a file of the new artifact whose bytes are all of one kind. +type Range struct { + Offset int64 `json:"offset"` + Length int64 `json:"length"` + Kind string `json:"kind"` + // for elsewhere: a base file and offset that hold the same bytes + SourceFile string `json:"source_file,omitempty"` + SourceOffset *int64 `json:"source_offset,omitempty"` } // DiffOptions controls a diff. @@ -102,15 +126,18 @@ type DiffOptions struct { // DiffStats sums a diff over the files. type DiffStats struct { - Files []FileDiff `json:"files"` - Size int64 `json:"size"` - Same int64 `json:"same"` - Moved int64 `json:"moved"` - New int64 `json:"new"` - NewStored int64 `json:"new_stored"` // stored (compressed) bytes of the distinct new chunks - SameBlocks int64 `json:"same_blocks"` - ChangedRuns int `json:"changed_runs"` - LargestRun int64 `json:"largest_run"` + Schema string `json:"schema"` // SwitchPlanSchema + BaseManifest string `json:"base_manifest"` // what the plan applies to + Manifest string `json:"manifest"` // what it produces + Files []FileDiff `json:"files"` + Size int64 `json:"size"` + Same int64 `json:"same"` + Moved int64 `json:"moved"` + New int64 `json:"new"` + NewStored int64 `json:"new_stored"` // stored (compressed) bytes of the distinct new chunks + SameBlocks int64 `json:"same_blocks"` + ChangedRuns int `json:"changed_runs"` + LargestRun int64 `json:"largest_run"` } // pairFiles pairs each file of m with a file of base: the same path, else an unpaired base file of @@ -160,7 +187,7 @@ func abs(x int64) int64 { // Diff compares artifact m against base, both in s. func (s *Store) Diff(ctx context.Context, base, m digest.Digest, o DiffOptions) (DiffStats, error) { - var st DiffStats + st := DiffStats{Schema: SwitchPlanSchema, BaseManifest: base.String(), Manifest: m.String()} bm, err := s.ReadManifest(ctx, base) if err != nil { return st, err @@ -192,6 +219,21 @@ func (s *Store) Diff(ctx context.Context, base, m digest.Digest, o DiffOptions) slices.SortFunc(held, cmp) held = slices.Compact(held) inBase := func(d digest.Digest) bool { _, ok := slices.BinarySearchFunc(held, d, cmp); return ok } + type place struct { + file int + off int64 + } + var where map[digest.Digest]place // the first place in the base of each chunk, for the range plan + if o.Ranges { + where = make(map[digest.Digest]place, len(held)) + for j, es := range be { + for _, e := range es { + if _, ok := where[e.Chunk]; !ok { + where[e.Chunk] = place{j, e.Off} + } + } + } + } pair := pairFiles(bm, nm) for i, f := range nm.Files { @@ -218,6 +260,25 @@ func (s *Store) Diff(ctx context.Context, base, m digest.Digest, o DiffOptions) } sameFrom = -1 } + addRange := func(e Extent, kind string) { + if !o.Ranges { + return + } + r := Range{Offset: e.Off, Length: int64(e.Len), Kind: kind} + if kind == RangeElsewhere { + p := where[e.Chunk] + r.SourceFile, r.SourceOffset = bm.Files[p.file].Path, &p.off + } + if n := len(fd.Ranges); n > 0 { + last := &fd.Ranges[n-1] + if last.Kind == kind && last.Offset+last.Length == r.Offset && (kind != RangeElsewhere || + last.SourceFile == r.SourceFile && *last.SourceOffset+last.Length == *r.SourceOffset) { + last.Length += r.Length + return + } + } + fd.Ranges = append(fd.Ranges, r) + } closeChanged := func(end int64) { if changedFrom < 0 { return @@ -235,6 +296,7 @@ func (s *Store) Diff(ctx context.Context, base, m digest.Digest, o DiffOptions) } end := e.Off + int64(e.Len) if k < len(bes) && bes[k].Off == e.Off && bes[k].Chunk == e.Chunk { + addRange(e, RangeSame) fd.Same += int64(e.Len) closeChanged(e.Off) if sameFrom < 0 { @@ -248,8 +310,10 @@ func (s *Store) Diff(ctx context.Context, base, m digest.Digest, o DiffOptions) changedFrom = e.Off } if inBase(e.Chunk) { + addRange(e, RangeElsewhere) fd.Moved += int64(e.Len) } else { + addRange(e, RangeNew) fd.New += int64(e.Len) } } diff --git a/pkg/store/diff_test.go b/pkg/store/diff_test.go index c413f0a..d72ecbc 100644 --- a/pkg/store/diff_test.go +++ b/pkg/store/diff_test.go @@ -85,3 +85,71 @@ func TestDiff(t *testing.T) { t.Fatalf("rearranged: %+v", f) } } + +// The range plan (ADR-0010) covers every file in order with runs of one kind; an elsewhere run +// names base bytes equal to its own, and the sums match the totals. +func TestRangePlan(t *testing.T) { + ctx := context.Background() + s := openTwoLevel(t, t.TempDir()) + w := random(31, 8<<20) + bm := pushDir(t, s, map[string][]byte{"decode_B2.rbln": w}) + v := append([]byte(nil), w...) + copy(v[5<<20:], random(32, 100<<10)) + r := shuffled(w, 33) + nm := pushDir(t, s, map[string][]byte{"decode_B2.rbln": v, "prefill.rbln": r}) + st, err := s.Diff(ctx, bm, nm, DiffOptions{Ranges: true}) + if err != nil { + t.Fatal(err) + } + if st.Schema != SwitchPlanSchema || st.BaseManifest != bm.String() || st.Manifest != nm.String() { + t.Fatalf("plan header: %q %q %q", st.Schema, st.BaseManifest, st.Manifest) + } + data := map[string][]byte{"decode_B2.rbln": v, "prefill.rbln": r} + kinds := map[string]int{} + for _, f := range st.Files { + var end int64 + sum := map[string]int64{} + for i, rg := range f.Ranges { + if rg.Offset != end || rg.Length <= 0 { + t.Fatalf("%s: range %d at %d, want %d", f.Path, i, rg.Offset, end) + } + if i > 0 && f.Ranges[i-1].Kind == rg.Kind && rg.Kind != RangeElsewhere { + t.Fatalf("%s: adjacent %s ranges not merged", f.Path, rg.Kind) + } + end += rg.Length + sum[rg.Kind] += rg.Length + kinds[rg.Kind]++ + got := data[f.Path][rg.Offset : rg.Offset+rg.Length] + switch rg.Kind { + case RangeSame: + if !bytes.Equal(got, w[rg.Offset:rg.Offset+rg.Length]) { + t.Fatalf("%s: same range at %d differs from the base", f.Path, rg.Offset) + } + case RangeElsewhere: + if rg.SourceFile != "decode_B2.rbln" || rg.SourceOffset == nil { + t.Fatalf("%s: elsewhere range without a source: %+v", f.Path, rg) + } + if !bytes.Equal(got, w[*rg.SourceOffset:*rg.SourceOffset+rg.Length]) { + t.Fatalf("%s: elsewhere range at %d differs from its source", f.Path, rg.Offset) + } + case RangeNew: + if rg.SourceOffset != nil { + t.Fatalf("%s: new range with a source", f.Path) + } + default: + t.Fatalf("unknown kind %q", rg.Kind) + } + } + if end != f.Size || sum[RangeSame] != f.Same || sum[RangeElsewhere] != f.Moved || sum[RangeNew] != f.New { + t.Fatalf("%s: plan covers %d of %d B; sums %v vs %d/%d/%d", f.Path, end, f.Size, sum, f.Same, f.Moved, f.New) + } + } + if kinds[RangeSame] == 0 || kinds[RangeElsewhere] == 0 || kinds[RangeNew] == 0 { + t.Fatalf("kinds %v; the test should produce all three", kinds) + } + // without --ranges the plan is not listed, and the old fields are unchanged + st2, err := s.Diff(ctx, bm, nm, DiffOptions{}) + if err != nil || st2.Files[0].Ranges != nil || st2.Files[0].Changed != nil || st2.Same != st.Same || st2.Schema != SwitchPlanSchema { + t.Fatalf("without ranges: %+v %v", st2.Files[0], err) + } +}