diff --git a/bench/README.md b/bench/README.md new file mode 100644 index 0000000..c331840 --- /dev/null +++ b/bench/README.md @@ -0,0 +1,36 @@ +# Nightly binding benchmarks (Refs project-minigraf/minigraf#776 P16) + +Run from this worktree after building the local binding. No released package +is loaded as a fallback. Build on Linux: + +```sh +python3 -m venv target/bench-venv +source target/bench-venv/bin/activate +pip install "maturin>=1.5,<2" +export CARGO_TARGET_DIR="$PWD/target/p16" +cargo build --bin uniffi-bindgen +maturin develop --release +target/bench-venv/bin/python bench/nightly.py --test +``` + +`--test` invokes each operation exactly once, checks fixed answers and reads +no timer. Omit it only in GitHub Actions to collect median latency (nanoseconds) +in `binding-results.json`. Do not run timed cases on a developer laptop or +on the occupied benchmark machine. + +IDs use `binding/python/` and the six suffixes +`prepared_execute/1k`, `cursor_batch_1/1k`, `cursor_batch_256/1k`, +`set_roundtrip/2`, `predicate/1k`, `aggregate/1k`. + +Fixtures contain 1,000 deterministic integer facts and one two-element set. +Setup, prepare, registration and correctness checks are outside measurement; +foreign result decoding and cursor close are included. Set binding containers +are created per call. Callbacks are implemented in this host language. + +Routing, build provenance, dedicated Bencher testbeds, provisional 5–15 minute +allowance plus setup per job, and manual coverage dispositions are documented +in [the core binding benchmark guide](https://github.com/project-minigraf/minigraf/blob/v3/docs/BINDING_BENCHMARKS.md). +The core `bindings` matrix builds this shim against the measured core SHA +using a temporary Cargo patch; this repository's production dependency pin is +unchanged. Land this PR before the core workflow PRs. Phase B performs the +serial workflow dispatch and upload/history validation. diff --git a/bench/nightly.py b/bench/nightly.py new file mode 100644 index 0000000..4ddf6a5 --- /dev/null +++ b/bench/nightly.py @@ -0,0 +1,106 @@ +"""#776 P16: real Python/UniFFI crossings; --test never reads a timer. + +Fixture/prepare/registration and correctness checks are outside measurement. +Each sample averages 20 foreign API round trips, including JSON decode. +""" +import argparse +import json +import statistics +import time +from pathlib import Path + +from minigraf import Accumulator, Aggregate, MiniGrafDb, Predicate, Value + +N = 1000 + + +def rows(raw): + return json.loads(raw)["results"] + + +def summary(answer): + return [len(answer), sum(row[0] for row in answer)] + + +class Even(Predicate): + def test(self, value): + return value.value % 2 == 0 + + +class TotalAcc(Accumulator): + def __init__(self): + self.total = 0 + + def step(self, value): + self.total += value.value + + def finish(self, count): + return Value.INT64(self.total) + + +class Total(Aggregate): + def accumulator(self): + return TotalAcc() + + +def cases(): + db = MiniGrafDb.open_in_memory() + db.execute("(transact [" + " ".join(f"[:e{i} :n {i}]" for i in range(N)) + "])") + db.execute('(transact [[:tags :tags #{1 "x"}]])') + db.register_predicate("bench-even?", Even()) + db.register_aggregate("bench-total", Total()) + scan = db.prepare("(query [:find ?n :where [?e :n ?n] [(>= ?n $min)]])") + point = db.prepare("(query [:find ?n :where [?e :n $n] [?e :n ?n]])") + sets = db.prepare("(query [:find ?s :where [:tags :tags $s] [:tags :tags ?s]])") + predicate = db.prepare("(query [:find ?n :where [?e :n ?n] [(bench-even? ?n)]])") + aggregate = db.prepare("(query [:find (bench-total ?n) :where [?e :n ?n]])") + + def drain(batch_size): + answer = [] + with scan.query(min=0) as cursor: + while (batch := cursor.next_batch(batch_size)) is not None: + answer.extend(json.loads(batch)) + return answer + + return [ + ("prepared_execute/1k", lambda: rows(point.execute(n=500)), [[500]]), + ("cursor_batch_1/1k", lambda: drain(1), [N, N * (N - 1) // 2]), + ("cursor_batch_256/1k", lambda: drain(256), [N, N * (N - 1) // 2]), + ("set_roundtrip/2", lambda: rows(sets.execute(s={1, "x"})), [[["x", 1]]]), + ("predicate/1k", lambda: rows(predicate.execute()), [500, 249500]), + ("aggregate/1k", lambda: rows(aggregate.execute()), [[499500]]), + ] + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("--test", action="store_true") + parser.add_argument("--output", default="binding-results.json") + args = parser.parse_args() + metrics = {} + for name, operation, expected in cases(): + name = "binding/python/" + name + check = summary if "cursor_batch" in name or "/predicate/" in name else lambda x: x + samples = [] + if not args.test: + for _ in range(3): + assert check(operation()) == expected, f"incorrect result: {name}" + for _ in range(1 if args.test else 10): + if args.test: + actual = operation() + else: + start = time.perf_counter_ns() + for _ in range(20): + actual = operation() + samples.append((time.perf_counter_ns() - start) / 20) + assert check(actual) == expected, f"incorrect result: {name}" + print("ok: " + name, flush=True) + if samples: + metrics[name] = {"latency": {"value": statistics.median(samples)}} + # Test mode writes no fake timing data. + if not args.test: + Path(args.output).write_text(json.dumps(metrics, indent=2) + "\n") + + +if __name__ == "__main__": + main()