From f1c0a4599318a9ab6baa6cea115a44b43f440eeb Mon Sep 17 00:00:00 2001 From: Aditya Mukhopadhyay Date: Wed, 4 Nov 2026 19:51:45 +0530 Subject: [PATCH] bench: add correctness-checked JVM release timing harness --- bench/manual/.gitignore | 3 + bench/manual/build.gradle.kts | 16 ++ bench/manual/build.sh | 52 ++++ bench/manual/gradle.properties | 2 + bench/manual/settings.gradle.kts | 2 + .../project_minigraf/benchmark/Benchmark.kt | 241 ++++++++++++++++++ .../benchmark/ResultChecks.kt | 9 + .../benchmark/ResultChecksTest.kt | 24 ++ 8 files changed, 349 insertions(+) create mode 100644 bench/manual/.gitignore create mode 100644 bench/manual/build.gradle.kts create mode 100644 bench/manual/build.sh create mode 100644 bench/manual/gradle.properties create mode 100644 bench/manual/settings.gradle.kts create mode 100644 bench/manual/src/main/kotlin/io/github/project_minigraf/benchmark/Benchmark.kt create mode 100644 bench/manual/src/main/kotlin/io/github/project_minigraf/benchmark/ResultChecks.kt create mode 100644 bench/manual/src/test/kotlin/io/github/project_minigraf/benchmark/ResultChecksTest.kt diff --git a/bench/manual/.gitignore b/bench/manual/.gitignore new file mode 100644 index 0000000..275c704 --- /dev/null +++ b/bench/manual/.gitignore @@ -0,0 +1,3 @@ +/.gradle/ +/build/ +/results/ diff --git a/bench/manual/build.gradle.kts b/bench/manual/build.gradle.kts new file mode 100644 index 0000000..fee2140 --- /dev/null +++ b/bench/manual/build.gradle.kts @@ -0,0 +1,16 @@ +plugins { + kotlin("jvm") version "2.0.21" + application +} +repositories { mavenCentral() } +kotlin { jvmToolchain(17) } +dependencies { + implementation(files("../../java/build/libs/minigraf-jvm-0.0.0-local.jar")) + implementation("net.java.dev.jna:jna:5.14.0") + implementation("org.json:json:20240303") + testImplementation("junit:junit:4.13.2") +} +application { + mainClass.set("io.github.project_minigraf.benchmark.BenchmarkKt") + applicationDefaultJvmArgs = listOf("-Xms512m", "-Xmx512m", "-XX:+UseG1GC") +} diff --git a/bench/manual/build.sh b/bench/manual/build.sh new file mode 100644 index 0000000..9483002 --- /dev/null +++ b/bench/manual/build.sh @@ -0,0 +1,52 @@ +#!/usr/bin/env bash +# Run in disposable binding/core checkouts. No benchmark executes during build. +set -euo pipefail +binding_dir=$(cd "$(dirname "$0")/../.." && pwd) +core_dir=$(cd "${1:?usage: build.sh CORE_CHECKOUT}" && pwd) +cd "$binding_dir" +core_revision=${CORE_REVISION:-$(git -C "$core_dir" rev-parse HEAD)} +binding_revision=${BINDING_REVISION:-$(git rev-parse HEAD)} +export CARGO_BUILD_JOBS=${CARGO_BUILD_JOBS:-8} +test ! -e .cargo/config.toml +mkdir -p .cargo bench/manual/build/evidence +python3 - "$core_dir" <<'PY' +import json, pathlib, sys +pathlib.Path('.cargo/config.toml').write_text( + '[patch."https://github.com/project-minigraf/minigraf"]\n' + 'minigraf = { path = ' + json.dumps(sys.argv[1]) + ' }\n') +PY +cargo update -p minigraf +cargo metadata --format-version 1 > bench/manual/build/evidence/cargo-metadata.json +python3 - "$core_dir" <<'PY' +import json, pathlib, sys +p = [p for p in json.load(open('bench/manual/build/evidence/cargo-metadata.json'))['packages'] if p['name'] == 'minigraf'] +assert len(p) == 1 and p[0]['source'] is None +assert pathlib.Path(p[0]['manifest_path']).parent == pathlib.Path(sys.argv[1]) +PY +cargo test --lib +cargo build --release --lib --bin uniffi-bindgen +# Separate invocation: copyLocalNative writes processResources inputs without +# declaring a Gradle dependency, so selecting both in one graph is rejected. +bash java/gradlew -p java --no-daemon --max-workers 8 copyLocalNative +bash java/gradlew -p java --no-daemon --max-workers 8 test jar +bash java/gradlew -p bench/manual --no-daemon test installDist +python3 - "$core_revision" "$binding_revision" <<'PY' +import hashlib, json, pathlib, subprocess, sys +root = pathlib.Path('bench/manual') +digest = hashlib.sha256() +for p in sorted(p for p in root.rglob('*') if p.is_file() and + not any(part in ('build', '.gradle', 'results') for part in p.relative_to(root).parts)): + digest.update(str(p.relative_to(root)).encode() + b'\0' + p.read_bytes() + b'\0') +def sha(p): return hashlib.sha256(pathlib.Path(p).read_bytes()).hexdigest() +record = dict(core_revision=sys.argv[1], binding_revision=sys.argv[2], + evidence_kind='source-build', harness_sha256=digest.hexdigest(), + binding_jar_sha256=sha('java/build/libs/minigraf-jvm-0.0.0-local.jar'), + native_sha256=sha('target/release/libminigraf_ffi.so'), + cargo_lock_sha256=sha('Cargo.lock'), + rustc=subprocess.check_output(['rustc','--version'], text=True).strip(), + java=subprocess.check_output(['java','-version'], stderr=subprocess.STDOUT, text=True).splitlines()[0], + profile='release', abi='x86_64-unknown-linux-gnu', gradle='8.11', + kotlin='2.0.21', jna='5.14.0', uniffi='0.31.1', json='20240303') +(root / 'build/evidence/build.json').write_text(json.dumps(record,indent=2)+'\n') +PY +cp Cargo.lock bench/manual/build/evidence/Cargo.lock diff --git a/bench/manual/gradle.properties b/bench/manual/gradle.properties new file mode 100644 index 0000000..d20c739 --- /dev/null +++ b/bench/manual/gradle.properties @@ -0,0 +1,2 @@ +org.gradle.workers.max=8 +org.gradle.jvmargs=-Xmx2g diff --git a/bench/manual/settings.gradle.kts b/bench/manual/settings.gradle.kts new file mode 100644 index 0000000..66c76da --- /dev/null +++ b/bench/manual/settings.gradle.kts @@ -0,0 +1,2 @@ +pluginManagement { repositories { gradlePluginPortal(); mavenCentral() } } +rootProject.name = "minigraf-jvm-benchmark" diff --git a/bench/manual/src/main/kotlin/io/github/project_minigraf/benchmark/Benchmark.kt b/bench/manual/src/main/kotlin/io/github/project_minigraf/benchmark/Benchmark.kt new file mode 100644 index 0000000..a0d7826 --- /dev/null +++ b/bench/manual/src/main/kotlin/io/github/project_minigraf/benchmark/Benchmark.kt @@ -0,0 +1,241 @@ +package io.github.project_minigraf.benchmark + +import io.github.project_minigraf.minigraf.* +import java.io.File +import org.json.JSONArray +import org.json.JSONObject + +/** Source-build qualification for minigraf#839; no benchmark framework in the library. */ +class Benchmark(private val mode: String, private val directory: File, private val provenance: File) { + private val expected = (0L until 1000L).toList() + private val scalarQuery = "(query [:find ?n :where [?e :bench/n ?n]])" + private val pointQuery = "(query [:find ?n :where [?e :bench/n \$n] [?e :bench/n ?n]])" + private val transaction = "(transact [" + (0 until 1000).joinToString(" ") { + "[:bench/e$it :bench/n $it]" + } + "])" + private val output = File(directory, "minigraf-bench.jsonl") + private var timed = false + private val completed = mutableListOf() + + private fun emit(record: JSONObject) { + val line = record.toString() + require(line.toByteArray(Charsets.UTF_8).size < 3800) { "log record too large" } + output.appendText(line + "\n") + println(line) + } + + private fun integers(rows: JSONArray): List = (0 until rows.length()).map { + val row = rows.getJSONArray(it) + require(row.length() == 1) { "unexpected projection" } + val value = row.get(0) + require(value is Int || value is Long) { "expected integer result" } + (value as Number).toLong() + } + + private fun result(json: String) = integers(JSONObject(json).getJSONArray("results")) + + private fun drain(cursor: MiniGrafCursor, batch: Int): List = cursor.use { + val values = mutableListOf() + while (true) { + val json = cursor.nextBatch(batch) ?: break + val rows = JSONArray(json) + require(rows.length() in 1..batch) { "invalid cursor batch size" } + values += integers(rows) + } + values + } + + private fun records(db: MiniGrafDb): List = + db.factLog(FactFilter(attributes = listOf(":bench/n"))).use { log -> + val all = mutableListOf() + while (true) { + val batch = log.nextBatch(256) ?: break + require(batch.size in 1..256) { "invalid fact-log batch size" } + all += batch + } + all + } + + private fun checkRecords(rows: List) { + ResultChecks.numbers(rows.map { (it.value as MiniGrafValue.Int64).value }, expected) + require(rows.all { it.asserted && it.attribute == ":bench/n" && it.txCount == 1L }) + require(rows.map { it.entity }.toSet().size == 1000) + } + + private val even = object : MiniGrafPredicate { + override fun test(value: MiniGrafValue) = (value as MiniGrafValue.Int64).value % 2 == 0L + } + private val sum = object : MiniGrafAggregate { + override fun accumulator() = object : MiniGrafAccumulator { + private var total = 0L + override fun step(value: MiniGrafValue) { total += (value as MiniGrafValue.Int64).value } + override fun finish(count: Long): MiniGrafValue { + check(count == 1000L) { "unexpected aggregate callback count" } + return MiniGrafValue.Int64(total) + } + } + } + + // Each operation is timed independently. Validation, reporting, setup and + // teardown run outside the timer. No synthetic timings are emitted in smoke mode. + private fun measure( + name: String, iterations: Int = 20, setup: () -> S, + action: (S) -> T, validate: (T) -> Unit, teardown: (S) -> Unit = {}, + ) { + val id = "binding/jvm/$name" + fun invoke(): Long { + val state = setup() + try { + val start = if (timed) System.nanoTime() else 0L + val value = action(state) + val elapsed = if (timed) System.nanoTime() - start else 0L + validate(value) + return elapsed + } finally { teardown(state) } + } + if (timed) { + repeat(if (iterations == 1) 5 else 100) { invoke() } + repeat(10) { sample -> + var total = 0L + repeat(iterations) { total += invoke() } + emit(JSONObject().put("type", "sample").put("id", id) + .put("sample", sample).put("iterations", iterations) + .put("total_ns", total).put("ns_per_op", total.toDouble() / iterations)) + } + } else { invoke() } + completed += id + emit(JSONObject().put("type", "checked").put("id", id)) + } + + private fun memory(name: String, action: () -> T, validate: (T) -> Unit) = + measure(name, setup = { Unit }, action = { action() }, validate = validate) + + fun verifyAndMeasure() { + require(mode == "smoke" || mode == "benchmark") { "mode must be smoke or benchmark" } + timed = mode == "benchmark" + output.writeText("") + completed.clear() + val build = JSONObject(provenance.readText()) + emit(JSONObject().put("type", "environment").put("mode", mode).put("build", build) + .put("vm", System.getProperty("java.vm.name")) + .put("vm_version", System.getProperty("java.vm.version")) + .put("java_version", System.getProperty("java.version")) + .put("java_vendor", System.getProperty("java.vendor")) + .put("jvm_arguments", JSONArray(java.lang.management.ManagementFactory.getRuntimeMXBean().inputArguments)) + .put("os", System.getProperty("os.name")).put("arch", System.getProperty("os.arch")) + .put("warmup_memory", 100).put("warmup_storage", 5) + .put("storage", "run-directory").put("rss_verified", false)) + + val dir = File(directory, "scratch-${System.nanoTime()}") + check(dir.mkdirs()) + try { + MiniGrafDb.openInMemory().use { db -> + db.execute(transaction) + db.execute("(transact [[:bench/set :bench/tags #{1 2}]])") + db.registerPredicate("bench-even?", even) + db.registerAggregate("bench-sum", sum) + db.prepare(pointQuery).use { point -> + memory("prepared_execute/1k", { result(point.execute("n" to 42L)) }) { + ResultChecks.numbers(it, listOf(42L)) + } + } + db.prepare(scalarQuery).use { scan -> + for (batch in listOf(1, 256)) { + memory("cursor_batch_$batch/1k", { drain(scan.query(), batch) }) { + ResultChecks.numbers(it, expected) + } + } + } + db.prepare("(query [:find ?s :where [?e :bench/tags \$s] [?e :bench/tags ?s]])").use { set -> + memory("set_roundtrip/2", { + val rows = JSONObject(set.execute("s" to setOf(1L, 2L))).getJSONArray("results") + require(rows.length() == 1 && rows.getJSONArray(0).length() == 1) + val elements = rows.getJSONArray(0).getJSONArray(0) + (0 until elements.length()).map { elements.getLong(it) } + }) { ResultChecks.numbers(it, listOf(1L, 2L)) } + } + memory("predicate/1k", { + result(db.execute("(query [:find ?n :where [?e :bench/n ?n] [(bench-even? ?n)]])")) + }) { ResultChecks.numbers(it, expected.filter { n -> n % 2 == 0L }) } + memory("aggregate/1k", { + result(db.execute("(query [:find (bench-sum ?n) :where [?e :bench/n ?n]])")) + }) { ResultChecks.numbers(it, listOf(499500L)) } + memory("prepare/point", { db.prepare(pointQuery) }) { point -> + point.use { ResultChecks.numbers(result(it.execute("n" to 42L)), listOf(42L)) } + } + measure("register_predicate/first", + setup = { MiniGrafDb.openInMemory().apply { execute(transaction) } }, + action = { fresh -> fresh.apply { registerPredicate("bench-even?", even) } }, + validate = { fresh -> ResultChecks.numbers(result(fresh.execute( + "(query [:find ?n :where [?e :bench/n ?n] [(bench-even? ?n)]])" + )), expected.filter { n -> n % 2 == 0L }) }, + teardown = { it.close() }) + measure("register_aggregate/first", + setup = { MiniGrafDb.openInMemory().apply { execute(transaction) } }, + action = { fresh -> fresh.apply { registerAggregate("bench-sum", sum) } }, + validate = { fresh -> ResultChecks.numbers(result(fresh.execute( + "(query [:find (bench-sum ?n) :where [?e :bench/n ?n]])" + )), listOf(499500L)) }, + teardown = { it.close() }) + memory("fact_log_batch_256/1k", { records(db) }) { checkRecords(it) } + memory("options_error/negative_cache", { + try { + MiniGrafDb.openWithOptions(File(dir, "invalid.graph").path, + OpenOptions(pageCacheSize = -1)).use { error("negative cache accepted") } + } catch (error: MiniGrafException.Api) { error.msg.take(9) } + }) { require(it == "[API-017]") } + + val seed = File(dir, "seed.graph") + MiniGrafDb.open(seed.path).use { it.execute(transaction); it.checkpoint() } + memory("options_readonly_open/1k", { + MiniGrafDb.openWithOptions(seed.path, OpenOptions(readOnly = true, + pageCacheSize = 64, queryMemoryLimit = 16L * 1024 * 1024)) + }) { opened -> opened.use { ResultChecks.numbers(result(it.execute(scalarQuery)), expected) } } + + measure("file_transact/1k", iterations = 1, + setup = { + val path = File(dir, "write-${System.nanoTime()}.graph").path + path to MiniGrafDb.open(path) + }, + action = { (_, writer) -> writer.execute(transaction) }, + validate = { require(JSONObject(it).getLong("transacted") > 0L) }, + teardown = { (path, writer) -> + writer.close() + MiniGrafDb.open(path).use { reopened -> + ResultChecks.numbers(result(reopened.execute(scalarQuery)), expected) + } + }) + + val source = records(db) + measure("log_writer_append_finish/1k", iterations = 1, + setup = { + val path = File(dir, "log-${System.nanoTime()}.graph").path + path to MiniGrafLogWriter.create(path, OpenOptions()) + }, + action = { (_, writer) -> writer.appendBatch(source); writer.finish() }, + validate = {}, + teardown = { (path, writer) -> + writer.close() + MiniGrafDb.open(path).use { reopened -> + require(records(reopened).sortedBy { it.entity } == source.sortedBy { it.entity }) { + "fact-log round trip changed records" + } + ResultChecks.numbers(result(reopened.execute(scalarQuery)), expected) + } + }) + } + require(completed.size == 14 && completed.toSet().size == 14) + emit(JSONObject().put("type", "complete").put("mode", mode) + .put("ids", JSONArray(completed)).put("count", completed.size)) + } finally { + check(dir.deleteRecursively()) { "scratch cleanup failed" } + } + } +} + +fun main(args: Array) { + require(args.size == 3) { "usage: benchmark smoke|benchmark OUTPUT_DIRECTORY BUILD_JSON" } + val directory = File(args[1]) + require(directory.mkdirs()) { "output directory must be fresh" } + Benchmark(args[0], directory, File(args[2])).verifyAndMeasure() +} diff --git a/bench/manual/src/main/kotlin/io/github/project_minigraf/benchmark/ResultChecks.kt b/bench/manual/src/main/kotlin/io/github/project_minigraf/benchmark/ResultChecks.kt new file mode 100644 index 0000000..881e7aa --- /dev/null +++ b/bench/manual/src/main/kotlin/io/github/project_minigraf/benchmark/ResultChecks.kt @@ -0,0 +1,9 @@ +package io.github.project_minigraf.benchmark + +internal object ResultChecks { + fun numbers(actual: List, expected: List) { + require(actual.size == expected.size) { "unexpected row count" } + require(actual.sum() == expected.sum()) { "unexpected row sum" } + require(actual.sorted() == expected.sorted()) { "unexpected row values" } + } +} diff --git a/bench/manual/src/test/kotlin/io/github/project_minigraf/benchmark/ResultChecksTest.kt b/bench/manual/src/test/kotlin/io/github/project_minigraf/benchmark/ResultChecksTest.kt new file mode 100644 index 0000000..d4adc5c --- /dev/null +++ b/bench/manual/src/test/kotlin/io/github/project_minigraf/benchmark/ResultChecksTest.kt @@ -0,0 +1,24 @@ +package io.github.project_minigraf.benchmark + +import org.junit.Assert.assertThrows +import org.junit.Test + +class ResultChecksTest { + private val expected = listOf(0L, 1L, 2L, 3L) + + @Test fun acceptsCompleteUnorderedResult() { + ResultChecks.numbers(listOf(3L, 1L, 0L, 2L), expected) + } + + @Test fun rejectsMissingRows() { + assertThrows(IllegalArgumentException::class.java) { + ResultChecks.numbers(emptyList(), expected) + } + } + + @Test fun rejectsWrongValuesDespiteEqualCountAndSum() { + assertThrows(IllegalArgumentException::class.java) { + ResultChecks.numbers(listOf(0L, 0L, 3L, 3L), expected) + } + } +}