diff --git a/docs/research/go-vs-rust-benchmark-2026-11-06/1-go-model_check.log b/docs/research/go-vs-rust-benchmark-2026-11-06/1-go-model_check.log new file mode 100644 index 000000000..941e16000 --- /dev/null +++ b/docs/research/go-vs-rust-benchmark-2026-11-06/1-go-model_check.log @@ -0,0 +1,13 @@ +$ /tmp/thirdshift-benchmark-sl4wkj08/go-tests '-test.run=^TestPickupModel' -test.count=1 -test.v +=== RUN TestPickupModelSafety + pickup_model_test.go:267: ADR-0017: 1993250 unique states +--- PASS: TestPickupModelSafety (4.15s) +=== RUN TestPickupModelMutations +=== RUN TestPickupModelMutations/WithoutClaimCount + pickup_model_test.go:285: WithoutClaimCount: 119601 states; shortest dispatch trace (19): [ListReady(0) SelectHistory(0) CreateReservation(0) RefreshIssue(0) RefreshHistory(0) ListReady(1) SelectHistory(1) PutClaimLabels(0) PrepareWorktree(0) Dispatch(0) KeepClaimWithOriginWork(0) ReleaseReservation(0) CreateReservation(1) DayHandsBack RefreshIssue(1) RefreshHistory(1) PutClaimLabels(1) PrepareWorktree(1) Dispatch(1)]; shortest claimed trace (19): [ListReady(0) SelectHistory(0) CreateReservation(0) RefreshIssue(0) RefreshHistory(0) ListReady(1) SelectHistory(1) PutClaimLabels(0) PrepareWorktree(0) Dispatch(0) KeepClaimWithOriginWork(0) ReleaseReservation(0) CreateReservation(1) DayHandsBack RefreshIssue(1) RefreshHistory(1) PutClaimLabels(1) PrepareWorktree(1) Dispatch(1)] +=== RUN TestPickupModelMutations/UnconditionalCreate + pickup_model_test.go:285: UnconditionalCreate: 223036 states; shortest dispatch trace (16): [ListReady(0) SelectHistory(0) CreateReservation(0) RefreshIssue(0) RefreshHistory(0) ListReady(1) SelectHistory(1) CreateReservation(1) RefreshIssue(1) RefreshHistory(1) PutClaimLabels(0) PrepareWorktree(0) Dispatch(0) PutClaimLabels(1) PrepareWorktree(1) Dispatch(1)]; shortest claimed trace (16): [ListReady(0) SelectHistory(0) CreateReservation(0) RefreshIssue(0) RefreshHistory(0) ListReady(1) SelectHistory(1) CreateReservation(1) RefreshIssue(1) RefreshHistory(1) PutClaimLabels(0) PrepareWorktree(0) Dispatch(0) PutClaimLabels(1) PrepareWorktree(1) Dispatch(1)] +--- PASS: TestPickupModelMutations (0.76s) + --- PASS: TestPickupModelMutations/WithoutClaimCount (0.26s) + --- PASS: TestPickupModelMutations/UnconditionalCreate (0.50s) +PASS diff --git a/docs/research/go-vs-rust-benchmark-2026-11-06/1-rust-model_check.log b/docs/research/go-vs-rust-benchmark-2026-11-06/1-rust-model_check.log new file mode 100644 index 000000000..dfa3c8c5a --- /dev/null +++ b/docs/research/go-vs-rust-benchmark-2026-11-06/1-rust-model_check.log @@ -0,0 +1,69 @@ +$ /tmp/thirdshift-benchmark-sl4wkj08/rust-target/debug/deps/pickup_model-1d72f02ce05acf2e --test-threads=1 --nocapture + +running 5 tests +test assert_requested_property ... ignored, known pre-Claim dispatch finding; run explicitly to see the failure +test current_protocol_reports_pre_claim_dispatch_counterexample ... Current, Dispatch: 7093 unique states +KNOWN FINDING: the literal dispatch property fails: Path[14]: +- ListReady(0) +- SelectHistory(0) +- CreateReservation(0) +- RefreshIssue(0) +- RefreshHistory(0) +- Dispatch(0) +- FailBeforeClaim(0) +- ReleaseReservation(0) +- ListReady(1) +- SelectHistory(1) +- CreateReservation(1) +- RefreshIssue(1) +- RefreshHistory(1) +- Dispatch(1) + +ok +test pickup_claimed_dispatch_safety ... Current, ClaimedDispatch: 86745 unique states +ok +test removing_claim_count_recheck_is_detected ... WithoutClaimCount, ClaimedDispatch: 20675 unique states +Claim-count mutation counterexample: Path[19]: +- ListReady(0) +- SelectHistory(0) +- CreateReservation(0) +- RefreshIssue(0) +- RefreshHistory(0) +- Dispatch(0) +- ReadClaimLabels(0) +- ListReady(1) +- SelectHistory(1) +- PutClaimLabels(0) +- KeepClaim(0) +- ReleaseReservation(0) +- CreateReservation(1) +- DayHandsBack +- RefreshIssue(1) +- RefreshHistory(1) +- Dispatch(1) +- ReadClaimLabels(1) +- PutClaimLabels(1) + +ok +test removing_empty_create_lease_is_detected ... UnconditionalCreate, ClaimedDispatch: 22520 unique states +Empty-lease mutation counterexample: Path[15]: +- ListReady(0) +- SelectHistory(0) +- CreateReservation(0) +- RefreshIssue(0) +- RefreshHistory(0) +- Dispatch(0) +- ReadClaimLabels(0) +- ListReady(1) +- SelectHistory(1) +- CreateReservation(1) +- RefreshIssue(1) +- RefreshHistory(1) +- PutClaimLabels(0) +- Dispatch(1) +- ReadClaimLabels(1) + +ok + +test result: ok. 4 passed; 0 failed; 1 ignored; 0 measured; 0 filtered out; finished in 1.12s + diff --git a/docs/research/go-vs-rust-benchmark-2026-11-06/2-go-model_check.log b/docs/research/go-vs-rust-benchmark-2026-11-06/2-go-model_check.log new file mode 100644 index 000000000..117272ea8 --- /dev/null +++ b/docs/research/go-vs-rust-benchmark-2026-11-06/2-go-model_check.log @@ -0,0 +1,13 @@ +$ /tmp/thirdshift-benchmark-f7shrf8g/go-tests '-test.run=^TestPickupModel' -test.count=1 -test.v +=== RUN TestPickupModelSafety + pickup_model_test.go:267: ADR-0017: 1993250 unique states +--- PASS: TestPickupModelSafety (4.15s) +=== RUN TestPickupModelMutations +=== RUN TestPickupModelMutations/WithoutClaimCount + pickup_model_test.go:285: WithoutClaimCount: 119601 states; shortest dispatch trace (19): [ListReady(0) SelectHistory(0) CreateReservation(0) RefreshIssue(0) RefreshHistory(0) ListReady(1) SelectHistory(1) PutClaimLabels(0) PrepareWorktree(0) Dispatch(0) KeepClaimWithOriginWork(0) ReleaseReservation(0) CreateReservation(1) DayHandsBack RefreshIssue(1) RefreshHistory(1) PutClaimLabels(1) PrepareWorktree(1) Dispatch(1)]; shortest claimed trace (19): [ListReady(0) SelectHistory(0) CreateReservation(0) RefreshIssue(0) RefreshHistory(0) ListReady(1) SelectHistory(1) PutClaimLabels(0) PrepareWorktree(0) Dispatch(0) KeepClaimWithOriginWork(0) ReleaseReservation(0) CreateReservation(1) DayHandsBack RefreshIssue(1) RefreshHistory(1) PutClaimLabels(1) PrepareWorktree(1) Dispatch(1)] +=== RUN TestPickupModelMutations/UnconditionalCreate + pickup_model_test.go:285: UnconditionalCreate: 223036 states; shortest dispatch trace (16): [ListReady(0) SelectHistory(0) CreateReservation(0) RefreshIssue(0) RefreshHistory(0) ListReady(1) SelectHistory(1) CreateReservation(1) RefreshIssue(1) RefreshHistory(1) PutClaimLabels(0) PrepareWorktree(0) Dispatch(0) PutClaimLabels(1) PrepareWorktree(1) Dispatch(1)]; shortest claimed trace (16): [ListReady(0) SelectHistory(0) CreateReservation(0) RefreshIssue(0) RefreshHistory(0) ListReady(1) SelectHistory(1) CreateReservation(1) RefreshIssue(1) RefreshHistory(1) PutClaimLabels(0) PrepareWorktree(0) Dispatch(0) PutClaimLabels(1) PrepareWorktree(1) Dispatch(1)] +--- PASS: TestPickupModelMutations (0.65s) + --- PASS: TestPickupModelMutations/WithoutClaimCount (0.20s) + --- PASS: TestPickupModelMutations/UnconditionalCreate (0.46s) +PASS diff --git a/docs/research/go-vs-rust-benchmark-2026-11-06/2-rust-model_check.log b/docs/research/go-vs-rust-benchmark-2026-11-06/2-rust-model_check.log new file mode 100644 index 000000000..c0fe886b5 --- /dev/null +++ b/docs/research/go-vs-rust-benchmark-2026-11-06/2-rust-model_check.log @@ -0,0 +1,69 @@ +$ /tmp/thirdshift-benchmark-f7shrf8g/rust-target/debug/deps/pickup_model-1d72f02ce05acf2e --test-threads=1 --nocapture + +running 5 tests +test assert_requested_property ... ignored, known pre-Claim dispatch finding; run explicitly to see the failure +test current_protocol_reports_pre_claim_dispatch_counterexample ... Current, Dispatch: 7093 unique states +KNOWN FINDING: the literal dispatch property fails: Path[14]: +- ListReady(0) +- SelectHistory(0) +- CreateReservation(0) +- RefreshIssue(0) +- RefreshHistory(0) +- Dispatch(0) +- FailBeforeClaim(0) +- ReleaseReservation(0) +- ListReady(1) +- SelectHistory(1) +- CreateReservation(1) +- RefreshIssue(1) +- RefreshHistory(1) +- Dispatch(1) + +ok +test pickup_claimed_dispatch_safety ... Current, ClaimedDispatch: 86745 unique states +ok +test removing_claim_count_recheck_is_detected ... WithoutClaimCount, ClaimedDispatch: 20675 unique states +Claim-count mutation counterexample: Path[19]: +- ListReady(0) +- SelectHistory(0) +- CreateReservation(0) +- RefreshIssue(0) +- RefreshHistory(0) +- Dispatch(0) +- ReadClaimLabels(0) +- ListReady(1) +- SelectHistory(1) +- PutClaimLabels(0) +- KeepClaim(0) +- ReleaseReservation(0) +- CreateReservation(1) +- DayHandsBack +- RefreshIssue(1) +- RefreshHistory(1) +- Dispatch(1) +- ReadClaimLabels(1) +- PutClaimLabels(1) + +ok +test removing_empty_create_lease_is_detected ... UnconditionalCreate, ClaimedDispatch: 22520 unique states +Empty-lease mutation counterexample: Path[15]: +- ListReady(0) +- SelectHistory(0) +- CreateReservation(0) +- RefreshIssue(0) +- RefreshHistory(0) +- Dispatch(0) +- ReadClaimLabels(0) +- ListReady(1) +- SelectHistory(1) +- CreateReservation(1) +- RefreshIssue(1) +- RefreshHistory(1) +- PutClaimLabels(0) +- Dispatch(1) +- ReadClaimLabels(1) + +ok + +test result: ok. 4 passed; 0 failed; 1 ignored; 0 measured; 0 filtered out; finished in 1.12s + diff --git a/docs/research/go-vs-rust-benchmark-2026-11-06/results.json b/docs/research/go-vs-rust-benchmark-2026-11-06/results.json new file mode 100644 index 000000000..7da86c56d --- /dev/null +++ b/docs/research/go-vs-rust-benchmark-2026-11-06/results.json @@ -0,0 +1,559 @@ +{ + "revision": "82cc4dc2f9bad8a93dd7ae111cce1afe34ec30de", + "timestamp_utc": "2026-11-06T23:12:51.890391+00:00", + "host": "Linux-6.8.0-147-generic-x86_64-with-glibc2.39", + "cpu_count": 4, + "load_average": [ + 0.59765625, + 0.5009765625, + 0.4638671875 + ], + "jobs": 2, + "rust_static_target": "x86_64-unknown-linux-musl", + "tools": { + "go": "go version go1.24.5 linux/amd64", + "rustc": "rustc 1.99.1 (58cf8f9c3 2026-10-13)", + "cargo": "cargo 1.99.1 (5f94df478 2026-08-27)", + "git": "git version 2.43.0", + "readelf": "GNU readelf (GNU Binutils for Ubuntu) 2.42\nCopyright (C) 2024 Free Software Foundation, Inc.\nThis program is free software; you may redistribute it under the terms of\nthe GNU General Public License version 3 or (at your option) any later version.\nThis program has absolutely no warranty." + }, + "samples": [ + { + "go": { + "seconds": { + "cold_build": 10.197248228942044, + "incremental_build": 0.6308893270324916, + "test_compile": 17.080289611942135, + "test_execution": 154.08379880001303, + "model_check": 5.298803306999616, + "static_build": 0.24893545906525105 + }, + "commands": [ + { + "metric": "cold_build", + "argv": [ + "go", + "build", + "-trimpath", + "-o", + "/tmp/thirdshift-benchmark-sl4wkj08/thirdshift-go", + "." + ], + "cwd": "/tmp/thirdshift-benchmark-sl4wkj08/worktree/prototype/go", + "log": "1-go-cold_build.log" + }, + { + "metric": "incremental_build", + "argv": [ + "go", + "build", + "-trimpath", + "-o", + "/tmp/thirdshift-benchmark-sl4wkj08/thirdshift-go", + "." + ], + "cwd": "/tmp/thirdshift-benchmark-sl4wkj08/worktree/prototype/go", + "log": "1-go-incremental_build.log" + }, + { + "metric": "test_compile", + "argv": [ + "go", + "test", + "-c", + "-o", + "/tmp/thirdshift-benchmark-sl4wkj08/go-tests", + "." + ], + "cwd": "/tmp/thirdshift-benchmark-sl4wkj08/worktree/prototype/go", + "log": "1-go-test_compile.log" + }, + { + "metric": "test_execution", + "argv": [ + "/tmp/thirdshift-benchmark-sl4wkj08/go-tests", + "-test.skip=^TestBenchmark", + "-test.count=1", + "-test.parallel=2" + ], + "cwd": "/tmp/thirdshift-benchmark-sl4wkj08/worktree/prototype/go", + "log": "1-go-test_execution.log" + }, + { + "metric": "model_check", + "argv": [ + "/tmp/thirdshift-benchmark-sl4wkj08/go-tests", + "-test.run=^TestPickupModel", + "-test.count=1", + "-test.v" + ], + "cwd": "/tmp/thirdshift-benchmark-sl4wkj08/worktree/prototype/go", + "log": "1-go-model_check.log" + }, + { + "metric": "static_build", + "argv": [ + "go", + "build", + "-trimpath", + "-ldflags=-s -w", + "-o", + "/tmp/thirdshift-benchmark-sl4wkj08/thirdshift-go", + "." + ], + "cwd": "/tmp/thirdshift-benchmark-sl4wkj08/worktree/prototype/go", + "log": "1-go-static_build.log" + } + ], + "static_bytes": 3289272 + }, + "rust": { + "seconds": { + "cold_build": 166.9769773290027, + "incremental_build": 9.292317796964198, + "test_compile": 81.40013830002863, + "test_execution": 262.2594986030599, + "model_check": 1.1230978259118274, + "static_build": 357.9989838870242 + }, + "commands": [ + { + "metric": "cold_build", + "argv": [ + "cargo", + "build", + "--locked", + "--bin", + "thirdshift" + ], + "cwd": "/tmp/thirdshift-benchmark-sl4wkj08/worktree", + "log": "1-rust-cold_build.log" + }, + { + "metric": "incremental_build", + "argv": [ + "cargo", + "build", + "--locked", + "--bin", + "thirdshift" + ], + "cwd": "/tmp/thirdshift-benchmark-sl4wkj08/worktree", + "log": "1-rust-incremental_build.log" + }, + { + "metric": "test_compile", + "argv": [ + "cargo", + "test", + "--locked", + "--no-run", + "--message-format=json", + "--test", + "fresh_run", + "--test", + "pickup_run", + "--test", + "harness", + "--test", + "ci", + "--test", + "pickup_model" + ], + "cwd": "/tmp/thirdshift-benchmark-sl4wkj08/worktree", + "log": "1-rust-test_compile.log" + }, + { + "metric": "test_execution", + "argv": [ + "/tmp/thirdshift-benchmark-sl4wkj08/rust-target/debug/deps/fresh_run-8cd13fbb94404779", + "--test-threads=2" + ], + "cwd": "/tmp/thirdshift-benchmark-sl4wkj08/worktree", + "log": "1-rust-test_execution.log" + }, + { + "metric": "test_execution", + "argv": [ + "/tmp/thirdshift-benchmark-sl4wkj08/rust-target/debug/deps/pickup_run-0d766899e6782ad1", + "--test-threads=2" + ], + "cwd": "/tmp/thirdshift-benchmark-sl4wkj08/worktree", + "log": "1-rust-test_execution.log" + }, + { + "metric": "test_execution", + "argv": [ + "/tmp/thirdshift-benchmark-sl4wkj08/rust-target/debug/deps/harness-123ea558a3d165ad", + "--test-threads=2" + ], + "cwd": "/tmp/thirdshift-benchmark-sl4wkj08/worktree", + "log": "1-rust-test_execution.log" + }, + { + "metric": "test_execution", + "argv": [ + "/tmp/thirdshift-benchmark-sl4wkj08/rust-target/debug/deps/ci-54219ff6ab5ab934", + "--test-threads=2" + ], + "cwd": "/tmp/thirdshift-benchmark-sl4wkj08/worktree", + "log": "1-rust-test_execution.log" + }, + { + "metric": "test_execution", + "argv": [ + "/tmp/thirdshift-benchmark-sl4wkj08/rust-target/debug/deps/pickup_model-1d72f02ce05acf2e", + "--test-threads=2" + ], + "cwd": "/tmp/thirdshift-benchmark-sl4wkj08/worktree", + "log": "1-rust-test_execution.log" + }, + { + "metric": "model_check", + "argv": [ + "/tmp/thirdshift-benchmark-sl4wkj08/rust-target/debug/deps/pickup_model-1d72f02ce05acf2e", + "--test-threads=1", + "--nocapture" + ], + "cwd": "/tmp/thirdshift-benchmark-sl4wkj08/worktree", + "log": "1-rust-model_check.log" + }, + { + "metric": "static_build", + "argv": [ + "cargo", + "build", + "--locked", + "--release", + "--target", + "x86_64-unknown-linux-musl", + "--bin", + "thirdshift" + ], + "cwd": "/tmp/thirdshift-benchmark-sl4wkj08/worktree", + "log": "1-rust-static_build.log" + } + ], + "static_bytes": 21210280 + } + }, + { + "go": { + "seconds": { + "cold_build": 9.65935393399559, + "incremental_build": 0.6011072089895606, + "test_compile": 15.55158239195589, + "test_execution": 152.8258832580177, + "model_check": 5.214108006097376, + "static_build": 0.2694852700224146 + }, + "commands": [ + { + "metric": "cold_build", + "argv": [ + "go", + "build", + "-trimpath", + "-o", + "/tmp/thirdshift-benchmark-f7shrf8g/thirdshift-go", + "." + ], + "cwd": "/tmp/thirdshift-benchmark-f7shrf8g/worktree/prototype/go", + "log": "2-go-cold_build.log" + }, + { + "metric": "incremental_build", + "argv": [ + "go", + "build", + "-trimpath", + "-o", + "/tmp/thirdshift-benchmark-f7shrf8g/thirdshift-go", + "." + ], + "cwd": "/tmp/thirdshift-benchmark-f7shrf8g/worktree/prototype/go", + "log": "2-go-incremental_build.log" + }, + { + "metric": "test_compile", + "argv": [ + "go", + "test", + "-c", + "-o", + "/tmp/thirdshift-benchmark-f7shrf8g/go-tests", + "." + ], + "cwd": "/tmp/thirdshift-benchmark-f7shrf8g/worktree/prototype/go", + "log": "2-go-test_compile.log" + }, + { + "metric": "test_execution", + "argv": [ + "/tmp/thirdshift-benchmark-f7shrf8g/go-tests", + "-test.skip=^TestBenchmark", + "-test.count=1", + "-test.parallel=2" + ], + "cwd": "/tmp/thirdshift-benchmark-f7shrf8g/worktree/prototype/go", + "log": "2-go-test_execution.log" + }, + { + "metric": "model_check", + "argv": [ + "/tmp/thirdshift-benchmark-f7shrf8g/go-tests", + "-test.run=^TestPickupModel", + "-test.count=1", + "-test.v" + ], + "cwd": "/tmp/thirdshift-benchmark-f7shrf8g/worktree/prototype/go", + "log": "2-go-model_check.log" + }, + { + "metric": "static_build", + "argv": [ + "go", + "build", + "-trimpath", + "-ldflags=-s -w", + "-o", + "/tmp/thirdshift-benchmark-f7shrf8g/thirdshift-go", + "." + ], + "cwd": "/tmp/thirdshift-benchmark-f7shrf8g/worktree/prototype/go", + "log": "2-go-static_build.log" + } + ], + "static_bytes": 3289272 + }, + "rust": { + "seconds": { + "cold_build": 160.73648330604192, + "incremental_build": 8.886530747055076, + "test_compile": 80.00961200799793, + "test_execution": 298.1343800970353, + "model_check": 1.1185202499618754, + "static_build": 387.148877888103 + }, + "commands": [ + { + "metric": "cold_build", + "argv": [ + "cargo", + "build", + "--locked", + "--bin", + "thirdshift" + ], + "cwd": "/tmp/thirdshift-benchmark-f7shrf8g/worktree", + "log": "2-rust-cold_build.log" + }, + { + "metric": "incremental_build", + "argv": [ + "cargo", + "build", + "--locked", + "--bin", + "thirdshift" + ], + "cwd": "/tmp/thirdshift-benchmark-f7shrf8g/worktree", + "log": "2-rust-incremental_build.log" + }, + { + "metric": "test_compile", + "argv": [ + "cargo", + "test", + "--locked", + "--no-run", + "--message-format=json", + "--test", + "fresh_run", + "--test", + "pickup_run", + "--test", + "harness", + "--test", + "ci", + "--test", + "pickup_model" + ], + "cwd": "/tmp/thirdshift-benchmark-f7shrf8g/worktree", + "log": "2-rust-test_compile.log" + }, + { + "metric": "test_execution", + "argv": [ + "/tmp/thirdshift-benchmark-f7shrf8g/rust-target/debug/deps/fresh_run-8cd13fbb94404779", + "--test-threads=2" + ], + "cwd": "/tmp/thirdshift-benchmark-f7shrf8g/worktree", + "log": "2-rust-test_execution.log" + }, + { + "metric": "test_execution", + "argv": [ + "/tmp/thirdshift-benchmark-f7shrf8g/rust-target/debug/deps/pickup_run-0d766899e6782ad1", + "--test-threads=2" + ], + "cwd": "/tmp/thirdshift-benchmark-f7shrf8g/worktree", + "log": "2-rust-test_execution.log" + }, + { + "metric": "test_execution", + "argv": [ + "/tmp/thirdshift-benchmark-f7shrf8g/rust-target/debug/deps/harness-123ea558a3d165ad", + "--test-threads=2" + ], + "cwd": "/tmp/thirdshift-benchmark-f7shrf8g/worktree", + "log": "2-rust-test_execution.log" + }, + { + "metric": "test_execution", + "argv": [ + "/tmp/thirdshift-benchmark-f7shrf8g/rust-target/debug/deps/ci-54219ff6ab5ab934", + "--test-threads=2" + ], + "cwd": "/tmp/thirdshift-benchmark-f7shrf8g/worktree", + "log": "2-rust-test_execution.log" + }, + { + "metric": "test_execution", + "argv": [ + "/tmp/thirdshift-benchmark-f7shrf8g/rust-target/debug/deps/pickup_model-1d72f02ce05acf2e", + "--test-threads=2" + ], + "cwd": "/tmp/thirdshift-benchmark-f7shrf8g/worktree", + "log": "2-rust-test_execution.log" + }, + { + "metric": "model_check", + "argv": [ + "/tmp/thirdshift-benchmark-f7shrf8g/rust-target/debug/deps/pickup_model-1d72f02ce05acf2e", + "--test-threads=1", + "--nocapture" + ], + "cwd": "/tmp/thirdshift-benchmark-f7shrf8g/worktree", + "log": "2-rust-model_check.log" + }, + { + "metric": "static_build", + "argv": [ + "cargo", + "build", + "--locked", + "--release", + "--target", + "x86_64-unknown-linux-musl", + "--bin", + "thirdshift" + ], + "cwd": "/tmp/thirdshift-benchmark-f7shrf8g/worktree", + "log": "2-rust-static_build.log" + } + ], + "static_bytes": 21210280 + } + } + ], + "cpu": "DO-Premium-AMD", + "source_lines": { + "go": { + "lines": 1615, + "files": [ + "prototype/go/ci.go", + "prototype/go/claim.go", + "prototype/go/delivery.go", + "prototype/go/github.go", + "prototype/go/main.go", + "prototype/go/pickup.go", + "prototype/go/process.go", + "prototype/go/pull_request.go", + "prototype/go/ready.go", + "prototype/go/reservation.go", + "prototype/go/run.go", + "prototype/go/salvage.go", + "prototype/go/session.go", + "prototype/go/skills.go", + "prototype/go/states.go" + ] + }, + "rust": { + "lines": 29948, + "files": [ + "src/ci.rs", + "src/claim.rs", + "src/delivery/repair_loop.rs", + "src/delivery/stage.rs", + "src/delivery.rs", + "src/harness/adapter.rs", + "src/harness/agy/catalog.rs", + "src/harness/agy/events.rs", + "src/harness/agy/stream.rs", + "src/harness/agy.rs", + "src/harness/built_with.rs", + "src/harness/claude.rs", + "src/harness/codex/catalog.rs", + "src/harness/codex/events.rs", + "src/harness/codex/limits.rs", + "src/harness/codex/patches.rs", + "src/harness/codex/stream.rs", + "src/harness/codex.rs", + "src/harness/grok/catalog.rs", + "src/harness/grok.rs", + "src/harness/instructions.rs", + "src/harness/interpretation/event.rs", + "src/harness/interpretation/limits.rs", + "src/harness/interpretation/models.rs", + "src/harness/interpretation/security.rs", + "src/harness/interpretation/stream/events.rs", + "src/harness/interpretation/stream.rs", + "src/harness/interpretation.rs", + "src/harness/muse/catalog.rs", + "src/harness/muse/events.rs", + "src/harness/muse/log.rs", + "src/harness/muse/stream.rs", + "src/harness/muse.rs", + "src/harness/opencode/events.rs", + "src/harness/opencode/export.rs", + "src/harness/opencode/stream.rs", + "src/harness/opencode.rs", + "src/harness/process.rs", + "src/harness/settings.rs", + "src/harness/skill_load.rs", + "src/harness/summary.rs", + "src/harness/version.rs", + "src/harness.rs", + "src/interrupt.rs", + "src/main.rs", + "src/pickup.rs", + "src/process/faults.rs", + "src/process/session.rs", + "src/process/tree.rs", + "src/process.rs", + "src/pull_request/findings.rs", + "src/pull_request.rs", + "src/ready.rs", + "src/reservation.rs", + "src/run.rs", + "src/session/kind.rs", + "src/session/limits.rs", + "src/session/private.rs", + "src/session/recording.rs", + "src/session/review_reports.rs", + "src/session/trace/attributes.rs", + "src/session/trace/calls.rs", + "src/session/trace/export.rs", + "src/session/trace.rs", + "src/session/usage.rs", + "src/session.rs" + ] + } + }, + "input_sha256": { + "Cargo.lock": "ba795bd473f7264313428c8ad9c0c805226e5d86edc2298a3ad6e82de9e3da2d", + "prototype/go/go.mod": "c610c110713f1062c22ee371a1f017f1dbbd879bd6759cf56ec8e5e91348955d", + "prototype/go/factory-skills.json": "4efbb5bb35cff8a190e86feffe902daae74b14c40c3844c583ff0646be436f34" + } +} diff --git a/docs/research/go-vs-rust-prototype.md b/docs/research/go-vs-rust-prototype.md new file mode 100644 index 000000000..6b41694e8 --- /dev/null +++ b/docs/research/go-vs-rust-prototype.md @@ -0,0 +1,216 @@ +# Go prototype versus Rust: empirical comparison + +Issue #1835, under #1830. ADR-0001 remains unchanged; choosing a production +language belongs to the Day shift after reviewing the evidence. + +## Reproduce + +The measured revision and host, tool versions, input hashes, exact commands, +per-sample wall times and counted files are in +[results.json](go-vs-rust-benchmark-2026-11-06/results.json). + +```sh +python3 prototype/go/benchmark.py --ref 82cc4dc2 --samples 2 --jobs 2 --output /tmp/thirdshift-benchmark-results +``` + +See [setup and runner semantics](../../prototype/go/README.md#reproducible-comparison). +The timed workloads are pinned to that revision; later changes add the report +and its contract checks. +Use a new/empty output directory. A rerun on another host produces new numbers. +Measured commands run sequentially, with caps of two build jobs, Go processors +and test threads (Go tests without `t.Parallel` remain serial); ADR-0017 Rust +test cases run one at a time. No external network or LLM calls +occur during test execution. Cargo downloads are prepared outside the timings. + +## Scope and method + +The Go executable implements direct Ticket runs, streamed Claude sessions, +Claim ending, Delivery, bounded Repairs and Pickup reservation/dispatch. The +corresponding Rust code is compiled as the production executable: it also +contains the Console, experiments, scoring, multiple Harnesses, Specs, security +and other features. Rust has no independently buildable library for just this +slice. Its build time and binary size therefore include that larger product. +This measures the actual development cost of changing those Rust owners; it +does not measure equal-feature implementations or establish a language-only +speedup. Go's own README lists its unsupported behavior. + +| Behavior | Go owners | Rust owners / selected tests | +| --- | --- | --- | +| Ticket run and Claim ending | `run.go`, `claim.go`, `salvage.go` | `src/run.rs`, `src/claim.rs`; `fresh_run` | +| Delivery, Repair and CI | `delivery.go`, `ci.go`, `pull_request.go` | `src/delivery/`, `src/ci.rs`, `src/pull_request.rs`; `fresh_run`, `ci` | +| Harness and child lifetime | `session.go`, `process.go` | `src/session.rs`, `src/process/`, `src/harness/`; `harness` | +| Pickup and reservation | `pickup.go`, `ready.go`, `reservation.go` | `src/pickup.rs`, `src/ready.rs`, `src/reservation.rs`; `pickup_run`, `pickup_model` | + +Each sample starts with a fresh detached Worktree and empty `GOCACHE`, +`GOMODCACHE` and `CARGO_TARGET_DIR`. Rust registry downloads and toolchains are +reused; OS filesystem caches are not flushed. Go's cold build compiles its +standard library cache; Rust reuses the standard library supplied by rustup. +Cold development builds use `go build -trimpath` with `CGO_ENABLED=0` and +`cargo build --locked --bin thirdshift`. An incremental build appends a comment +to each entry point, rebuilds with the same cache, then restores the source. +This is an edited package rebuild, rather than a no-change cache lookup. + +`test_compile` follows those builds: its test artifacts have not been built, +but it is not the first command against an empty cache. Go uses `go test -c` +with default path handling, which differs from the development build's +`-trimpath` and can require separate cache entries. Rust compiles only +`fresh_run`, `pickup_run`, `harness`, `ci` +and `pickup_model`. `test_execution` invokes those compiled binaries directly; +Go's `TestMain` still performs its normal warm `go build`. The Go benchmark +contract is excluded to prevent recursion. Rust tests exercise the equivalent +Ticket/Delivery/Pickup/Harness/CI seams but are more numerous and cover a +larger product. These are real existing suites, not matched microbenchmarks. + +`static_build` separately builds stripped executables: Go `-ldflags='-s -w'`, +Rust `--release --target x86_64-unknown-linux-musl` with +`CARGO_PROFILE_RELEASE_STRIP=symbols`. It uses the sample's caches, but Rust's +release/musl artifacts have not been built yet. `readelf -lW` and `-dW` must +show neither `INTERP` nor `DT_NEEDED`; a dynamic result fails the runner. +The Rust release profile differs from the distribution profile's thin LTO. +This is a static executable size comparison, not release packaging size. + +## Measurements + +Host: DO-Premium-AMD; 4 visible CPUs; go version go1.24.5 linux/amd64; rustc 1.99.1 (58cf8f9c3 2026-10-13). Native GNU/Linux development builds; x86_64 musl Rust static builds. + +Wall-clock seconds (each column is one fresh Worktree): + +| Metric | Go 1 | Go 2 | Rust 1 | Rust 2 | +| --- | ---: | ---: | ---: | ---: | +| cold_build | 10.197 | 9.659 | 166.977 | 160.736 | +| incremental_build | 0.631 | 0.601 | 9.292 | 8.887 | +| test_compile | 17.080 | 15.552 | 81.400 | 80.010 | +| test_execution | 154.084 | 152.826 | 262.259 | 298.134 | +| model_check | 5.299 | 5.214 | 1.123 | 1.119 | +| static_build | 0.249 | 0.269 | 357.999 | 387.149 | + +| Inventory | Go | Rust | +| --- | ---: | ---: | +| Physical source lines in manifest | 1615 | 29948 | +| Stripped static bytes, sample 1 | 3289272 | 21210280 | +| Stripped static bytes, sample 2 | 3289272 | 21210280 | + +For these artifacts on this host, the Go prototype builds substantially +faster and yields a smaller executable. Its existing tests still take about +2.5 minutes, including real Delivery grace periods. Rust's broader selected +suite takes 4.4–5.0 minutes. The build difference is useful evidence of current +development cost, while the unequal feature and test scope prevents estimating +the benefit of a complete Go rewrite from these numbers. + +Two samples on a shared host describe observed cost, without confidence +intervals or a claim that their ratio is a language benchmark. Full logs from +the run are retained at `/tmp/thirdshift-1835-benchmark.PgYbxF/`; the JSON +retains command/log names even after disposable Worktrees have been removed. + +Source counts are physical lines, including blank lines and comments. The +JSON manifest lists every selected file: Go's production `*.go` files; Rust's +entry point and the `run`, `delivery`, `pickup`, `ready`, `reservation`, `claim`, +`process`, `session`, `harness`, `ci`, `pull_request` and `interrupt` owners. +Separate test/support files are excluded; Rust's inline tests remain counted. +Dependencies, embedded Factory-skill data, Go's benchmark tooling and Rust +owners outside this manifest are excluded. The Rust manifest includes more +Harnesses and lifecycle features, and omits transitive shared helpers: its +line count is an explicitly scoped inventory, not a normalized complexity +measure or a count of all code linked into the executable. + +## Subprocess and signal ergonomics + +Go's [`process.go`](../../prototype/go/process.go) uses `os/exec`, one goroutine +waiting on `Cmd.Wait`, and a `select` over completion and context cancellation. +It assigns a process group, sends SIGINT then SIGTERM with two-second grace +periods, then SIGKILL, and waits for the child. The deferred group kill handles +members left after the leader exits. [`session.go`](../../prototype/go/session.go) +streams JSON through a pipe while supervision runs concurrently. Claim ending +and salvage receive independent contexts after interruption. This is compact, +but descendant processes that escape the group are outside that mechanism. +`WaitDelay` addresses pipe/child waiting in `os/exec`; it does not implement +the escalation policy. See the [Go Cmd contract](https://pkg.go.dev/os/exec#Cmd). + +Rust's [`process.rs`](../../src/process.rs) owns the child and I/O worker threads, +uses completion channels and RAII cleanup, and propagates errors from partial +setup. [`process/tree.rs`](../../src/process/tree.rs) snapshots descendants and +groups, with ten-second signal grace periods; +[`process/session.rs`](../../src/process/session.rs) also sweeps work by session +marker or scope after a leader exits. Those guarantees account for some of +the extra implementation. The production CLI's child supervision uses Rust +threads; `tokio` supplies the Console server runtime and blocking-task bridge +([`console.rs`](../../src/console.rs)), rather than replacing that supervisor. +Goroutines make the simple wait/cancel flow shorter; they do not remove the +need to implement process ownership, escaped-child cleanup and cancellation +semantics. The comparison is an inspection of these implementations, not an +ergonomics score inferred from timing. + +## Exhaustiveness guarantees + +Go can model closed alternatives with a package-private marker interface and +distinct variant structs, or with a named integer type and constants. Neither +pattern makes the compiler require every type/constant in a switch; interface +closure also depends on the package discipline. The prototype uses the latter: +four named state types in [`states.go`](../../prototype/go/states.go), plus +[`exhaustiveness_test.go`](../../prototype/go/exhaustiveness_test.go)'s typed-AST +check. It rejects missing constants even with a `default`, and includes negative +fixtures for a newly added variant. This gives a useful test-time gate for +this package's expression switches. It is not general sum-type checking: +runtime integer values can be invalid, payloads are not variant-specific, and +other dispatch forms or packages would require checker coverage. The +[Go specification](https://go.dev/ref/spec#Switch_statements) defines switch +behavior without an exhaustiveness requirement. + +Rust's payload-bearing `enum` and `match` make missing cases a compile error +([Rust book](https://doc.rust-lang.org/book/ch06-02-match.html)). A wildcard +can absorb future variants, so [`Cargo.toml`](../../Cargo.toml) enables +[`clippy::wildcard_enum_match_arm`](https://clippy.rust-lang.org/master/index.html#wildcard_enum_match_arm) +and CI runs clippy with `-D warnings`; +documented exceptions use `#[expect]`. Rust therefore offers compiler and +lint guarantees without maintaining a project-specific syntax checker. +The prototype demonstrates mitigation, rather than parity of guarantees. + +## Model checking ADR-0017 + +Go's [`pickup_model_test.go`](../../prototype/go/pickup_model_test.go) implements +a deterministic BFS with comparable states, a visited set and predecessor/action +links. Its tests check the guarded protocol and require shortest counterexamples +for `WithoutClaimCount` (19 actions) and `UnconditionalCreate` (16). The +[request mapping and bounds](../../prototype/go/PICKUP-MODEL.md) are part of +the evidence. Maintaining the exploration engine, witness replay and sensitivity +tests is project work; no third-party model checker is used. + +Rust's [`tests/pickup_model.rs`](../../tests/pickup_model.rs) uses +`stateright = 0.31.0` with `spawn_bfs().join()`, property assertions and +discovery paths ([stateright Checker](https://docs.rs/stateright/0.31.0/stateright/trait.Checker.html)). +Both models bound competition to three hosts and test reservation leases and +the Claim-count generation check. Their state graphs and dispatch markers +differ: Go claims before Harness dispatch; Rust's model retains the known +pre-Claim literal dispatch counterexample and separately checks claimed-dispatch +safety. Rust's ignored negative assertion stays ignored in these timings; +the passing regression requires the known counterexample to remain visible. +Go's guarded protocol checks both dispatch predicates. + +Retained model output: [Go sample 1](go-vs-rust-benchmark-2026-11-06/1-go-model_check.log), +[Go sample 2](go-vs-rust-benchmark-2026-11-06/2-go-model_check.log), +[Rust sample 1](go-vs-rust-benchmark-2026-11-06/1-rust-model_check.log), +[Rust sample 2](go-vs-rust-benchmark-2026-11-06/2-rust-model_check.log). + +In the first sample, Go's guarded search visited 1,993,250 states and its +mutations visited 119,601 / 223,036 states. Rust's claimed-dispatch safety +search visited 86,745 states, its known literal-dispatch counterexample search +7,093, and its mutations 20,675 / 22,520. These are different exploration +workloads; state totals cannot be treated as matched checker throughput. + +`model_check` records the compiled model-test command's complete wall time, +including Go `TestMain`'s warm build, test startup, safety exploration, +mutation/counterexample exploration and output. Rust runs four ordinary tests; +Go runs three (one has two mutation subtests). It is not pure checker-kernel +time, and different explored states preclude attributing a timing ratio to +BFS implementation quality. Both remain bounded evidence, not proof for +unbounded hosts or arbitrary GitHub consistency. Neither model is changed by +this work. + +## Decision boundary: ADR-0001 + +This report supplies reproducible build/test/size observations and concrete +tradeoffs for the Day shift. A full Go migration still needs product parity, +an agreed replacement for Rust's exhaustiveness gate, and a maintainable +verification strategy. The two small-sample suites and inventories cannot +price that rewrite. [ADR-0001](../adr/0001-rust-binary-with-embedded-skills.md) +is deliberately left unchanged. diff --git a/prototype/go/README.md b/prototype/go/README.md index 1f68d70b6..d1d76bca9 100644 --- a/prototype/go/README.md +++ b/prototype/go/README.md @@ -18,6 +18,39 @@ CGO_ENABLED=0 go build -o /tmp/thirdshift-go . The build produces a single static Linux binary; the CLI tests inspect its ELF program headers and imported libraries. No Go dependencies are needed. + +## Reproducible comparison + +The [empirical Go/Rust report](../../docs/research/go-vs-rust-prototype.md) +records the measured revision, scope and limitations for #1835. From the +repository root, on native GNU/Linux with Go on `PATH`, the native Rust musl +target installed (`rustup target add x86_64-unknown-linux-musl` on x86_64), +its C linker (`musl-tools` on Ubuntu), and binutils (`readelf`): + +```sh +python3 prototype/go/benchmark.py --ref HEAD --samples 3 --jobs 2 --output /tmp/thirdshift-benchmark-results +``` + +The output directory must be new or empty. Only committed sources at `--ref` +are measured. Each sample gets a fresh detached Worktree, private Go build/module +caches and a private Cargo target directory. Rust dependencies are downloaded +before timing. Timings exclude Worktree creation and cleanup, toolchain setup +and downloads; they include compiler/linker processes and their children. +Cold and edited incremental builds use development profiles. Test compilation, +compiled test execution, ADR-0017 checks and stripped static release builds +are separate metrics. JSON records every command and the source-count manifest; +logs retain full tool output. Failed commands fail the runner and remove its +Worktree, leaving logs but no completed `results.json`. No GitHub/LLM credentials +are needed. Allow several minutes per sample and several GB of temporary disk. + +Rust builds the production executable that owns the equivalent slices; the +prototype does not have feature parity. Rust test execution selects `fresh_run`, +`pickup_run`, `harness`, `ci` and `pickup_model`. Go executes its prototype tests, +excluding `TestBenchmark*` to prevent recursion. The report explains why these +numbers cannot estimate a complete rewrite's speedup. + +## Running a Ticket + Run it from the Launch directory of a disposable repository: ```sh diff --git a/prototype/go/benchmark.py b/prototype/go/benchmark.py new file mode 100644 index 000000000..85d5f6e4a --- /dev/null +++ b/prototype/go/benchmark.py @@ -0,0 +1,200 @@ +#!/usr/bin/env python3 +"""Measure the committed Go vertical slice and its Rust owners in fresh worktrees. + +Requires Linux, Python 3, Git, Go, Cargo, the native Rust musl target/linker, +and readelf. Dependency downloads are prepared outside timed regions. +""" +import argparse +from datetime import datetime, timezone +import hashlib +import json +import os +from pathlib import Path +import platform +import shlex +import subprocess +import tempfile +import time + + +RUST_TESTS = ["fresh_run", "pickup_run", "harness", "ci", "pickup_model"] +RUST_OWNERS = ["main", "run", "delivery", "pickup", "ready", "reservation", + "claim", "process", "session", "harness", "ci", "pull_request", "interrupt"] + + +def capture(command, cwd, env=None): + return subprocess.check_output(command, cwd=cwd, env=env, text=True, stderr=subprocess.STDOUT).strip() + + +def run_logged(command, cwd, env, log): + try: + subprocess.run(command, cwd=cwd, env=env, stdout=log, stderr=subprocess.STDOUT, check=True) + except subprocess.CalledProcessError as error: + log.flush() + error.output = Path(log.name).read_text()[-4000:] + raise + + +def source_lines(worktree): + go = sorted((worktree / "prototype/go").glob("*.go")) + rust = [] + for owner in RUST_OWNERS: + rust.extend((worktree / "src").glob(owner + ".rs")) + rust.extend((worktree / "src" / owner).rglob("*.rs")) + return { + language: {"lines": sum(len(path.read_text().splitlines()) for path in paths), + "files": [str(path.relative_to(worktree)) for path in paths]} + for language, paths in { + "go": [path for path in go if not path.name.endswith("_test.go")], + "rust": sorted(path for path in rust if not path.stem.endswith(("test", "tests", "test_support", "testing"))), + }.items() + } + + +class Sample: + def __init__(self, worktree, output, index, jobs, target, scratch): + self.worktree, self.output, self.index = worktree, output, index + self.jobs, self.target, self.scratch = jobs, target, scratch + self.env = os.environ.copy() + for name in ["RUSTFLAGS", "CARGO_ENCODED_RUSTFLAGS", "CARGO_BUILD_TARGET", + "RUSTC_WRAPPER", "RUSTC_WORKSPACE_WRAPPER", "GOFLAGS", "GOOS", "GOARCH"]: + self.env.pop(name, None) + self.env.update({"CGO_ENABLED": "0", "GOMAXPROCS": str(jobs), "GOFLAGS": "-p=" + str(jobs), + "GOCACHE": str(scratch / "go-cache"), "GOMODCACHE": str(scratch / "go-mod"), + "CARGO_TARGET_DIR": str(scratch / "rust-target"), "CARGO_BUILD_JOBS": str(jobs), + "CARGO_PROFILE_RELEASE_STRIP": "symbols"}) + self.rows = {language: {"seconds": {}, "commands": []} for language in ["go", "rust"]} + + def measure(self, language, metric, commands): + row = self.rows[language] + log_path = self.output / f"{self.index}-{language}-{metric}.log" + cwd = self.worktree / "prototype/go" if language == "go" else self.worktree + print(f"sample {self.index}: {language} {metric} -> {log_path}", flush=True) + started = time.perf_counter() + with log_path.open("w") as log: + for command in commands: + row["commands"].append({"metric": metric, "argv": list(map(str, command)), + "cwd": str(cwd), "log": log_path.name}) + log.write("$ " + shlex.join(map(str, command)) + "\n") + log.flush() + run_logged(command, cwd, self.env, log) + row["seconds"][metric] = time.perf_counter() - started + return log_path + + def verify_static(self, language, binary): + headers = capture(["readelf", "-lW", str(binary)], self.worktree, self.env) + dynamic = capture(["readelf", "-dW", str(binary)], self.worktree, self.env) + if "INTERP" in headers or "(NEEDED)" in dynamic: + raise ValueError(f"{language} binary is not static: {binary}") + (self.output / f"{self.index}-{language}-static.log").write_text(headers + "\n" + dynamic + "\n") + self.rows[language]["static_bytes"] = binary.stat().st_size + + def run(self): + go_binary = self.scratch / "thirdshift-go" + go_build = ["go", "build", "-trimpath", "-o", str(go_binary), "."] + self.measure("go", "cold_build", [go_build]) + rust_build = ["cargo", "build", "--locked", "--bin", "thirdshift"] + self.measure("rust", "cold_build", [rust_build]) + # A content edit forces package recompilation; an unchanged build is a + # cache-hit benchmark instead. Only this disposable worktree is edited. + for language, source, command in [("go", "prototype/go/main.go", go_build), + ("rust", "src/main.rs", rust_build)]: + path = self.worktree / source + original = path.read_bytes() + path.write_bytes(original + b"\n// benchmark: incremental source edit\n") + try: + self.measure(language, "incremental_build", [command]) + finally: + path.write_bytes(original) + + go_tests = self.scratch / "go-tests" + self.measure("go", "test_compile", [["go", "test", "-c", "-o", str(go_tests), "."]]) + # Exclude the runner/report contract to prevent recursive benchmarking. + self.measure("go", "test_execution", [[str(go_tests), "-test.skip=^TestBenchmark", "-test.count=1", + f"-test.parallel={self.jobs}"]]) + self.measure("go", "model_check", [[str(go_tests), "-test.run=^TestPickupModel", "-test.count=1", "-test.v"]]) + test_command = ["cargo", "test", "--locked", "--no-run", "--message-format=json"] + for name in RUST_TESTS: + test_command += ["--test", name] + compiled = self.measure("rust", "test_compile", [test_command]) + executables = {} + for line in compiled.read_text().splitlines(): + if line.startswith("{"): + message = json.loads(line) + if message.get("reason") == "compiler-artifact" and message.get("executable") and message.get("profile", {}).get("test"): + executables[message["target"]["name"]] = message["executable"] + if set(executables) != set(RUST_TESTS): + raise ValueError(f"expected Rust test targets {RUST_TESTS}, got {sorted(executables)}") + self.measure("rust", "test_execution", [[executables[name], f"--test-threads={self.jobs}"] for name in RUST_TESTS]) + self.measure("rust", "model_check", [[executables["pickup_model"], "--test-threads=1", "--nocapture"]]) + self.measure("go", "static_build", [["go", "build", "-trimpath", "-ldflags=-s -w", "-o", str(go_binary), "."]]) + self.verify_static("go", go_binary) + self.measure("rust", "static_build", [["cargo", "build", "--locked", "--release", "--target", self.target, "--bin", "thirdshift"]]) + self.verify_static("rust", self.scratch / "rust-target" / self.target / "release/thirdshift") + return self.rows + + +def positive(value): + number = int(value) + if number < 1: + raise argparse.ArgumentTypeError("must be positive") + return number + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--repo", type=Path, default=Path(__file__).resolve().parents[2]) + parser.add_argument("--ref", default="HEAD", help="committed revision to measure (uncommitted edits excluded)") + parser.add_argument("--output", type=Path, required=True, help="new or empty results directory") + parser.add_argument("--samples", type=positive, default=3) + parser.add_argument("--jobs", type=positive, default=2) + args = parser.parse_args() + repo = args.repo.resolve() + revision = capture(["git", "rev-parse", "--verify", args.ref + "^{commit}"], repo) + host = capture(["rustc", "-vV"], repo).split("host: ")[-1].splitlines()[0] + target = host.replace("-gnu", "-musl") + if platform.system() != "Linux" or target == host: + raise ValueError("run on native GNU/Linux with its Rust musl target installed") + output = args.output.resolve() + output.mkdir(parents=True, exist_ok=True) + if any(output.iterdir()): + raise ValueError("output directory must be empty") + data = {"revision": revision, "timestamp_utc": datetime.now(timezone.utc).isoformat(), + "host": platform.platform(), "cpu_count": os.cpu_count(), "load_average": os.getloadavg(), + "jobs": args.jobs, "rust_static_target": target, + "tools": {name: capture(command, repo) for name, command in { + "go": ["go", "version"], "rustc": ["rustc", "--version"], "cargo": ["cargo", "--version"], + "git": ["git", "--version"], "readelf": ["readelf", "--version"]}.items()}, + "samples": []} + cpuinfo = Path("/proc/cpuinfo").read_text() + data["cpu"] = next((line.split(":", 1)[1].strip() for line in cpuinfo.splitlines() if line.startswith("model name")), platform.machine()) + for index in range(1, args.samples + 1): + with tempfile.TemporaryDirectory(prefix="thirdshift-benchmark-") as temp: + scratch = Path(temp) + worktree = scratch / "worktree" + subprocess.run(["git", "worktree", "add", "--detach", str(worktree), revision], cwd=repo, check=True, + stdout=subprocess.DEVNULL) + try: + if index == 1: + data["source_lines"] = source_lines(worktree) + data["input_sha256"] = {path: hashlib.sha256((worktree / path).read_bytes()).hexdigest() + for path in ["Cargo.lock", "prototype/go/go.mod", "prototype/go/factory-skills.json"] + if (worktree / path).exists()} + # Downloads do not contaminate cold compiler timings. + with (output / "dependency-fetch.log").open("w") as log: + run_logged(["cargo", "fetch", "--locked"], worktree, None, log) + sample = Sample(worktree, output, index, args.jobs, target, scratch) + data["samples"].append(sample.run()) + finally: + subprocess.run(["git", "worktree", "remove", "--force", str(worktree)], cwd=repo, check=True) + (output / "results.json").write_text(json.dumps(data, indent=2) + "\n") + print(f"completed: {output / 'results.json'}", flush=True) + + +if __name__ == "__main__": + try: + main() + except subprocess.CalledProcessError as error: + raise SystemExit(error.output or f"command failed: {error.cmd}; see output logs") + except (OSError, ValueError) as error: + raise SystemExit(str(error)) diff --git a/prototype/go/benchmark_test.go b/prototype/go/benchmark_test.go new file mode 100644 index 000000000..75aa652d5 --- /dev/null +++ b/prototype/go/benchmark_test.go @@ -0,0 +1,13 @@ +package main_test + +import ( + "os/exec" + "testing" +) + +func TestBenchmarkRunner(t *testing.T) { + cmd := exec.Command("python3", "-m", "unittest", "discover", "-p", "benchmark_test.py", "-v") + if output, err := cmd.CombinedOutput(); err != nil { + t.Fatalf("benchmark contract: %v\n%s", err, output) + } +} diff --git a/prototype/go/benchmark_test.py b/prototype/go/benchmark_test.py new file mode 100644 index 000000000..3ef5c420a --- /dev/null +++ b/prototype/go/benchmark_test.py @@ -0,0 +1,144 @@ +"""Contract tests at the benchmark CLI/tool and published-report seams.""" +import json +import os +from pathlib import Path +import re +import subprocess +import tempfile +import unittest + + +RUNNER = Path(__file__).with_name("benchmark.py") + + +class BenchmarkCLI(unittest.TestCase): + def setUp(self): + self.temp = tempfile.TemporaryDirectory(prefix="benchmark-contract-") + self.addCleanup(self.temp.cleanup) + self.root = Path(self.temp.name) + self.repo = self.root / "repo" + self.repo.mkdir() + self.env = dict(os.environ, GIT_CONFIG_GLOBAL="/dev/null", GIT_CONFIG_NOSYSTEM="1") + for args in [("init", "-q"), ("config", "user.name", "Test"), + ("config", "user.email", "test@example.com")]: + self.git(*args) + for name in ["prototype/go/main.go", "src/main.rs", "tests/pickup_model.rs"]: + path = self.repo / name + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text("// fixture\n") + self.git("add", ".") + self.git("-c", "core.hooksPath=/dev/null", "commit", "-qm", "fixture") + self.tools = self.root / "tools" + self.tools.mkdir() + # Only external tool processes are faked. Git worktrees, logs, timings, + # source counting, serialization and cleanup run for real. + tool = self.tools / "tool" + tool.write_text('''#!/usr/bin/env python3 +import json, os, pathlib, sys +name = pathlib.Path(sys.argv[0]).name +args = sys.argv[1:] +with open(os.environ["BENCH_CALLS"], "a") as out: + out.write(json.dumps({"tool": name, "args": args, "cwd": os.getcwd(), + "gocache": os.environ.get("GOCACHE"), "target": os.environ.get("CARGO_TARGET_DIR")}) + "\\n") +if "--version" in args or args == ["version"] or args == ["-vV"]: + print("host: x86_64-unknown-linux-gnu" if args == ["-vV"] else "fixture 1.0") +elif os.environ.get("BENCH_FAIL") == name and "build" in args: + print("intentional tool failure", file=sys.stderr) + sys.exit(19) +elif name == "readelf": + print("INTERP" if os.environ.get("BENCH_DYNAMIC") else "ELF fixture; no dynamic section") +elif name == "go": + output = pathlib.Path(args[args.index("-o") + 1]) + output.write_text("#!/bin/sh\\nexit 0\\n") + output.chmod(0o755) +elif name == "cargo": + if "fetch" in args: + sys.exit(0) + target = pathlib.Path(os.environ["CARGO_TARGET_DIR"]) + if "test" in args: + for test in ["fresh_run", "pickup_run", "harness", "ci", "pickup_model"]: + output = target / "debug" / test + output.parent.mkdir(parents=True, exist_ok=True) + output.write_text("#!/bin/sh\\nexit 0\\n") + output.chmod(0o755) + print(json.dumps({"reason": "compiler-artifact", "target": {"name": test}, + "profile": {"test": True}, "executable": str(output)})) + elif "--release" in args: + output = target / args[args.index("--target") + 1] / "release" / "thirdshift" + output.parent.mkdir(parents=True, exist_ok=True) + output.write_bytes(b"static fixture") +''') + tool.chmod(0o755) + for name in ["go", "cargo", "rustc", "readelf"]: + (self.tools / name).symlink_to(tool) + self.calls = self.root / "calls.jsonl" + self.env.update(PATH=str(self.tools) + ":" + os.environ["PATH"], BENCH_CALLS=str(self.calls)) + + def git(self, *args): + return subprocess.check_output(["git", "-C", str(self.repo), *args], env=self.env, text=True) + + def run_benchmark(self, **environment): + return subprocess.run(["python3", str(RUNNER), "--repo", str(self.repo), + "--samples", "2", "--output", str(self.root / "results")], + env=dict(self.env, **environment), text=True, capture_output=True) + + def test_runner_records_both_languages_and_removes_worktrees(self): + result = self.run_benchmark() + self.assertEqual(result.returncode, 0, result.stderr) + data = json.loads((self.root / "results/results.json").read_text()) + self.assertEqual(len(data["samples"]), 2) + for sample in data["samples"]: + for language in ["go", "rust"]: + row = sample[language] + for metric in ["cold_build", "incremental_build", "test_compile", "test_execution", "model_check", "static_build"]: + self.assertGreaterEqual(row["seconds"][metric], 0) + self.assertGreater(row["static_bytes"], 0) + self.assertEqual(data["source_lines"]["go"]["lines"], 1) + calls = [json.loads(line) for line in self.calls.read_text().splitlines()] + caches = {call["gocache"] for call in calls if call["tool"] == "go" and "build" in call["args"]} + self.assertEqual(len(caches), 2, "each sample needs a private cache") + self.assertTrue(all(str(self.repo) != call["cwd"] for call in calls if call["tool"] == "go" and "build" in call["args"])) + self.assertEqual(self.git("worktree", "list", "--porcelain").count("worktree "), 1) + self.assertEqual(self.git("status", "--porcelain"), "") + self.assertTrue(list((self.root / "results").glob("*.log"))) + + def test_tool_failure_is_nonzero_and_cleans_up(self): + result = self.run_benchmark(BENCH_FAIL="cargo") + self.assertNotEqual(result.returncode, 0) + self.assertIn("intentional tool failure", result.stderr) + self.assertFalse((self.root / "results/results.json").exists()) + self.assertEqual(self.git("worktree", "list", "--porcelain").count("worktree "), 1) + + def test_dynamic_binary_is_rejected(self): + result = self.run_benchmark(BENCH_DYNAMIC="1") + self.assertNotEqual(result.returncode, 0) + self.assertIn("static", result.stderr) + + +class PublishedReport(unittest.TestCase): + def test_report_is_complete_and_numbers_match_measured_artifact(self): + root = RUNNER.parents[2] + report = (root / "docs/research/go-vs-rust-prototype.md").read_text() + artifact = root / "docs/research/go-vs-rust-benchmark-2026-11-06/results.json" + data = json.loads(artifact.read_text()) + self.assertIn("go-vs-rust-benchmark-2026-11-06/results.json", report) + for topic in ["Reproduce", "Scope", "Subprocess", "Exhaustiveness", "Model checking", "ADR-0001"]: + self.assertIn(topic, report) + self.assertRegex(data["revision"], r"^[0-9a-f]{40}$") + self.assertGreaterEqual(len(data["samples"]), 2) + for metric in ["cold_build", "incremental_build", "test_compile", "test_execution", "model_check", "static_build"]: + row = re.search(r"^\| " + metric + r" \| (.+) \|$", report, re.MULTILINE) + self.assertIsNotNone(row, metric) + for language in ["go", "rust"]: + for sample in data["samples"]: + value = sample[language]["seconds"][metric] + self.assertGreater(value, 0) + self.assertIn(f"{value:.3f}", row.group(1), metric) + for language in ["go", "rust"]: + self.assertIn(str(data["source_lines"][language]["lines"]), report) + for sample in data["samples"]: + self.assertIn(str(sample[language]["static_bytes"]), report) + + +if __name__ == "__main__": + unittest.main()