From cbf97fafd46c22d697f6b71b1f7bdad871061d3e Mon Sep 17 00:00:00 2001 From: forhappy Date: Sun, 18 Oct 2026 10:14:23 -0700 Subject: [PATCH 1/2] fix(build): align SQLite dependency locks Keep the bundled WAL-reset fix while restoring locked CI builds. Align nested examples, cookbook, fuzz and native probes with 0.39. Refs #79 --- cookbook/Cargo.lock | 45 +++-- .../application-builder-service/Cargo.lock | 45 +++-- .../application-builder-service/Cargo.toml | 2 +- crates/cellule-ltx/fuzz/Cargo.lock | 163 ++++++++++------- crates/cellule-ltx/perf/cellule/Cargo.lock | 152 +++++++++++---- .../cellule-ltx/perf/replica-cost/Cargo.lock | 173 ++++++++++-------- 6 files changed, 381 insertions(+), 199 deletions(-) diff --git a/cookbook/Cargo.lock b/cookbook/Cargo.lock index d3aa2715..7a5c540d 100644 --- a/cookbook/Cargo.lock +++ b/cookbook/Cargo.lock @@ -1190,9 +1190,9 @@ checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" [[package]] name = "foldhash" -version = "0.1.5" +version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2" +checksum = "77ce24cb58228fbb8aa041425bb1050850ac19177686ea6e0f41a70416f56fdb" [[package]] name = "form_urlencoded" @@ -1340,9 +1340,9 @@ dependencies = [ [[package]] name = "hashbrown" -version = "0.15.5" +version = "0.16.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1" +checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100" dependencies = [ "foldhash", ] @@ -1355,11 +1355,11 @@ checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" [[package]] name = "hashlink" -version = "0.10.0" +version = "0.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7382cf6263419f2d8df38c55d7da83da5c18aef87fc7a7fc1fb1e344edfe14c1" +checksum = "824e001ac4f3012dd16a264bec811403a67ca9deb6c102fc5049b32c4574b35f" dependencies = [ - "hashbrown 0.15.5", + "hashbrown 0.16.1", ] [[package]] @@ -1754,9 +1754,9 @@ checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2" [[package]] name = "libsqlite3-sys" -version = "0.32.0" +version = "0.37.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fbb8270bb4060bd76c6e96f20c52d80620f1d82a3470885694e41e0f81ef6fe7" +checksum = "b1f111c8c41e7c61a49cd34e44c7619462967221a6443b0ec299e0ac30cfb9b1" dependencies = [ "cc", "pkg-config", @@ -2488,11 +2488,21 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "rsqlite-vfs" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c51c9ae4df8a7fba42103df5c621fa3c37eccf3a3c650879e90fc48b11cc192c" +dependencies = [ + "hashbrown 0.16.1", + "thiserror", +] + [[package]] name = "rusqlite" -version = "0.34.0" +version = "0.39.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "37e34486da88d8e051c7c0e23c3f15fd806ea8546260aa2fec247e97242ec143" +checksum = "a0d2b0146dd9661bf67bb107c0bb2a55064d556eeb3fc314151b957f313bcd4e" dependencies = [ "bitflags", "fallible-iterator", @@ -2500,6 +2510,7 @@ dependencies = [ "hashlink", "libsqlite3-sys", "smallvec", + "sqlite-wasm-rs", ] [[package]] @@ -2877,6 +2888,18 @@ dependencies = [ "der", ] +[[package]] +name = "sqlite-wasm-rs" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc3efc0da82635d7e1ced0053bbbfa8c7ab9645d0bf36ceb4f7127bb85315d75" +dependencies = [ + "cc", + "js-sys", + "rsqlite-vfs", + "wasm-bindgen", +] + [[package]] name = "stable_deref_trait" version = "1.2.1" diff --git a/crates/cellule-axum/examples/application-builder-service/Cargo.lock b/crates/cellule-axum/examples/application-builder-service/Cargo.lock index fadb5955..d5bfa558 100644 --- a/crates/cellule-axum/examples/application-builder-service/Cargo.lock +++ b/crates/cellule-axum/examples/application-builder-service/Cargo.lock @@ -660,9 +660,9 @@ checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" [[package]] name = "foldhash" -version = "0.1.5" +version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2" +checksum = "77ce24cb58228fbb8aa041425bb1050850ac19177686ea6e0f41a70416f56fdb" [[package]] name = "form_urlencoded" @@ -810,9 +810,9 @@ dependencies = [ [[package]] name = "hashbrown" -version = "0.15.5" +version = "0.16.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1" +checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100" dependencies = [ "foldhash", ] @@ -825,11 +825,11 @@ checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" [[package]] name = "hashlink" -version = "0.10.0" +version = "0.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7382cf6263419f2d8df38c55d7da83da5c18aef87fc7a7fc1fb1e344edfe14c1" +checksum = "824e001ac4f3012dd16a264bec811403a67ca9deb6c102fc5049b32c4574b35f" dependencies = [ - "hashbrown 0.15.5", + "hashbrown 0.16.1", ] [[package]] @@ -1207,9 +1207,9 @@ checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2" [[package]] name = "libsqlite3-sys" -version = "0.32.0" +version = "0.37.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fbb8270bb4060bd76c6e96f20c52d80620f1d82a3470885694e41e0f81ef6fe7" +checksum = "b1f111c8c41e7c61a49cd34e44c7619462967221a6443b0ec299e0ac30cfb9b1" dependencies = [ "cc", "pkg-config", @@ -1874,11 +1874,21 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "rsqlite-vfs" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c51c9ae4df8a7fba42103df5c621fa3c37eccf3a3c650879e90fc48b11cc192c" +dependencies = [ + "hashbrown 0.16.1", + "thiserror", +] + [[package]] name = "rusqlite" -version = "0.34.0" +version = "0.39.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "37e34486da88d8e051c7c0e23c3f15fd806ea8546260aa2fec247e97242ec143" +checksum = "a0d2b0146dd9661bf67bb107c0bb2a55064d556eeb3fc314151b957f313bcd4e" dependencies = [ "bitflags", "fallible-iterator", @@ -1886,6 +1896,7 @@ dependencies = [ "hashlink", "libsqlite3-sys", "smallvec", + "sqlite-wasm-rs", ] [[package]] @@ -2247,6 +2258,18 @@ dependencies = [ "der", ] +[[package]] +name = "sqlite-wasm-rs" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc3efc0da82635d7e1ced0053bbbfa8c7ab9645d0bf36ceb4f7127bb85315d75" +dependencies = [ + "cc", + "js-sys", + "rsqlite-vfs", + "wasm-bindgen", +] + [[package]] name = "stable_deref_trait" version = "1.2.1" diff --git a/crates/cellule-axum/examples/application-builder-service/Cargo.toml b/crates/cellule-axum/examples/application-builder-service/Cargo.toml index 5def077e..9ab9fac6 100644 --- a/crates/cellule-axum/examples/application-builder-service/Cargo.toml +++ b/crates/cellule-axum/examples/application-builder-service/Cargo.toml @@ -18,7 +18,7 @@ cellule-cookbook-support = { path = "../../../../cookbook/support" } cellule-runtime = { path = "../../../cellule-runtime" } cellule-store = { path = "../../../cellule-store" } object_store = { version = "0.14.1", default-features = false } -rusqlite = { version = "0.34", features = ["bundled"] } +rusqlite = { version = "0.39", features = ["bundled", "fallible_uint"] } serde_json = "1" tempfile = "3" tokio = { version = "1.49", features = ["macros", "net", "rt-multi-thread", "signal"] } diff --git a/crates/cellule-ltx/fuzz/Cargo.lock b/crates/cellule-ltx/fuzz/Cargo.lock index 04525ef2..55f8ffe2 100644 --- a/crates/cellule-ltx/fuzz/Cargo.lock +++ b/crates/cellule-ltx/fuzz/Cargo.lock @@ -133,6 +133,65 @@ dependencies = [ "shlex", ] +[[package]] +name = "cellule-ltx" +version = "0.2.0" +dependencies = [ + "async-trait", + "blake3", + "bytes", + "cellule-store", + "crc-fast", + "futures-util", + "lz4_flex", + "object_store", + "rusqlite", + "serde", + "serde_json", + "tempfile", + "thiserror", + "tokio", + "tokio-util", +] + +[[package]] +name = "cellule-ltx-fuzz" +version = "0.0.0" +dependencies = [ + "cellule-ltx", + "libfuzzer-sys", +] + +[[package]] +name = "cellule-store" +version = "0.2.0" +dependencies = [ + "async-trait", + "blake3", + "bytes", + "cellule-types", + "futures-util", + "hyper", + "object_store", + "rand 0.9.5", + "reqwest 0.12.28", + "serde", + "serde_json", + "thiserror", + "tokio", + "tokio-util", + "tracing", + "url", +] + +[[package]] +name = "cellule-types" +version = "0.2.0" +dependencies = [ + "schemars", + "serde", +] + [[package]] name = "cfg-if" version = "1.0.5" @@ -218,65 +277,6 @@ dependencies = [ "libc", ] -[[package]] -name = "cellule-ltx" -version = "0.2.0" -dependencies = [ - "async-trait", - "blake3", - "bytes", - "cellule-store", - "crc-fast", - "futures-util", - "lz4_flex", - "object_store", - "rusqlite", - "serde", - "serde_json", - "tempfile", - "thiserror", - "tokio", - "tokio-util", -] - -[[package]] -name = "cellule-ltx-fuzz" -version = "0.0.0" -dependencies = [ - "cellule-ltx", - "libfuzzer-sys", -] - -[[package]] -name = "cellule-store" -version = "0.2.0" -dependencies = [ - "async-trait", - "blake3", - "bytes", - "cellule-types", - "futures-util", - "hyper", - "object_store", - "rand 0.9.5", - "reqwest 0.12.28", - "serde", - "serde_json", - "thiserror", - "tokio", - "tokio-util", - "tracing", - "url", -] - -[[package]] -name = "cellule-types" -version = "0.2.0" -dependencies = [ - "schemars", - "serde", -] - [[package]] name = "crc-fast" version = "1.10.0" @@ -402,9 +402,9 @@ checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" [[package]] name = "foldhash" -version = "0.1.5" +version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2" +checksum = "77ce24cb58228fbb8aa041425bb1050850ac19177686ea6e0f41a70416f56fdb" [[package]] name = "form_urlencoded" @@ -552,9 +552,9 @@ dependencies = [ [[package]] name = "hashbrown" -version = "0.15.5" +version = "0.16.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1" +checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100" dependencies = [ "foldhash", ] @@ -567,11 +567,11 @@ checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" [[package]] name = "hashlink" -version = "0.10.0" +version = "0.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7382cf6263419f2d8df38c55d7da83da5c18aef87fc7a7fc1fb1e344edfe14c1" +checksum = "824e001ac4f3012dd16a264bec811403a67ca9deb6c102fc5049b32c4574b35f" dependencies = [ - "hashbrown 0.15.5", + "hashbrown 0.16.1", ] [[package]] @@ -935,9 +935,9 @@ dependencies = [ [[package]] name = "libsqlite3-sys" -version = "0.32.0" +version = "0.37.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fbb8270bb4060bd76c6e96f20c52d80620f1d82a3470885694e41e0f81ef6fe7" +checksum = "b1f111c8c41e7c61a49cd34e44c7619462967221a6443b0ec299e0ac30cfb9b1" dependencies = [ "cc", "pkg-config", @@ -1394,11 +1394,21 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "rsqlite-vfs" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c51c9ae4df8a7fba42103df5c621fa3c37eccf3a3c650879e90fc48b11cc192c" +dependencies = [ + "hashbrown 0.16.1", + "thiserror", +] + [[package]] name = "rusqlite" -version = "0.34.0" +version = "0.39.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "37e34486da88d8e051c7c0e23c3f15fd806ea8546260aa2fec247e97242ec143" +checksum = "a0d2b0146dd9661bf67bb107c0bb2a55064d556eeb3fc314151b957f313bcd4e" dependencies = [ "bitflags", "fallible-iterator", @@ -1406,6 +1416,7 @@ dependencies = [ "hashlink", "libsqlite3-sys", "smallvec", + "sqlite-wasm-rs", ] [[package]] @@ -1716,6 +1727,18 @@ version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "023a211cb3138dbc438680b32560ad89f699977624c9f8dbb95a47d5b4c07dd3" +[[package]] +name = "sqlite-wasm-rs" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc3efc0da82635d7e1ced0053bbbfa8c7ab9645d0bf36ceb4f7127bb85315d75" +dependencies = [ + "cc", + "js-sys", + "rsqlite-vfs", + "wasm-bindgen", +] + [[package]] name = "stable_deref_trait" version = "1.2.1" diff --git a/crates/cellule-ltx/perf/cellule/Cargo.lock b/crates/cellule-ltx/perf/cellule/Cargo.lock index 1d5cf5c5..d9cb95f1 100644 --- a/crates/cellule-ltx/perf/cellule/Cargo.lock +++ b/crates/cellule-ltx/perf/cellule/Cargo.lock @@ -27,6 +27,12 @@ dependencies = [ "cpufeatures", ] +[[package]] +name = "bumpalo" +version = "3.20.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649" + [[package]] name = "cc" version = "1.4.7" @@ -37,27 +43,6 @@ dependencies = [ "shlex", ] -[[package]] -name = "cfg-if" -version = "1.0.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4e7648175b45a9a48536d676f68d918270699102aa8dab5496df06904c914600" - -[[package]] -name = "constant_time_eq" -version = "0.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3d52eff69cd5e647efe296129160853a42795992097e8af39800e1060caeea9b" - -[[package]] -name = "cpufeatures" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5ca28b0ae3115b884660db4118d803791fd6756b6e88f39c0f3f7859060d7566" -dependencies = [ - "libc", -] - [[package]] name = "cellule-ltx" version = "0.2.0" @@ -80,6 +65,27 @@ dependencies = [ "tempfile", ] +[[package]] +name = "cfg-if" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4e7648175b45a9a48536d676f68d918270699102aa8dab5496df06904c914600" + +[[package]] +name = "constant_time_eq" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d52eff69cd5e647efe296129160853a42795992097e8af39800e1060caeea9b" + +[[package]] +name = "cpufeatures" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5ca28b0ae3115b884660db4118d803791fd6756b6e88f39c0f3f7859060d7566" +dependencies = [ + "libc", +] + [[package]] name = "crc-fast" version = "1.10.0" @@ -145,9 +151,9 @@ checksum = "ef25905e51abafe4dcea6c15fec58c57b601cdbd0ee53d22ea1d3016c587d39b" [[package]] name = "foldhash" -version = "0.1.5" +version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2" +checksum = "77ce24cb58228fbb8aa041425bb1050850ac19177686ea6e0f41a70416f56fdb" [[package]] name = "generic-array" @@ -172,18 +178,18 @@ dependencies = [ [[package]] name = "hashbrown" -version = "0.15.5" +version = "0.16.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1" +checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100" dependencies = [ "foldhash", ] [[package]] name = "hashlink" -version = "0.10.0" +version = "0.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7382cf6263419f2d8df38c55d7da83da5c18aef87fc7a7fc1fb1e344edfe14c1" +checksum = "824e001ac4f3012dd16a264bec811403a67ca9deb6c102fc5049b32c4574b35f" dependencies = [ "hashbrown", ] @@ -194,6 +200,16 @@ version = "1.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" +[[package]] +name = "js-sys" +version = "0.3.106" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7883d941dae510fb2d978fc3fe018c71c9e2892fd38854de3e8b92c2e5ad9cc5" +dependencies = [ + "cfg-if", + "wasm-bindgen", +] + [[package]] name = "libc" version = "0.2.189" @@ -202,9 +218,9 @@ checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2" [[package]] name = "libsqlite3-sys" -version = "0.32.0" +version = "0.37.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fbb8270bb4060bd76c6e96f20c52d80620f1d82a3470885694e41e0f81ef6fe7" +checksum = "b1f111c8c41e7c61a49cd34e44c7619462967221a6443b0ec299e0ac30cfb9b1" dependencies = [ "cc", "pkg-config", @@ -268,11 +284,21 @@ version = "6.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" +[[package]] +name = "rsqlite-vfs" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c51c9ae4df8a7fba42103df5c621fa3c37eccf3a3c650879e90fc48b11cc192c" +dependencies = [ + "hashbrown", + "thiserror", +] + [[package]] name = "rusqlite" -version = "0.34.0" +version = "0.39.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "37e34486da88d8e051c7c0e23c3f15fd806ea8546260aa2fec247e97242ec143" +checksum = "a0d2b0146dd9661bf67bb107c0bb2a55064d556eeb3fc314151b957f313bcd4e" dependencies = [ "bitflags", "fallible-iterator", @@ -280,6 +306,7 @@ dependencies = [ "hashlink", "libsqlite3-sys", "smallvec", + "sqlite-wasm-rs", ] [[package]] @@ -295,6 +322,12 @@ dependencies = [ "windows-sys", ] +[[package]] +name = "rustversion" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf54715a573b99ac80df0bc206da022bcd442c974952c7b9720069370852e21f" + [[package]] name = "serde" version = "1.0.229" @@ -356,6 +389,18 @@ version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "023a211cb3138dbc438680b32560ad89f699977624c9f8dbb95a47d5b4c07dd3" +[[package]] +name = "sqlite-wasm-rs" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc3efc0da82635d7e1ced0053bbbfa8c7ab9645d0bf36ceb4f7127bb85315d75" +dependencies = [ + "cc", + "js-sys", + "rsqlite-vfs", + "wasm-bindgen", +] + [[package]] name = "syn" version = "3.0.6" @@ -430,6 +475,51 @@ version = "0.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" +[[package]] +name = "wasm-bindgen" +version = "0.2.129" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9bb54f33acc68fd454578d9820b0bde1a1a3d17aa17bb7b6595806d02886d409" +dependencies = [ + "cfg-if", + "once_cell", + "rustversion", + "wasm-bindgen-macro", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.129" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e29d0c35b16e224a7eeb5cd2d25e3e1968fbd65604117b44d3b789d00ee8535" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.129" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6f501a8bc3719dba86ef8ae4728879c08001bea749eb1333ac5b91e040e2a6b7" +dependencies = [ + "bumpalo", + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.129" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23f0c9c52aa7cd7d77769a4cfe2a9adb1b331f489a41d912ce14513d5ab995c6" +dependencies = [ + "unicode-ident", +] + [[package]] name = "windows-link" version = "0.2.1" diff --git a/crates/cellule-ltx/perf/replica-cost/Cargo.lock b/crates/cellule-ltx/perf/replica-cost/Cargo.lock index fef9a404..c8b56374 100644 --- a/crates/cellule-ltx/perf/replica-cost/Cargo.lock +++ b/crates/cellule-ltx/perf/replica-cost/Cargo.lock @@ -127,6 +127,70 @@ dependencies = [ "shlex", ] +[[package]] +name = "cellule-ltx" +version = "0.2.0" +dependencies = [ + "async-trait", + "blake3", + "bytes", + "cellule-store", + "crc-fast", + "futures-util", + "lz4_flex", + "object_store", + "rusqlite", + "serde", + "serde_json", + "tempfile", + "thiserror", + "tokio", + "tokio-util", +] + +[[package]] +name = "cellule-ltx-replica-cost" +version = "0.1.0" +dependencies = [ + "cellule-ltx", + "cellule-store", + "object_store", + "serde", + "serde_json", + "tempfile", + "tokio", +] + +[[package]] +name = "cellule-store" +version = "0.2.0" +dependencies = [ + "async-trait", + "blake3", + "bytes", + "cellule-types", + "futures-util", + "hyper", + "object_store", + "rand 0.9.5", + "reqwest 0.12.28", + "serde", + "serde_json", + "thiserror", + "tokio", + "tokio-util", + "tracing", + "url", +] + +[[package]] +name = "cellule-types" +version = "0.2.0" +dependencies = [ + "schemars", + "serde", +] + [[package]] name = "cfg-if" version = "1.0.5" @@ -212,70 +276,6 @@ dependencies = [ "libc", ] -[[package]] -name = "cellule-ltx" -version = "0.2.0" -dependencies = [ - "async-trait", - "blake3", - "bytes", - "cellule-store", - "crc-fast", - "futures-util", - "lz4_flex", - "object_store", - "rusqlite", - "serde", - "serde_json", - "tempfile", - "thiserror", - "tokio", - "tokio-util", -] - -[[package]] -name = "cellule-ltx-replica-cost" -version = "0.1.0" -dependencies = [ - "cellule-ltx", - "cellule-store", - "object_store", - "serde", - "serde_json", - "tempfile", - "tokio", -] - -[[package]] -name = "cellule-store" -version = "0.2.0" -dependencies = [ - "async-trait", - "blake3", - "bytes", - "cellule-types", - "futures-util", - "hyper", - "object_store", - "rand 0.9.5", - "reqwest 0.12.28", - "serde", - "serde_json", - "thiserror", - "tokio", - "tokio-util", - "tracing", - "url", -] - -[[package]] -name = "cellule-types" -version = "0.2.0" -dependencies = [ - "schemars", - "serde", -] - [[package]] name = "crc-fast" version = "1.10.0" @@ -401,9 +401,9 @@ checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" [[package]] name = "foldhash" -version = "0.1.5" +version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2" +checksum = "77ce24cb58228fbb8aa041425bb1050850ac19177686ea6e0f41a70416f56fdb" [[package]] name = "form_urlencoded" @@ -551,9 +551,9 @@ dependencies = [ [[package]] name = "hashbrown" -version = "0.15.5" +version = "0.16.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1" +checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100" dependencies = [ "foldhash", ] @@ -566,11 +566,11 @@ checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" [[package]] name = "hashlink" -version = "0.10.0" +version = "0.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7382cf6263419f2d8df38c55d7da83da5c18aef87fc7a7fc1fb1e344edfe14c1" +checksum = "824e001ac4f3012dd16a264bec811403a67ca9deb6c102fc5049b32c4574b35f" dependencies = [ - "hashbrown 0.15.5", + "hashbrown 0.16.1", ] [[package]] @@ -924,9 +924,9 @@ checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2" [[package]] name = "libsqlite3-sys" -version = "0.32.0" +version = "0.37.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fbb8270bb4060bd76c6e96f20c52d80620f1d82a3470885694e41e0f81ef6fe7" +checksum = "b1f111c8c41e7c61a49cd34e44c7619462967221a6443b0ec299e0ac30cfb9b1" dependencies = [ "cc", "pkg-config", @@ -1383,11 +1383,21 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "rsqlite-vfs" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c51c9ae4df8a7fba42103df5c621fa3c37eccf3a3c650879e90fc48b11cc192c" +dependencies = [ + "hashbrown 0.16.1", + "thiserror", +] + [[package]] name = "rusqlite" -version = "0.34.0" +version = "0.39.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "37e34486da88d8e051c7c0e23c3f15fd806ea8546260aa2fec247e97242ec143" +checksum = "a0d2b0146dd9661bf67bb107c0bb2a55064d556eeb3fc314151b957f313bcd4e" dependencies = [ "bitflags", "fallible-iterator", @@ -1395,6 +1405,7 @@ dependencies = [ "hashlink", "libsqlite3-sys", "smallvec", + "sqlite-wasm-rs", ] [[package]] @@ -1705,6 +1716,18 @@ version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "023a211cb3138dbc438680b32560ad89f699977624c9f8dbb95a47d5b4c07dd3" +[[package]] +name = "sqlite-wasm-rs" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc3efc0da82635d7e1ced0053bbbfa8c7ab9645d0bf36ceb4f7127bb85315d75" +dependencies = [ + "cc", + "js-sys", + "rsqlite-vfs", + "wasm-bindgen", +] + [[package]] name = "stable_deref_trait" version = "1.2.1" From fbc6f775020191adbfdfe5cd726728bb061c863a Mon Sep 17 00:00:00 2001 From: forhappy Date: Sun, 18 Oct 2026 10:15:54 -0700 Subject: [PATCH 2/2] perf(runtime): batch durable Cell enrollment Serial catalog publication queues cold Cells behind repeated I/O. Share catalog updates while preserving original pins and recovery. Join accepted callbacks across cancellation and respect metadata caps. --- .../examples/fleet/authority/mod.rs | 64 +- .../examples/fleet/authority/renewal_tests.rs | 56 +- .../src/node/bundle/binding.rs | 221 ++- .../src/node/bundle/index/mod.rs | 36 + .../src/node/bundle/index/tests/admission.rs | 52 + crates/cellule-runtime/src/node/bundle/mod.rs | 1 + .../src/node/bundle/tests/actor.rs | 47 +- .../src/node/bundle/tests/bootstrap_cost.rs | 153 ++ .../src/node/bundle/tests/enrollment.rs | 386 +++++ .../src/node/bundle/tests/enrollment_cost.rs | 140 ++ .../src/node/bundle/tests/mod.rs | 3 + .../src/node/durability/enrollment/mod.rs | 185 ++ .../src/node/durability/enrollment/tests.rs | 226 +++ .../src/node/durability/mod.rs | 35 +- docs/bundle-coverage-implementation.md | 3 +- docs/cell-bootstrap-latency.md | 123 ++ docs/measurements/cell-bootstrap-latency.json | 1493 +++++++++++++++++ docs/quickstart.md | 5 + 18 files changed, 3131 insertions(+), 98 deletions(-) create mode 100644 crates/cellule-runtime/src/node/bundle/tests/bootstrap_cost.rs create mode 100644 crates/cellule-runtime/src/node/bundle/tests/enrollment.rs create mode 100644 crates/cellule-runtime/src/node/bundle/tests/enrollment_cost.rs create mode 100644 crates/cellule-runtime/src/node/durability/enrollment/mod.rs create mode 100644 crates/cellule-runtime/src/node/durability/enrollment/tests.rs create mode 100644 docs/cell-bootstrap-latency.md create mode 100644 docs/measurements/cell-bootstrap-latency.json diff --git a/crates/cellule-axum/examples/fleet/authority/mod.rs b/crates/cellule-axum/examples/fleet/authority/mod.rs index 2c35d67f..3227187f 100644 --- a/crates/cellule-axum/examples/fleet/authority/mod.rs +++ b/crates/cellule-axum/examples/fleet/authority/mod.rs @@ -233,23 +233,67 @@ impl Authority { } } +impl Authority { + async fn bind_cells( + &self, + cells: &[( + &cellule_runtime::control::authority::CellAuthority, + &cellule_runtime::control::authority::VersionedControl, + )], + ) -> Result> { + let _catalog = self.catalog.lock().await; + let mut state = self.live_state().await?; + let current = self.current(&state, 1).await?; + let (next, pinned) = self + .directory + .bind_bundle_cells(¤t, cells, clock()?) + .await?; + state.observed = next; + self.lease.check()?; + Ok(pinned) + } +} + impl NodeBundleAuthority for Authority { + fn bind_batch<'a>( + &'a self, + cells: &'a [( + &'a cellule_runtime::control::authority::CellAuthority, + &'a cellule_runtime::control::authority::VersionedControl, + )], + ) -> BoxFuture<'a, Option>>> + { + Box::pin(async move { + match self.bind_cells(cells).await { + // Declining releases both locks before individual callbacks; + // queued cells remain admissible under the original 4-MiB cap. + Err(Error::Capacity( + "bundle selected shard bytes" | "bundle enrollment pin metadata", + )) => None, + Ok(pinned) => Some(pinned.into_iter().map(Ok).collect()), + Err(source) if cells.len() == 1 => Some(vec![Err(source)]), + Err(source) => { + let source = Arc::new(source); + Some( + cells + .iter() + .map(|_| Err(Error::Shared(source.clone()))) + .collect(), + ) + } + } + }) + } fn bind<'a>( &'a self, authority: &'a cellule_runtime::control::authority::CellAuthority, observed: &'a cellule_runtime::control::authority::VersionedControl, ) -> BoxFuture<'a, Result> { Box::pin(async move { - let _catalog = self.catalog.lock().await; - let mut state = self.live_state().await?; - let current = self.current(&state, 1).await?; - let (next, pinned) = self - .directory - .bind_bundle_cell(¤t, authority, observed, clock()?) - .await?; - state.observed = next; - self.lease.check()?; - Ok(pinned) + self.bind_cells(&[(authority, observed)]) + .await? + .pop() + .ok_or(Error::Node("bundle enrollment lacks Cell")) }) } diff --git a/crates/cellule-axum/examples/fleet/authority/renewal_tests.rs b/crates/cellule-axum/examples/fleet/authority/renewal_tests.rs index 000e0f02..1c44b193 100644 --- a/crates/cellule-axum/examples/fleet/authority/renewal_tests.rs +++ b/crates/cellule-axum/examples/fleet/authority/renewal_tests.rs @@ -22,25 +22,49 @@ async fn authority() -> Arc { code, code, ); - let enrollment = Enrollment::start( + // Keep the production 30-second advertisement lifetime, but make its + // issue time unambiguously older than the queued renewal. Creating and + // renewing in the same wall-clock millisecond cannot advance authority. + let now = clock().unwrap(); + let observed = directory + .create( + NodeAdvertisement::sign( + node(0), + SessionId::from_bytes([11; 16]), + "https://localhost:8081".into(), + tls.fleet(), + tls.certificate(), + code, + code, + tls.signing_key(), + 1, + now - 1_000, + now + 29_000, + vec![code], + vec![1], + NodeFailureDomain::default(), + capacity(None), + ) + .unwrap(), + now, + ) + .await + .unwrap(); + // A shorter local deadline remains conservative relative to that signed + // advertisement. The 20 queued jobs must still cross this one-second bound. + Arc::new(Authority { directory, tls, - 0, - SessionId::from_bytes([11; 16]), - "https://localhost:8081".into(), code, - None, - ) - .await - .unwrap(); - enrollment.stop.send(true).unwrap(); - enrollment.heartbeat.await.unwrap().unwrap(); - // A shorter local deadline is conservative relative to the signed - // advertisement. It reproduces a nearly exhausted renewal window. - let mut authority = Arc::try_unwrap(enrollment.authority).ok().unwrap(); - let now = clock().unwrap(); - authority.lease = NodeLeaseGuard::new(now, now + 1_000).unwrap(); - Arc::new(authority) + lease: NodeLeaseGuard::new(now, now + 1_000).unwrap(), + catalog: Mutex::new(()), + state: Mutex::new(State { + observed, + closed: None, + }), + follower: None, + shutdown_error: std::sync::OnceLock::new(), + }) } #[tokio::test] diff --git a/crates/cellule-runtime/src/node/bundle/binding.rs b/crates/cellule-runtime/src/node/bundle/binding.rs index edcc3fea..aec91265 100644 --- a/crates/cellule-runtime/src/node/bundle/binding.rs +++ b/crates/cellule-runtime/src/node/bundle/binding.rs @@ -4,6 +4,9 @@ use crate::control::authority::{CellAuthority, VersionedControl}; use crate::control::{ControlState, Transition}; use crate::identity::ApplicationId; use crate::node::{NodeDirectory, VersionedNodeAdvertisement}; +use futures_util::StreamExt as _; + +pub(crate) const MAX_BUNDLE_ENROLLMENTS: usize = 8; impl NodeDirectory { /// Creates an empty bundle lane before native issuance starts for this boot. @@ -36,9 +39,7 @@ impl NodeDirectory { self.select_catalog(observed, &prepared, now_ms).await } - /// Reserves a provisional catalog entry before pinning the original Cell, - /// then opens it only after the Cell CAS is confirmed. Cancellation or an - /// ambiguous failure leaves a catalog obligation that blocks maintenance. + /// Reserves and pins one original Cell before opening SQL admission. pub async fn bind_bundle_cell( &self, observed: &VersionedNodeAdvertisement, @@ -46,18 +47,161 @@ impl NodeDirectory { control: &VersionedControl, now_ms: i64, ) -> Result<(VersionedNodeAdvertisement, VersionedControl)> { + let (node, mut pinned) = self + .bind_bundle_cells(observed, &[(authority, control)], now_ms) + .await?; + Ok(( + node, + pinned + .pop() + .ok_or(Error::Node("bundle enrollment lacks Cell"))?, + )) + } + + /// Enrolls at most eight Cells in one provisional/open catalog pair. + /// + /// The provisional inventory CAS precedes every original Cell pin. All pin + /// attempts join before the Open catalog is selected. A failed or cancelled + /// cohort leaves discoverable provisional obligations, never SQL admission. + /// Results retain input order. Callers serialize the two catalog selections + /// with publication and reload node/Cell authority after ambiguous failure. + pub async fn bind_bundle_cells( + &self, + observed: &VersionedNodeAdvertisement, + cells: &[(&CellAuthority, &VersionedControl)], + now_ms: i64, + ) -> Result<(VersionedNodeAdvertisement, Vec)> { self.validate(&observed.advertisement, now_ms)?; + if cells.is_empty() || cells.len() > MAX_BUNDLE_ENROLLMENTS { + return Err(Error::Node("invalid bundle enrollment cohort")); + } let head = observed .advertisement .bundle .ok_or(Error::Node("bundle lane is absent"))?; - let value = control.value(); - let cells = [(*authority.layout().application_id(), *value.cell.as_bytes())] - .into_iter() - .collect(); + let keys = cells + .iter() + .map(|(authority, control)| { + ( + *authority.layout().application_id(), + *control.value().cell.as_bytes(), + ) + }) + .collect::>(); + if keys.len() != cells.len() { + return Err(Error::Node("duplicate bundle enrollment Cell")); + } let mut catalog = - store::load_catalog_cells(&self.layout, observed.advertisement.session, head, &cells) + store::load_catalog_cells(&self.layout, observed.advertisement.session, head, &keys) .await?; + let mut plans = Vec::with_capacity(cells.len()); + let mut reserve = false; + let mut opening = false; + for (authority, control) in cells { + let (pin, next, new) = self.plan_binding(&catalog, authority, control)?; + let exists = catalog.bindings.iter().any(|binding| { + binding.control.bundle_binding == Some(pin) && binding.phase == BindingPhase::Open + }); + opening |= !exists; + if new { + let base = next + .ltx_root() + .ok_or(Error::Node("bundle binding lacks published base"))?; + catalog.bindings.push(Binding { + application: ApplicationId::from_bytes(*authority.layout().application_id()), + first_commit: base.commit_sequence, + control: next.clone(), + phase: BindingPhase::Provisional, + terminal: None, + selected_sequence: 0, + selected_commit: base.commit_sequence, + selected_position: base.position, + locators: Vec::new(), + }); + reserve = true; + } + plans.push((pin, next)); + } + if !opening { + return Ok(( + observed.clone(), + cells + .iter() + .map(|(_, control)| (*control).clone()) + .collect(), + )); + } + let (reserved, pin_concurrency) = if reserve { + catalog.bindings.sort_unstable_by_key(|binding| { + binding + .control + .bundle_binding + .map(|pin| *pin.digest.as_bytes()) + }); + let staged = self.stage_catalog(Some(head), catalog.clone(), &[], Vec::new())?; + let concurrency = index::enrollment_pin_concurrency(&catalog, Some(&staged.body))?; + if concurrency < cells.len() { + return Err(Error::Capacity("bundle enrollment pin metadata")); + } + let prepared = self.upload_node_bundle(staged).await?; + ( + self.select_catalog(observed, &prepared, now_ms).await?, + concurrency, + ) + } else { + let concurrency = index::enrollment_pin_concurrency(&catalog, None)?; + if concurrency < cells.len() { + return Err(Error::Capacity("bundle enrollment pin metadata")); + } + (observed.clone(), concurrency) + }; + // Join accepted sibling pins even if one fails. Their provisional + // catalog entries remain the complete recovery/maintenance inventory. + let pin_checks = cells + .iter() + .zip(&plans) + .map(|((authority, control), (_, next))| async move { + if control.value().bundle_binding.is_some() { + return Ok((*control).clone()); + } + match authority + .transition(control, next.clone(), Transition::BindBundle) + .await + { + Ok(pinned) => Ok(pinned), + Err(source) => match authority.load(control.value().cell).await { + Ok(Some(current)) if current.value() == next => Ok(current), + _ => Err(source), + }, + } + }) + .collect::>(); + let results = futures_util::stream::iter(pin_checks) + .buffered(pin_concurrency) + .collect::>() + .await; + let pinned = results.into_iter().collect::>>()?; + for ((pin, _), control) in plans.iter().zip(&pinned) { + let binding = catalog.binding_mut(pin.digest)?; + binding.control = control.value().clone(); + binding.phase = BindingPhase::Open; + } + let prepared = self + .upload_catalog(reserved.advertisement.bundle, catalog, &[]) + .await?; + Ok(( + self.select_catalog(&reserved, &prepared, now_ms).await?, + pinned, + )) + } + + fn plan_binding( + &self, + catalog: &Catalog, + authority: &CellAuthority, + control: &VersionedControl, + ) -> Result<(BundleBindingRef, Control, bool)> { + let value = control.value(); if value.state != ControlState::Serving || value.recovery.is_some() || value @@ -94,7 +238,11 @@ impl NodeDirectory { if binding.control.bundle_binding != value.bundle_binding { return Err(Error::Fenced); } - return Ok((observed.clone(), control.clone())); + return Ok(( + binding.control.bundle_binding.ok_or(Error::Fenced)?, + value.clone(), + false, + )); } if binding.phase != BindingPhase::Provisional { return Err(Error::Fenced); @@ -141,59 +289,6 @@ impl NodeDirectory { .ok_or(Error::Control("progress overflow"))?; next.bundle_binding = Some(pin); } - // Select the inventory obligation first. Otherwise cancellation after - // Cell pinning could leave a pin absent from the complete node catalog. - let reserved = if existing.is_none() { - let base = next - .ltx_root() - .ok_or(Error::Node("bundle binding lacks published base"))?; - catalog.bindings.push(Binding { - application, - first_commit: base.commit_sequence, - control: next.clone(), - phase: BindingPhase::Provisional, - terminal: None, - selected_sequence: 0, - selected_commit: base.commit_sequence, - selected_position: base.position, - locators: Vec::new(), - }); - catalog.bindings.sort_unstable_by_key(|binding| { - binding - .control - .bundle_binding - .map(|pin| *pin.digest.as_bytes()) - }); - let prepared = self - .upload_catalog(Some(head), catalog.clone(), &[]) - .await?; - self.select_catalog(observed, &prepared, now_ms).await? - } else { - observed.clone() - }; - let pinned = if value.bundle_binding.is_some() { - control.clone() - } else { - match authority - .transition(control, next.clone(), Transition::BindBundle) - .await - { - Ok(pinned) => pinned, - Err(source) => match authority.load(value.cell).await { - Ok(Some(current)) if current.value() == &next => current, - _ => return Err(source), - }, - } - }; - let binding = catalog.binding_mut(pin.digest)?; - binding.control = pinned.value().clone(); - binding.phase = BindingPhase::Open; - let prepared = self - .upload_catalog(reserved.advertisement.bundle, catalog, &[]) - .await?; - Ok(( - self.select_catalog(&reserved, &prepared, now_ms).await?, - pinned, - )) + Ok((pin, next, existing.is_none())) } } diff --git a/crates/cellule-runtime/src/node/bundle/index/mod.rs b/crates/cellule-runtime/src/node/bundle/index/mod.rs index 0536eeca..5301229c 100644 --- a/crates/cellule-runtime/src/node/bundle/index/mod.rs +++ b/crates/cellule-runtime/src/node/bundle/index/mod.rs @@ -34,6 +34,42 @@ pub(super) fn history_extent(catalog: &Catalog, pin: Digest) -> Option } pub(super) use io::{binding_inventory, ensure_drained, load_cells}; +/// Concurrent original-pin checks retain at most the old single-operation +/// metadata allowance. Provisional Cells have no detached native history. +pub(super) fn enrollment_pin_concurrency( + catalog: &Catalog, + prepared: Option<&[u8]>, +) -> Result { + let index = catalog + .index + .as_ref() + .ok_or(Error::Node("enrollment index is absent"))?; + let encoded = prepared + .map(|prepared| { + let header = prepared + .get(..HEADER_BYTES) + .ok_or(Error::Node("enrollment header is truncated"))?; + codec::decode(header) + }) + .transpose()?; + let root = encoded.as_ref().unwrap_or(&index.root); + let largest = index + .loaded + .keys() + .filter_map(|id| root.shards.get(usize::from(*id)).and_then(Option::as_ref)) + .map(|shard| shard.extent.bytes) + .max() + .unwrap_or(0); + let overhead = HEADER_BYTES as u64 + crate::node::MAX_NODE_BYTES; + let per_pin = largest + .checked_add(overhead) + .ok_or(Error::Capacity("enrollment pin metadata"))?; + Ok( + ((MAX_BUNDLE_BYTES + overhead) / per_pin).clamp(1, super::MAX_BUNDLE_ENROLLMENTS as u64) + as usize, + ) +} + pub(super) const HEADER_BYTES: usize = 32 << 10; const SHARDS: usize = 256; const MAGIC: &[u8] = b"cellule.node-bundle.v1\0"; diff --git a/crates/cellule-runtime/src/node/bundle/index/tests/admission.rs b/crates/cellule-runtime/src/node/bundle/index/tests/admission.rs index 153b9055..ed7f4ff1 100644 --- a/crates/cellule-runtime/src/node/bundle/index/tests/admission.rs +++ b/crates/cellule-runtime/src/node/bundle/index/tests/admission.rs @@ -1,5 +1,57 @@ use super::*; +#[test] +fn enrollment_pin_fanout_stays_within_one_original_metadata_read_budget() { + let session = SessionId::from_bytes([1; 16]); + let root = Root { + session, + epoch: 2, + predecessor: None, + selected_through: 0, + object_bytes: HEADER_BYTES as u64, + shards: vec![None; SHARDS], + frames: Vec::new(), + }; + let mut catalog = Catalog { + session, + epoch: 2, + predecessor: None, + selected_through: 0, + bindings: Vec::new(), + index: Some(LoadedIndex { + root, + loaded: [(0, Vec::new())].into_iter().collect(), + histories: BTreeMap::new(), + }), + }; + for bytes in [1, 1 << 20, MAX_BUNDLE_BYTES] { + catalog.index.as_mut().unwrap().root.shards[0] = Some(Shard { + extent: Locator { + object: Some(Digest::from_bytes([7; 32])), + offset: HEADER_BYTES as u64, + bytes, + frame_digest: Digest::from_bytes([8; 32]), + }, + bindings: 1, + }); + let concurrency = enrollment_pin_concurrency(&catalog, None).unwrap(); + let overhead = HEADER_BYTES as u64 + crate::node::MAX_NODE_BYTES; + assert!( + concurrency * (bytes + overhead) as usize <= (MAX_BUNDLE_BYTES + overhead) as usize + ); + assert_eq!( + concurrency, + if bytes == 1 { + 8 + } else if bytes == 1 << 20 { + 3 + } else { + 1 + } + ); + } +} + #[tokio::test] async fn aggregate_history_budget_is_checked_before_the_first_history_read() { let counted = Arc::new(CountingObjectStore::new(Arc::new(InMemory::new()))); diff --git a/crates/cellule-runtime/src/node/bundle/mod.rs b/crates/cellule-runtime/src/node/bundle/mod.rs index c40fa51a..3accf192 100644 --- a/crates/cellule-runtime/src/node/bundle/mod.rs +++ b/crates/cellule-runtime/src/node/bundle/mod.rs @@ -42,6 +42,7 @@ use crate::{Error, Result}; use bytes::Bytes; mod binding; +pub(crate) use binding::MAX_BUNDLE_ENROLLMENTS; mod closure; mod codec; mod continuation; diff --git a/crates/cellule-runtime/src/node/bundle/tests/actor.rs b/crates/cellule-runtime/src/node/bundle/tests/actor.rs index 9671349b..4668b994 100644 --- a/crates/cellule-runtime/src/node/bundle/tests/actor.rs +++ b/crates/cellule-runtime/src/node/bundle/tests/actor.rs @@ -26,20 +26,53 @@ pub(super) struct Authority { pub(super) observed: tokio::sync::Mutex, } +impl Authority { + async fn bind_cells( + &self, + cells: &[(&CellAuthority, &VersionedControl)], + ) -> Result> { + let mut node = self.observed.lock().await; + let (next, pinned) = self.directory.bind_bundle_cells(&node, cells, NOW).await?; + *node = next; + Ok(pinned) + } +} + impl NodeBundleAuthority for Authority { + fn bind_batch<'a>( + &'a self, + cells: &'a [(&'a CellAuthority, &'a VersionedControl)], + ) -> BoxFuture<'a, Option>>> { + Box::pin(async move { + match self.bind_cells(cells).await { + // The metadata preflight has performed no authority mutation. + Err(Error::Capacity( + "bundle selected shard bytes" | "bundle enrollment pin metadata", + )) => None, + Ok(pinned) => Some(pinned.into_iter().map(Ok).collect()), + Err(source) if cells.len() == 1 => Some(vec![Err(source)]), + Err(source) => { + let source = Arc::new(source); + Some( + cells + .iter() + .map(|_| Err(Error::Shared(source.clone()))) + .collect(), + ) + } + } + }) + } fn bind<'a>( &'a self, authority: &'a CellAuthority, observed: &'a VersionedControl, ) -> BoxFuture<'a, Result> { Box::pin(async move { - let mut node = self.observed.lock().await; - let (next, pinned) = self - .directory - .bind_bundle_cell(&node, authority, observed, NOW) - .await?; - *node = next; - Ok(pinned) + self.bind_cells(&[(authority, observed)]) + .await? + .pop() + .ok_or(Error::Node("bundle enrollment lacks Cell")) }) } fn close<'a>( diff --git a/crates/cellule-runtime/src/node/bundle/tests/bootstrap_cost.rs b/crates/cellule-runtime/src/node/bundle/tests/bootstrap_cost.rs new file mode 100644 index 00000000..f5b946b3 --- /dev/null +++ b/crates/cellule-runtime/src/node/bundle/tests/bootstrap_cost.rs @@ -0,0 +1,153 @@ +//! Full fresh-Cell catalog/owner/SQLite/root/enrollment/first-read diagnostic. +use super::*; +use crate::cell::{ + actor::CellRuntime, + catalog::{CatalogEntry, CatalogRole, CellCatalog}, + worker::SqlWorkerPool, +}; +use crate::identity::{CellTarget, NamespaceId, TenantId}; +use crate::node::{ + durability::{NodeBundleAuthority, NodeDurability}, + log_shipper::NodeLogShipper, +}; +use object_store::throttle::{ThrottleConfig, ThrottledStore}; +use std::time::{Duration, Instant}; + +#[tokio::test] +#[ignore = "manual fresh SQLite/bootstrap/first-read diagnostic, not production capacity"] +async fn bootstrap_first_read_comparison() { + for delay in [5, 20] { + for count in [1, 8] { + for sample in 0..3 { + for coalesce in if sample % 2 == 0 { + [false, true] + } else { + [true, false] + } { + measure(delay, count, sample, coalesce).await; + } + } + } + } +} + +async fn measure(delay: u64, count: u8, sample: usize, coalesce: bool) { + let backend = Arc::new(ThrottledStore::new( + InMemory::new(), + ThrottleConfig { + wait_get_per_call: Duration::from_millis(delay), + wait_put_per_call: Duration::from_millis(delay), + ..Default::default() + }, + )); + let mut f = Fixture::with_store(backend).await; + super::coverage::enroll(&mut f).await; + let original = Arc::new(super::actor::Authority { + directory: f.directory.clone(), + observed: tokio::sync::Mutex::new(f.node.clone()), + }); + let authority: Arc = if coalesce { + original.clone() + } else { + Arc::new(super::enrollment_cost::Serial(original.clone())) + }; + let peers = super::actor::transport(&f, true); + let shipper = NodeLogShipper::new(f.gate.clone(), peers.clone(), Limits::default()).unwrap(); + let durability = Arc::new(NodeDurability::new( + f.gate.clone(), + shipper, + original, + peers, + f.lease.clone(), + )); + let feed = durability.enable_bundle_publication(authority).unwrap(); + let runtime = CellRuntime::new_with_replica_host_requiring_node_lease( + SqlWorkerPool::new(2, count as usize).unwrap(), + 32 << 20, + SessionId::from_bytes([1; 16]), + cellule_ltx::Host::default(), + ) + .unwrap(); + runtime.install_node_lease(f.lease.clone()).unwrap(); + runtime + .install_node_durability(ApplicationId::from_bytes([9; 16]), durability.clone()) + .unwrap(); + let catalog = CellCatalog::new(f.layout.clone(), TenantId::from_bytes([1; 16])); + let cell_authority = CellAuthority::new(f.layout.clone()); + f.count.reset(); + let started = Instant::now(); + let results = futures_util::future::join_all((0..count).map(|number| { + let runtime = &runtime; + let catalog = &catalog; + let cell_authority = &cell_authority; + let f = &f; + async move { + let target = CellTarget::new(TenantId::from_bytes([1;16]), ApplicationId::from_bytes([9;16]), NamespaceId::from_bytes([13;16]), &[number]).unwrap(); + let proof = catalog.provision(CatalogEntry::new(&target, CatalogRole::Application, Digest::from_bytes([12;32]), 1).unwrap()).await.unwrap(); + assert!(cell_authority.load(target.cell_id()).await.unwrap().is_none()); + let control = cell_authority.create_initial(&proof, IncarnationId::from_bytes([number+10;16]), Owner { + session: SessionId::from_bytes([1;16]), endpoint: "https://bundle.internal:8081".into(), + }).await.unwrap(); + let replica = CellReplica::new(f.layout.clone(), *target.cell_id().as_bytes(), *control.value().incarnation.as_bytes(), Limits::default()).unwrap(); + let handle = runtime.bootstrap(proof, replica.clone(), cell_authority.clone(), control, f.scratch.path().join(format!("first-{number}.sqlite")), |tx| { + tx.execute_batch("CREATE TABLE counter(value INTEGER NOT NULL); INSERT INTO counter VALUES(0)")?; + Ok(()) + }).await.unwrap(); + let value = handle.query(0, 8, |db| { + let value: u64 = db.query_row("SELECT value FROM counter", [], |row| row.get(0))?; + Ok(value.to_le_bytes().to_vec()) + }).await.unwrap(); + assert_eq!(value, 0_u64.to_le_bytes()); + (handle, replica, started.elapsed().as_secs_f64()*1000.0) + } + })).await; + let elapsed = started.elapsed().as_secs_f64() * 1000.0; + let puts = f.count.put_requests(); + let reads = f.count.counts(); + let latencies = results.iter().map(|(_, _, ms)| *ms).collect::>(); + f.count.reset(); + for (handle, _, _) in &results { + assert_eq!(handle.query(0, 1, |_| Ok(vec![0])).await.unwrap(), vec![0]); + } + assert_eq!( + f.count.put_requests() + f.count.counts().body_requests() + f.count.counts().heads, + 0, + "warm leased reads must add no object-store work" + ); + eprintln!( + "bootstrap_measurement {}", + serde_json::json!({ + "mode": if coalesce { "cohort" } else { "serial" }, "delay_ms": delay, + "cells": count, "sample": sample, "elapsed_ms": elapsed, + "latencies_ms": latencies, "puts": puts, "gets": reads.body_requests(), + "warm_read_object_operations": 0, + }) + ); + for (handle, replica, _) in results { + handle.drain().await.unwrap(); + let control = cell_authority + .load(handle.cell_id()) + .await + .unwrap() + .unwrap(); + let path = f.scratch.path().join(format!( + "restored-{}.sqlite", + crate::identity::encode_hex(handle.cell_id().as_bytes()) + )); + replica + .open_root(&control.value().ltx_root().unwrap()) + .await + .unwrap() + .restore(&path) + .await + .unwrap(); + let db = rusqlite::Connection::open(path).unwrap(); + let value: u64 = db + .query_row("SELECT value FROM counter", [], |row| row.get(0)) + .unwrap(); + assert_eq!(value, 0); + } + runtime.shutdown().await.unwrap(); + drop(feed); + durability.shutdown().await.unwrap(); +} diff --git a/crates/cellule-runtime/src/node/bundle/tests/enrollment.rs b/crates/cellule-runtime/src/node/bundle/tests/enrollment.rs new file mode 100644 index 00000000..5c933c94 --- /dev/null +++ b/crates/cellule-runtime/src/node/bundle/tests/enrollment.rs @@ -0,0 +1,386 @@ +//! Concurrent enrollment must share catalog work without admitting an unpinned Cell. +use super::*; +use crate::node::durability::{NodeBundleAuthority, NodeDurability}; +use crate::node::log_shipper::NodeLogShipper; + +#[tokio::test] +async fn simultaneous_enrollment_amortizes_catalog_writes() { + let mut f = Fixture::new().await; + super::coverage::enroll(&mut f).await; + let mut cells = Vec::new(); + for number in 4..12 { + cells.push(f.unbound_cell_at_commit(number, [9; 16], 0).await); + } + let authority = Arc::new(super::actor::Authority { + directory: f.directory.clone(), + observed: tokio::sync::Mutex::new(f.node.clone()), + }); + let peers = super::actor::transport(&f, true); + let shipper = NodeLogShipper::new(f.gate.clone(), peers.clone(), Limits::default()).unwrap(); + let durability = Arc::new(NodeDurability::new( + f.gate.clone(), + shipper, + authority.clone(), + peers, + f.lease.clone(), + )); + let feed = durability + .enable_bundle_publication(authority.clone()) + .unwrap(); + f.count.reset(); + let results = futures_util::future::join_all( + cells + .iter() + .map(|cell| durability.bind_bundle_cell(&cell.authority, &cell.control)), + ) + .await; + let puts = f.count.put_requests(); + for (cell, control) in cells.iter_mut().zip(results) { + cell.control = control.unwrap(); + assert!(cell.control.value().bundle_binding.is_some()); + } + eprintln!("concurrent_enrollment cells={} puts={puts}", cells.len()); + for cell in &cells { + let issued = f + .gate + .close_cell_issuance( + Fixture::scope(cell), + cell.control.value().ltx_root().unwrap(), + ) + .unwrap(); + authority + .close(&cell.authority, &cell.control, issued) + .await + .unwrap(); + } + drop(feed); + durability.shutdown().await.unwrap(); + assert_eq!( + puts, + cells.len() + 4, + "eight pins share two catalog PUTs and two node CASes" + ); +} + +#[tokio::test] +async fn cohort_enrollment_is_ordered_idempotent_and_separates_applications() { + let mut f = Fixture::new().await; + let a = f.unbound_cell_at_commit(4, [9; 16], 0).await; + let b = f.unbound_cell_at_commit(4, [8; 16], 0).await; + let input = [(&a.authority, &a.control), (&b.authority, &b.control)]; + f.count.reset(); + let (node, pinned) = f + .directory + .bind_bundle_cells(&f.node, &input, NOW) + .await + .unwrap(); + assert_eq!(f.count.put_requests(), 6); + assert_ne!( + pinned[0].value().bundle_binding, + pinned[1].value().bundle_binding + ); + let input = [(&a.authority, &pinned[0]), (&b.authority, &pinned[1])]; + f.count.reset(); + let (same, replay) = f + .directory + .bind_bundle_cells(&node, &input, NOW) + .await + .unwrap(); + assert_eq!(same.advertisement(), node.advertisement()); + assert_eq!(f.count.put_requests(), 0); + for (actual, expected) in replay.iter().zip(&pinned) { + assert_eq!(actual.value(), expected.value()); + } + let catalog = load_catalog( + &f.layout, + SessionId::from_bytes([1; 16]), + node.advertisement().bundle_head().unwrap(), + ) + .await + .unwrap(); + assert_eq!(catalog.bindings.len(), 2); + assert!( + catalog + .bindings + .iter() + .all(|binding| binding.phase == BindingPhase::Open) + ); + for cell in [a, b] { + let path = f.scratch.path().join(format!( + "enrolled-{}.sqlite", + cell.authority.layout().application_id()[0] + )); + cell.replica + .open_root(&cell.control.value().ltx_root().unwrap()) + .await + .unwrap() + .restore(&path) + .await + .unwrap(); + let db = rusqlite::Connection::open(path).unwrap(); + let value: String = db + .query_row( + "SELECT result FROM outcomes WHERE request='seed'", + [], + |row| row.get(0), + ) + .unwrap(); + assert_eq!(value, "original"); + } +} + +#[tokio::test] +async fn invalid_enrollment_cohorts_publish_nothing() { + let mut f = Fixture::new().await; + let cell = f.unbound_cell_at_commit(4, [9; 16], 0).await; + f.count.reset(); + assert!( + f.directory + .bind_bundle_cells(&f.node, &[], NOW) + .await + .is_err() + ); + let duplicate = vec![(&cell.authority, &cell.control); 2]; + assert!( + f.directory + .bind_bundle_cells(&f.node, &duplicate, NOW) + .await + .is_err() + ); + let excessive = vec![(&cell.authority, &cell.control); 9]; + assert!( + f.directory + .bind_bundle_cells(&f.node, &excessive, NOW) + .await + .is_err() + ); + assert_eq!(f.count.put_requests(), 0); + assert!( + cell.authority + .load(cell.control.value().cell) + .await + .unwrap() + .unwrap() + .value() + .bundle_binding + .is_none() + ); +} + +#[tokio::test] +async fn wide_catalog_enrollment_declines_the_cohort_without_refusing_individual_cells() { + use crate::node::bundle::index as catalog_index; + let mut f = Fixture::new().await; + let a = f.cell(4).await; + let shard_a = catalog_index::shard(&[9; 16], &[4; 32]); + let byte_b = (5..=200) + .find(|byte| catalog_index::shard(&[9; 16], &[*byte; 32]) != shard_a) + .unwrap(); + let b = f.cell(byte_b).await; + for seed in [&a, &b] { + let cell = seed.control.value().cell; + let wanted = catalog_index::shard(&[9; 16], cell.as_bytes()); + let head = f.node.advertisement().bundle_head().unwrap(); + let keys = [([9; 16], *cell.as_bytes())].into_iter().collect(); + let mut catalog = + store::load_catalog_cells(&f.layout, SessionId::from_bytes([1; 16]), head, &keys) + .await + .unwrap(); + let template = catalog.bindings[0].clone(); + // As in the density fixtures, unrelated rows have no origin roots and + // grant no native authority. Enrollment must inspect only its targets. + for number in 0_u64..2_000_000 { + let mut id = [255; 32]; + id[..8].copy_from_slice(&number.to_le_bytes()); + if catalog_index::shard(&[9; 16], &id) != wanted { + continue; + } + let mut row = template.clone(); + row.control.cell = CellId::from_bytes(id); + row.control.owner.as_mut().unwrap().endpoint = format!("https://{}", "a".repeat(504)); + row.control.bundle_binding.as_mut().unwrap().digest = + Digest::from_bytes(*blake3::hash(&id).as_bytes()); + catalog.bindings.push(row); + if catalog.bindings.len() == 2_000 { + break; + } + } + assert_eq!(catalog.bindings.len(), 2_000); + catalog + .bindings + .sort_unstable_by_key(|row| *row.control.bundle_binding.unwrap().digest.as_bytes()); + let prepared = f + .directory + .upload_catalog(Some(head), catalog, &[]) + .await + .unwrap(); + f.node = f + .directory + .select_catalog(&f.node, &prepared, NOW) + .await + .unwrap(); + } + let application_for = |cell: CellId, skip: usize| { + let wanted = catalog_index::shard(&[9; 16], cell.as_bytes()); + (10_u64..100_000) + .map(|number| { + let mut app = [0; 16]; + app[..8].copy_from_slice(&number.to_le_bytes()); + app + }) + .filter(|app| catalog_index::shard(app, cell.as_bytes()) == wanted) + .nth(skip) + .unwrap() + }; + let cells = [ + f.unbound_cell_at_commit(4, application_for(a.control.value().cell, 0), 0) + .await, + f.unbound_cell_at_commit(byte_b, application_for(b.control.value().cell, 0), 0) + .await, + f.unbound_cell_at_commit(4, application_for(a.control.value().cell, 1), 0) + .await, + f.unbound_cell_at_commit(4, application_for(a.control.value().cell, 2), 0) + .await, + ]; + let authority = super::actor::Authority { + directory: f.directory.clone(), + observed: tokio::sync::Mutex::new(f.node.clone()), + }; + for (pair, reason) in [ + (&cells[..2], "bundle selected shard bytes"), + (&cells[2..], "bundle enrollment pin metadata"), + ] { + let input = pair + .iter() + .map(|cell| (&cell.authority, &cell.control)) + .collect::>(); + let node = authority.observed.lock().await.clone(); + f.count.reset(); + assert!( + matches!(f.directory.bind_bundle_cells(&node, &input, NOW).await, Err(Error::Capacity(actual)) if actual == reason) + ); + assert!(authority.bind_batch(&input).await.is_none()); + assert_eq!( + f.count.put_requests(), + 0, + "declining must not mutate authority or upload a proposal" + ); + let pinned = + futures_util::future::join_all(input.into_iter().map(|(a, c)| authority.bind(a, c))) + .await; + assert!( + pinned + .into_iter() + .all(|control| control.unwrap().value().bundle_binding.is_some()) + ); + assert_eq!( + f.count.put_requests(), + 10, + "two individually admitted catalog cohorts" + ); + } +} + +#[tokio::test] +async fn partial_pin_failure_preserves_every_provisional_cell_and_retries_exactly() { + cohort_reply_failure(3).await; +} + +#[tokio::test] +async fn failed_open_selection_preserves_all_pins_and_retries_exactly() { + cohort_reply_failure(4).await; +} + +#[tokio::test] +async fn lost_cohort_selection_and_pin_replies_reconcile_without_duplicate_bindings() { + cohort_reply_failure(2).await; + cohort_reply_failure(8).await; +} + +async fn cohort_reply_failure(mode: u8) { + use std::sync::atomic::Ordering; + let backend = Arc::new(super::faults::ReplyFault::default()); + let mut f = Fixture::with_store(backend.clone()).await; + let mut cells = Vec::new(); + for number in 4..12 { + cells.push(f.unbound_cell_at_commit(number, [9; 16], 0).await); + } + let input = cells + .iter() + .map(|cell| (&cell.authority, &cell.control)) + .collect::>(); + backend.mode.store(mode, Ordering::SeqCst); + let result = f.directory.bind_bundle_cells(&f.node, &input, NOW).await; + assert_eq!(result.is_ok(), matches!(mode, 2 | 8)); + drop(input); + backend.mode.store(0, Ordering::SeqCst); + f.node = f + .directory + .load(SessionId::from_bytes([1; 16]), NOW) + .await + .unwrap() + .unwrap(); + let catalog = load_catalog( + &f.layout, + SessionId::from_bytes([1; 16]), + f.node.advertisement().bundle_head().unwrap(), + ) + .await + .unwrap(); + assert_eq!(catalog.bindings.len(), 8); + for cell in &mut cells { + cell.control = cell + .authority + .load(cell.control.value().cell) + .await + .unwrap() + .unwrap(); + } + if matches!(mode, 3 | 4) { + assert!( + catalog + .bindings + .iter() + .all(|binding| binding.phase == BindingPhase::Provisional) + ); + assert_eq!( + cells + .iter() + .filter(|cell| cell.control.value().bundle_binding.is_some()) + .count(), + if mode == 3 { 7 } else { 8 } + ); + } + let pins = catalog + .bindings + .iter() + .map(|binding| binding.control.bundle_binding.unwrap()) + .collect::>(); + let input = cells + .iter() + .map(|cell| (&cell.authority, &cell.control)) + .collect::>(); + let (node, pinned) = f + .directory + .bind_bundle_cells(&f.node, &input, NOW) + .await + .unwrap(); + assert_eq!(pinned.len(), 8); + for control in pinned { + assert!(pins.contains(&control.value().bundle_binding.unwrap())); + } + let catalog = load_catalog( + &f.layout, + SessionId::from_bytes([1; 16]), + node.advertisement().bundle_head().unwrap(), + ) + .await + .unwrap(); + assert_eq!(catalog.bindings.len(), 8); + assert!( + catalog + .bindings + .iter() + .all(|binding| binding.phase == BindingPhase::Open) + ); +} diff --git a/crates/cellule-runtime/src/node/bundle/tests/enrollment_cost.rs b/crates/cellule-runtime/src/node/bundle/tests/enrollment_cost.rs new file mode 100644 index 00000000..e38e315c --- /dev/null +++ b/crates/cellule-runtime/src/node/bundle/tests/enrollment_cost.rs @@ -0,0 +1,140 @@ +//! Manual measured comparison of the original serial adapter and bounded cohorts. +use super::*; +use crate::fleet::resource::{ResourceCost, ResourceLedger}; +use crate::node::durability::{NodeBundleAuthority, NodeDurability}; +use crate::node::log_shipper::NodeLogShipper; +use futures_util::future::BoxFuture; +use object_store::throttle::{ThrottleConfig, ThrottledStore}; +use std::time::{Duration, Instant}; + +pub(super) struct Serial(pub(super) Arc); + +impl NodeBundleAuthority for Serial { + fn bind<'a>( + &'a self, + authority: &'a CellAuthority, + control: &'a VersionedControl, + ) -> BoxFuture<'a, Result> { + self.0.bind(authority, control) + } + fn close<'a>( + &'a self, + authority: &'a CellAuthority, + control: &'a VersionedControl, + issued: crate::node::log::CellIssuedRange, + ) -> BoxFuture<'a, Result<()>> { + self.0.close(authority, control, issued) + } +} + +#[tokio::test] +#[ignore = "manual controlled-latency enrollment diagnostic, not production capacity"] +async fn enrollment_latency_comparison() { + for delay in [0, 5, 20] { + for cells in [1, 8] { + for sample in 0..3 { + // Alternate ordering to keep host drift from favoring a mode. + for coalesce in if sample % 2 == 0 { + [false, true] + } else { + [true, false] + } { + measure(delay, cells, sample, coalesce).await; + } + } + } + } +} + +async fn measure(delay: u64, count: u8, sample: usize, coalesce: bool) { + let backend = Arc::new(ThrottledStore::new( + InMemory::new(), + ThrottleConfig { + wait_get_per_call: Duration::from_millis(delay), + wait_put_per_call: Duration::from_millis(delay), + ..Default::default() + }, + )); + let mut f = Fixture::with_store(backend).await; + super::coverage::enroll(&mut f).await; + let mut cells = Vec::new(); + for number in 4..4 + count { + cells.push(f.unbound_cell_at_commit(number, [9; 16], 0).await); + } + let original = Arc::new(super::actor::Authority { + directory: f.directory.clone(), + observed: tokio::sync::Mutex::new(f.node.clone()), + }); + let authority: Arc = if coalesce { + original.clone() + } else { + Arc::new(Serial(original.clone())) + }; + let peers = super::actor::transport(&f, true); + let shipper = NodeLogShipper::new(f.gate.clone(), peers.clone(), Limits::default()).unwrap(); + let durability = Arc::new(NodeDurability::new( + f.gate.clone(), + shipper, + original.clone(), + peers, + f.lease.clone(), + )); + let resources = ResourceLedger::new(ResourceCost::zero().with_retained_bytes(64 << 20)); + durability + .attach_selection_resources(resources.clone()) + .unwrap(); + let feed = durability.enable_bundle_publication(authority).unwrap(); + f.count.reset(); + let started = Instant::now(); + let results = futures_util::future::join_all(cells.iter().map(|cell| { + let durability = &durability; + async move { + let result = durability + .bind_bundle_cell(&cell.authority, &cell.control) + .await; + (result.unwrap(), started.elapsed().as_secs_f64() * 1000.0) + } + })) + .await; + let elapsed = started.elapsed().as_secs_f64() * 1000.0; + let puts = f.count.put_requests(); + let reads = f.count.counts(); + let latencies = results + .iter() + .map(|(_, latency)| *latency) + .collect::>(); + assert_eq!( + puts, + if coalesce { + count as usize + 4 + } else { + 5 * count as usize + } + ); + eprintln!( + "enrollment_measurement {}", + serde_json::json!({ + "mode": if coalesce { "cohort" } else { "serial" }, + "delay_ms": delay, "cells": count, "sample": sample, + "elapsed_ms": elapsed, "latencies_ms": latencies, + "puts": puts, "gets": reads.body_requests(), "heads": reads.heads, + }) + ); + for (cell, (control, _)) in cells.iter_mut().zip(results) { + cell.control = control; + let issued = f + .gate + .close_cell_issuance( + Fixture::scope(cell), + cell.control.value().ltx_root().unwrap(), + ) + .unwrap(); + original + .close(&cell.authority, &cell.control, issued) + .await + .unwrap(); + } + drop(feed); + durability.shutdown().await.unwrap(); + assert_eq!(resources.snapshot().unwrap().used.retained_bytes(), 0); +} diff --git a/crates/cellule-runtime/src/node/bundle/tests/mod.rs b/crates/cellule-runtime/src/node/bundle/tests/mod.rs index 60f8664f..f473c732 100644 --- a/crates/cellule-runtime/src/node/bundle/tests/mod.rs +++ b/crates/cellule-runtime/src/node/bundle/tests/mod.rs @@ -16,10 +16,13 @@ use std::sync::Arc; const NOW: i64 = 1_000_000; const EPOCH: u64 = 2; mod actor; +mod bootstrap_cost; mod checkpoint_credit; mod checkpoint_pipeline; mod composed; mod coverage; +mod enrollment; +mod enrollment_cost; mod faults; mod index; mod lifecycle; diff --git a/crates/cellule-runtime/src/node/durability/enrollment/mod.rs b/crates/cellule-runtime/src/node/durability/enrollment/mod.rs new file mode 100644 index 00000000..2955dfe9 --- /dev/null +++ b/crates/cellule-runtime/src/node/durability/enrollment/mod.rs @@ -0,0 +1,185 @@ +//! Bounded, epoch-owned enrollment cohorts; no timer on the first cold Cell. +use super::NodeBundleAuthority; +use crate::control::authority::{CellAuthority, VersionedControl}; +use crate::node::bundle::MAX_BUNDLE_ENROLLMENTS; +use crate::node::lease::NodeLeaseGuard; +use crate::{Error, Result}; +use futures_util::{ + FutureExt, + future::{BoxFuture, Shared}, +}; +use std::sync::{Arc, Mutex}; +use tokio::sync::{mpsc, oneshot}; + +type Completion = Shared>>>; + +#[derive(Default)] +pub(super) struct Enrollment { + state: Mutex, +} + +#[derive(Default)] +struct State { + sender: Option>, + completion: Option, + joining: bool, +} + +struct Request { + authority: CellAuthority, + observed: VersionedControl, + reply: oneshot::Sender>, +} + +impl Enrollment { + pub(super) async fn bind( + &self, + authority: &Arc, + lease: &NodeLeaseGuard, + resources: Option<&crate::fleet::resource::ResourceLedger>, + cell_authority: &CellAuthority, + observed: &VersionedControl, + ) -> Result { + lease.check()?; + let sender = { + let mut state = self + .state + .lock() + .map_err(|_| Error::Node("bundle enrollment lock poisoned"))?; + if state.joining { + return Err(Error::RuntimeClosed); + } + if state.sender.is_none() { + let runtime = tokio::runtime::Handle::try_current().map_err(Error::RuntimeStart)?; + // Two bounded cohorts (running and queued), including owned + // decoded metadata and bookkeeping. Standalone manual users + // without an attached runtime ledger own their host admission. + let working = resources + .map(|resources| { + resources.try_reserve_in( + crate::fleet::resource::RetentionStage::Publisher, + crate::fleet::resource::ResourceCost::zero().with_retained_bytes( + 2 * MAX_BUNDLE_ENROLLMENTS * crate::control::MAX_CONTROL_BYTES * 4 + + (64 << 10), + ), + ) + }) + .transpose()?; + let (sender, receiver) = mpsc::channel(MAX_BUNDLE_ENROLLMENTS); + let authority = authority.clone(); + let lease = lease.clone(); + let worker = runtime.spawn(async move { + let _working = working; + run(authority, lease, receiver).await; + }); + let completion = async move { + worker + .await + .map_err(|source| Arc::new(Error::FollowerWorkerJoin(source))) + } + .boxed() + .shared(); + state.sender = Some(sender); + state.completion = Some(completion); + } + state.sender.as_ref().cloned().ok_or(Error::RuntimeClosed)? + }; + // Reserve before cloning bounded control metadata. A waiting caller + // retains its original activation reservation, not another queued copy. + let permit = tokio::select! { + permit = sender.reserve() => permit.map_err(|_| Error::RuntimeClosed)?, + () = lease.wait_fenced() => return Err(Error::Fenced), + }; + lease.check()?; + let (reply, response) = oneshot::channel(); + permit.send(Request { + authority: cell_authority.clone(), + observed: observed.clone(), + reply, + }); + drop(sender); + // Once accepted, a caller disconnect cannot abandon catalog obligations + // or drop a sibling's CAS. Epoch shutdown joins the retained worker. + response.await.map_err(|_| Error::RuntimeClosed)? + } + + pub(super) async fn join(&self) -> Result<()> { + let completion = { + let mut state = self + .state + .lock() + .map_err(|_| Error::Node("bundle enrollment lock poisoned"))?; + state.joining = true; + state.sender.take(); + state.completion.clone() + }; + if let Some(completion) = completion { + completion.await.map_err(Error::Shared)?; + } + Ok(()) + } +} + +async fn run( + authority: Arc, + lease: NodeLeaseGuard, + mut receiver: mpsc::Receiver, +) { + while let Some(first) = receiver.recv().await { + let mut requests = Vec::with_capacity(MAX_BUNDLE_ENROLLMENTS); + requests.push(first); + // One scheduler yield coalesces ready activations without a batching + // timer or a requirement to wait for every Cell on the node. + tokio::task::yield_now().await; + while requests.len() < MAX_BUNDLE_ENROLLMENTS { + match receiver.try_recv() { + Ok(request) => requests.push(request), + Err(_) => break, + } + } + let cells = requests + .iter() + .map(|request| (&request.authority, &request.observed)) + .collect::>(); + if lease.check().is_err() { + for request in requests { + let _ = request.reply.send(Err(Error::Fenced)); + } + continue; + } + let Some(results) = authority.bind_batch(&cells).await else { + // Legacy adapters release each response after its own callback, + // without imposing a cohort barrier on independent successes. + let pending = requests + .into_iter() + .map(|request| { + let authority = authority.clone(); + let lease = lease.clone(); + async move { + let result = match lease.check() { + Ok(()) => authority.bind(&request.authority, &request.observed).await, + Err(_) => Err(Error::Fenced), + }; + let _ = request.reply.send(result); + } + }) + .collect::>(); + futures_util::future::join_all(pending).await; + continue; + }; + if results.len() != requests.len() { + for request in requests { + let _ = request + .reply + .send(Err(Error::Node("bundle enrollment result count differs"))); + } + } else { + for (request, result) in requests.into_iter().zip(results) { + let _ = request.reply.send(result); + } + } + } +} + +#[cfg(test)] +mod tests; diff --git a/crates/cellule-runtime/src/node/durability/enrollment/tests.rs b/crates/cellule-runtime/src/node/durability/enrollment/tests.rs new file mode 100644 index 00000000..ee389bbe --- /dev/null +++ b/crates/cellule-runtime/src/node/durability/enrollment/tests.rs @@ -0,0 +1,226 @@ +//! Callback ownership and memory admission survive cancelled waiters and joins. +use super::*; +use crate::cell::catalog::{CatalogEntry, CatalogRole, CellCatalog}; +use crate::control::Owner; +use crate::fleet::resource::{ResourceCost, ResourceLedger}; +use crate::identity::{ + ApplicationId, CellTarget, Digest, IncarnationId, NamespaceId, SessionId, TenantId, +}; +use cellule_ltx::CellStorageLayout; +use cellule_store::Store; +use object_store::{memory::InMemory, path::Path}; +use std::sync::atomic::{AtomicUsize, Ordering}; +use tokio::sync::Notify; + +#[derive(Default)] +struct Held { + entered: Notify, + resume: Notify, + calls: AtomicUsize, + completed: AtomicUsize, + cohort: AtomicUsize, + serial: tokio::sync::Mutex<()>, +} + +impl NodeBundleAuthority for Held { + fn bind_batch<'a>( + &'a self, + cells: &'a [(&'a CellAuthority, &'a VersionedControl)], + ) -> BoxFuture<'a, Option>>> { + Box::pin(async move { + self.cohort.store(cells.len(), Ordering::SeqCst); + None + }) + } + + fn bind<'a>( + &'a self, + _: &'a CellAuthority, + observed: &'a VersionedControl, + ) -> BoxFuture<'a, Result> { + Box::pin(async move { + let _serial = self.serial.lock().await; + self.calls.fetch_add(1, Ordering::SeqCst); + self.entered.notify_one(); + self.resume.notified().await; + self.completed.fetch_add(1, Ordering::SeqCst); + Ok(observed.clone()) + }) + } + fn close<'a>( + &'a self, + _: &'a CellAuthority, + _: &'a VersionedControl, + _: crate::node::log::CellIssuedRange, + ) -> BoxFuture<'a, Result<()>> { + Box::pin(async { Ok(()) }) + } +} + +async fn fixture() -> (CellAuthority, VersionedControl) { + let layout = CellStorageLayout::new( + Store::new(Arc::new(InMemory::new())), + Path::from("enrollment-lifetime"), + [1; 16], + ); + let target = CellTarget::new( + TenantId::from_bytes([2; 16]), + ApplicationId::from_bytes([1; 16]), + NamespaceId::from_bytes([3; 16]), + b"cell", + ) + .unwrap(); + let proof = CellCatalog::new(layout.clone(), target.tenant()) + .provision( + CatalogEntry::new( + &target, + CatalogRole::Application, + Digest::from_bytes([4; 32]), + 1, + ) + .unwrap(), + ) + .await + .unwrap(); + let authority = CellAuthority::new(layout); + let control = authority + .create_initial( + &proof, + IncarnationId::from_bytes([5; 16]), + Owner { + session: SessionId::from_bytes([6; 16]), + endpoint: "https://enrollment.internal".into(), + }, + ) + .await + .unwrap(); + (authority, control) +} + +#[tokio::test] +async fn cancelled_waiter_and_cancelled_join_cannot_abandon_the_accepted_callback() { + let (cell_authority, control) = fixture().await; + let enrollment = Arc::new(Enrollment::default()); + let original = Arc::new(Held::default()); + let authority: Arc = original.clone(); + let resources = ResourceLedger::new(ResourceCost::zero().with_retained_bytes(2 << 20)); + let lease = NodeLeaseGuard::new(1_000, 31_000).unwrap(); + let caller = { + let enrollment = enrollment.clone(); + let authority = authority.clone(); + let lease = lease.clone(); + let resources = resources.clone(); + let cell_authority = cell_authority.clone(); + let control = control.clone(); + tokio::spawn(async move { + enrollment + .bind( + &authority, + &lease, + Some(&resources), + &cell_authority, + &control, + ) + .await + }) + }; + original.entered.notified().await; + caller.abort(); + assert!(matches!(caller.await, Err(source) if source.is_cancelled())); + assert_eq!(original.completed.load(Ordering::SeqCst), 0); + assert!(resources.snapshot().unwrap().used.retained_bytes() > 0); + let joining = { + let enrollment = enrollment.clone(); + tokio::spawn(async move { enrollment.join().await }) + }; + // Join must close admission before awaiting the still-held callback. + loop { + if enrollment.state.lock().unwrap().joining { + break; + } + tokio::task::yield_now().await; + } + assert!(!joining.is_finished()); + joining.abort(); + assert!(joining.await.unwrap_err().is_cancelled()); + assert!(matches!( + enrollment + .bind( + &authority, + &lease, + Some(&resources), + &cell_authority, + &control + ) + .await, + Err(Error::RuntimeClosed) + )); + original.resume.notify_one(); + enrollment.join().await.unwrap(); + assert_eq!(original.calls.load(Ordering::SeqCst), 1); + assert_eq!(original.completed.load(Ordering::SeqCst), 1); + assert_eq!(resources.snapshot().unwrap().used.retained_bytes(), 0); +} + +#[tokio::test] +async fn insufficient_memory_refuses_before_accepting_callback_or_cloning_a_queue() { + let (cell_authority, control) = fixture().await; + let enrollment = Enrollment::default(); + let original = Arc::new(Held::default()); + let authority: Arc = original.clone(); + let resources = ResourceLedger::new(ResourceCost::zero().with_retained_bytes(1)); + let lease = NodeLeaseGuard::new(1_000, 31_000).unwrap(); + assert!(matches!( + enrollment + .bind( + &authority, + &lease, + Some(&resources), + &cell_authority, + &control + ) + .await, + Err(Error::Capacity(_)) + )); + assert_eq!(original.calls.load(Ordering::SeqCst), 0); + assert_eq!(resources.snapshot().unwrap().used.retained_bytes(), 0); + enrollment.join().await.unwrap(); +} + +#[tokio::test] +async fn legacy_callback_releases_first_result_while_the_second_callback_is_held() { + let (cell_authority, control) = fixture().await; + let enrollment = Arc::new(Enrollment::default()); + let original = Arc::new(Held::default()); + let authority: Arc = original.clone(); + let lease = NodeLeaseGuard::new(1_000, 31_000).unwrap(); + let spawn = || { + let enrollment = enrollment.clone(); + let authority = authority.clone(); + let lease = lease.clone(); + let cell_authority = cell_authority.clone(); + let control = control.clone(); + tokio::spawn(async move { + enrollment + .bind(&authority, &lease, None, &cell_authority, &control) + .await + }) + }; + let first = spawn(); + let second = spawn(); + original.entered.notified().await; + assert_eq!(original.cohort.load(Ordering::SeqCst), 2); + original.resume.notify_one(); + original.entered.notified().await; + assert!( + tokio::time::timeout(std::time::Duration::from_secs(5), first) + .await + .unwrap() + .unwrap() + .is_ok() + ); + assert!(!second.is_finished()); + original.resume.notify_one(); + assert!(second.await.unwrap().is_ok()); + enrollment.join().await.unwrap(); +} diff --git a/crates/cellule-runtime/src/node/durability/mod.rs b/crates/cellule-runtime/src/node/durability/mod.rs index a5d99730..62724cf7 100644 --- a/crates/cellule-runtime/src/node/durability/mod.rs +++ b/crates/cellule-runtime/src/node/durability/mod.rs @@ -16,6 +16,7 @@ use crate::node::log_transport::NodeLogTransport; use crate::{Error, Result}; mod activation; +mod enrollment; mod object_coverage; use object_coverage::ObjectCoverage; mod publication; @@ -40,6 +41,23 @@ pub trait NodeBundleAuthority: Send + Sync { observed: &'a crate::control::authority::VersionedControl, ) -> BoxFuture<'a, Result>; + /// Optionally enrolls one bounded cohort, returning one result per input in + /// input order. Implement with `NodeDirectory::bind_bundle_cells` under the + /// original catalog lock to share its provisional/Open publication pair. + /// Return `None` before enrollment publication or Cell pinning to decline + /// a cohort whose metadata exceeds the per-operation bound. Independent + /// heartbeat renewal is allowed. The default declines; individual `bind` + /// callbacks then deliver each result promptly. + fn bind_batch<'a>( + &'a self, + _cells: &'a [( + &'a crate::control::authority::CellAuthority, + &'a crate::control::authority::VersionedControl, + )], + ) -> BoxFuture<'a, Option>>> { + Box::pin(async { None }) + } + /// Joins complete issued coverage, materialization/checkpoint and catalog /// closure after SQL/capture tasks close, including earlier Fleet ACKs. fn close<'a>( @@ -197,6 +215,7 @@ pub struct NodeDurability { transport: Arc, node_lease: NodeLeaseGuard, activation: activation::Activation, + enrollment: enrollment::Enrollment, object_coverage: ObjectCoverage, shutdown: Mutex<()>, retirement: std::sync::Mutex>>, @@ -247,6 +266,7 @@ impl NodeDurability { transport, node_lease, activation: activation::Activation::default(), + enrollment: enrollment::Enrollment::default(), object_coverage: ObjectCoverage::default(), shutdown: Mutex::new(()), retirement: std::sync::Mutex::new(None), @@ -408,7 +428,16 @@ impl NodeDurability { return Ok(observed.clone()); }; self.node_lease.check()?; - let bound = bundle.bind(authority, observed).await?; + let bound = self + .enrollment + .bind( + bundle, + &self.node_lease, + self.selection_resources.get(), + authority, + observed, + ) + .await?; self.node_lease.check()?; observed .value() @@ -666,6 +695,9 @@ impl NodeDurability { } return Ok(proof); } + // Enrollment owns accepted catalog/Cell CAS callbacks independently of + // caller cancellation. Join them before closing this epoch's producer. + let enrollment = self.enrollment.join().await; let shipping = self.shipper.shutdown().await; let publication = if let Some(publisher) = self.publisher.get() { // Join both tasks even when fencing stopped the shipper. Preserve @@ -678,6 +710,7 @@ impl NodeDurability { // and closure use the authority adapter's acknowledged version, even // when shipping or publication has already failed. let activation = self.activation.join().await; + enrollment?; publication?; shipping?; activation?; diff --git a/docs/bundle-coverage-implementation.md b/docs/bundle-coverage-implementation.md index 295b748c..5c5be5e1 100644 --- a/docs/bundle-coverage-implementation.md +++ b/docs/bundle-coverage-implementation.md @@ -54,7 +54,7 @@ Uploading an immutable object supplies no coverage proof. | API or boundary | Behavior | | --- | --- | -| `NodeDirectory::initialize_bundle_lane` / `bind_bundle_cell` | Establish a boot/epoch catalog; reserve a provisional inventory entry, pin the original Cell's base/code/schema/writer, then open it before issuing bundled commands | +| `NodeDirectory::initialize_bundle_lane` / `bind_bundle_cell` / `bind_bundle_cells` | Establish a boot/epoch catalog; up to eight ready Cells share provisional/Open inventory publication while retaining their individual original base/code/schema/writer pins before SQL admission | | `NodeLogShipper::submit_assigned` / `NodeDurability::submit_assigned` | Use the existing bounded native shipping lane and return an opaque witness for every frame in a complete capture | | `prepare_node_bundle` / `select_node_bundle` | Reject missing, overlapping, unassigned or cross-binding ranges; verify origin extents; select the bundle and enrolled native coverage in one node CAS; reconcile only the exact head and coverage under the original lease | | `NodeDurability::confirm_bundle` | Confirm only the exact original assigned captures and lease locally, with no storage I/O or second authority CAS; reject cold reconstruction proofs and replacement gates or guards | @@ -229,6 +229,7 @@ rewrite existing objects or discard recovery obligations. | Selected shard plus history encoded metadata | 4 MiB, checked before the respective reads | | Authenticated index header | 32 KiB / 256 shard references | | Original bindings per boot/epoch | 4,096 | +| Enrollment cohort / waiting queue | 8 / 8 Cells | | Native frames per selection | 64 | | Detached exact frame references per Cell | 256 | | One encoded history | 32 KiB | diff --git a/docs/cell-bootstrap-latency.md b/docs/cell-bootstrap-latency.md new file mode 100644 index 00000000..8606db1c --- /dev/null +++ b/docs/cell-bootstrap-latency.md @@ -0,0 +1,123 @@ +# Cell creation and first-request latency + +Creating a Cell performs durable work before its handle can serve a request. +Putting creation inside an HTTP handler moves that work onto the handler's +critical path. It does not remove the publication or ownership checks. + +## Required ordering + +| Stage | Framework boundary | Required result | +| --- | --- | --- | +| Catalog | `CellCatalog::provision` | Immutable entry selected by its tenant/application shard head | +| Ownership reservation | `CellAuthority::create_initial` | Rootless, Recovering control with one original owner | +| SQLite initialization | `CellRuntime::bootstrap` | Runtime metadata and application schema committed and captured on a SQL worker | +| Initial publication | `prepare_initial` / `publish_prepared` | Verified initial root selected by the fenced Cell control; state becomes Serving | +| Bundle enrollment, when installed | `NodeBundleAuthority` | Provisional node inventory selected, original Cell pinned, then Open inventory selected | +| Serving | Actor activation completion | Handle returned only after publication and enrollment complete | + +Ownership is reserved **before** SQLite initialization. Serving authority follows +initial root publication. Bundle enrollment applies when the node has installed +that authority; an ordinary object-only bootstrap does not add it automatically. + +The ordinary Cell catalog has 256 shards and no framework-wide async mutex. +The node bundle catalog is a separate inventory. Its authority adapter orders +enrollment, publication and closure around one selected node head. An embedding +can introduce additional serialization outside these framework boundaries. + +Independent activations and immutable LTX uploads already overlap. The remaining +enrollment sequence has dependent publication steps, so simply spawning more +requests against a serialized adapter does not eliminate its object-store cost. + +## Bounded enrollment cohorts + +`NodeDirectory::bind_bundle_cells` enrolls up to eight ready Cells in one +provisional/Open catalog pair. The existing single-Cell method uses this same +implementation. Each Cell still receives its own original control CAS. + +For a fresh cohort, without retries: + +| Eight ready enrollments | Immutable catalog PUTs | Node selector CASes | Cell pin CASes | Total PUTs | +| --- | ---: | ---: | ---: | ---: | +| Original per-Cell adapter | 16 | 16 | 8 | 40 | +| One bounded cohort | 2 | 2 | 8 | 12 | + +Initial SQLite/root publication is additional work in both rows. A lone Cell +still performs five enrollment PUTs. Fresh node and catalog reads remain part +of each pin's authority check; cached metadata does not replace them. + +The runtime retains one epoch-owned worker with an eight-entry queue and an +eight-Cell running cohort. One scheduler yield collects ready activations, +without a batching timer or a barrier over the node's entire Cell population. +The runtime ledger reserves 576 KiB for queued/running control metadata and +bookkeeping. Cohort admission checks authenticated shard sizes so simultaneous +pin checks stay within the original single-pin raw metadata allowance. If the +cohort cannot fit, the reference adapter declines it before enrollment uploads +or Cell pinning and invokes individual callbacks. A maximal shard therefore +remains admissible individually, and heartbeat work can run between callbacks. + +Implement the optional `NodeBundleAuthority::bind_batch` hook using the canonical +cohort method under the original catalog ordering. The +[reference adapter](../crates/cellule-axum/examples/fleet/authority/mod.rs) +does this. Adapters without the hook continue through individual `bind` +callbacks and receive each result as soon as that callback completes. Different +Cells must reach activation concurrently to benefit: an embedding that holds +one creation-map lock through all bootstrap I/O must narrow that lock first. +Sharing catalog publication does not combine the Cells' SQLite transactions. + +All pin attempts finish before the Open catalog is selected. A failed or +cancelled attempt can leave a durable provisional obligation; it cannot open +SQL admission or erase a pinned sibling. Retries reload authority and reuse +the original bindings. Epoch shutdown joins accepted work independently of +caller cancellation and releases the worker's reservation. + +## Fresh controlled measurements + +The diagnostics run real SQLite initialization, capture, root publication, +control CAS, enrollment, and first reads through the runtime APIs. After each +run they drain and restore the authoritative roots and verify initialized +state. Repeated warm leased reads must perform zero object-store operations. + +Both adapters run in the same test binary to isolate enrollment behavior. +The fixture uses an in-memory object store with controlled per-call delays, +temporary local SQLite files, two SQL workers, and three fresh runs per setting. +Each small Cell initializes one counter table to zero, plus runtime metadata. +The shared workstation also runs other workloads. Results describe this local +diagnostic; provider, HTTP, three-VM capacity and ordinary write throughput +require their separate qualification runs. + +Run with Rust 1.97.0: + +```sh +CARGO_PROFILE_TEST_DEBUG=0 cargo +1.97.0 test -p cellule-runtime --lib comparison \ + --locked -- --ignored --nocapture --test-threads=1 +``` + +`enrollment_measurement` reports ready-to-enroll latency and object operations. +`bootstrap_measurement` reports catalog provisioning through the first SQL read, +including owner creation, SQLite, initial root and enrollment. Each record +contains the adapter, delay, population, sample number and per-Cell latencies. + +Fresh full-path results, including the first SQL read (milliseconds): + +| Per-call delay | Concurrent new Cells | Serial p50 / p95 | Cohort p50 / p95 | +| --- | ---: | ---: | ---: | +| 5 ms | 1 | 136.6 / 139.5 | 133.8 / 157.9 | +| 5 ms | 8 | 351.5 / 639.3 | 212.1 / 220.6 | +| 20 ms | 1 | 386.7 / 428.6 | 386.4 / 425.3 | +| 20 ms | 8 | 1,121.5 / 2,009.4 | 640.7 / 665.6 | + +These are nearest-rank sample percentiles: three observations for a lone Cell, +24 for each eight-Cell setting. The full eight-Cell runs used two enrollment +cohorts as initialization completed, reducing total creation PUTs from 96 to +72 and GETs from 48 to 42. When all eight Cells were already ready to enroll, +the isolated stage used one cohort: 40 to 12 PUTs and 32 to 25 GETs. With 20 ms +per-call delay, that stage's p95 was 1,689.0 versus 251.0 ms. + +The single-Cell path keeps its mandatory durable work. Cohorts reduce burst +queueing; a response in a cohort still waits for all its pin checks and the +shared Open selection. See the [raw observations and source checksums](measurements/cell-bootstrap-latency.json). + +For known populations, an embedding can provision expected Cells with bounded +concurrency before admitting external traffic. Lazy creation still pays the +durable bootstrap cost on its first request. Neither approach changes the +first write's normal durability/output gate. diff --git a/docs/measurements/cell-bootstrap-latency.json b/docs/measurements/cell-bootstrap-latency.json new file mode 100644 index 00000000..e609ee60 --- /dev/null +++ b/docs/measurements/cell-bootstrap-latency.json @@ -0,0 +1,1493 @@ +{ + "base_revision": "aec0e2b73528bdb0471cfe713b9b3ce30c6f37cb", + "source_sha256": { + "crates/cellule-runtime/src/node/bundle/binding.rs": "e41002bc453fe78a270c4ca30f9dfd95dacaa5e52ac2a397e846a56be07a7f76", + "crates/cellule-runtime/src/node/bundle/index/mod.rs": "0e5048439ea9ed9aafb722be9283d9dbee37f7943685430b5c8eb2886b8cb9bc", + "crates/cellule-runtime/src/node/bundle/mod.rs": "9dbe7b079bbcfb067375f8f4023b7c9cc6c5da863173d7701db490c45288f77c", + "crates/cellule-runtime/src/node/durability/mod.rs": "01352634814ad10100fec1d5e9e48ef50caa7910f4d05e32f9200812b41f7e7c", + "crates/cellule-runtime/src/node/durability/enrollment/mod.rs": "cb32ea6f1dd0662fcf0b2d6b49f9be676fb7c12d9985e7040e0e4e911423ac23", + "crates/cellule-axum/examples/fleet/authority/mod.rs": "0286b6ef2b7afd7b9615822f2541dde8d8e56cbb0ca7aa47bf9c755f32b3bf28", + "crates/cellule-runtime/src/node/bundle/tests/bootstrap_cost.rs": "e71811917a31bc9e2dc56670d5e1a0f00ede46fb64447e882355c05ba16f29d5", + "crates/cellule-runtime/src/node/bundle/tests/enrollment_cost.rs": "84b5f02ac05ebc3d3decfd1f2a5824240fe7ff626ef8d9f44ace4e4a3982ad23", + "crates/cellule-runtime/src/node/bundle/tests/actor.rs": "d4d7c25f8d5cd501b10770256755f46612a53cb8d01f492ee97d56d4a8461385" + }, + "environment": { + "toolchain": "Rust 1.97.0", + "test_profile": "unoptimized, debug=0", + "host": "shared macOS/aarch64 workstation with other workloads", + "object_store": "fresh InMemory behind ThrottledStore per fixture", + "per_call_delay_ms": [ + 0, + 5, + 20 + ], + "sqlite": "fresh temporary local SQLite databases", + "sql_workers_for_full_bootstrap": 2, + "repetitions_per_setting": 3, + "workload": "Small Cells with one counter initialized to zero plus runtime metadata; no concurrent application write commands", + "node_cpu_or_memory_quota": "none; shared native host" + }, + "comparison": "Both adapters run in one binary. Serial uses individual canonical bind callbacks; cohort implements the optional bounded bind_batch hook.", + "command": "CARGO_PROFILE_TEST_DEBUG=0 cargo +1.97.0 test -p cellule-runtime --lib comparison --locked -- --ignored --nocapture --test-threads=1", + "postmeasurement_checks": { + "all_initialized_roots_restored_and_counter_zero": true, + "all_warm_leased_reads_zero_object_operations": true, + "both_manual_diagnostics_exit_code": 0 + }, + "scope": "Framework API bootstrap and enrollment under controlled delays; provider/HTTP/three-VM capacity and ordinary write throughput remain separate qualifications.", + "percentiles": "nearest rank over per-Cell latency observations", + "summary": [ + { + "kind": "bootstrap_measurement", + "delay_ms": 5, + "cells": 1, + "mode": "cohort", + "runs": 3, + "observations": 3, + "p50_ms": 133.840875, + "p95_ms": 157.89666599999998, + "puts_per_run": [ + 12, + 12, + 12 + ], + "gets_per_run": [ + 6, + 6, + 6 + ] + }, + { + "kind": "bootstrap_measurement", + "delay_ms": 5, + "cells": 1, + "mode": "serial", + "runs": 3, + "observations": 3, + "p50_ms": 136.60566599999999, + "p95_ms": 139.472542, + "puts_per_run": [ + 12, + 12, + 12 + ], + "gets_per_run": [ + 6, + 6, + 6 + ] + }, + { + "kind": "bootstrap_measurement", + "delay_ms": 5, + "cells": 8, + "mode": "cohort", + "runs": 3, + "observations": 24, + "p50_ms": 212.0875, + "p95_ms": 220.632, + "puts_per_run": [ + 72, + 72, + 72 + ], + "gets_per_run": [ + 42, + 42, + 42 + ] + }, + { + "kind": "bootstrap_measurement", + "delay_ms": 5, + "cells": 8, + "mode": "serial", + "runs": 3, + "observations": 24, + "p50_ms": 351.481334, + "p95_ms": 639.325209, + "puts_per_run": [ + 96, + 96, + 96 + ], + "gets_per_run": [ + 48, + 48, + 48 + ] + }, + { + "kind": "bootstrap_measurement", + "delay_ms": 20, + "cells": 1, + "mode": "cohort", + "runs": 3, + "observations": 3, + "p50_ms": 386.435208, + "p95_ms": 425.33824999999996, + "puts_per_run": [ + 12, + 12, + 12 + ], + "gets_per_run": [ + 6, + 6, + 6 + ] + }, + { + "kind": "bootstrap_measurement", + "delay_ms": 20, + "cells": 1, + "mode": "serial", + "runs": 3, + "observations": 3, + "p50_ms": 386.68875, + "p95_ms": 428.591541, + "puts_per_run": [ + 12, + 12, + 12 + ], + "gets_per_run": [ + 6, + 6, + 6 + ] + }, + { + "kind": "bootstrap_measurement", + "delay_ms": 20, + "cells": 8, + "mode": "cohort", + "runs": 3, + "observations": 24, + "p50_ms": 640.663709, + "p95_ms": 665.60475, + "puts_per_run": [ + 72, + 72, + 72 + ], + "gets_per_run": [ + 42, + 42, + 42 + ] + }, + { + "kind": "bootstrap_measurement", + "delay_ms": 20, + "cells": 8, + "mode": "serial", + "runs": 3, + "observations": 24, + "p50_ms": 1121.455042, + "p95_ms": 2009.354334, + "puts_per_run": [ + 96, + 96, + 96 + ], + "gets_per_run": [ + 48, + 48, + 48 + ] + }, + { + "kind": "enrollment_measurement", + "delay_ms": 0, + "cells": 1, + "mode": "cohort", + "runs": 3, + "observations": 3, + "p50_ms": 8.079042, + "p95_ms": 9.012625, + "puts_per_run": [ + 5, + 5, + 5 + ], + "gets_per_run": [ + 4, + 4, + 4 + ] + }, + { + "kind": "enrollment_measurement", + "delay_ms": 0, + "cells": 1, + "mode": "serial", + "runs": 3, + "observations": 3, + "p50_ms": 7.062208, + "p95_ms": 12.232083, + "puts_per_run": [ + 5, + 5, + 5 + ], + "gets_per_run": [ + 4, + 4, + 4 + ] + }, + { + "kind": "enrollment_measurement", + "delay_ms": 0, + "cells": 8, + "mode": "cohort", + "runs": 3, + "observations": 24, + "p50_ms": 17.975250000000003, + "p95_ms": 18.097917000000002, + "puts_per_run": [ + 12, + 12, + 12 + ], + "gets_per_run": [ + 25, + 25, + 25 + ] + }, + { + "kind": "enrollment_measurement", + "delay_ms": 0, + "cells": 8, + "mode": "serial", + "runs": 3, + "observations": 24, + "p50_ms": 56.288625, + "p95_ms": 56.736292, + "puts_per_run": [ + 40, + 40, + 40 + ], + "gets_per_run": [ + 32, + 32, + 32 + ] + }, + { + "kind": "enrollment_measurement", + "delay_ms": 5, + "cells": 1, + "mode": "cohort", + "runs": 3, + "observations": 3, + "p50_ms": 94.38050000000001, + "p95_ms": 106.501333, + "puts_per_run": [ + 5, + 5, + 5 + ], + "gets_per_run": [ + 4, + 4, + 4 + ] + }, + { + "kind": "enrollment_measurement", + "delay_ms": 5, + "cells": 1, + "mode": "serial", + "runs": 3, + "observations": 3, + "p50_ms": 85.301917, + "p95_ms": 96.316625, + "puts_per_run": [ + 5, + 5, + 5 + ], + "gets_per_run": [ + 4, + 4, + 4 + ] + }, + { + "kind": "enrollment_measurement", + "delay_ms": 5, + "cells": 8, + "mode": "cohort", + "runs": 3, + "observations": 24, + "p50_ms": 101.3045, + "p95_ms": 104.84845800000001, + "puts_per_run": [ + 12, + 12, + 12 + ], + "gets_per_run": [ + 25, + 25, + 25 + ] + }, + { + "kind": "enrollment_measurement", + "delay_ms": 5, + "cells": 8, + "mode": "serial", + "runs": 3, + "observations": 24, + "p50_ms": 372.474333, + "p95_ms": 726.781083, + "puts_per_run": [ + 40, + 40, + 40 + ], + "gets_per_run": [ + 32, + 32, + 32 + ] + }, + { + "kind": "enrollment_measurement", + "delay_ms": 20, + "cells": 1, + "mode": "cohort", + "runs": 3, + "observations": 3, + "p50_ms": 244.017958, + "p95_ms": 244.2795, + "puts_per_run": [ + 5, + 5, + 5 + ], + "gets_per_run": [ + 4, + 4, + 4 + ] + }, + { + "kind": "enrollment_measurement", + "delay_ms": 20, + "cells": 1, + "mode": "serial", + "runs": 3, + "observations": 3, + "p50_ms": 245.225125, + "p95_ms": 250.705791, + "puts_per_run": [ + 5, + 5, + 5 + ], + "gets_per_run": [ + 4, + 4, + 4 + ] + }, + { + "kind": "enrollment_measurement", + "delay_ms": 20, + "cells": 8, + "mode": "cohort", + "runs": 3, + "observations": 24, + "p50_ms": 218.660292, + "p95_ms": 251.006542, + "puts_per_run": [ + 12, + 12, + 12 + ], + "gets_per_run": [ + 25, + 25, + 25 + ] + }, + { + "kind": "enrollment_measurement", + "delay_ms": 20, + "cells": 8, + "mode": "serial", + "runs": 3, + "observations": 24, + "p50_ms": 957.888709, + "p95_ms": 1689.0104999999999, + "puts_per_run": [ + 40, + 40, + 40 + ], + "gets_per_run": [ + 32, + 32, + 32 + ] + } + ], + "observations": [ + { + "cells": 1, + "delay_ms": 5, + "elapsed_ms": 139.477083, + "gets": 6, + "latencies_ms": [ + 139.472542 + ], + "mode": "serial", + "puts": 12, + "sample": 0, + "warm_read_object_operations": 0, + "kind": "bootstrap_measurement" + }, + { + "cells": 1, + "delay_ms": 5, + "elapsed_ms": 133.846708, + "gets": 6, + "latencies_ms": [ + 133.840875 + ], + "mode": "cohort", + "puts": 12, + "sample": 0, + "warm_read_object_operations": 0, + "kind": "bootstrap_measurement" + }, + { + "cells": 1, + "delay_ms": 5, + "elapsed_ms": 130.922166, + "gets": 6, + "latencies_ms": [ + 130.91687499999998 + ], + "mode": "cohort", + "puts": 12, + "sample": 1, + "warm_read_object_operations": 0, + "kind": "bootstrap_measurement" + }, + { + "cells": 1, + "delay_ms": 5, + "elapsed_ms": 136.61041600000001, + "gets": 6, + "latencies_ms": [ + 136.60566599999999 + ], + "mode": "serial", + "puts": 12, + "sample": 1, + "warm_read_object_operations": 0, + "kind": "bootstrap_measurement" + }, + { + "cells": 1, + "delay_ms": 5, + "elapsed_ms": 136.58999999999997, + "gets": 6, + "latencies_ms": [ + 136.5835 + ], + "mode": "serial", + "puts": 12, + "sample": 2, + "warm_read_object_operations": 0, + "kind": "bootstrap_measurement" + }, + { + "cells": 1, + "delay_ms": 5, + "elapsed_ms": 157.904875, + "gets": 6, + "latencies_ms": [ + 157.89666599999998 + ], + "mode": "cohort", + "puts": 12, + "sample": 2, + "warm_read_object_operations": 0, + "kind": "bootstrap_measurement" + }, + { + "cells": 8, + "delay_ms": 5, + "elapsed_ms": 640.4045000000001, + "gets": 48, + "latencies_ms": [ + 135.712875, + 351.481334, + 209.289917, + 423.7625, + 280.006375, + 495.30654200000004, + 568.5350420000001, + 640.391209 + ], + "mode": "serial", + "puts": 96, + "sample": 0, + "warm_read_object_operations": 0, + "kind": "bootstrap_measurement" + }, + { + "cells": 8, + "delay_ms": 5, + "elapsed_ms": 220.644792, + "gets": 42, + "latencies_ms": [ + 142.84199999999998, + 220.577958, + 219.96925000000002, + 220.579917, + 220.096167, + 220.630625, + 220.632, + 220.633333 + ], + "mode": "cohort", + "puts": 72, + "sample": 0, + "warm_read_object_operations": 0, + "kind": "bootstrap_measurement" + }, + { + "cells": 8, + "delay_ms": 5, + "elapsed_ms": 209.011292, + "gets": 42, + "latencies_ms": [ + 131.59316700000002, + 208.936959, + 208.33545900000001, + 208.99525, + 208.42054199999998, + 208.99675000000002, + 208.998209, + 208.999459 + ], + "mode": "cohort", + "puts": 72, + "sample": 1, + "warm_read_object_operations": 0, + "kind": "bootstrap_measurement" + }, + { + "cells": 8, + "delay_ms": 5, + "elapsed_ms": 639.339334, + "gets": 48, + "latencies_ms": [ + 132.973917, + 275.673709, + 204.37675, + 420.750917, + 346.333125, + 493.449584, + 567.3947499999999, + 639.325209 + ], + "mode": "serial", + "puts": 96, + "sample": 1, + "warm_read_object_operations": 0, + "kind": "bootstrap_measurement" + }, + { + "cells": 8, + "delay_ms": 5, + "elapsed_ms": 633.860833, + "gets": 48, + "latencies_ms": [ + 131.536417, + 274.872208, + 203.5575, + 415.222542, + 345.5575, + 489.310917, + 560.457958, + 633.840208 + ], + "mode": "serial", + "puts": 96, + "sample": 2, + "warm_read_object_operations": 0, + "kind": "bootstrap_measurement" + }, + { + "cells": 8, + "delay_ms": 5, + "elapsed_ms": 212.594334, + "gets": 42, + "latencies_ms": [ + 135.989042, + 212.52158400000002, + 211.951459, + 212.579417, + 212.0875, + 212.580792, + 212.581917, + 212.583084 + ], + "mode": "cohort", + "puts": 72, + "sample": 2, + "warm_read_object_operations": 0, + "kind": "bootstrap_measurement" + }, + { + "cells": 1, + "delay_ms": 20, + "elapsed_ms": 379.043708, + "gets": 6, + "latencies_ms": [ + 379.03858299999996 + ], + "mode": "serial", + "puts": 12, + "sample": 0, + "warm_read_object_operations": 0, + "kind": "bootstrap_measurement" + }, + { + "cells": 1, + "delay_ms": 20, + "elapsed_ms": 386.441625, + "gets": 6, + "latencies_ms": [ + 386.435208 + ], + "mode": "cohort", + "puts": 12, + "sample": 0, + "warm_read_object_operations": 0, + "kind": "bootstrap_measurement" + }, + { + "cells": 1, + "delay_ms": 20, + "elapsed_ms": 378.38883400000003, + "gets": 6, + "latencies_ms": [ + 378.383334 + ], + "mode": "cohort", + "puts": 12, + "sample": 1, + "warm_read_object_operations": 0, + "kind": "bootstrap_measurement" + }, + { + "cells": 1, + "delay_ms": 20, + "elapsed_ms": 386.71337500000004, + "gets": 6, + "latencies_ms": [ + 386.68875 + ], + "mode": "serial", + "puts": 12, + "sample": 1, + "warm_read_object_operations": 0, + "kind": "bootstrap_measurement" + }, + { + "cells": 1, + "delay_ms": 20, + "elapsed_ms": 428.601083, + "gets": 6, + "latencies_ms": [ + 428.591541 + ], + "mode": "serial", + "puts": 12, + "sample": 2, + "warm_read_object_operations": 0, + "kind": "bootstrap_measurement" + }, + { + "cells": 1, + "delay_ms": 20, + "elapsed_ms": 425.40462499999995, + "gets": 6, + "latencies_ms": [ + 425.33824999999996 + ], + "mode": "cohort", + "puts": 12, + "sample": 2, + "warm_read_object_operations": 0, + "kind": "bootstrap_measurement" + }, + { + "cells": 8, + "delay_ms": 20, + "elapsed_ms": 1996.6422499999999, + "gets": 48, + "latencies_ms": [ + 654.957792, + 425.86775, + 1113.8025, + 880.162542, + 1562.903875, + 1336.02575, + 1777.422375, + 1996.607833 + ], + "mode": "serial", + "puts": 96, + "sample": 0, + "warm_read_object_operations": 0, + "kind": "bootstrap_measurement" + }, + { + "cells": 8, + "delay_ms": 20, + "elapsed_ms": 665.714583, + "gets": 42, + "latencies_ms": [ + 415.156125, + 665.533083, + 664.380792, + 665.540125, + 664.679583, + 665.543542, + 665.60475, + 665.684375 + ], + "mode": "cohort", + "puts": 72, + "sample": 0, + "warm_read_object_operations": 0, + "kind": "bootstrap_measurement" + }, + { + "cells": 8, + "delay_ms": 20, + "elapsed_ms": 642.124917, + "gets": 42, + "latencies_ms": [ + 397.104709, + 397.114084, + 640.663709, + 641.797084, + 640.897375, + 641.8022500000001, + 641.92625, + 642.087209 + ], + "mode": "cohort", + "puts": 72, + "sample": 1, + "warm_read_object_operations": 0, + "kind": "bootstrap_measurement" + }, + { + "cells": 8, + "delay_ms": 20, + "elapsed_ms": 2026.079541, + "gets": 48, + "latencies_ms": [ + 406.952166, + 648.4657080000001, + 876.666791, + 1109.7785, + 1575.5675, + 1334.472458, + 1807.4270000000001, + 2026.0642080000002 + ], + "mode": "serial", + "puts": 96, + "sample": 1, + "warm_read_object_operations": 0, + "kind": "bootstrap_measurement" + }, + { + "cells": 8, + "delay_ms": 20, + "elapsed_ms": 2009.3984169999999, + "gets": 48, + "latencies_ms": [ + 423.07129199999997, + 655.801709, + 1121.455042, + 885.560542, + 1564.038209, + 1350.173542, + 1777.5794170000001, + 2009.354334 + ], + "mode": "serial", + "puts": 96, + "sample": 2, + "warm_read_object_operations": 0, + "kind": "bootstrap_measurement" + }, + { + "cells": 8, + "delay_ms": 20, + "elapsed_ms": 600.520708, + "gets": 42, + "latencies_ms": [ + 387.320916, + 387.323125, + 599.970375, + 600.412625, + 600.085375, + 600.414125, + 600.4500830000001, + 600.505125 + ], + "mode": "cohort", + "puts": 72, + "sample": 2, + "warm_read_object_operations": 0, + "kind": "bootstrap_measurement" + }, + { + "cells": 1, + "delay_ms": 0, + "elapsed_ms": 12.23575, + "gets": 4, + "heads": 0, + "latencies_ms": [ + 12.232083 + ], + "mode": "serial", + "puts": 5, + "sample": 0, + "kind": "enrollment_measurement" + }, + { + "cells": 1, + "delay_ms": 0, + "elapsed_ms": 9.015334, + "gets": 4, + "heads": 0, + "latencies_ms": [ + 9.012625 + ], + "mode": "cohort", + "puts": 5, + "sample": 0, + "kind": "enrollment_measurement" + }, + { + "cells": 1, + "delay_ms": 0, + "elapsed_ms": 8.081084, + "gets": 4, + "heads": 0, + "latencies_ms": [ + 8.079042 + ], + "mode": "cohort", + "puts": 5, + "sample": 1, + "kind": "enrollment_measurement" + }, + { + "cells": 1, + "delay_ms": 0, + "elapsed_ms": 7.0635, + "gets": 4, + "heads": 0, + "latencies_ms": [ + 7.062208 + ], + "mode": "serial", + "puts": 5, + "sample": 1, + "kind": "enrollment_measurement" + }, + { + "cells": 1, + "delay_ms": 0, + "elapsed_ms": 6.921167, + "gets": 4, + "heads": 0, + "latencies_ms": [ + 6.92 + ], + "mode": "serial", + "puts": 5, + "sample": 2, + "kind": "enrollment_measurement" + }, + { + "cells": 1, + "delay_ms": 0, + "elapsed_ms": 7.423125, + "gets": 4, + "heads": 0, + "latencies_ms": [ + 7.420833 + ], + "mode": "cohort", + "puts": 5, + "sample": 2, + "kind": "enrollment_measurement" + }, + { + "cells": 8, + "delay_ms": 0, + "elapsed_ms": 56.300334, + "gets": 32, + "heads": 0, + "latencies_ms": [ + 56.28129199999999, + 56.284124999999996, + 56.286417, + 56.288625, + 56.290791999999996, + 56.293042, + 56.295417, + 56.29775 + ], + "mode": "serial", + "puts": 40, + "sample": 0, + "kind": "enrollment_measurement" + }, + { + "cells": 8, + "delay_ms": 0, + "elapsed_ms": 18.102458, + "gets": 25, + "heads": 0, + "latencies_ms": [ + 18.087208, + 18.089458, + 18.09125, + 18.092875000000003, + 18.094542, + 18.096167, + 18.097917000000002, + 18.099583 + ], + "mode": "cohort", + "puts": 12, + "sample": 0, + "kind": "enrollment_measurement" + }, + { + "cells": 8, + "delay_ms": 0, + "elapsed_ms": 17.897750000000002, + "gets": 25, + "heads": 0, + "latencies_ms": [ + 17.8825, + 17.884708, + 17.886499999999998, + 17.888208, + 17.889833, + 17.891417, + 17.893042, + 17.894625 + ], + "mode": "cohort", + "puts": 12, + "sample": 1, + "kind": "enrollment_measurement" + }, + { + "cells": 8, + "delay_ms": 0, + "elapsed_ms": 55.5655, + "gets": 32, + "heads": 0, + "latencies_ms": [ + 55.550166999999995, + 55.552499999999995, + 55.554292, + 55.556042, + 55.557792, + 55.559459, + 55.561125000000004, + 55.562833999999995 + ], + "mode": "serial", + "puts": 40, + "sample": 1, + "kind": "enrollment_measurement" + }, + { + "cells": 8, + "delay_ms": 0, + "elapsed_ms": 56.741167000000004, + "gets": 32, + "heads": 0, + "latencies_ms": [ + 56.720167000000004, + 56.727125, + 56.729042, + 56.730958, + 56.732791999999996, + 56.734583, + 56.736292, + 56.738 + ], + "mode": "serial", + "puts": 40, + "sample": 2, + "kind": "enrollment_measurement" + }, + { + "cells": 8, + "delay_ms": 0, + "elapsed_ms": 17.98725, + "gets": 25, + "heads": 0, + "latencies_ms": [ + 17.968, + 17.970833, + 17.973083000000003, + 17.975250000000003, + 17.977415999999998, + 17.979541, + 17.981832999999998, + 17.983958 + ], + "mode": "cohort", + "puts": 12, + "sample": 2, + "kind": "enrollment_measurement" + }, + { + "cells": 1, + "delay_ms": 5, + "elapsed_ms": 77.513458, + "gets": 4, + "heads": 0, + "latencies_ms": [ + 77.5085 + ], + "mode": "serial", + "puts": 5, + "sample": 0, + "kind": "enrollment_measurement" + }, + { + "cells": 1, + "delay_ms": 5, + "elapsed_ms": 83.4935, + "gets": 4, + "heads": 0, + "latencies_ms": [ + 83.482917 + ], + "mode": "cohort", + "puts": 5, + "sample": 0, + "kind": "enrollment_measurement" + }, + { + "cells": 1, + "delay_ms": 5, + "elapsed_ms": 94.387417, + "gets": 4, + "heads": 0, + "latencies_ms": [ + 94.38050000000001 + ], + "mode": "cohort", + "puts": 5, + "sample": 1, + "kind": "enrollment_measurement" + }, + { + "cells": 1, + "delay_ms": 5, + "elapsed_ms": 85.30908299999999, + "gets": 4, + "heads": 0, + "latencies_ms": [ + 85.301917 + ], + "mode": "serial", + "puts": 5, + "sample": 1, + "kind": "enrollment_measurement" + }, + { + "cells": 1, + "delay_ms": 5, + "elapsed_ms": 96.322833, + "gets": 4, + "heads": 0, + "latencies_ms": [ + 96.316625 + ], + "mode": "serial", + "puts": 5, + "sample": 2, + "kind": "enrollment_measurement" + }, + { + "cells": 1, + "delay_ms": 5, + "elapsed_ms": 106.51733300000001, + "gets": 4, + "heads": 0, + "latencies_ms": [ + 106.501333 + ], + "mode": "cohort", + "puts": 5, + "sample": 2, + "kind": "enrollment_measurement" + }, + { + "cells": 8, + "delay_ms": 5, + "elapsed_ms": 746.7623329999999, + "gets": 32, + "heads": 0, + "latencies_ms": [ + 88.61770800000001, + 194.467333, + 278.928875, + 372.474333, + 464.07329200000004, + 558.77425, + 657.048958, + 746.754833 + ], + "mode": "serial", + "puts": 40, + "sample": 0, + "kind": "enrollment_measurement" + }, + { + "cells": 8, + "delay_ms": 5, + "elapsed_ms": 101.3265, + "gets": 25, + "heads": 0, + "latencies_ms": [ + 101.290916, + 101.29625, + 101.30045799999999, + 101.3045, + 101.308708, + 101.312666, + 101.31675, + 101.320791 + ], + "mode": "cohort", + "puts": 12, + "sample": 0, + "kind": "enrollment_measurement" + }, + { + "cells": 8, + "delay_ms": 5, + "elapsed_ms": 104.86045800000001, + "gets": 25, + "heads": 0, + "latencies_ms": [ + 104.820291, + 104.82600000000001, + 104.83049999999999, + 104.83516599999999, + 104.839708, + 104.843833, + 104.84845800000001, + 104.852666 + ], + "mode": "cohort", + "puts": 12, + "sample": 1, + "kind": "enrollment_measurement" + }, + { + "cells": 8, + "delay_ms": 5, + "elapsed_ms": 591.016875, + "gets": 32, + "heads": 0, + "latencies_ms": [ + 74.26275, + 148.113875, + 221.43633300000002, + 295.03175, + 369.69383300000004, + 444.381917, + 517.800833, + 591.013708 + ], + "mode": "serial", + "puts": 40, + "sample": 1, + "kind": "enrollment_measurement" + }, + { + "cells": 8, + "delay_ms": 5, + "elapsed_ms": 726.79025, + "gets": 32, + "heads": 0, + "latencies_ms": [ + 91.545125, + 180.87029099999998, + 273.025333, + 377.63345799999996, + 469.573333, + 546.0595000000001, + 630.703625, + 726.781083 + ], + "mode": "serial", + "puts": 40, + "sample": 2, + "kind": "enrollment_measurement" + }, + { + "cells": 8, + "delay_ms": 5, + "elapsed_ms": 85.4805, + "gets": 25, + "heads": 0, + "latencies_ms": [ + 85.46008400000001, + 85.4635, + 85.465834, + 85.468084, + 85.470292, + 85.472459, + 85.47458400000001, + 85.476459 + ], + "mode": "cohort", + "puts": 12, + "sample": 2, + "kind": "enrollment_measurement" + }, + { + "cells": 1, + "delay_ms": 20, + "elapsed_ms": 240.991625, + "gets": 4, + "heads": 0, + "latencies_ms": [ + 240.97987500000002 + ], + "mode": "serial", + "puts": 5, + "sample": 0, + "kind": "enrollment_measurement" + }, + { + "cells": 1, + "delay_ms": 20, + "elapsed_ms": 234.553166, + "gets": 4, + "heads": 0, + "latencies_ms": [ + 234.5415 + ], + "mode": "cohort", + "puts": 5, + "sample": 0, + "kind": "enrollment_measurement" + }, + { + "cells": 1, + "delay_ms": 20, + "elapsed_ms": 244.036583, + "gets": 4, + "heads": 0, + "latencies_ms": [ + 244.017958 + ], + "mode": "cohort", + "puts": 5, + "sample": 1, + "kind": "enrollment_measurement" + }, + { + "cells": 1, + "delay_ms": 20, + "elapsed_ms": 250.71933299999998, + "gets": 4, + "heads": 0, + "latencies_ms": [ + 250.705791 + ], + "mode": "serial", + "puts": 5, + "sample": 1, + "kind": "enrollment_measurement" + }, + { + "cells": 1, + "delay_ms": 20, + "elapsed_ms": 245.238, + "gets": 4, + "heads": 0, + "latencies_ms": [ + 245.225125 + ], + "mode": "serial", + "puts": 5, + "sample": 2, + "kind": "enrollment_measurement" + }, + { + "cells": 1, + "delay_ms": 20, + "elapsed_ms": 244.29625, + "gets": 4, + "heads": 0, + "latencies_ms": [ + 244.2795 + ], + "mode": "cohort", + "puts": 5, + "sample": 2, + "kind": "enrollment_measurement" + }, + { + "cells": 8, + "delay_ms": 20, + "elapsed_ms": 1894.428792, + "gets": 32, + "heads": 0, + "latencies_ms": [ + 251.24625, + 486.57516699999996, + 719.293125, + 957.888709, + 1193.526167, + 1425.2148339999999, + 1659.493417, + 1894.41525 + ], + "mode": "serial", + "puts": 40, + "sample": 0, + "kind": "enrollment_measurement" + }, + { + "cells": 8, + "delay_ms": 20, + "elapsed_ms": 251.031292, + "gets": 25, + "heads": 0, + "latencies_ms": [ + 250.957625, + 250.966167, + 250.973583, + 250.98054199999999, + 250.99325, + 251.00020800000001, + 251.006542, + 251.012958 + ], + "mode": "cohort", + "puts": 12, + "sample": 0, + "kind": "enrollment_measurement" + }, + { + "cells": 8, + "delay_ms": 20, + "elapsed_ms": 215.328125, + "gets": 25, + "heads": 0, + "latencies_ms": [ + 215.308458, + 215.311375, + 215.31341600000002, + 215.31537500000002, + 215.317416, + 215.319333, + 215.32137500000002, + 215.32333300000002 + ], + "mode": "cohort", + "puts": 12, + "sample": 1, + "kind": "enrollment_measurement" + }, + { + "cells": 8, + "delay_ms": 20, + "elapsed_ms": 1689.0139159999999, + "gets": 32, + "heads": 0, + "latencies_ms": [ + 210.1215, + 420.50320800000003, + 628.585458, + 841.459291, + 1051.940208, + 1265.216541, + 1475.42525, + 1689.0104999999999 + ], + "mode": "serial", + "puts": 40, + "sample": 1, + "kind": "enrollment_measurement" + }, + { + "cells": 8, + "delay_ms": 20, + "elapsed_ms": 1683.03325, + "gets": 32, + "heads": 0, + "latencies_ms": [ + 206.590875, + 411.713834, + 623.81525, + 840.6524589999999, + 1049.962167, + 1261.7890419999999, + 1471.175334, + 1683.029667 + ], + "mode": "serial", + "puts": 40, + "sample": 2, + "kind": "enrollment_measurement" + }, + { + "cells": 8, + "delay_ms": 20, + "elapsed_ms": 218.672333, + "gets": 25, + "heads": 0, + "latencies_ms": [ + 218.651917, + 218.655417, + 218.658083, + 218.660292, + 218.662333, + 218.6645, + 218.666417, + 218.66825 + ], + "mode": "cohort", + "puts": 12, + "sample": 2, + "kind": "enrollment_measurement" + } + ] +} diff --git a/docs/quickstart.md b/docs/quickstart.md index 9b52f860..bf36c802 100644 --- a/docs/quickstart.md +++ b/docs/quickstart.md @@ -36,6 +36,11 @@ checks both Cell types against their modules and prints the descriptor digest. The example provisions a fenced owner for each Cell, bootstraps managed SQLite, and binds both handles to one `ApplicationHandle`. +Creation publishes durable state before returning a handle. For many new Cells, +use bounded activation and an enrollment adapter that shares catalog publication. +See [Cell creation latency](cell-bootstrap-latency.md) for the required ordering, +bounded cohort API, and reproducible measurements. + `KvNamespace::atomic` writes a setting in one scope; the returned receipt gates `get`. `QueueNamespace::send` creates an at-least-once job. The example claims a lease, validates it on the owner, and acknowledges the message.