From 6597c4c0ece7a0d2553a9f5bed8808e5816856d0 Mon Sep 17 00:00:00 2001 From: forkwright Date: Tue, 25 Aug 2026 13:36:20 -0500 Subject: [PATCH] deps: move to snafu 0.9 and fjall 3 snafu 0.9 makes Location a type alias for std's, which has no public constructor, so eight manual Location::default() sites stop compiling. Location::caller() records the identical site: 0.8's default() was itself track_caller, and these helpers are not, so the chain terminated at the same place either way. fjall 3 swapped its vocabulary -- the whole store is now Database, and what was PartitionHandle is now Keyspace -- and moved persist from the partition handle to the store. The data API is byte-identical either side; only the type names and the open and persist entry points moved. fjall 3 does change the on-disk format, and it fails closed rather than silently: recover() checks the version marker before any journal or manifest recovery runs, so the destructive path that drops segments absent from the manifest never executes against a 2.x store -- it errors and names the migration tool instead. A comment claiming this pin matched kanon's archeion crate is corrected: archeion still pins fjall 2, so the two now diverge. --- Cargo.lock | 129 ++++++++----------- Cargo.toml | 11 +- crates/atmis/src/lib.rs | 4 +- crates/heurema/tests/persistence_contract.rs | 4 +- crates/thesauros/src/lib.rs | 44 ++++--- crates/thesauros/tests/persistence_fjall.rs | 8 +- 6 files changed, 93 insertions(+), 107 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 1c39394..d246825 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -19,16 +19,16 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b588b76d00fde79687d7646a9b5bdf3cc0f655e0bbd080335a95d7e96f3587da" [[package]] -name = "byteorder" -version = "1.5.0" +name = "byteorder-lite" +version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" +checksum = "8f1fe948ff07f4bd06c30984e69f5b4899c516a3ef74f34df92a2df2ab535495" [[package]] name = "byteview" -version = "0.6.1" +version = "0.10.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6236364b88b9b6d0bc181ba374cf1ab55ba3ef97a1cb6f8cddad48a273767fb5" +checksum = "0d74937895e761d5984206f82ca8ff7725d0fd8021921011921894b41ab1b9f7" [[package]] name = "cfg-if" @@ -81,12 +81,6 @@ dependencies = [ "parking_lot_core", ] -[[package]] -name = "double-ended-peekable" -version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c0d05e1c0dbad51b52c38bda7adceef61b9efc2baf04acfe8726a8c4630a6f57" - [[package]] name = "enum_dispatch" version = "0.3.13" @@ -123,21 +117,30 @@ checksum = "da7c62ceae207dd37ea5b845da6a0696c799f85e97da1ab5b7910be3c1c80223" [[package]] name = "fjall" -version = "2.11.2" +version = "3.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b25ad44cd4360a0448a9b5a0a6f1c7a621101cca4578706d43c9a821418aebc" +checksum = "d5427b70d8592043024955a4a40b5cf44af323fdad7c1f62848a01ec7806709e" dependencies = [ - "byteorder", + "byteorder-lite", "byteview", "dashmap", + "flume", "log", "lsm-tree", - "path-absolutize", - "std-semaphore", + "lz4_flex", "tempfile", "xxhash-rust", ] +[[package]] +name = "flume" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e139bc46ca777eb5efaf62df0ab8cc5fd400866427e56c68b22e414e53bd3be" +dependencies = [ + "spin", +] + [[package]] name = "getrandom" version = "0.4.3" @@ -149,12 +152,6 @@ dependencies = [ "r-efi", ] -[[package]] -name = "guardian" -version = "1.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "17e2ac29387b1aa07a1e448f7bb4f35b500787971e965b02842b900afa5c8f6f" - [[package]] name = "hashbrown" version = "0.14.5" @@ -226,33 +223,34 @@ checksum = "0ceec5bc11778974d1bcb055b18002eba7f4b3518b6a0081b3af5f21666da9ad" [[package]] name = "lsm-tree" -version = "2.10.4" +version = "3.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "799399117a2bfb37660e08be33f470958babb98386b04185288d829df362ea15" +checksum = "03c4da84fa3a8638fb26d4432bfeb69a1b560905ee2980134d8993e2cfa0dbf3" dependencies = [ - "byteorder", + "byteorder-lite", + "byteview", "crossbeam-skiplist", - "double-ended-peekable", "enum_dispatch", - "guardian", "interval-heap", "log", "lz4_flex", - "path-absolutize", "quick_cache", "rustc-hash", "self_cell", + "sfa", "tempfile", - "value-log", "varint-rs", "xxhash-rust", ] [[package]] name = "lz4_flex" -version = "0.11.6" +version = "0.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "373f5eceeeab7925e0c1098212f2fbc4d416adec9d35051a6ab251e824c1854a" +checksum = "7ef0d4ed8669f8f8826eb00dc878084aa8f253506c4fd5e8f58f5bce72ddb97e" +dependencies = [ + "twox-hash", +] [[package]] name = "memchr" @@ -279,24 +277,6 @@ dependencies = [ "windows-link", ] -[[package]] -name = "path-absolutize" -version = "3.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e4af381fe79fa195b4909485d99f73a80792331df0625188e707854f0b3383f5" -dependencies = [ - "path-dedot", -] - -[[package]] -name = "path-dedot" -version = "3.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "07ba0ad7e047712414213ff67533e6dd477af0a4e1d14fb52343e53d30ea9397" -dependencies = [ - "once_cell", -] - [[package]] name = "proc-macro2" version = "1.0.106" @@ -414,6 +394,17 @@ dependencies = [ "zmij", ] +[[package]] +name = "sfa" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1296838937cab56cd6c4eeeb8718ec777383700c33f060e2869867bd01d1175" +dependencies = [ + "byteorder-lite", + "log", + "xxhash-rust", +] + [[package]] name = "smallvec" version = "1.15.2" @@ -422,18 +413,18 @@ checksum = "8ed6a63f02c8539c91a8685a86f4099661ba3da017932f6ebbea6de3f0fa7c90" [[package]] name = "snafu" -version = "0.8.9" +version = "0.9.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e84b3f4eacbf3a1ce05eac6763b4d629d60cbc94d632e4092c54ade71f1e1a2" +checksum = "e45cb604038abb7b926b679887b3226d8d0f23874b66623625a0454be425a4b7" dependencies = [ "snafu-derive", ] [[package]] name = "snafu-derive" -version = "0.8.9" +version = "0.9.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c1c97747dbf44bb1ca44a561ece23508e99cb592e862f22222dcf42f51d1e451" +checksum = "287f59010008f0d7cf5e3b03196d666c1acc46c8d3e9cf34c28a1a7157601e72" dependencies = [ "heck", "proc-macro2", @@ -442,10 +433,13 @@ dependencies = [ ] [[package]] -name = "std-semaphore" -version = "0.1.0" +name = "spin" +version = "0.9.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "33ae9eec00137a8eed469fb4148acd9fc6ac8c3f9b110f52cd34698c8b5bfa0e" +checksum = "3763264f6b73151db08c50ff20d7d8a0b8796e021cdea7ceedad07b80155fa0e" +dependencies = [ + "lock_api", +] [[package]] name = "syn" @@ -495,27 +489,16 @@ dependencies = [ ] [[package]] -name = "unicode-ident" -version = "1.0.24" +name = "twox-hash" +version = "2.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" +checksum = "8464ec13c3691491391d9fce00f6416c9a48e46972f72d7865688be2080192c9" [[package]] -name = "value-log" -version = "1.9.0" +name = "unicode-ident" +version = "1.0.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "62fc7c4ce161f049607ecea654dca3f2d727da5371ae85e2e4f14ce2b98ed67c" -dependencies = [ - "byteorder", - "byteview", - "interval-heap", - "log", - "path-absolutize", - "rustc-hash", - "tempfile", - "varint-rs", - "xxhash-rust", -] +checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" [[package]] name = "varint-rs" diff --git a/Cargo.toml b/Cargo.toml index 7a9295a..f7bc363 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -31,17 +31,18 @@ atmis = { version = "0.2.0", path = "crates/atmis" } # WHY: snafu is the fleet-wide error library per kanon RUST.md. Pinned at the # workspace root so the single heurema crate inherits via `workspace = true`, # matching the inheritance pattern adapter sub-crates will use later. -snafu = "0.8" +snafu = "0.9" # WHY: heurema's PersistenceBackend bound (crates/heurema/src/persistence.rs) # needs Serialize on save and DeserializeOwned on load, so heurema itself # depends on serde; the adapter crates add serde_json (atmis, thesauros) to -# turn that into snapshot bytes. fjall is thesauros's storage engine, pinned -# at the same major version kanon's own archeion crate already uses -# fleet-wide. +# turn that into snapshot bytes. fjall is thesauros's storage engine. +# NOTE: kanon's archeion crate still pins fjall 2 (workspace root +# `fjall = "2"`) — this bump is no longer at the "same major version" +# fleet-wide; the two now diverge until archeion moves too. serde = { version = "1", features = ["derive"] } serde_json = "1" -fjall = "2" +fjall = "3" [workspace.lints.clippy] pedantic = { level = "warn", priority = -1 } diff --git a/crates/atmis/src/lib.rs b/crates/atmis/src/lib.rs index 748e424..aa36e80 100644 --- a/crates/atmis/src/lib.rs +++ b/crates/atmis/src/lib.rs @@ -42,14 +42,14 @@ impl AtmisBackend { fn not_found(name: &str) -> HeuremaError { HeuremaError::IndexNotFound { name: name.to_owned(), - location: snafu::Location::default(), + location: std::panic::Location::caller(), } } fn codec_error(source: serde_json::Error) -> HeuremaError { HeuremaError::Persistence { source: PersistenceSource::new(source), - location: snafu::Location::default(), + location: std::panic::Location::caller(), } } diff --git a/crates/heurema/tests/persistence_contract.rs b/crates/heurema/tests/persistence_contract.rs index 9846ecb..99f5278 100644 --- a/crates/heurema/tests/persistence_contract.rs +++ b/crates/heurema/tests/persistence_contract.rs @@ -47,7 +47,7 @@ impl RecordingBackend { fn not_found(name: &str) -> HeuremaError { HeuremaError::IndexNotFound { name: name.to_owned(), - location: snafu::Location::default(), + location: std::panic::Location::caller(), } } } @@ -91,7 +91,7 @@ impl FailingBackend { fn storage_failure() -> HeuremaError { HeuremaError::Persistence { source: PersistenceSource::new(MidError::default()), - location: snafu::Location::default(), + location: std::panic::Location::caller(), } } } diff --git a/crates/thesauros/src/lib.rs b/crates/thesauros/src/lib.rs index b3bd37e..704350b 100644 --- a/crates/thesauros/src/lib.rs +++ b/crates/thesauros/src/lib.rs @@ -2,18 +2,18 @@ //! [`PersistenceBackend`] adapter for heurema indexes. //! //! Every save encodes the index through `serde_json`, writes it to a fjall -//! partition, then fsyncs the keyspace journal +//! keyspace, then fsyncs the database journal //! (`fjall::PersistMode::SyncAll`) before returning `Ok`. A save that //! returns `Ok` is durable on disk at that point, not merely buffered. That //! is this crate's entire reason to exist over `atmis`, and the property -//! its test suite proves by closing and reopening the keyspace rather than +//! its test suite proves by closing and reopening the database rather than //! trusting an in-process value. #![deny(missing_docs)] use std::path::Path; -use fjall::PartitionCreateOptions; +use fjall::KeyspaceCreateOptions; use heurema::{FtsIndex, HeuremaError, PersistenceBackend, PersistenceSource, VectorIndex}; use serde::Serialize; use serde::de::DeserializeOwned; @@ -22,36 +22,38 @@ const VECTOR_PARTITION: &str = "vector_indexes"; const FTS_PARTITION: &str = "fts_indexes"; /// fjall-backed [`PersistenceBackend`]. Vector-index and FTS-index -/// snapshots live in separate fjall partitions so the two name spaces the +/// snapshots live in separate fjall keyspaces so the two name spaces the /// trait itself keeps independent stay independent on disk. pub struct ThesaurosBackend { - keyspace: fjall::Keyspace, - vector_indexes: fjall::PartitionHandle, - fts_indexes: fjall::PartitionHandle, + db: fjall::Database, + vector_indexes: fjall::Keyspace, + fts_indexes: fjall::Keyspace, } impl ThesaurosBackend { - /// Open (or create) a keyspace at `path`. + /// Open (or create) a database at `path`. /// - /// WARNING: fjall keyspaces are single-process — opening the same + /// WARNING: fjall databases are single-process — opening the same /// `path` from two live backends at once is a fleet-wide known hazard /// (see kanon's `archeion` crate CLAUDE.md), not specific to this /// adapter. The caller owns process-level exclusivity over `path`. /// /// # Errors /// - /// Returns [`HeuremaError::Persistence`] if the keyspace or either - /// partition fails to open. + /// Returns [`HeuremaError::Persistence`] if the database or either + /// keyspace fails to open. pub fn open>(path: P) -> Result { - let keyspace = fjall::Config::new(path).open().map_err(Self::fjall_error)?; - let vector_indexes = keyspace - .open_partition(VECTOR_PARTITION, PartitionCreateOptions::default()) + let db = fjall::Database::builder(path) + .open() .map_err(Self::fjall_error)?; - let fts_indexes = keyspace - .open_partition(FTS_PARTITION, PartitionCreateOptions::default()) + let vector_indexes = db + .keyspace(VECTOR_PARTITION, KeyspaceCreateOptions::default) + .map_err(Self::fjall_error)?; + let fts_indexes = db + .keyspace(FTS_PARTITION, KeyspaceCreateOptions::default) .map_err(Self::fjall_error)?; Ok(Self { - keyspace, + db, vector_indexes, fts_indexes, }) @@ -60,21 +62,21 @@ impl ThesaurosBackend { fn not_found(name: &str) -> HeuremaError { HeuremaError::IndexNotFound { name: name.to_owned(), - location: snafu::Location::default(), + location: std::panic::Location::caller(), } } fn fjall_error(source: fjall::Error) -> HeuremaError { HeuremaError::Persistence { source: PersistenceSource::new(source), - location: snafu::Location::default(), + location: std::panic::Location::caller(), } } fn codec_error(source: serde_json::Error) -> HeuremaError { HeuremaError::Persistence { source: PersistenceSource::new(source), - location: snafu::Location::default(), + location: std::panic::Location::caller(), } } @@ -85,7 +87,7 @@ impl ThesaurosBackend { // `PersistMode` later; Phase 3 keeps it unconditional because no // caller has asked for the weaker mode yet. fn sync(&self) -> Result<(), HeuremaError> { - self.keyspace + self.db .persist(fjall::PersistMode::SyncAll) .map_err(Self::fjall_error) } diff --git a/crates/thesauros/tests/persistence_fjall.rs b/crates/thesauros/tests/persistence_fjall.rs index 04f7655..98c0f56 100644 --- a/crates/thesauros/tests/persistence_fjall.rs +++ b/crates/thesauros/tests/persistence_fjall.rs @@ -1,8 +1,8 @@ //! Integration tests for [`thesauros::ThesaurosBackend`] against the //! `heurema::PersistenceBackend` contract. //! -//! WHY every round trip here drops the backend and reopens the keyspace at -//! the same path before loading: `fjall::Keyspace::open` auto-recovery +//! WHY every round trip here drops the backend and reopens the database at +//! the same path before loading: `fjall::Database::open` auto-recovery //! deletes segments absent from the levels manifest, so the only way to //! prove data survived is to actually close the process's handle and come //! back through the disk, exactly as a crash-restart would. An in-process @@ -47,7 +47,7 @@ fn fts_config() -> FtsConfig { fn io_error(source: std::io::Error) -> HeuremaError { HeuremaError::Persistence { source: PersistenceSource::new(source), - location: snafu::Location::default(), + location: std::panic::Location::caller(), } } @@ -58,7 +58,7 @@ fn vector_index_survives_close_and_reopen() -> Result<(), HeuremaError> { { let backend = ThesaurosBackend::open(dir.path())?; backend.save_vector_index("embeddings", &HnswIndex::::new(vector_config()))?; - } // WHY: the backend, its Keyspace, and both PartitionHandles drop here — + } // WHY: the backend, its Database, and both Keyspaces drop here — // the only handle onto this fjall database goes away before reopening. let reopened = ThesaurosBackend::open(dir.path())?;