diff --git a/README.md b/README.md
index a6d65e2..7051a5f 100644
--- a/README.md
+++ b/README.md
@@ -127,13 +127,13 @@ forgetting, and encryption are part of the product contract. See
- [x] Episode-to-fact consolidation via injected LLM (any local runner, no hard dependency)
- [x] Hardware-aware provisioning — no silent model downloads, explicit setup command
- [x] Encryption at rest via native envelope (ADR-030); centralized passphrase resolution (ADR-034)
-- [x] Per-agent isolation enforced structurally by key layout — cross-agent leakage is a security invariant
+- [x] Per-agent result isolation across retrieval surfaces — record, FTS and graph namespaces are structurally keyed by agent; the vector (ANN) index is currently searched globally and post-filtered by agent, full physical search-space isolation is targeted by V2-08 ([ADR-069](docs/adr/ADR-069-structural-project-agent-memory-scope.md))
- [x] MCP server (stdio + HTTP), CLI (`basemyai`), REST sidecar (axum), Python SDK (PyO3), Node SDK (NAPI-RS), native Rust crate
P1 Public Proofs
- [Benchmark harness: BaseMyAI local vs Mem0 + Qdrant local](docs/benchmarks/local-memory-vs-mem0-qdrant.md)
-- [Adversarial isolation test](crates/basemyai/tests/p1_isolation_adversarial.rs)
+- [Adversarial isolation test](crates/basemyai/tests/p1_isolation_adversarial.rs) — proves no cross-agent leakage in returned *results* (record/FTS/vector/hybrid recall); does not prove ANN search-space isolation (see [Security](#security))
- [Temporal replacement demo](crates/basemyai/examples/temporal_replacement.rs)
- [Zero network after setup](docs/zero-network-after-setup.md)
- [BaseMyAI is not a vector DB](docs/not-a-vector-db.md)
@@ -167,7 +167,7 @@ BaseMyAI is a **Cargo workspace** with two publishable crates (`basemyai-core`,
**`basemyai-engine`** is the durable storage layer: crash-consistent LSM, LM-DiskANN vector index, inverted FTS index, graph traversal, and at-rest encryption — all in pure Rust, no libSQL/SQLite dependency.
-**`basemyai`** is the memory product built on top: the four layers, temporal RAG, per-agent isolation enforced structurally in the key layout, and all language binding surfaces.
+**`basemyai`** is the memory product built on top: the four layers, temporal RAG, per-agent result isolation (structural for records/FTS/graph, post-filtered today for the vector index — see [Security](#security)), and all language binding surfaces.
Since **[ADR-032](docs/adr/ADR-032-native-only.md)** (2026-07-08), the native engine is the **only** active backend. libSQL/V1 compatibility paths have been removed from the workspace.
@@ -609,7 +609,7 @@ runs is the failure mode this guard exists to make impossible.
For security issues, kindly email us at [security@basemyai.com](mailto:security@basemyai.com) instead of posting a public issue on GitHub.
- **100 % local** — no data leaves your machine, no telemetry by default
-- **Per-agent isolation** — every access is scoped structurally by `agent_id`; cross-agent leakage is a security invariant, not a config option
+- **Per-agent result isolation** — record, FTS and graph namespaces are scoped structurally by `agent_id`; cross-agent leakage of *results* is a security invariant, not a config option. The ANN vector index is currently searched globally and post-filtered by agent — the query's candidate set, beam and expansions are not yet scope-isolated. Full physical search-space isolation is targeted by V2-08 ([ADR-069](docs/adr/ADR-069-structural-project-agent-memory-scope.md)); see also the adversarial isolation test note below, which proves result-level non-leakage, not search-space isolation.
- **Encrypted at rest** — native envelope (ADR-030); passphrase resolved per ADR-034, never stored in config
- **No silent network** — the embedder receives a local model path and never auto-downloads
diff --git a/crates/basemyai-engine/src/crypto.rs b/crates/basemyai-engine/src/crypto.rs
index 84f6f1d..6e02d13 100644
--- a/crates/basemyai-engine/src/crypto.rs
+++ b/crates/basemyai-engine/src/crypto.rs
@@ -413,6 +413,10 @@ pub(crate) fn publish_staged_meta_tracked(
if let Err(error) = crate::failpoint_result!("after_crypto_meta_write") {
return DurablePublish::Unknown(error);
}
+ tracker.directory_sync_started();
+ if let Err(error) = crate::failpoint_result!("during_crypto_meta_directory_sync") {
+ return DurablePublish::Unknown(error);
+ }
// ENG-DUR-003: see `crate::fs_util`.
if let Err(error) = crate::fs_util::sync_dir(dir) {
return DurablePublish::Unknown(error);
diff --git a/crates/basemyai-engine/src/error.rs b/crates/basemyai-engine/src/error.rs
index aaa00c8..aba6a72 100644
--- a/crates/basemyai-engine/src/error.rs
+++ b/crates/basemyai-engine/src/error.rs
@@ -74,6 +74,17 @@ pub enum EngineError {
#[error("sequence space exhausted after {last_allocated}: cannot reserve {requested} contiguous mutation(s)")]
SequenceSpaceExhausted { last_allocated: u64, requested: usize },
+ /// A WAL write was attempted but its complete absence or durability
+ /// could not be proved. The writer is terminal until the store is
+ /// reopened; callers must not retry the logical mutation blindly.
+ #[error("WAL append outcome is unknown and the writer must be reopened: {cause}")]
+ WalAppendOutcomeUnknown { cause: String },
+
+ /// The WAL bytes were appended but their `sync_all` outcome could not be
+ /// proved. The writer is terminal until the store is reopened.
+ #[error("WAL sync outcome is unknown and the writer must be reopened: {cause}")]
+ WalSyncOutcomeUnknown { cause: String },
+
/// A point-in-time read snapshot is process-local and cannot be used
/// after the engine instance that created it has closed or dropped.
#[error("read snapshot belongs to a closed store")]
@@ -237,6 +248,13 @@ pub enum EngineError {
#[error("memory record already exists for agent {agent:?}, id {id:?}")]
DuplicateMemoryId { agent: String, id: String },
+ /// The monotonic memory/vector-id allocator cannot represent the cursor
+ /// immediately after the requested contiguous range. Rejected before
+ /// staging reaches the WAL or mutates index RAM; wraparound and id reuse
+ /// are never permitted.
+ #[error("vector id space exhausted at next id {next}: cannot reserve {requested} contiguous id(s)")]
+ VectorIdSpaceExhausted { next: u64, requested: usize },
+
/// A string handed to an FTS-index key encoder (`key::fts_index`) would
/// overflow that field's `u32` length prefix. Sibling of
/// [`EngineError::GraphKeyTooLong`]/[`EngineError::MemoryKeyTooLong`],
diff --git a/crates/basemyai-engine/src/failpoint.rs b/crates/basemyai-engine/src/failpoint.rs
index c4af183..5d9ce39 100644
--- a/crates/basemyai-engine/src/failpoint.rs
+++ b/crates/basemyai-engine/src/failpoint.rs
@@ -10,6 +10,10 @@
//!
//! ```text
//! after_wal_append after the WAL record's write_all, before fsync
+//! before_wal_append_attempt immediately before ADR-070's phase-aware batch
+//! write; errors here are provably `Aborted`
+//! during_wal_sync after append succeeds, immediately before
+//! `sync_all`; errors are `OutcomeUnknown(Sync)`
//! after_wal_fsync after the WAL record's sync_all
//! after_sst_tmp_write after the SST tmp file's write_all, before fsync
//! after_sst_tmp_fsync after the SST tmp file's sync_all, before rename
@@ -25,10 +29,16 @@
//! during_wal_segment_removal after catalog.meta publishes a new active
//! WAL (old-segment unlink retried + counted)
//! after_crypto_meta_write after crypto.meta's atomic replace (rotation)
+//! during_crypto_meta_directory_sync after crypto.meta replacement, at the
+//! directory-durability boundary (ADR-070)
//! after_full_rotation_new_dek after the next generation's fresh DEK wrap
//! after_full_rotation_sst_write after the merged SST is durable
//! before_full_rotation_publish after all content fsyncs, before pointer rename
//! after_full_rotation_publish after pointer rename, before old-generation GC
+//! before_full_rotation_install after durable generation publication, before
+//! the prebuilt RAM image is installed (ADR-070)
+//! after_light_rotation_publish after crypto.meta publication is complete;
+//! finalisation panic cannot demote Committed
//! during_full_rotation_gc immediately before best-effort old-generation GC
//! during_generation_gc_removal one simulated failed removal attempt per hit
//! (retried up to 3x, then counted via
@@ -59,6 +69,9 @@
//! root directory fsync (ADR-066) — same class as
//! `after_catalog_rename`, on the pointer that selects
//! which generation directory a reopen adopts
+//! during_generation_directory_sync after generation.meta replacement, at
+//! the root directory-durability boundary
+//! (ADR-070)
//! before_seal_candidate_wal_fsync immediately before the replacement WAL's
//! initial fsync, with no reader-facing or
//! durable-publication lock held (V2-01B)
diff --git a/crates/basemyai-engine/src/idx/graph/persistent.rs b/crates/basemyai-engine/src/idx/graph/persistent.rs
index c82d8e7..4ddba33 100644
--- a/crates/basemyai-engine/src/idx/graph/persistent.rs
+++ b/crates/basemyai-engine/src/idx/graph/persistent.rs
@@ -38,14 +38,10 @@
//! ## Atomicity
//!
//! [`PersistentGraph::upsert_entity`] and [`PersistentGraph::upsert_edge`]
-//! each write exactly **one** KV record via a plain [`crate::store::Engine::put`].
-//! `Engine::put` is already durable and atomic per key (WAL-fsync-then-
-//! memtable, see `store::engine`'s own doc) — wrapping a single `put` in a
-//! `Batch`/`apply_batch` would add nothing here. This is the one structural
-//! difference from the vector index's inserts, which always touch multiple
-//! records (the new node plus every re-pruned neighbor plus shared
-//! metadata) and therefore need `apply_batch`'s all-or-nothing guarantee; a
-//! graph upsert never touches more than the one record it names.
+//! each write exactly **one** KV record. Their legacy wrappers apply the
+//! [`Batch`] produced by the corresponding `stage_*` primitive, so callers
+//! that own a wider logical transaction can compose graph records into that
+//! same commit without giving this stateless index a second write authority.
//! [`PersistentGraph::traverse`] is read-only.
use super::edge::{self, GraphEdgeMeta};
@@ -53,7 +49,161 @@ use super::entity::{self, GraphEntity};
use super::traverse::{self, GraphProvider, OutEdge, Reached};
use crate::error::{EngineError, Result};
use crate::key::graph_index;
-use crate::store::{Engine, EngineRead};
+use crate::store::{Batch, Engine, EngineCommitOutcome, EngineRead, SequenceRange, WalCommitPhase};
+
+/// Explicit admission bounds for one graph-purge WAL record.
+#[doc(hidden)]
+#[derive(Clone, Copy, Debug)]
+pub struct GraphPurgeChunkOptions {
+ pub max_items: usize,
+ pub max_wal_bytes: usize,
+}
+
+impl Default for GraphPurgeChunkOptions {
+ fn default() -> Self {
+ Self {
+ max_items: 256,
+ max_wal_bytes: 4 * 1024 * 1024,
+ }
+ }
+}
+
+#[derive(Clone, Copy, Debug, Eq, PartialEq)]
+enum GraphPurgePhase {
+ Entities,
+ Edges,
+}
+
+/// Opaque resume point. It is process-local planning state, not a durable
+/// idempotency key and not a caller-controlled physical key.
+#[doc(hidden)]
+#[derive(Clone, Debug, Eq, PartialEq)]
+pub struct GraphPurgeCursor {
+ agent: String,
+ phase: GraphPurgePhase,
+ start: Vec,
+}
+
+/// One owned bounded read-set. Keys and values stay private so callers can
+/// neither widen the delete set nor forge revalidation evidence.
+#[doc(hidden)]
+#[derive(Debug)]
+pub struct GraphPurgeChunkPlan {
+ agent: String,
+ phase: GraphPurgePhase,
+ range_start: Vec,
+ range_end: Vec,
+ records: Vec<(Vec, Vec)>,
+ resume: Option,
+ approx_wal_bytes: usize,
+}
+
+impl GraphPurgeChunkPlan {
+ #[must_use]
+ pub fn len(&self) -> usize {
+ self.records.len()
+ }
+
+ #[must_use]
+ pub fn is_empty(&self) -> bool {
+ self.records.is_empty()
+ }
+
+ #[must_use]
+ pub fn approx_wal_bytes(&self) -> usize {
+ self.approx_wal_bytes
+ }
+
+ #[must_use]
+ pub fn resume(&self) -> Option<&GraphPurgeCursor> {
+ self.resume.as_ref()
+ }
+}
+
+#[doc(hidden)]
+#[derive(Debug, thiserror::Error)]
+pub enum GraphPurgePlanError {
+ #[error("graph purge chunk bounds must both be non-zero")]
+ ZeroBound,
+ #[error("graph purge cursor belongs to a different agent")]
+ WrongAgent,
+ #[error("graph purge cursor is outside its agent keyspace")]
+ InvalidCursor,
+ #[error("one graph purge tombstone needs approximately {needed} WAL bytes, above chunk cap {max}")]
+ ItemTooLarge { needed: usize, max: usize },
+ #[error("graph purge read-set changed before staging")]
+ StaleReadSet,
+ #[error("a new graph record appeared inside the planned purge interval")]
+ PhantomInRange,
+ #[error(transparent)]
+ Engine(#[from] EngineError),
+}
+
+/// Closed phase-preserving result for a staged graph purge.
+#[doc(hidden)]
+#[derive(Debug)]
+pub enum GraphPurgeCommitOutcome {
+ Aborted {
+ burned_sequence_range: Option,
+ cause: Option,
+ },
+ Committed {
+ sequence_range: SequenceRange,
+ removed: u64,
+ },
+ OutcomeUnknown {
+ phase: WalCommitPhase,
+ cause: EngineError,
+ },
+ DurableReopenRequired {
+ sequence_range: SequenceRange,
+ cause: EngineError,
+ },
+}
+
+#[doc(hidden)]
+#[derive(Debug)]
+pub struct StagedGraphPurgeChunk {
+ batch: Batch,
+ removed: u64,
+}
+
+impl StagedGraphPurgeChunk {
+ #[doc(hidden)]
+ pub fn commit(self, engine: &mut Engine) -> GraphPurgeCommitOutcome {
+ let Self { batch, removed } = self;
+ match engine.commit_batch(batch) {
+ EngineCommitOutcome::Aborted {
+ burned_sequence_range,
+ cause,
+ } => GraphPurgeCommitOutcome::Aborted {
+ burned_sequence_range,
+ cause,
+ },
+ EngineCommitOutcome::Durable(receipt) => {
+ let sequence_range = receipt.sequence_range();
+ match engine.install_committed_batch_with(receipt, |_| Ok(())) {
+ Ok(()) => GraphPurgeCommitOutcome::Committed {
+ sequence_range,
+ removed,
+ },
+ Err(cause) => GraphPurgeCommitOutcome::DurableReopenRequired { sequence_range, cause },
+ }
+ }
+ EngineCommitOutcome::OutcomeUnknown { phase, cause } => {
+ GraphPurgeCommitOutcome::OutcomeUnknown { phase, cause }
+ }
+ }
+ }
+}
+
+fn prefix_end(prefix: &[u8]) -> Option> {
+ let mut end = prefix.to_vec();
+ let index = end.iter().rposition(|byte| *byte != u8::MAX)?;
+ end[index] += 1;
+ end.truncate(index + 1);
+ Some(end)
+}
/// Stateless handle over the KV-persisted graph index — holds no cached
/// state at all (see the module doc for why no metadata record is needed).
@@ -69,19 +219,201 @@ impl PersistentGraph {
Self
}
- /// Inserts or overwrites the entity `(agent, id)`: one durable
- /// `Engine::put` (see the module doc's atomicity note). Takes a whole
+ /// Plans one bounded, read-only page of an agent graph purge.
+ ///
+ /// The returned plan owns the exact key/value read-set and an opaque
+ /// resume cursor. Planning never stages or commits a mutation. Because
+ /// the graph format has no scope epoch, this can only protect the
+ /// interval represented by this chunk; a coordinator-level mutation
+ /// epoch remains necessary to prove a whole multi-chunk purge had no
+ /// insertion behind an already-committed cursor.
+ #[doc(hidden)]
+ pub fn plan_purge_chunk(
+ &self,
+ source: &R,
+ agent: &str,
+ cursor: Option<&GraphPurgeCursor>,
+ options: GraphPurgeChunkOptions,
+ ) -> std::result::Result {
+ if options.max_items == 0 || options.max_wal_bytes == 0 {
+ return Err(GraphPurgePlanError::ZeroBound);
+ }
+
+ let (phase, start) = match cursor {
+ Some(cursor) => {
+ if cursor.agent != agent {
+ return Err(GraphPurgePlanError::WrongAgent);
+ }
+ (cursor.phase, cursor.start.clone())
+ }
+ None => (GraphPurgePhase::Entities, graph_index::entity_agent_prefix(agent)?),
+ };
+ let prefix = match phase {
+ GraphPurgePhase::Entities => graph_index::entity_agent_prefix(agent)?,
+ GraphPurgePhase::Edges => graph_index::edge_agent_prefix(agent)?,
+ };
+ let end = prefix_end(&prefix).ok_or(GraphPurgePlanError::InvalidCursor)?;
+ if start < prefix || start >= end {
+ return Err(GraphPurgePlanError::InvalidCursor);
+ }
+
+ let page = source.scan_range_page(&start, &end, options.max_items)?;
+ let mut records = Vec::with_capacity(page.entries.len());
+ let mut approx_wal_bytes = 0_usize;
+ let mut capped_resume = None;
+ for (key, value) in page.entries {
+ let mut item = Batch::new();
+ item.delete(key.as_bytes());
+ let needed = item.approx_wire_bytes();
+ if records.is_empty() && needed > options.max_wal_bytes {
+ return Err(GraphPurgePlanError::ItemTooLarge {
+ needed,
+ max: options.max_wal_bytes,
+ });
+ }
+ let next_bytes = approx_wal_bytes.saturating_add(needed);
+ if next_bytes > options.max_wal_bytes {
+ capped_resume = Some(key.as_bytes().to_vec());
+ break;
+ }
+ approx_wal_bytes = next_bytes;
+ records.push((key.as_bytes().to_vec(), value));
+ }
+
+ let (range_end, resume) = if let Some(start) = capped_resume {
+ (
+ start.clone(),
+ Some(GraphPurgeCursor {
+ agent: agent.to_string(),
+ phase,
+ start,
+ }),
+ )
+ } else if let Some(start) = page.next_start {
+ (
+ start.clone(),
+ Some(GraphPurgeCursor {
+ agent: agent.to_string(),
+ phase,
+ start,
+ }),
+ )
+ } else if phase == GraphPurgePhase::Entities {
+ let edge_start = graph_index::edge_agent_prefix(agent)?;
+ (
+ end,
+ Some(GraphPurgeCursor {
+ agent: agent.to_string(),
+ phase: GraphPurgePhase::Edges,
+ start: edge_start,
+ }),
+ )
+ } else {
+ (end, None)
+ };
+
+ Ok(GraphPurgeChunkPlan {
+ agent: agent.to_string(),
+ phase,
+ range_start: start,
+ range_end,
+ records,
+ resume,
+ approx_wal_bytes,
+ })
+ }
+
+ /// Revalidates and stages one previously planned purge chunk.
+ ///
+ /// This method is inert: it returns an owned batch but never writes a
+ /// WAL record. Missing/changed planned members and new members inside
+ /// the exact planned interval are rejected before a batch can commit.
+ #[doc(hidden)]
+ pub fn stage_purge_chunk(
+ &self,
+ source: &R,
+ plan: GraphPurgeChunkPlan,
+ ) -> std::result::Result {
+ let expected_prefix = match plan.phase {
+ GraphPurgePhase::Entities => graph_index::entity_agent_prefix(&plan.agent)?,
+ GraphPurgePhase::Edges => graph_index::edge_agent_prefix(&plan.agent)?,
+ };
+ let expected_end = prefix_end(&expected_prefix).ok_or(GraphPurgePlanError::InvalidCursor)?;
+ if plan.range_start < expected_prefix || plan.range_end > expected_end || plan.range_start > plan.range_end {
+ return Err(GraphPurgePlanError::InvalidCursor);
+ }
+
+ let current = source.scan_range(&plan.range_start, &plan.range_end)?;
+ let mut expected = plan.records.iter();
+ let mut observed = current.iter();
+ loop {
+ match (expected.next(), observed.next()) {
+ (None, None) => break,
+ (Some(_), None) => return Err(GraphPurgePlanError::StaleReadSet),
+ (None, Some(_)) => return Err(GraphPurgePlanError::PhantomInRange),
+ (Some((expected_key, expected_value)), Some((observed_key, observed_value))) => {
+ match observed_key.as_bytes().cmp(expected_key) {
+ std::cmp::Ordering::Less => return Err(GraphPurgePlanError::PhantomInRange),
+ std::cmp::Ordering::Greater => return Err(GraphPurgePlanError::StaleReadSet),
+ std::cmp::Ordering::Equal if observed_value != expected_value => {
+ return Err(GraphPurgePlanError::StaleReadSet);
+ }
+ std::cmp::Ordering::Equal => {}
+ }
+ }
+ }
+ }
+
+ let mut batch = Batch::new();
+ for (key, _) in plan.records {
+ batch.delete(&key);
+ }
+ debug_assert_eq!(batch.approx_wire_bytes(), plan.approx_wal_bytes);
+ Ok(StagedGraphPurgeChunk {
+ removed: u64::try_from(batch.len()).unwrap_or(u64::MAX),
+ batch,
+ })
+ }
+
+ /// Stages an insert or overwrite of entity `(agent, id)` into `batch`.
+ /// No engine mutation occurs, so a caller can compose sibling-index
+ /// changes before one caller-owned commit.
+ pub fn stage_upsert_entity(&self, agent: &str, id: &str, entity: &GraphEntity, batch: &mut Batch) -> Result<()> {
+ let key = graph_index::entity_key(agent, id)?;
+ batch.put(key.as_bytes(), &entity::encode(entity)?);
+ Ok(())
+ }
+
+ /// Inserts or overwrites the entity `(agent, id)` in one durable batch.
+ /// Takes a whole
/// [`GraphEntity`] rather than its individual fields, both to keep the
/// argument count small and because "kind + label + validity" is
/// exactly the block this writes — no partial-field update exists.
pub fn upsert_entity(&self, engine: &mut Engine, agent: &str, id: &str, entity: GraphEntity) -> Result<()> {
- let key = graph_index::entity_key(agent, id)?;
- let value = entity::encode(&entity)?;
- engine.put(key.as_bytes(), &value)
+ let mut batch = Batch::new();
+ self.stage_upsert_entity(agent, id, &entity, &mut batch)?;
+ engine.apply_batch(&batch)
+ }
+
+ /// Stages an insert or overwrite of directed edge
+ /// `(agent, src) --relation--> dst` into `batch`, without mutating the
+ /// engine.
+ pub fn stage_upsert_edge(
+ &self,
+ agent: &str,
+ src: &str,
+ relation: &str,
+ dst: &str,
+ meta: &GraphEdgeMeta,
+ batch: &mut Batch,
+ ) -> Result<()> {
+ let key = graph_index::edge_key(agent, src, relation, dst)?;
+ batch.put(key.as_bytes(), &edge::encode(meta));
+ Ok(())
}
/// Inserts or overwrites the directed edge `(agent, src) --relation--> dst`:
- /// one durable `Engine::put`. `relation`/`dst` stay separate parameters
+ /// one durable batch. `relation`/`dst` stay separate parameters
/// (they belong in the *key*, see `key::graph_index`), while the edge's
/// own attributes travel together as a [`GraphEdgeMeta`].
pub fn upsert_edge(
@@ -93,9 +425,9 @@ impl PersistentGraph {
dst: &str,
meta: GraphEdgeMeta,
) -> Result<()> {
- let key = graph_index::edge_key(agent, src, relation, dst)?;
- let value = edge::encode(&meta);
- engine.put(key.as_bytes(), &value)
+ let mut batch = Batch::new();
+ self.stage_upsert_edge(agent, src, relation, dst, &meta, &mut batch)?;
+ engine.apply_batch(&batch)
}
/// Breadth-first traversal — see [`traverse::run`] for the exact,
@@ -202,25 +534,39 @@ impl PersistentGraph {
Ok(Some(edge::decode(&bytes)?))
}
- /// Removes **every** entity and edge of `agent`, in one atomic batch
- /// (unlike the memory index's per-item purge, nothing here needs to ride
- /// a vector-index mutation — plain KV deletes suffice). Returns the
- /// number of records removed. A no-op (empty batch skipped) when the
- /// agent has no graph.
- pub fn purge_agent(&self, engine: &mut Engine, agent: &str) -> Result {
- let mut batch = crate::store::Batch::new();
+ /// Stages deletion of every entity and edge of `agent` into `batch` and
+ /// returns the number of graph records named by the current read view.
+ /// No mutation occurs until the caller applies the batch.
+ ///
+ /// This is the unbounded staging primitive retained for the legacy
+ /// whole-agent wrapper. A bounded coordinator must not call it for an
+ /// arbitrary agent: it must plan a capped page and submit one owned
+ /// purge chunk per logical commit, then resume from a fresh read view.
+ pub fn stage_purge_agent(&self, source: &R, agent: &str, batch: &mut Batch) -> Result {
+ let mut removed = 0_u64;
for prefix in [
graph_index::entity_agent_prefix(agent)?,
graph_index::edge_agent_prefix(agent)?,
] {
- for (key, _) in engine.scan_prefix(&prefix)? {
+ for (key, _) in source.scan_prefix(&prefix)? {
batch.delete(key.as_bytes());
+ removed += 1;
}
}
+ Ok(removed)
+ }
+
+ /// Removes **every** entity and edge of `agent`, in one atomic batch
+ /// (unlike the memory index's per-item purge, nothing here needs to ride
+ /// a vector-index mutation — plain KV deletes suffice). Returns the
+ /// number of records removed. A no-op (empty batch skipped) when the
+ /// agent has no graph.
+ pub fn purge_agent(&self, engine: &mut Engine, agent: &str) -> Result {
+ let mut batch = Batch::new();
+ let removed = self.stage_purge_agent(engine, agent, &mut batch)?;
if batch.is_empty() {
return Ok(0);
}
- let removed = batch.len() as u64;
engine.apply_batch(&batch)?;
Ok(removed)
}
@@ -264,3 +610,325 @@ impl GraphProvider for EngineGraphProvider<'_, R> {
Ok(out)
}
}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+ use crate::idx::graph::GraphSource;
+
+ fn entity(label: &str) -> GraphEntity {
+ GraphEntity {
+ kind: "person".to_string(),
+ label: label.to_string(),
+ valid_from: 10,
+ valid_until: None,
+ source: GraphSource::User,
+ }
+ }
+
+ fn edge(weight: f64) -> GraphEdgeMeta {
+ GraphEdgeMeta {
+ weight,
+ valid_from: 10,
+ valid_until: None,
+ source: GraphSource::User,
+ }
+ }
+
+ #[test]
+ fn staged_upserts_are_inert_then_commit_together_once() {
+ let directory = tempfile::tempdir().expect("temporary graph store");
+ let mut engine = Engine::open(directory.path()).expect("engine opens");
+ let graph = PersistentGraph::new();
+ let mut batch = Batch::new();
+
+ graph
+ .stage_upsert_entity("agent-a", "alice", &entity("Alice"), &mut batch)
+ .expect("entity stages");
+ graph
+ .stage_upsert_edge("agent-a", "alice", "knows", "bob", &edge(0.75), &mut batch)
+ .expect("edge stages");
+
+ assert!(
+ graph
+ .entity(&engine, "agent-a", "alice")
+ .expect("entity lookup")
+ .is_none()
+ );
+ assert!(
+ graph
+ .edge_meta(&engine, "agent-a", "alice", "knows", "bob")
+ .expect("edge lookup")
+ .is_none()
+ );
+ let before = engine.stats().expect("stats before commit").wal_records;
+ engine.apply_batch(&batch).expect("composed graph batch commits");
+ let after = engine.stats().expect("stats after commit").wal_records;
+
+ assert_eq!(after - before, 1, "both graph records must share one WAL commit");
+ assert_eq!(
+ graph.entity(&engine, "agent-a", "alice").expect("entity lookup"),
+ Some(entity("Alice"))
+ );
+ assert_eq!(
+ graph
+ .edge_meta(&engine, "agent-a", "alice", "knows", "bob")
+ .expect("edge lookup"),
+ Some(edge(0.75))
+ );
+ }
+
+ #[test]
+ fn legacy_upsert_wrappers_keep_one_commit_and_behavior() {
+ let directory = tempfile::tempdir().expect("temporary graph store");
+ let mut engine = Engine::open(directory.path()).expect("engine opens");
+ let graph = PersistentGraph::new();
+ let before = engine.stats().expect("initial stats").wal_records;
+
+ graph
+ .upsert_entity(&mut engine, "agent-a", "alice", entity("Alice"))
+ .expect("legacy entity upsert");
+ let after_entity = engine.stats().expect("entity stats").wal_records;
+ graph
+ .upsert_edge(&mut engine, "agent-a", "alice", "knows", "bob", edge(0.5))
+ .expect("legacy edge upsert");
+ let after_edge = engine.stats().expect("edge stats").wal_records;
+
+ assert_eq!(after_entity - before, 1);
+ assert_eq!(after_edge - after_entity, 1);
+ assert_eq!(
+ graph.entity(&engine, "agent-a", "alice").expect("entity lookup"),
+ Some(entity("Alice"))
+ );
+ assert_eq!(
+ graph
+ .edge_meta(&engine, "agent-a", "alice", "knows", "bob")
+ .expect("edge lookup"),
+ Some(edge(0.5))
+ );
+ }
+
+ #[test]
+ fn staged_purge_is_inert_scoped_and_commits_once() {
+ let directory = tempfile::tempdir().expect("temporary graph store");
+ let mut engine = Engine::open(directory.path()).expect("engine opens");
+ let graph = PersistentGraph::new();
+ graph
+ .upsert_entity(&mut engine, "agent-a", "alice", entity("Alice"))
+ .expect("first entity");
+ graph
+ .upsert_edge(&mut engine, "agent-a", "alice", "knows", "bob", edge(1.0))
+ .expect("first edge");
+ graph
+ .upsert_entity(&mut engine, "agent-b", "alice", entity("Other Alice"))
+ .expect("other agent entity");
+
+ let mut batch = Batch::new();
+ assert_eq!(
+ graph
+ .stage_purge_agent(&engine, "agent-a", &mut batch)
+ .expect("purge stages"),
+ 2
+ );
+ assert_eq!(graph.entities(&engine, "agent-a").expect("still visible").len(), 1);
+ assert_eq!(graph.edges(&engine, "agent-a").expect("still visible").len(), 1);
+
+ let before = engine.stats().expect("stats before purge").wal_records;
+ engine.apply_batch(&batch).expect("purge batch commits");
+ let after = engine.stats().expect("stats after purge").wal_records;
+ assert_eq!(after - before, 1);
+ assert!(graph.entities(&engine, "agent-a").expect("purged entities").is_empty());
+ assert!(graph.edges(&engine, "agent-a").expect("purged edges").is_empty());
+ assert_eq!(
+ graph.entities(&engine, "agent-b").expect("other agent retained").len(),
+ 1
+ );
+ }
+
+ fn seed_purge_fixture(engine: &mut Engine, graph: PersistentGraph) {
+ for id in ["a", "b", "c"] {
+ graph
+ .upsert_entity(engine, "agent-a", id, entity(id))
+ .expect("fixture entity");
+ }
+ for dst in ["a", "b", "c"] {
+ graph
+ .upsert_edge(engine, "agent-a", "root", "knows", dst, edge(1.0))
+ .expect("fixture edge");
+ }
+ graph
+ .upsert_entity(engine, "agent-b", "retained", entity("retained"))
+ .expect("other agent fixture");
+ }
+
+ #[test]
+ fn purge_plan_honors_item_and_byte_caps() {
+ let directory = tempfile::tempdir().expect("temporary graph store");
+ let mut engine = Engine::open(directory.path()).expect("engine opens");
+ let graph = PersistentGraph::new();
+ seed_purge_fixture(&mut engine, graph);
+
+ let item_bounded = graph
+ .plan_purge_chunk(
+ &engine,
+ "agent-a",
+ None,
+ GraphPurgeChunkOptions {
+ max_items: 2,
+ max_wal_bytes: usize::MAX,
+ },
+ )
+ .expect("item-bounded plan");
+ assert_eq!(item_bounded.len(), 2);
+ assert!(item_bounded.resume().is_some());
+
+ let first_key = graph_index::entity_key("agent-a", "a").expect("fixture key");
+ let mut one_delete = Batch::new();
+ one_delete.delete(first_key.as_bytes());
+ let one_item_bytes = one_delete.approx_wire_bytes();
+ let byte_bounded = graph
+ .plan_purge_chunk(
+ &engine,
+ "agent-a",
+ None,
+ GraphPurgeChunkOptions {
+ max_items: 10,
+ max_wal_bytes: one_item_bytes,
+ },
+ )
+ .expect("byte-bounded plan");
+ assert_eq!(byte_bounded.len(), 1);
+ assert!(byte_bounded.approx_wal_bytes() <= one_item_bytes);
+
+ let before = engine.stats().expect("stats before rejection").wal_records;
+ assert!(matches!(
+ graph.plan_purge_chunk(
+ &engine,
+ "agent-a",
+ None,
+ GraphPurgeChunkOptions {
+ max_items: 10,
+ max_wal_bytes: one_item_bytes - 1,
+ },
+ ),
+ Err(GraphPurgePlanError::ItemTooLarge { .. })
+ ));
+ assert_eq!(engine.stats().expect("stats after rejection").wal_records, before);
+ }
+
+ #[test]
+ fn purge_staging_is_inert_and_rejects_stale_or_phantom_pre_wal() {
+ let directory = tempfile::tempdir().expect("temporary graph store");
+ let mut engine = Engine::open(directory.path()).expect("engine opens");
+ let graph = PersistentGraph::new();
+ graph
+ .upsert_entity(&mut engine, "agent-a", "a", entity("old"))
+ .expect("fixture entity");
+ let stale = graph
+ .plan_purge_chunk(&engine, "agent-a", None, GraphPurgeChunkOptions::default())
+ .expect("stale candidate plan");
+ graph
+ .upsert_entity(&mut engine, "agent-a", "a", entity("new"))
+ .expect("concurrent update");
+ let before_stale = engine.stats().expect("stats before stale stage").wal_records;
+ assert!(matches!(
+ graph.stage_purge_chunk(&engine, stale),
+ Err(GraphPurgePlanError::StaleReadSet)
+ ));
+ assert_eq!(
+ engine.stats().expect("stats after stale stage").wal_records,
+ before_stale
+ );
+
+ let phantom = graph
+ .plan_purge_chunk(&engine, "agent-a", None, GraphPurgeChunkOptions::default())
+ .expect("phantom candidate plan");
+ graph
+ .upsert_entity(&mut engine, "agent-a", "b", entity("late"))
+ .expect("concurrent insert");
+ let before_phantom = engine.stats().expect("stats before phantom stage").wal_records;
+ assert!(matches!(
+ graph.stage_purge_chunk(&engine, phantom),
+ Err(GraphPurgePlanError::PhantomInRange)
+ ));
+ assert_eq!(
+ engine.stats().expect("stats after phantom stage").wal_records,
+ before_phantom
+ );
+ }
+
+ #[test]
+ fn one_staged_purge_chunk_is_inert_then_exactly_one_wal() {
+ let directory = tempfile::tempdir().expect("temporary graph store");
+ let mut engine = Engine::open(directory.path()).expect("engine opens");
+ let graph = PersistentGraph::new();
+ seed_purge_fixture(&mut engine, graph);
+ let plan = graph
+ .plan_purge_chunk(
+ &engine,
+ "agent-a",
+ None,
+ GraphPurgeChunkOptions {
+ max_items: 2,
+ max_wal_bytes: usize::MAX,
+ },
+ )
+ .expect("chunk plan");
+ let staged = graph.stage_purge_chunk(&engine, plan).expect("chunk stages");
+ assert_eq!(graph.entities(&engine, "agent-a").expect("still visible").len(), 3);
+ let before = engine.stats().expect("stats before chunk").wal_records;
+ let outcome = staged.commit(&mut engine);
+ let after = engine.stats().expect("stats after chunk").wal_records;
+ assert!(matches!(outcome, GraphPurgeCommitOutcome::Committed { removed: 2, .. }));
+ assert_eq!(after - before, 1);
+ assert_eq!(graph.entities(&engine, "agent-a").expect("remaining entities").len(), 1);
+ }
+
+ #[test]
+ fn purge_cursor_resumes_to_legacy_parity_without_cross_agent_deletes() {
+ let chunked_directory = tempfile::tempdir().expect("chunked temporary store");
+ let legacy_directory = tempfile::tempdir().expect("legacy temporary store");
+ let mut chunked = Engine::open(chunked_directory.path()).expect("chunked engine opens");
+ let mut legacy = Engine::open(legacy_directory.path()).expect("legacy engine opens");
+ let graph = PersistentGraph::new();
+ seed_purge_fixture(&mut chunked, graph);
+ seed_purge_fixture(&mut legacy, graph);
+
+ let mut cursor = None;
+ loop {
+ let plan = graph
+ .plan_purge_chunk(
+ &chunked,
+ "agent-a",
+ cursor.as_ref(),
+ GraphPurgeChunkOptions {
+ max_items: 2,
+ max_wal_bytes: usize::MAX,
+ },
+ )
+ .expect("resumed plan");
+ let next = plan.resume().cloned();
+ if !plan.is_empty() {
+ let staged = graph.stage_purge_chunk(&chunked, plan).expect("resumed stage");
+ assert!(matches!(
+ staged.commit(&mut chunked),
+ GraphPurgeCommitOutcome::Committed { .. }
+ ));
+ }
+ cursor = next;
+ if cursor.is_none() {
+ break;
+ }
+ }
+
+ assert_eq!(graph.purge_agent(&mut legacy, "agent-a").expect("legacy purge"), 6);
+ assert_eq!(
+ chunked.scan_prefix(b"idx/graph/").expect("chunked graph scan"),
+ legacy.scan_prefix(b"idx/graph/").expect("legacy graph scan")
+ );
+ assert_eq!(
+ graph.entities(&chunked, "agent-b").expect("other agent retained").len(),
+ 1
+ );
+ }
+}
diff --git a/crates/basemyai-engine/src/idx/memory/mod.rs b/crates/basemyai-engine/src/idx/memory/mod.rs
index 2eda102..74b0950 100644
--- a/crates/basemyai-engine/src/idx/memory/mod.rs
+++ b/crates/basemyai-engine/src/idx/memory/mod.rs
@@ -18,6 +18,9 @@ pub mod persistent;
pub mod record;
pub mod vecmap;
-pub use persistent::{ForgetBatchOptions, NewMemoryRecord, PersistentMemoryIndex};
+pub use persistent::{
+ ForgetBatchOptions, MemoryCommitOutcome, NewMemoryRecord, PersistentMemoryIndex, PurgeChunkCommit, PurgeChunkPlan,
+ PurgeCursor, StagedForgetChunk, StagedMemoryBatch, StagedMemoryPut, StagedPurgeChunk, StagedPurgeFinalize,
+};
pub use record::MemoryRecord;
pub use vecmap::VecMapEntry;
diff --git a/crates/basemyai-engine/src/idx/memory/persistent.rs b/crates/basemyai-engine/src/idx/memory/persistent.rs
index 06db6e9..88c3955 100644
--- a/crates/basemyai-engine/src/idx/memory/persistent.rs
+++ b/crates/basemyai-engine/src/idx/memory/persistent.rs
@@ -34,9 +34,9 @@ use std::collections::HashSet;
use crate::error::{EngineError, Result};
use crate::idx::fts::PersistentFts;
-use crate::idx::vector::PersistentVectorIndex;
+use crate::idx::vector::{PersistentVectorIndex, StagedVectorDelete, StagedVectorInsert};
use crate::key::{agent_registry, memory_index, temporal_index, vector_index};
-use crate::store::{Batch, Engine, EngineRead};
+use crate::store::{Batch, Engine, EngineCommitOutcome, EngineRead, WalCommitPhase};
use super::meta::{self, MemoryIndexMeta};
use super::record::{self, MemoryRecord};
@@ -76,6 +76,375 @@ pub struct ForgetBatchOptions {
pub max_wal_bytes: usize,
}
+/// Owned preparation of one memory multi-insert. The embedded vector stage
+/// owns the exact atomic KV batch; the allocator transition remains private
+/// until the same durable callback installs both vector RAM and
+/// `next_vec_id`.
+#[doc(hidden)]
+#[derive(Debug)]
+pub struct StagedMemoryPut {
+ vector: StagedVectorInsert,
+ expected_next_vec_id: u64,
+ next_vec_id: u64,
+ allocated: Vec,
+}
+
+/// Owned preparation of one bounded forget chunk. It is deliberately a
+/// single vector stage and therefore a single WAL commit.
+#[doc(hidden)]
+#[derive(Debug)]
+pub struct StagedForgetChunk {
+ vector: StagedVectorDelete,
+ removed: u64,
+}
+
+/// One purge-only chunk carrying the mutation-epoch precondition through the
+/// final pre-WAL boundary.
+#[doc(hidden)]
+#[derive(Debug)]
+pub struct StagedPurgeChunk {
+ inner: StagedForgetChunk,
+ agent: String,
+ resume_after: Option,
+ expected_sequence: crate::store::SequenceNumber,
+}
+
+/// Registry deletion prepared from an empty terminal purge plan. It retains
+/// the same epoch until commit so a late insertion cannot be hidden by marker
+/// removal.
+#[doc(hidden)]
+#[derive(Debug)]
+pub struct StagedPurgeFinalize {
+ batch: Batch,
+ expected_sequence: crate::store::SequenceNumber,
+}
+
+/// Owned read-set for one bounded, coordinator-sized agent purge commit.
+/// Fields stay private so transports cannot forge `(id, vec_id, validity)`
+/// tuples; only [`PersistentMemoryIndex::plan_purge_chunk`] constructs it.
+#[doc(hidden)]
+#[derive(Debug)]
+pub struct PurgeChunkPlan {
+ agent: String,
+ records: Vec,
+ resume_after: Option,
+ exhausted: bool,
+ estimated_wal_bytes: usize,
+ expected_sequence: crate::store::SequenceNumber,
+}
+
+/// Opaque, linear continuation of one live purge session. The global engine
+/// sequence is deliberately used as a conservative mutation epoch: under the
+/// single product owner, only the preceding purge chunk may advance it. Any
+/// intervening mutation, including an insertion behind the id cursor,
+/// invalidates the continuation before another WAL append.
+#[doc(hidden)]
+#[derive(Debug)]
+pub struct PurgeCursor {
+ agent: String,
+ after_id: Option,
+ expected_sequence: crate::store::SequenceNumber,
+}
+
+/// Successful result of exactly one purge chunk. The continuation is minted
+/// only from the durable receipt, so callers cannot advance a purge cursor
+/// after an aborted or ambiguous commit.
+#[doc(hidden)]
+#[derive(Debug)]
+pub struct PurgeChunkCommit {
+ removed: u64,
+ cursor: PurgeCursor,
+}
+
+impl PurgeChunkCommit {
+ #[must_use]
+ pub fn removed(&self) -> u64 {
+ self.removed
+ }
+
+ #[must_use]
+ pub fn into_cursor(self) -> PurgeCursor {
+ self.cursor
+ }
+}
+
+#[derive(Debug)]
+struct PurgeChunkRecord {
+ id: String,
+ vec_id: u64,
+ valid_until: Option,
+}
+
+impl PurgeChunkPlan {
+ #[must_use]
+ pub fn len(&self) -> usize {
+ self.records.len()
+ }
+
+ #[must_use]
+ pub fn is_empty(&self) -> bool {
+ self.records.is_empty()
+ }
+
+ #[must_use]
+ pub fn resume_after(&self) -> Option<&str> {
+ self.resume_after.as_deref()
+ }
+
+ #[must_use]
+ pub fn exhausted(&self) -> bool {
+ self.exhausted
+ }
+
+ #[must_use]
+ pub fn estimated_wal_bytes(&self) -> usize {
+ self.estimated_wal_bytes
+ }
+}
+
+/// Owned record-only mutation (update/touch). No product RAM is published;
+/// the type still closes direct access to its batch and commits through the
+/// phase-aware engine lifecycle.
+#[doc(hidden)]
+#[derive(Debug)]
+pub struct StagedMemoryBatch {
+ batch: Batch,
+}
+
+/// Phase-preserving result exposed to the future product coordinator. Unlike
+/// the legacy `Result`, it never flattens an unknown WAL phase or a durable
+/// product-install failure.
+#[doc(hidden)]
+#[derive(Debug)]
+pub enum MemoryCommitOutcome {
+ Aborted {
+ burned_sequence_range: Option,
+ cause: Option,
+ },
+ Committed {
+ sequence_range: crate::store::SequenceRange,
+ value: T,
+ },
+ OutcomeUnknown {
+ phase: WalCommitPhase,
+ cause: EngineError,
+ },
+ DurableReopenRequired {
+ sequence_range: crate::store::SequenceRange,
+ cause: EngineError,
+ },
+}
+
+fn legacy_commit_error(phase: WalCommitPhase, cause: EngineError) -> EngineError {
+ match phase {
+ WalCommitPhase::Append => EngineError::WalAppendOutcomeUnknown {
+ cause: cause.to_string(),
+ },
+ WalCommitPhase::Sync => EngineError::WalSyncOutcomeUnknown {
+ cause: cause.to_string(),
+ },
+ }
+}
+
+impl StagedMemoryPut {
+ #[doc(hidden)]
+ pub fn commit(
+ self,
+ engine: &mut Engine,
+ memory: &mut PersistentMemoryIndex,
+ vectors: &mut PersistentVectorIndex,
+ ) -> MemoryCommitOutcome> {
+ let Self {
+ vector,
+ expected_next_vec_id,
+ next_vec_id,
+ allocated,
+ } = self;
+ match vector.commit(engine) {
+ EngineCommitOutcome::Aborted {
+ burned_sequence_range,
+ cause,
+ } => MemoryCommitOutcome::Aborted {
+ burned_sequence_range,
+ cause,
+ },
+ EngineCommitOutcome::Durable(receipt) => {
+ let sequence_range = receipt.sequence_range();
+ match engine.install_committed_batch_with(receipt, |vector_install| {
+ if memory.next_vec_id != expected_next_vec_id {
+ return Err(EngineError::InvalidOptions {
+ reason: "staged memory allocator installation no longer matches RAM state".to_owned(),
+ });
+ }
+ vectors.install_staged_insert_many(vector_install)?;
+ memory.next_vec_id = next_vec_id;
+ Ok(())
+ }) {
+ Ok(()) => MemoryCommitOutcome::Committed {
+ sequence_range,
+ value: allocated,
+ },
+ Err(cause) => MemoryCommitOutcome::DurableReopenRequired { sequence_range, cause },
+ }
+ }
+ EngineCommitOutcome::OutcomeUnknown { phase, cause } => {
+ MemoryCommitOutcome::OutcomeUnknown { phase, cause }
+ }
+ }
+ }
+}
+
+impl StagedForgetChunk {
+ #[doc(hidden)]
+ pub fn commit(self, engine: &mut Engine, vectors: &mut PersistentVectorIndex) -> MemoryCommitOutcome {
+ let Self { vector, removed } = self;
+ match vector.commit(engine) {
+ EngineCommitOutcome::Aborted {
+ burned_sequence_range,
+ cause,
+ } => MemoryCommitOutcome::Aborted {
+ burned_sequence_range,
+ cause,
+ },
+ EngineCommitOutcome::Durable(receipt) => {
+ let sequence_range = receipt.sequence_range();
+ match engine.install_committed_batch_with(receipt, |vector_install| {
+ vectors.install_staged_delete_many(vector_install).map(|_| ())
+ }) {
+ Ok(()) => MemoryCommitOutcome::Committed {
+ sequence_range,
+ value: removed,
+ },
+ Err(cause) => MemoryCommitOutcome::DurableReopenRequired { sequence_range, cause },
+ }
+ }
+ EngineCommitOutcome::OutcomeUnknown { phase, cause } => {
+ MemoryCommitOutcome::OutcomeUnknown { phase, cause }
+ }
+ }
+ }
+}
+
+fn stale_purge_epoch(expected: crate::store::SequenceNumber, actual: crate::store::SequenceNumber) -> EngineError {
+ EngineError::InvalidOptions {
+ reason: format!("stale purge mutation epoch: expected visible sequence {expected}, found {actual}"),
+ }
+}
+
+impl StagedPurgeChunk {
+ #[doc(hidden)]
+ pub fn commit(
+ self,
+ engine: &mut Engine,
+ vectors: &mut PersistentVectorIndex,
+ ) -> MemoryCommitOutcome {
+ let actual = engine.snapshot().visible_sequence();
+ if actual != self.expected_sequence {
+ return MemoryCommitOutcome::Aborted {
+ burned_sequence_range: None,
+ cause: Some(stale_purge_epoch(self.expected_sequence, actual)),
+ };
+ }
+ let Self {
+ inner,
+ agent,
+ resume_after,
+ expected_sequence: _,
+ } = self;
+ match inner.commit(engine, vectors) {
+ MemoryCommitOutcome::Aborted {
+ burned_sequence_range,
+ cause,
+ } => MemoryCommitOutcome::Aborted {
+ burned_sequence_range,
+ cause,
+ },
+ MemoryCommitOutcome::Committed {
+ sequence_range,
+ value: removed,
+ } => MemoryCommitOutcome::Committed {
+ sequence_range,
+ value: PurgeChunkCommit {
+ removed,
+ cursor: PurgeCursor {
+ agent,
+ after_id: resume_after,
+ expected_sequence: sequence_range.last(),
+ },
+ },
+ },
+ MemoryCommitOutcome::OutcomeUnknown { phase, cause } => {
+ MemoryCommitOutcome::OutcomeUnknown { phase, cause }
+ }
+ MemoryCommitOutcome::DurableReopenRequired { sequence_range, cause } => {
+ MemoryCommitOutcome::DurableReopenRequired { sequence_range, cause }
+ }
+ }
+ }
+}
+
+impl StagedPurgeFinalize {
+ #[doc(hidden)]
+ pub fn commit(self, engine: &mut Engine) -> MemoryCommitOutcome<()> {
+ let actual = engine.snapshot().visible_sequence();
+ if actual != self.expected_sequence {
+ return MemoryCommitOutcome::Aborted {
+ burned_sequence_range: None,
+ cause: Some(stale_purge_epoch(self.expected_sequence, actual)),
+ };
+ }
+ StagedMemoryBatch { batch: self.batch }.commit(engine)
+ }
+}
+
+impl StagedMemoryBatch {
+ #[doc(hidden)]
+ pub fn commit(self, engine: &mut Engine) -> MemoryCommitOutcome<()> {
+ match engine.commit_batch(self.batch) {
+ EngineCommitOutcome::Aborted {
+ burned_sequence_range,
+ cause,
+ } => MemoryCommitOutcome::Aborted {
+ burned_sequence_range,
+ cause,
+ },
+ EngineCommitOutcome::Durable(receipt) => {
+ let sequence_range = receipt.sequence_range();
+ match engine.install_committed_batch_with(receipt, |_| Ok(())) {
+ Ok(()) => MemoryCommitOutcome::Committed {
+ sequence_range,
+ value: (),
+ },
+ Err(cause) => MemoryCommitOutcome::DurableReopenRequired { sequence_range, cause },
+ }
+ }
+ EngineCommitOutcome::OutcomeUnknown { phase, cause } => {
+ MemoryCommitOutcome::OutcomeUnknown { phase, cause }
+ }
+ }
+ }
+}
+
+fn legacy_memory_outcome(outcome: MemoryCommitOutcome, aborted_value: T) -> Result {
+ match outcome {
+ MemoryCommitOutcome::Aborted { cause: None, .. } => Ok(aborted_value),
+ MemoryCommitOutcome::Aborted { cause: Some(cause), .. }
+ | MemoryCommitOutcome::DurableReopenRequired { cause, .. } => Err(cause),
+ MemoryCommitOutcome::Committed { value, .. } => Ok(value),
+ MemoryCommitOutcome::OutcomeUnknown { phase, cause } => Err(legacy_commit_error(phase, cause)),
+ }
+}
+
+fn legacy_purge_chunk_outcome(outcome: MemoryCommitOutcome) -> Result> {
+ match outcome {
+ MemoryCommitOutcome::Aborted { cause: None, .. } => Ok(None),
+ MemoryCommitOutcome::Aborted { cause: Some(cause), .. }
+ | MemoryCommitOutcome::DurableReopenRequired { cause, .. } => Err(cause),
+ MemoryCommitOutcome::Committed { value, .. } => Ok(Some(value)),
+ MemoryCommitOutcome::OutcomeUnknown { phase, cause } => Err(legacy_commit_error(phase, cause)),
+ }
+}
+
impl Default for ForgetBatchOptions {
/// Order-of-magnitude defaults (same posture as the engine's block-cache
/// default: a sane starting point, not a measured optimum): 256 memories
@@ -117,6 +486,15 @@ pub struct PersistentMemoryIndex {
}
impl PersistentMemoryIndex {
+ const FORGET_CHUNK_OVERHEAD_BYTES: usize = 256;
+
+ fn reserve_vector_ids(next: u64, requested: usize) -> Result {
+ let requested_u64 =
+ u64::try_from(requested).map_err(|_| EngineError::VectorIdSpaceExhausted { next, requested })?;
+ next.checked_add(requested_u64)
+ .ok_or(EngineError::VectorIdSpaceExhausted { next, requested })
+ }
+
/// Opens the index stored in `engine`, or initializes an empty one.
///
/// The allocator record is loaded when present and valid; **healed from
@@ -152,7 +530,10 @@ impl PersistentMemoryIndex {
max_seen = Some(max_seen.map_or(id, |m| m.max(id)));
}
}
- Ok(max_seen.map_or(0, |m| m + 1))
+ match max_seen {
+ Some(max) => Self::reserve_vector_ids(max, 1),
+ None => Ok(0),
+ }
}
/// The next id [`Self::put`] will allocate. Exposed for tests and
@@ -215,8 +596,28 @@ impl PersistentMemoryIndex {
agent: &str,
items: &[(&str, NewMemoryRecord<'_>, Vec)],
) -> Result> {
+ let staged = self.stage_put_many(engine, vectors, fts, agent, items)?;
+ legacy_memory_outcome(staged.commit(engine, self, vectors), Vec::new())
+ }
+
+ /// Prepares one owned atomic memory/vector/FTS insert without publishing
+ /// vector RAM or advancing the in-RAM allocator.
+ #[doc(hidden)]
+ pub fn stage_put_many(
+ &self,
+ engine: &Engine,
+ vectors: &PersistentVectorIndex,
+ fts: &PersistentFts,
+ agent: &str,
+ items: &[(&str, NewMemoryRecord<'_>, Vec)],
+ ) -> Result {
if items.is_empty() {
- return Ok(Vec::new());
+ return Ok(StagedMemoryPut {
+ vector: vectors.stage_insert_many_with(engine, Vec::new(), &Batch::new())?,
+ expected_next_vec_id: self.next_vec_id,
+ next_vec_id: self.next_vec_id,
+ allocated: Vec::new(),
+ });
}
for (i, (id, _, _)) in items.iter().enumerate() {
let record_key = memory_index::record_key(agent, id)?;
@@ -229,11 +630,11 @@ impl PersistentMemoryIndex {
}
let first_vec_id = self.next_vec_id;
+ let next_vec_id = Self::reserve_vector_ids(first_vec_id, items.len())?;
let mut extra = Batch::new();
let mut fts_docs: Vec<(u64, &str)> = Vec::with_capacity(items.len());
let mut vector_items: Vec<(u64, Vec)> = Vec::with_capacity(items.len());
- for (offset, (id, new, vector)) in items.iter().enumerate() {
- let vec_id = first_vec_id + offset as u64;
+ for (vec_id, (id, new, vector)) in (first_vec_id..next_vec_id).zip(items.iter()) {
let stored = MemoryRecord {
layer: new.layer.to_string(),
content: new.content.to_string(),
@@ -257,17 +658,20 @@ impl PersistentMemoryIndex {
fts_docs.push((vec_id, new.content));
vector_items.push((vec_id, vector.clone()));
}
- let next_vec_id = first_vec_id + items.len() as u64;
extra.put(memory_index::META_KEY, &meta::encode(&MemoryIndexMeta { next_vec_id })?);
// Agent-registry marker (ADR-041 §7.5): staged with every insert
// batch — an idempotent overwrite of an empty value, so re-inserts
// cost one no-op-shaped op rather than a read-before-write.
extra.put(agent_registry::agent_key(agent)?.as_bytes(), &[]);
fts.stage_insert_many(engine, agent, &fts_docs, &mut extra)?;
- vectors.insert_many_with(engine, vector_items, &extra)?;
-
- self.next_vec_id = next_vec_id;
- Ok((first_vec_id..next_vec_id).collect())
+ let vector = vectors.stage_insert_many_with(engine, vector_items, &extra)?;
+
+ Ok(StagedMemoryPut {
+ vector,
+ expected_next_vec_id: self.next_vec_id,
+ next_vec_id,
+ allocated: (first_vec_id..next_vec_id).collect(),
+ })
}
/// The memory record `(agent, id)`, if any — regardless of its validity
@@ -295,6 +699,18 @@ impl PersistentMemoryIndex {
/// expiry). All of it — the record overwrite and the expiry delta —
/// rides **one** atomic batch, never two separate writes.
pub fn update(&self, engine: &mut Engine, agent: &str, id: &str, updated: &MemoryRecord) -> Result<()> {
+ legacy_memory_outcome(self.stage_update(engine, agent, id, updated)?.commit(engine), ())
+ }
+
+ /// Stages one record update and its temporal-index delta.
+ #[doc(hidden)]
+ pub fn stage_update(
+ &self,
+ engine: &Engine,
+ agent: &str,
+ id: &str,
+ updated: &MemoryRecord,
+ ) -> Result {
let key = memory_index::record_key(agent, id)?;
let mut batch = Batch::new();
batch.put(key.as_bytes(), &record::encode(updated)?);
@@ -303,7 +719,7 @@ impl PersistentMemoryIndex {
stage_expiry_delete(&mut batch, agent, id, previous_valid_until)?;
stage_expiry_insert(&mut batch, agent, id, updated.valid_until)?;
}
- engine.apply_batch(&batch)
+ Ok(StagedMemoryBatch { batch })
}
/// Rewrites `last_access = now` on every existing `(agent, id)` of `ids`
@@ -316,6 +732,21 @@ impl PersistentMemoryIndex {
ids: impl IntoIterator- ,
now: i64,
) -> Result<()> {
+ legacy_memory_outcome(
+ self.stage_touch_last_access(engine, agent, ids, now)?.commit(engine),
+ (),
+ )
+ }
+
+ /// Stages one atomic touch batch; absent ids remain no-ops.
+ #[doc(hidden)]
+ pub fn stage_touch_last_access<'a>(
+ &self,
+ engine: &Engine,
+ agent: &str,
+ ids: impl IntoIterator
- ,
+ now: i64,
+ ) -> Result {
let mut batch = Batch::new();
for id in ids {
if let Some(mut stored) = self.get(engine, agent, id)? {
@@ -324,10 +755,7 @@ impl PersistentMemoryIndex {
batch.put(key.as_bytes(), &record::encode(&stored)?);
}
}
- if batch.is_empty() {
- return Ok(());
- }
- engine.apply_batch(&batch)
+ Ok(StagedMemoryBatch { batch })
}
/// Physically forgets the memory `(agent, id)`: record, reverse mapping
@@ -349,14 +777,8 @@ impl PersistentMemoryIndex {
let Some(stored) = self.get(engine, agent, id)? else {
return Ok(false);
};
- let record_key = memory_index::record_key(agent, id)?;
- let mut extra = Batch::new();
- extra.delete(record_key.as_bytes());
- extra.delete(memory_index::vecmap_key(stored.vec_id).as_bytes());
- stage_expiry_delete(&mut extra, agent, id, stored.valid_until)?;
- fts.stage_delete(engine, agent, stored.vec_id, &mut extra)?;
- vectors.delete_with(engine, stored.vec_id, &extra)?;
- Ok(true)
+ let staged = Self::stage_forget_chunk(engine, vectors, fts, agent, &[(id, stored.vec_id, stored.valid_until)])?;
+ legacy_memory_outcome(staged.commit(engine, vectors), 0).map(|_| true)
}
/// Physically forgets **several** memories of one `agent`, in bounded
@@ -389,14 +811,12 @@ impl PersistentMemoryIndex {
let max_items = options.max_items.max(1);
// Fixed per-chunk overhead: the vector META rewrite + the FTS stats
// record the chunk stages once regardless of its item count.
- const CHUNK_OVERHEAD_BYTES: usize = 256;
-
let mut removed = 0u64;
let mut seen: HashSet<&str> = HashSet::with_capacity(ids.len());
// The pending chunk: (id, vec_id, valid_until) of each memory, plus
// its running byte estimate.
let mut chunk: Vec<(&str, u64, Option)> = Vec::new();
- let mut chunk_bytes = CHUNK_OVERHEAD_BYTES;
+ let mut chunk_bytes = Self::FORGET_CHUNK_OVERHEAD_BYTES;
for &id in ids {
if !seen.insert(id) {
@@ -419,7 +839,7 @@ impl PersistentMemoryIndex {
if !chunk.is_empty() && (chunk.len() >= max_items || chunk_bytes + item_bytes > options.max_wal_bytes) {
removed += Self::apply_forget_chunk(engine, vectors, *fts, agent, &chunk)?;
chunk.clear();
- chunk_bytes = CHUNK_OVERHEAD_BYTES;
+ chunk_bytes = Self::FORGET_CHUNK_OVERHEAD_BYTES;
}
chunk.push((id, stored.vec_id, stored.valid_until));
chunk_bytes += item_bytes;
@@ -440,6 +860,26 @@ impl PersistentMemoryIndex {
agent: &str,
chunk: &[(&str, u64, Option)],
) -> Result {
+ let staged = Self::stage_forget_chunk(engine, vectors, &fts, agent, chunk)?;
+ legacy_memory_outcome(staged.commit(engine, vectors), 0)
+ }
+
+ /// Prepares exactly one bounded forget chunk. It never loops or commits;
+ /// the future coordinator can admit each returned chunk independently.
+ ///
+ /// This workspace-internal surface consumes the trusted planner's owned
+ /// read-set `(id, vec_id, valid_until)`. Transports must never construct
+ /// these triples directly: the live handler first resolves records under
+ /// its product read gate, then submits this one bounded chunk for
+ /// revalidation/commit.
+ #[doc(hidden)]
+ pub fn stage_forget_chunk(
+ engine: &Engine,
+ vectors: &PersistentVectorIndex,
+ fts: &PersistentFts,
+ agent: &str,
+ chunk: &[(&str, u64, Option)],
+ ) -> Result {
let mut extra = Batch::new();
let mut vec_ids: Vec = Vec::with_capacity(chunk.len());
for &(id, vec_id, valid_until) in chunk {
@@ -449,8 +889,145 @@ impl PersistentMemoryIndex {
vec_ids.push(vec_id);
}
fts.stage_delete_many(engine, agent, &vec_ids, &mut extra)?;
- vectors.delete_many_with(engine, &vec_ids, &extra)?;
- Ok(chunk.len() as u64)
+ let vector = vectors.stage_delete_many_with(engine, &vec_ids, &extra)?;
+ Ok(StagedForgetChunk {
+ vector,
+ removed: chunk.len() as u64,
+ })
+ }
+
+ /// Plans at most one bounded purge chunk from a stable id cursor. This is
+ /// read-only: it neither stages a batch nor commits. A single item larger
+ /// than the byte target is admitted alone so every cursor can progress.
+ #[doc(hidden)]
+ pub fn plan_purge_chunk(
+ &self,
+ engine: &Engine,
+ vectors: &PersistentVectorIndex,
+ fts: &PersistentFts,
+ agent: &str,
+ cursor: Option,
+ options: ForgetBatchOptions,
+ ) -> Result {
+ let expected_sequence = engine.snapshot().visible_sequence();
+ let after_id = match cursor {
+ Some(cursor) => {
+ if cursor.agent != agent {
+ return Err(EngineError::InvalidOptions {
+ reason: "purge cursor belongs to a different agent".to_owned(),
+ });
+ }
+ if cursor.expected_sequence != expected_sequence {
+ return Err(stale_purge_epoch(cursor.expected_sequence, expected_sequence));
+ }
+ cursor.after_id
+ }
+ None => None,
+ };
+ let max_items = options.max_items.max(1);
+ let page = self.scan_agent_page(engine, agent, after_id.as_deref(), max_items)?;
+ let page_exhausted = page.len() < max_items;
+ let mut estimated_wal_bytes = Self::FORGET_CHUNK_OVERHEAD_BYTES;
+ let mut records = Vec::with_capacity(page.len());
+ let mut stopped_for_bytes = false;
+ for (id, stored) in page {
+ let record_key = memory_index::record_key(agent, &id)?;
+ let mut item_bytes = record_key.as_bytes().len()
+ + memory_index::vecmap_key(stored.vec_id).as_bytes().len()
+ + vectors.approx_tombstone_wire_bytes()
+ + fts.delete_footprint(engine, agent, stored.vec_id)?;
+ if let Some(until) = stored.valid_until {
+ item_bytes = item_bytes.saturating_add(temporal_index::expiry_key(agent, until, &id)?.as_bytes().len());
+ }
+ if !records.is_empty() && estimated_wal_bytes.saturating_add(item_bytes) > options.max_wal_bytes {
+ stopped_for_bytes = true;
+ break;
+ }
+ estimated_wal_bytes = estimated_wal_bytes.saturating_add(item_bytes);
+ records.push(PurgeChunkRecord {
+ id,
+ vec_id: stored.vec_id,
+ valid_until: stored.valid_until,
+ });
+ }
+ let resume_after = records.last().map(|record| record.id.clone()).or(after_id);
+ Ok(PurgeChunkPlan {
+ agent: agent.to_owned(),
+ records,
+ resume_after,
+ exhausted: page_exhausted && !stopped_for_bytes,
+ estimated_wal_bytes,
+ expected_sequence,
+ })
+ }
+
+ /// Revalidates and stages exactly one owned purge plan. No loop or commit
+ /// is hidden here; a stale read-set aborts before any WAL mutation.
+ #[doc(hidden)]
+ pub fn stage_purge_chunk(
+ &self,
+ engine: &Engine,
+ vectors: &PersistentVectorIndex,
+ fts: &PersistentFts,
+ plan: PurgeChunkPlan,
+ ) -> Result {
+ if plan.records.is_empty() {
+ return Err(EngineError::InvalidOptions {
+ reason: "an empty purge plan may only finalize the registry".to_owned(),
+ });
+ }
+ let actual_sequence = engine.snapshot().visible_sequence();
+ if actual_sequence != plan.expected_sequence {
+ return Err(stale_purge_epoch(plan.expected_sequence, actual_sequence));
+ }
+ for expected in &plan.records {
+ match self.get(engine, &plan.agent, &expected.id)? {
+ Some(actual) if actual.vec_id == expected.vec_id && actual.valid_until == expected.valid_until => {}
+ _ => {
+ return Err(EngineError::InvalidOptions {
+ reason: format!("stale purge read-set for memory {:?}", expected.id),
+ });
+ }
+ }
+ }
+ let chunk = plan
+ .records
+ .iter()
+ .map(|record| (record.id.as_str(), record.vec_id, record.valid_until))
+ .collect::>();
+ Ok(StagedPurgeChunk {
+ inner: Self::stage_forget_chunk(engine, vectors, fts, &plan.agent, &chunk)?,
+ agent: plan.agent,
+ resume_after: plan.resume_after,
+ expected_sequence: plan.expected_sequence,
+ })
+ }
+
+ /// Stages the final registry-marker deletion only from an empty,
+ /// exhausted plan whose opaque mutation epoch still matches. The plan is
+ /// consumed, so a non-empty page can never double as finalization proof.
+ #[doc(hidden)]
+ pub fn stage_finalize_purge(&self, engine: &Engine, plan: PurgeChunkPlan) -> Result {
+ if !plan.records.is_empty() || !plan.exhausted {
+ return Err(EngineError::InvalidOptions {
+ reason: "purge finalization requires an empty exhausted plan".to_owned(),
+ });
+ }
+ let actual_sequence = engine.snapshot().visible_sequence();
+ if actual_sequence != plan.expected_sequence {
+ return Err(stale_purge_epoch(plan.expected_sequence, actual_sequence));
+ }
+ if !self.scan_agent_page(engine, &plan.agent, None, 1)?.is_empty() {
+ return Err(EngineError::InvalidOptions {
+ reason: format!("cannot finalize purge for non-empty agent {:?}", plan.agent),
+ });
+ }
+ let mut batch = Batch::new();
+ batch.delete(agent_registry::agent_key(&plan.agent)?.as_bytes());
+ Ok(StagedPurgeFinalize {
+ batch,
+ expected_sequence: plan.expected_sequence,
+ })
}
/// Every agent id present in the registry (ADR-041 §7.5), in ascending
@@ -625,22 +1202,23 @@ impl PersistentMemoryIndex {
agent: &str,
) -> Result {
let mut purged = 0u64;
- for (id, stored) in self.scan_agent(engine, agent)? {
- let record_key = memory_index::record_key(agent, &id)?;
- let mut extra = Batch::new();
- extra.delete(record_key.as_bytes());
- extra.delete(memory_index::vecmap_key(stored.vec_id).as_bytes());
- stage_expiry_delete(&mut extra, agent, &id, stored.valid_until)?;
- fts.stage_delete(engine, agent, stored.vec_id, &mut extra)?;
- vectors.delete_with(engine, stored.vec_id, &extra)?;
- purged += 1;
+ let mut cursor = None;
+ loop {
+ let plan = self.plan_purge_chunk(engine, vectors, fts, agent, cursor, ForgetBatchOptions::default())?;
+ if plan.is_empty() {
+ // Registry entry last (ADR-041 §7.5). The empty plan and its
+ // epoch remain the finalization proof all the way to WAL.
+ legacy_memory_outcome(self.stage_finalize_purge(engine, plan)?.commit(engine), ())?;
+ return Ok(purged);
+ }
+ let staged = self.stage_purge_chunk(engine, vectors, fts, plan)?;
+ if let Some(committed) = legacy_purge_chunk_outcome(staged.commit(engine, vectors))? {
+ purged = purged.saturating_add(committed.removed());
+ cursor = Some(committed.into_cursor());
+ } else {
+ cursor = None;
+ }
}
- // Registry entry last (ADR-041 §7.5): a crash anywhere above leaves
- // the entry in place, and re-running the purge (the documented
- // recovery, ADR-027 §6) removes it — never the reverse order, which
- // could leave un-purged memories invisible to registry consumers.
- engine.delete(agent_registry::agent_key(agent)?.as_bytes())?;
- Ok(purged)
}
}
@@ -678,6 +1256,206 @@ mod tests {
(engine, vectors, memory, PersistentFts::new())
}
+ #[test]
+ fn staged_put_does_not_publish_allocator_or_vector_ram_before_commit_callback() {
+ let dir = tempfile::tempdir().expect("tempdir");
+ let (mut engine, mut vectors, mut memory, fts) = open_all(dir.path());
+ let record = new_record("staged", "episodic");
+ let items = [("m1", record, vec_for(1))];
+
+ let staged = memory
+ .stage_put_many(&engine, &vectors, &fts, "agent-a", &items)
+ .expect("stage put");
+ assert_eq!(memory.next_vec_id(), 0);
+ assert_eq!(vectors.len(), 0);
+ assert!(
+ memory
+ .get(&engine, "agent-a", "m1")
+ .expect("read before commit")
+ .is_none()
+ );
+
+ let outcome = staged.commit(&mut engine, &mut memory, &mut vectors);
+ let allocated = match outcome {
+ MemoryCommitOutcome::Committed { sequence_range, value } => {
+ assert_eq!(sequence_range.first(), 1);
+ assert_eq!(sequence_range.last(), engine.snapshot().visible_sequence());
+ value
+ }
+ other => panic!("expected committed memory put, got {other:?}"),
+ };
+ assert_eq!(allocated, vec![0]);
+ assert_eq!(memory.next_vec_id(), 1);
+ assert_eq!(vectors.len(), 1);
+ assert!(
+ memory
+ .get(&engine, "agent-a", "m1")
+ .expect("read after commit")
+ .is_some()
+ );
+ }
+
+ #[test]
+ fn vector_id_exhaustion_is_rejected_before_wal_or_ram_mutation() {
+ let dir = tempfile::tempdir().expect("tempdir");
+ let (engine, vectors, mut memory, fts) = open_all(dir.path());
+ memory.next_vec_id = u64::MAX;
+ let record = new_record("exhausted", "episodic");
+ let items = [("m-max", record, vec_for(1))];
+ let before = engine.stats().expect("stats before exhausted staging");
+
+ let error = memory
+ .stage_put_many(&engine, &vectors, &fts, "agent-a", &items)
+ .expect_err("an allocator at u64::MAX cannot persist a successor cursor");
+ assert!(matches!(
+ error,
+ EngineError::VectorIdSpaceExhausted {
+ next: u64::MAX,
+ requested: 1
+ }
+ ));
+ let after = engine.stats().expect("stats after exhausted staging");
+ assert_eq!(after.wal_records, before.wal_records);
+ assert_eq!(after.wal_bytes, before.wal_bytes);
+ assert_eq!(memory.next_vec_id(), u64::MAX);
+ assert_eq!(vectors.len(), 0);
+ assert!(
+ memory
+ .get(&engine, "agent-a", "m-max")
+ .expect("rejected record lookup")
+ .is_none()
+ );
+ }
+
+ #[test]
+ fn vector_id_max_minus_one_commits_once_then_exhausts_without_retry_append() {
+ let dir = tempfile::tempdir().expect("tempdir");
+ let (mut engine, mut vectors, mut memory, fts) = open_all(dir.path());
+ memory.next_vec_id = u64::MAX - 1;
+ let first = [("m-last", new_record("last", "episodic"), vec_for(1))];
+ let allocated = memory
+ .put_many(&mut engine, &mut vectors, &fts, "agent-a", &first)
+ .expect("the final representable allocation with a successor commits");
+ assert_eq!(allocated, vec![u64::MAX - 1]);
+ assert_eq!(memory.next_vec_id(), u64::MAX);
+ assert_eq!(vectors.len(), 1);
+
+ let before = engine.stats().expect("stats before exhausted retry");
+ let second = [("m-overflow", new_record("overflow", "episodic"), vec_for(2))];
+ let error = memory
+ .stage_put_many(&engine, &vectors, &fts, "agent-a", &second)
+ .expect_err("the next allocation is exhausted");
+ assert!(matches!(
+ error,
+ EngineError::VectorIdSpaceExhausted { next: u64::MAX, .. }
+ ));
+ let after = engine.stats().expect("stats after exhausted retry");
+ assert_eq!(after.wal_records, before.wal_records);
+ assert_eq!(after.wal_bytes, before.wal_bytes);
+ assert_eq!(memory.next_vec_id(), u64::MAX);
+ assert_eq!(vectors.len(), 1);
+ }
+
+ #[test]
+ fn allocator_healing_refuses_a_maximum_observed_vector_id() {
+ let dir = tempfile::tempdir().expect("tempdir");
+ let mut engine = Engine::open(dir.path()).expect("open engine");
+ engine
+ .put(vector_index::node_key(u64::MAX).as_bytes(), b"opaque")
+ .expect("seed maximum observed vector id");
+
+ let error =
+ PersistentMemoryIndex::open(&engine).expect_err("healing cannot wrap one past the maximum observed id");
+ assert!(matches!(
+ error,
+ EngineError::VectorIdSpaceExhausted {
+ next: u64::MAX,
+ requested: 1
+ }
+ ));
+ }
+
+ #[test]
+ fn staged_put_callback_error_leaves_engine_terminal() {
+ let dir = tempfile::tempdir().expect("tempdir");
+ let (mut engine, mut vectors, mut memory, fts) = open_all(dir.path());
+ let record = new_record("stale", "episodic");
+ let items = [("m1", record, vec_for(1))];
+ let staged = memory
+ .stage_put_many(&engine, &vectors, &fts, "agent-a", &items)
+ .expect("stage put");
+ memory.next_vec_id = 7;
+
+ let error = legacy_memory_outcome(staged.commit(&mut engine, &mut memory, &mut vectors), Vec::new())
+ .expect_err("stale allocator must reject install");
+ assert!(matches!(error, EngineError::InvalidOptions { .. }));
+ assert!(
+ matches!(
+ engine.put(b"after-terminal", b"blocked"),
+ Err(EngineError::WriterReconcileRequired { .. })
+ ),
+ "failed product callback must inherit the engine's terminal obligation"
+ );
+ }
+
+ #[test]
+ fn explicit_forget_chunk_is_one_wal_record_and_matches_legacy_state() {
+ let legacy_dir = tempfile::tempdir().expect("legacy tempdir");
+ let staged_dir = tempfile::tempdir().expect("staged tempdir");
+ let (mut legacy_engine, mut legacy_vectors, mut legacy_memory, legacy_fts) = open_all(legacy_dir.path());
+ let (mut staged_engine, mut staged_vectors, mut staged_memory, staged_fts) = open_all(staged_dir.path());
+ for (engine, vectors, memory, fts) in [
+ (&mut legacy_engine, &mut legacy_vectors, &mut legacy_memory, &legacy_fts),
+ (&mut staged_engine, &mut staged_vectors, &mut staged_memory, &staged_fts),
+ ] {
+ let items = [
+ ("m1", new_record("one", "episodic"), vec_for(1)),
+ ("m2", new_record("two", "episodic"), vec_for(2)),
+ ];
+ memory
+ .put_many(engine, vectors, fts, "agent-a", &items)
+ .expect("seed memories");
+ }
+
+ let legacy_before = legacy_engine.stats().expect("legacy stats").wal_records;
+ let removed = legacy_memory
+ .forget_many(
+ &mut legacy_engine,
+ &mut legacy_vectors,
+ &legacy_fts,
+ "agent-a",
+ &["m1", "m2"],
+ ForgetBatchOptions {
+ max_items: 2,
+ max_wal_bytes: usize::MAX,
+ },
+ )
+ .expect("legacy forget chunk");
+ assert_eq!(removed, 2);
+ assert_eq!(
+ legacy_engine.stats().expect("legacy stats").wal_records - legacy_before,
+ 1
+ );
+
+ let chunk = [("m1", 0, None), ("m2", 1, None)];
+ let staged_before = staged_engine.stats().expect("staged stats").wal_records;
+ let staged =
+ PersistentMemoryIndex::stage_forget_chunk(&staged_engine, &staged_vectors, &staged_fts, "agent-a", &chunk)
+ .expect("stage forget chunk");
+ assert_eq!(
+ legacy_memory_outcome(staged.commit(&mut staged_engine, &mut staged_vectors), 0).expect("commit chunk"),
+ 2
+ );
+ assert_eq!(
+ staged_engine.stats().expect("staged stats").wal_records - staged_before,
+ 1
+ );
+ assert_eq!(
+ legacy_engine.scan_prefix(b"").expect("legacy scan"),
+ staged_engine.scan_prefix(b"").expect("staged scan")
+ );
+ }
+
#[test]
fn put_get_search_resolve_roundtrip() {
let dir = tempfile::tempdir().expect("tempdir");
@@ -981,6 +1759,393 @@ mod tests {
assert_eq!(vectors.search_scored(&engine, &vec_for(3), 1).expect("search").len(), 1);
}
+ #[test]
+ fn purge_planner_commits_one_bounded_chunk_per_wal_record() {
+ let dir = tempfile::tempdir().expect("tempdir");
+ let (mut engine, mut vectors, mut memory, fts) = open_all(dir.path());
+ for (id, seed) in [("m1", 1), ("m2", 2), ("m3", 3)] {
+ memory
+ .put(
+ &mut engine,
+ &mut vectors,
+ &fts,
+ "agent-a",
+ id,
+ &new_record(id, "episodic"),
+ vec_for(seed),
+ )
+ .expect("seed memory");
+ }
+
+ let plan = memory
+ .plan_purge_chunk(
+ &engine,
+ &vectors,
+ &fts,
+ "agent-a",
+ None,
+ ForgetBatchOptions {
+ max_items: 2,
+ max_wal_bytes: usize::MAX,
+ },
+ )
+ .expect("plan first bounded chunk");
+ assert_eq!(plan.len(), 2);
+ assert_eq!(plan.resume_after(), Some("m2"));
+ assert!(!plan.exhausted(), "a full page conservatively requires another plan");
+ assert_eq!(
+ memory.scan_agent(&engine, "agent-a").expect("staging is inert").len(),
+ 3
+ );
+
+ let before = engine.stats().expect("stats before chunk").wal_records;
+ let staged = memory
+ .stage_purge_chunk(&engine, &vectors, &fts, plan)
+ .expect("stage exact purge read-set");
+ let committed = legacy_purge_chunk_outcome(staged.commit(&mut engine, &mut vectors))
+ .expect("commit chunk")
+ .expect("chunk is committed");
+ assert_eq!(committed.removed(), 2);
+ assert_eq!(engine.stats().expect("stats after chunk").wal_records - before, 1);
+ assert_eq!(memory.scan_agent(&engine, "agent-a").expect("one remains").len(), 1);
+
+ let tiny = memory
+ .plan_purge_chunk(
+ &engine,
+ &vectors,
+ &fts,
+ "agent-a",
+ None,
+ ForgetBatchOptions {
+ max_items: 256,
+ max_wal_bytes: 0,
+ },
+ )
+ .expect("oversized item progresses alone");
+ assert_eq!(
+ tiny.len(),
+ 1,
+ "byte target never creates an unbounded batch or stalls progress"
+ );
+ }
+
+ #[test]
+ fn purge_registry_marker_is_stageable_only_after_agent_is_empty() {
+ let dir = tempfile::tempdir().expect("tempdir");
+ let (mut engine, mut vectors, mut memory, fts) = open_all(dir.path());
+ memory
+ .put(
+ &mut engine,
+ &mut vectors,
+ &fts,
+ "agent-a",
+ "m1",
+ &new_record("one", "episodic"),
+ vec_for(1),
+ )
+ .expect("seed memory");
+ let non_empty = memory
+ .plan_purge_chunk(&engine, &vectors, &fts, "agent-a", None, ForgetBatchOptions::default())
+ .expect("plan non-empty agent");
+ assert!(matches!(
+ memory.stage_finalize_purge(&engine, non_empty),
+ Err(EngineError::InvalidOptions { .. })
+ ));
+ assert_eq!(memory.list_agents(&engine).expect("marker retained"), vec!["agent-a"]);
+
+ let plan = memory
+ .plan_purge_chunk(&engine, &vectors, &fts, "agent-a", None, ForgetBatchOptions::default())
+ .expect("plan only chunk");
+ let staged = memory
+ .stage_purge_chunk(&engine, &vectors, &fts, plan)
+ .expect("stage only chunk");
+ let committed = legacy_purge_chunk_outcome(staged.commit(&mut engine, &mut vectors))
+ .expect("commit only chunk")
+ .expect("chunk committed");
+ assert!(memory.scan_agent(&engine, "agent-a").expect("empty").is_empty());
+ assert_eq!(
+ memory.list_agents(&engine).expect("marker still present"),
+ vec!["agent-a"]
+ );
+
+ let empty = memory
+ .plan_purge_chunk(
+ &engine,
+ &vectors,
+ &fts,
+ "agent-a",
+ Some(committed.into_cursor()),
+ ForgetBatchOptions::default(),
+ )
+ .expect("plan terminal empty page");
+ let final_stage = memory
+ .stage_finalize_purge(&engine, empty)
+ .expect("empty agent may finalize marker");
+ assert_eq!(
+ memory.list_agents(&engine).expect("final staging inert"),
+ vec!["agent-a"]
+ );
+ let before = engine.stats().expect("stats before marker commit").wal_records;
+ legacy_memory_outcome(final_stage.commit(&mut engine), ()).expect("commit marker deletion");
+ assert_eq!(
+ engine.stats().expect("stats after marker commit").wal_records - before,
+ 1
+ );
+ assert!(memory.list_agents(&engine).expect("marker removed").is_empty());
+ }
+
+ #[test]
+ fn purge_chunk_revalidates_its_exact_read_set_before_wal() {
+ let dir = tempfile::tempdir().expect("tempdir");
+ let (mut engine, mut vectors, mut memory, fts) = open_all(dir.path());
+ memory
+ .put(
+ &mut engine,
+ &mut vectors,
+ &fts,
+ "agent-a",
+ "m1",
+ &record_until("one", "episodic", Some(10)),
+ vec_for(1),
+ )
+ .expect("seed memory");
+ let plan = memory
+ .plan_purge_chunk(&engine, &vectors, &fts, "agent-a", None, ForgetBatchOptions::default())
+ .expect("plan purge");
+
+ let mut changed = memory.get(&engine, "agent-a", "m1").expect("read").expect("present");
+ changed.valid_until = Some(20);
+ memory
+ .update(&mut engine, "agent-a", "m1", &changed)
+ .expect("concurrent logical change");
+ let before = engine.stats().expect("stats before stale stage").wal_records;
+ assert!(matches!(
+ memory.stage_purge_chunk(&engine, &vectors, &fts, plan),
+ Err(EngineError::InvalidOptions { .. })
+ ));
+ assert_eq!(engine.stats().expect("stats after stale stage").wal_records, before);
+ assert!(memory.get(&engine, "agent-a", "m1").expect("retained").is_some());
+ }
+
+ #[test]
+ fn purge_cursor_rejects_an_insertion_behind_the_committed_scan() {
+ let dir = tempfile::tempdir().expect("tempdir");
+ let (mut engine, mut vectors, mut memory, fts) = open_all(dir.path());
+ for (id, seed) in [("m2", 2), ("m3", 3)] {
+ memory
+ .put(
+ &mut engine,
+ &mut vectors,
+ &fts,
+ "agent-a",
+ id,
+ &new_record(id, "episodic"),
+ vec_for(seed),
+ )
+ .expect("seed memory");
+ }
+ let first = memory
+ .plan_purge_chunk(
+ &engine,
+ &vectors,
+ &fts,
+ "agent-a",
+ None,
+ ForgetBatchOptions {
+ max_items: 1,
+ max_wal_bytes: usize::MAX,
+ },
+ )
+ .expect("plan first chunk");
+ let committed = legacy_purge_chunk_outcome(
+ memory
+ .stage_purge_chunk(&engine, &vectors, &fts, first)
+ .expect("stage first chunk")
+ .commit(&mut engine, &mut vectors),
+ )
+ .expect("commit first chunk")
+ .expect("first chunk committed");
+
+ memory
+ .put(
+ &mut engine,
+ &mut vectors,
+ &fts,
+ "agent-a",
+ "m1",
+ &new_record("late behind cursor", "episodic"),
+ vec_for(1),
+ )
+ .expect("concurrent insertion");
+ let before = engine.stats().expect("stats before rejected resume").wal_records;
+ assert!(matches!(
+ memory.plan_purge_chunk(
+ &engine,
+ &vectors,
+ &fts,
+ "agent-a",
+ Some(committed.into_cursor()),
+ ForgetBatchOptions::default(),
+ ),
+ Err(EngineError::InvalidOptions { .. })
+ ));
+ assert_eq!(engine.stats().expect("stats after rejected resume").wal_records, before);
+ assert!(
+ memory
+ .get(&engine, "agent-a", "m1")
+ .expect("late member retained")
+ .is_some()
+ );
+ }
+
+ #[test]
+ fn purge_epoch_is_rechecked_after_staging_and_before_wal() {
+ let dir = tempfile::tempdir().expect("tempdir");
+ let (mut engine, mut vectors, mut memory, fts) = open_all(dir.path());
+ memory
+ .put(
+ &mut engine,
+ &mut vectors,
+ &fts,
+ "agent-a",
+ "m2",
+ &new_record("planned", "episodic"),
+ vec_for(2),
+ )
+ .expect("seed memory");
+ let plan = memory
+ .plan_purge_chunk(&engine, &vectors, &fts, "agent-a", None, ForgetBatchOptions::default())
+ .expect("plan purge");
+ let staged = memory
+ .stage_purge_chunk(&engine, &vectors, &fts, plan)
+ .expect("stage purge");
+ memory
+ .put(
+ &mut engine,
+ &mut vectors,
+ &fts,
+ "agent-a",
+ "m1",
+ &new_record("phantom", "episodic"),
+ vec_for(1),
+ )
+ .expect("insert after staging");
+ let before = engine.stats().expect("stats before stale commit").wal_records;
+ assert!(matches!(
+ staged.commit(&mut engine, &mut vectors),
+ MemoryCommitOutcome::Aborted {
+ cause: Some(EngineError::InvalidOptions { .. }),
+ ..
+ }
+ ));
+ assert_eq!(engine.stats().expect("stats after stale commit").wal_records, before);
+ assert_eq!(memory.scan_agent(&engine, "agent-a").expect("both retained").len(), 2);
+ }
+
+ #[test]
+ fn purge_finalization_epoch_rejects_a_late_insert_without_hiding_registry() {
+ let dir = tempfile::tempdir().expect("tempdir");
+ let (mut engine, mut vectors, mut memory, fts) = open_all(dir.path());
+ memory
+ .put(
+ &mut engine,
+ &mut vectors,
+ &fts,
+ "agent-a",
+ "m1",
+ &new_record("first", "episodic"),
+ vec_for(1),
+ )
+ .expect("seed marker");
+ memory
+ .forget(&mut engine, &mut vectors, &fts, "agent-a", "m1")
+ .expect("make agent empty while retaining marker");
+ let empty = memory
+ .plan_purge_chunk(&engine, &vectors, &fts, "agent-a", None, ForgetBatchOptions::default())
+ .expect("empty terminal plan");
+ let staged = memory
+ .stage_finalize_purge(&engine, empty)
+ .expect("stage marker deletion");
+ memory
+ .put(
+ &mut engine,
+ &mut vectors,
+ &fts,
+ "agent-a",
+ "late",
+ &new_record("late", "episodic"),
+ vec_for(2),
+ )
+ .expect("late insert");
+ let before = engine.stats().expect("stats before stale finalize").wal_records;
+ assert!(matches!(
+ staged.commit(&mut engine),
+ MemoryCommitOutcome::Aborted {
+ cause: Some(EngineError::InvalidOptions { .. }),
+ ..
+ }
+ ));
+ assert_eq!(engine.stats().expect("stats after stale finalize").wal_records, before);
+ assert_eq!(memory.list_agents(&engine).expect("registry retained"), vec!["agent-a"]);
+ }
+
+ #[test]
+ fn purge_restarts_from_a_fresh_plan_after_reopen() {
+ let dir = tempfile::tempdir().expect("tempdir");
+ {
+ let (mut engine, mut vectors, mut memory, fts) = open_all(dir.path());
+ for (id, seed) in [("m1", 1), ("m2", 2), ("m3", 3)] {
+ memory
+ .put(
+ &mut engine,
+ &mut vectors,
+ &fts,
+ "agent-a",
+ id,
+ &new_record(id, "episodic"),
+ vec_for(seed),
+ )
+ .expect("seed memory");
+ }
+ let first = memory
+ .plan_purge_chunk(
+ &engine,
+ &vectors,
+ &fts,
+ "agent-a",
+ None,
+ ForgetBatchOptions {
+ max_items: 1,
+ max_wal_bytes: usize::MAX,
+ },
+ )
+ .expect("plan one chunk");
+ assert!(matches!(
+ memory
+ .stage_purge_chunk(&engine, &vectors, &fts, first)
+ .expect("stage one chunk")
+ .commit(&mut engine, &mut vectors),
+ MemoryCommitOutcome::Committed { .. }
+ ));
+ engine.close().expect("close interrupted purge");
+ }
+
+ let (mut engine, mut vectors, memory, fts) = open_all(dir.path());
+ assert_eq!(
+ memory
+ .purge_agent(&mut engine, &mut vectors, &fts, "agent-a")
+ .expect("fresh purge resumes"),
+ 2
+ );
+ assert!(
+ memory
+ .scan_agent(&engine, "agent-a")
+ .expect("purged after reopen")
+ .is_empty()
+ );
+ assert!(memory.list_agents(&engine).expect("registry finalized").is_empty());
+ }
+
#[test]
fn allocator_is_monotonic_across_reopen_and_forgets() {
let dir = tempfile::tempdir().expect("tempdir");
diff --git a/crates/basemyai-engine/src/idx/vector/mod.rs b/crates/basemyai-engine/src/idx/vector/mod.rs
index 50d07a1..79f8069 100644
--- a/crates/basemyai-engine/src/idx/vector/mod.rs
+++ b/crates/basemyai-engine/src/idx/vector/mod.rs
@@ -39,5 +39,8 @@ pub mod trace;
pub use graph::VectorIndex;
pub use meta::{EncodingMigrationState, VectorIndexMeta, VectorIndexParams, VectorMetric};
pub use node::{NodeEncoding, VectorNode};
-pub use persistent::{PersistentVectorIndex, VectorCacheStats};
+pub use persistent::{
+ PersistentVectorIndex, StagedVectorDelete, StagedVectorInsert, VectorCacheStats, VectorDeleteInstallToken,
+ VectorInsertInstallToken,
+};
pub use trace::VectorInsertTrace;
diff --git a/crates/basemyai-engine/src/idx/vector/persistent.rs b/crates/basemyai-engine/src/idx/vector/persistent.rs
index b400346..2c3aa44 100644
--- a/crates/basemyai-engine/src/idx/vector/persistent.rs
+++ b/crates/basemyai-engine/src/idx/vector/persistent.rs
@@ -89,7 +89,7 @@ use super::node::{self, NodeEncoding, VectorNode};
use super::trace::VectorInsertTrace;
use crate::error::{EngineError, Result};
use crate::key::vector_index::{META_KEY, NODE_PREFIX, node_id, node_key};
-use crate::store::{Batch, Engine, ReadSnapshot};
+use crate::store::{Batch, Engine, EngineCommitOutcome, ReadSnapshot, WalCommitPhase};
/// Default byte budget for the decoded-node read-through cache (ADR-060
/// groundwork). Replaces the old fixed-entry-count `CACHE_CAP: usize = 4096`
@@ -420,6 +420,70 @@ struct ReadSnapshotProvider<'a> {
snapshot: &'a ReadSnapshot,
}
+/// Prepared vector insertion that has not crossed the engine durability
+/// boundary. Deliberately non-clonable and non-constructible by callers.
+#[doc(hidden)]
+#[derive(Debug)]
+pub struct StagedVectorInsert {
+ batch: Batch,
+ install: VectorInsertInstallToken,
+}
+
+impl StagedVectorInsert {
+ /// Commits the exact staged batch while carrying the private RAM plan
+ /// inside the engine-bound receipt. Only a durable receipt can release
+ /// the install token through `Engine::install_committed_batch_with`.
+ #[doc(hidden)]
+ pub fn commit(self, engine: &mut Engine) -> EngineCommitOutcome {
+ engine.commit_batch_with(self.batch, self.install)
+ }
+}
+
+/// Prepared vector deletion; see [`StagedVectorInsert`].
+#[doc(hidden)]
+#[derive(Debug)]
+pub struct StagedVectorDelete {
+ batch: Batch,
+ install: VectorDeleteInstallToken,
+}
+
+impl StagedVectorDelete {
+ #[doc(hidden)]
+ pub fn commit(self, engine: &mut Engine) -> EngineCommitOutcome {
+ engine.commit_batch_with(self.batch, self.install)
+ }
+}
+
+#[derive(Debug, Clone, Copy, PartialEq)]
+struct VectorRamPrecondition {
+ params: VectorIndexParams,
+ entry_point: Option,
+ epoch: u64,
+ count: u64,
+ encoding: NodeEncoding,
+ policy_persisted: bool,
+}
+
+/// Opaque proof payload released only after the engine installs the durable
+/// batch bound to its receipt. It cannot be forged or cloned.
+#[doc(hidden)]
+#[derive(Debug)]
+pub struct VectorInsertInstallToken {
+ expected: VectorRamPrecondition,
+ added: u64,
+ entry_point: Option,
+ changed: HashMap,
+}
+
+/// Opaque deletion counterpart of [`VectorInsertInstallToken`].
+#[doc(hidden)]
+#[derive(Debug)]
+pub struct VectorDeleteInstallToken {
+ expected: VectorRamPrecondition,
+ removed: u64,
+ changed: Vec<(u64, VectorNode)>,
+}
+
impl NodeProvider for ReadSnapshotProvider<'_> {
fn node(&mut self, id: u64) -> Result> {
let Some(bytes) = self.snapshot.get(node_key(id).as_bytes())? else {
@@ -648,6 +712,27 @@ impl PersistentVectorIndex {
self.epoch
}
+ fn ram_precondition(&self) -> VectorRamPrecondition {
+ VectorRamPrecondition {
+ params: self.params,
+ entry_point: self.entry_point,
+ epoch: self.epoch,
+ count: self.count,
+ encoding: self.encoding,
+ policy_persisted: self.policy_persisted,
+ }
+ }
+
+ fn validate_install_precondition(&self, expected: VectorRamPrecondition) -> Result<()> {
+ if self.ram_precondition() == expected {
+ Ok(())
+ } else {
+ Err(EngineError::InvalidOptions {
+ reason: "staged vector installation no longer matches the index RAM state".to_owned(),
+ })
+ }
+ }
+
/// Inserts `vector` under `id`, durably: the new node block, every
/// re-pruned neighbor block, and the refreshed metadata record travel in
/// **one** [`Engine::apply_batch`] — after a crash the whole insert is
@@ -801,11 +886,49 @@ impl PersistentVectorIndex {
/// holds on the error path too. An empty `items` applies only `extra`
/// (itself a no-op when empty).
pub fn insert_many_with(&mut self, engine: &mut Engine, items: Vec<(u64, Vec)>, extra: &Batch) -> Result<()> {
- if items.is_empty() {
- if !extra.is_empty() {
- engine.apply_batch(extra)?;
+ let staged = self.stage_insert_many_with(engine, items, extra)?;
+ match staged.commit(engine) {
+ EngineCommitOutcome::Aborted { cause: None, .. } => Ok(()),
+ EngineCommitOutcome::Aborted { cause: Some(cause), .. } => {
+ self.cache.clear();
+ Err(cause)
+ }
+ EngineCommitOutcome::Durable(receipt) => {
+ engine.install_committed_batch_with(receipt, |install| self.install_staged_insert_many(install))
+ }
+ EngineCommitOutcome::OutcomeUnknown { phase, cause } => {
+ self.cache.clear();
+ Err(match phase {
+ WalCommitPhase::Append => EngineError::WalAppendOutcomeUnknown {
+ cause: cause.to_string(),
+ },
+ WalCommitPhase::Sync => EngineError::WalSyncOutcomeUnknown {
+ cause: cause.to_string(),
+ },
+ })
}
- return Ok(());
+ }
+ }
+
+ /// Stages one atomic multi-insert without publishing planned RAM state.
+ #[doc(hidden)]
+ pub fn stage_insert_many_with(
+ &self,
+ engine: &Engine,
+ items: Vec<(u64, Vec)>,
+ extra: &Batch,
+ ) -> Result {
+ let expected = self.ram_precondition();
+ if items.is_empty() {
+ return Ok(StagedVectorInsert {
+ batch: extra.clone(),
+ install: VectorInsertInstallToken {
+ expected,
+ added: 0,
+ entry_point: self.entry_point,
+ changed: HashMap::new(),
+ },
+ });
}
let added = items.len() as u64;
let mut pending: HashMap = HashMap::new();
@@ -837,12 +960,29 @@ impl PersistentVectorIndex {
};
batch.put(META_KEY, &meta::encode(&new_meta)?);
batch.extend_from(extra);
- engine.apply_batch(&batch)?;
- self.count += added;
- self.entry_point = entry_point;
- self.policy_persisted = true;
- for (node_id, node) in pending {
+ Ok(StagedVectorInsert {
+ batch,
+ install: VectorInsertInstallToken {
+ expected,
+ added,
+ entry_point,
+ changed: pending,
+ },
+ })
+ }
+
+ /// Installs vector RAM only from the opaque payload released after the
+ /// engine has validated and installed its durable receipt.
+ #[doc(hidden)]
+ pub fn install_staged_insert_many(&mut self, install: VectorInsertInstallToken) -> Result<()> {
+ self.validate_install_precondition(install.expected)?;
+ self.count += install.added;
+ self.entry_point = install.entry_point;
+ if install.added > 0 {
+ self.policy_persisted = true;
+ }
+ for (node_id, node) in install.changed {
self.cache.put(node_id, node);
}
Ok(())
@@ -930,6 +1070,39 @@ impl PersistentVectorIndex {
/// the caller's companion deletes must not survive a no-op tombstone
/// pass. Returns how many ids *this call* tombstoned.
pub fn delete_many_with(&mut self, engine: &mut Engine, ids: &[u64], extra: &Batch) -> Result {
+ let staged = self.stage_delete_many_with(engine, ids, extra)?;
+ match staged.commit(engine) {
+ EngineCommitOutcome::Aborted { cause: None, .. } => Ok(0),
+ EngineCommitOutcome::Aborted { cause: Some(cause), .. } => {
+ self.cache.clear();
+ Err(cause)
+ }
+ EngineCommitOutcome::Durable(receipt) => {
+ let mut removed = None;
+ engine.install_committed_batch_with(receipt, |install| {
+ removed = Some(self.install_staged_delete_many(install)?);
+ Ok(())
+ })?;
+ Ok(removed.expect("successful vector delete installation records its count"))
+ }
+ EngineCommitOutcome::OutcomeUnknown { phase, cause } => {
+ self.cache.clear();
+ Err(match phase {
+ WalCommitPhase::Append => EngineError::WalAppendOutcomeUnknown {
+ cause: cause.to_string(),
+ },
+ WalCommitPhase::Sync => EngineError::WalSyncOutcomeUnknown {
+ cause: cause.to_string(),
+ },
+ })
+ }
+ }
+ }
+
+ /// Stages one atomic multi-delete without publishing tombstones/count.
+ #[doc(hidden)]
+ pub fn stage_delete_many_with(&self, engine: &Engine, ids: &[u64], extra: &Batch) -> Result {
+ let expected = self.ram_precondition();
let mut pending: Vec<(u64, VectorNode)> = Vec::new();
let mut seen: HashSet = HashSet::with_capacity(ids.len());
for &id in ids {
@@ -950,10 +1123,14 @@ impl PersistentVectorIndex {
}
}
if pending.is_empty() {
- if !extra.is_empty() {
- engine.apply_batch(extra)?;
- }
- return Ok(0);
+ return Ok(StagedVectorDelete {
+ batch: extra.clone(),
+ install: VectorDeleteInstallToken {
+ expected,
+ removed: 0,
+ changed: Vec::new(),
+ },
+ });
}
let removed = pending.len() as u64;
@@ -974,14 +1151,30 @@ impl PersistentVectorIndex {
};
batch.put(META_KEY, &meta::encode(&new_meta)?);
batch.extend_from(extra);
- engine.apply_batch(&batch)?;
- self.count = self.count.saturating_sub(removed);
- self.policy_persisted = true;
- for (id, tombstoned) in pending {
+ Ok(StagedVectorDelete {
+ batch,
+ install: VectorDeleteInstallToken {
+ expected,
+ removed,
+ changed: pending,
+ },
+ })
+ }
+
+ /// Installs delete RAM only after the engine has released the durable
+ /// receipt's opaque payload.
+ #[doc(hidden)]
+ pub fn install_staged_delete_many(&mut self, install: VectorDeleteInstallToken) -> Result {
+ self.validate_install_precondition(install.expected)?;
+ self.count = self.count.saturating_sub(install.removed);
+ if install.removed > 0 {
+ self.policy_persisted = true;
+ }
+ for (id, tombstoned) in install.changed {
self.cache.put(id, tombstoned);
}
- Ok(removed)
+ Ok(install.removed)
}
/// Approximate wire bytes one tombstone rewrite stages (node key + the
@@ -1365,6 +1558,107 @@ fn nearest_live_in_snapshot(nodes: &HashMap, reference: Option<
None => live.map(|(&id, _)| id).min(),
}
}
+#[cfg(test)]
+mod staging_tests {
+ use super::*;
+ use tempfile::tempdir;
+
+ fn params() -> VectorIndexParams {
+ VectorIndexParams::with_dim(2)
+ }
+
+ fn items() -> Vec<(u64, Vec)> {
+ vec![(1, vec![1.0, 0.0]), (2, vec![0.0, 1.0]), (3, vec![0.7, 0.7])]
+ }
+
+ fn install_durable_payload(
+ engine: &mut Engine,
+ outcome: EngineCommitOutcome,
+ install: impl FnOnce(T) -> Result<()>,
+ ) {
+ match outcome {
+ EngineCommitOutcome::Durable(receipt) => {
+ engine
+ .install_committed_batch_with(receipt, install)
+ .expect("install engine receipt and product RAM");
+ }
+ other => panic!("expected durable outcome, got {other:?}"),
+ }
+ }
+
+ #[test]
+ fn staging_does_not_publish_ram_before_durable_receipt_install() {
+ let dir = tempdir().expect("tempdir");
+ let mut engine = Engine::open(dir.path()).expect("open engine");
+ let mut index = PersistentVectorIndex::open(&mut engine, params()).expect("open index");
+
+ let staged = index
+ .stage_insert_many_with(&engine, items(), &Batch::new())
+ .expect("stage insert");
+ assert_eq!(index.len(), 0);
+ assert_eq!(index.entry_point, None);
+ assert_eq!(index.node_cache_stats().resident_bytes, 0);
+ assert!(
+ engine
+ .get(node_key(1).as_bytes())
+ .expect("read before commit")
+ .is_none()
+ );
+
+ let outcome = staged.commit(&mut engine);
+ assert_eq!(index.len(), 0, "WAL durability must not publish vector RAM");
+ assert_eq!(index.node_cache_stats().resident_bytes, 0);
+ install_durable_payload(&mut engine, outcome, |install| {
+ index.install_staged_insert_many(install)
+ });
+ assert_eq!(index.len(), 3);
+ assert!(index.entry_point.is_some());
+ assert!(index.node_cache_stats().resident_bytes > 0);
+ }
+
+ #[test]
+ fn legacy_wrappers_match_explicit_stage_commit_install() {
+ let legacy_dir = tempdir().expect("legacy tempdir");
+ let staged_dir = tempdir().expect("staged tempdir");
+ let mut legacy_engine = Engine::open(legacy_dir.path()).expect("open legacy engine");
+ let mut staged_engine = Engine::open(staged_dir.path()).expect("open staged engine");
+ let mut legacy = PersistentVectorIndex::open(&mut legacy_engine, params()).expect("open legacy index");
+ let mut staged = PersistentVectorIndex::open(&mut staged_engine, params()).expect("open staged index");
+
+ legacy
+ .insert_many_with(&mut legacy_engine, items(), &Batch::new())
+ .expect("legacy insert");
+ let prepared = staged
+ .stage_insert_many_with(&staged_engine, items(), &Batch::new())
+ .expect("stage insert");
+ let outcome = prepared.commit(&mut staged_engine);
+ install_durable_payload(&mut staged_engine, outcome, |token| {
+ staged.install_staged_insert_many(token)
+ });
+
+ let legacy_removed = legacy
+ .delete_many_with(&mut legacy_engine, &[1, 3, 3, 99], &Batch::new())
+ .expect("legacy delete");
+ let prepared = staged
+ .stage_delete_many_with(&staged_engine, &[1, 3, 3, 99], &Batch::new())
+ .expect("stage delete");
+ let outcome = prepared.commit(&mut staged_engine);
+ let mut staged_removed = None;
+ install_durable_payload(&mut staged_engine, outcome, |token| {
+ staged_removed = Some(staged.install_staged_delete_many(token)?);
+ Ok(())
+ });
+ let staged_removed = staged_removed.expect("successful staged delete records its count");
+
+ assert_eq!(legacy_removed, staged_removed);
+ assert_eq!(legacy.len(), staged.len());
+ assert_eq!(
+ legacy_engine.scan_prefix(b"").expect("scan legacy"),
+ staged_engine.scan_prefix(b"").expect("scan staged")
+ );
+ }
+}
+
#[cfg(test)]
mod cache_poison_tests {
use std::panic::{AssertUnwindSafe, catch_unwind};
diff --git a/crates/basemyai-engine/src/lib.rs b/crates/basemyai-engine/src/lib.rs
index 4e2f838..709672e 100644
--- a/crates/basemyai-engine/src/lib.rs
+++ b/crates/basemyai-engine/src/lib.rs
@@ -110,8 +110,10 @@ pub use idx::memory::{ForgetBatchOptions, MemoryRecord, NewMemoryRecord, Persist
pub use idx::vector::{PersistentVectorIndex, VectorCacheStats, VectorIndex, VectorIndexParams, VectorInsertTrace};
pub use key::Key;
pub use store::{
- Batch, CompactionJob, DEFAULT_BLOCK_SIZE, DEFAULT_METADATA_CACHE_CAPACITY_BYTES, Engine, EngineOptions, EngineRead,
- EngineStats, IntegrityIssue, InternalKey, IssueKind, LatencySummary, ReadSnapshot, RebuildReport, RepairAction,
- RepairPlan, ScanPage, SequenceNumber, Snapshot, Value, ValueKind, VerifyMode, VerifyReport, plan_repair,
- rebuild_indexes, verify_store, verify_store_with_passphrase,
+ Batch, CommitReceipt, CompactionJob, DEFAULT_BLOCK_SIZE, DEFAULT_METADATA_CACHE_CAPACITY_BYTES, Engine,
+ EngineCommitOutcome, EngineOptions, EngineRead, EngineStats, EngineTerminalHandle, IntegrityIssue, InternalKey,
+ IssueKind, LatencySummary, ReadSnapshot, RebuildReport, RepairAction, RepairPlan, ScanPage, SequenceNumber,
+ SequenceRange, Snapshot, StructuralCandidateOwnership, StructuralCommitOutcome, StructuralCommitPhase,
+ StructuralReceipt, Value, ValueKind, VerifyMode, VerifyReport, WalCommitPhase, plan_repair, rebuild_indexes,
+ verify_store, verify_store_with_passphrase,
};
diff --git a/crates/basemyai-engine/src/store/durable.rs b/crates/basemyai-engine/src/store/durable.rs
index f88bbd2..7c1fe76 100644
--- a/crates/basemyai-engine/src/store/durable.rs
+++ b/crates/basemyai-engine/src/store/durable.rs
@@ -53,6 +53,12 @@ use crate::error::EngineError;
/// state can diverge there.
pub(crate) trait DurablePublicationTracker {
fn pointer_replaced(&self);
+
+ /// Marks entry into the directory-durability phase after the pointer has
+ /// already been replaced. Most publishers only need the panic-aware
+ /// `pointer_replaced` transition; live structural rotations additionally
+ /// observe this boundary to report ADR-070's closed structural phase.
+ fn directory_sync_started(&self) {}
}
impl DurablePublicationTracker for () {
diff --git a/crates/basemyai-engine/src/store/engine/background.rs b/crates/basemyai-engine/src/store/engine/background.rs
index 4e5ad01..d76daca 100644
--- a/crates/basemyai-engine/src/store/engine/background.rs
+++ b/crates/basemyai-engine/src/store/engine/background.rs
@@ -83,6 +83,67 @@ pub(super) struct DurablePublicationGuard<'a> {
pointer_replaced: Cell,
}
+/// One-way capability used by the logical writer owner to make this exact
+/// engine instance terminal (ADR-070 §6).
+///
+/// This is deliberately workspace-private and non-forgeable: its constructor
+/// is private to this module and [`Engine`](super::Engine) creates the sole
+/// capability from the same publication lock and sticky health already used
+/// by every [`DurablePublicationGuard`]. It therefore introduces neither a
+/// second lock nor a second terminal verdict.
+///
+/// Terminalisation linearises while holding `catalog_commit_lock`, then wakes
+/// workers only after releasing it. A publisher that already owns a guard may
+/// finish before that point; every later guard acquisition observes terminal
+/// health under the same lock and is refused before `atomic_replace`.
+#[doc(hidden)]
+pub struct EngineTerminalHandle {
+ catalog_commit_lock: Arc>,
+ background: Arc,
+}
+
+impl EngineTerminalHandle {
+ fn new(catalog_commit_lock: Arc>, background: Arc) -> Self {
+ Self {
+ catalog_commit_lock,
+ background,
+ }
+ }
+
+ /// Escalates product/runtime failure to the engine's existing terminal
+ /// health. This method must be called outside a `DurablePublicationGuard`:
+ /// both intentionally serialize on the same non-reentrant mutex.
+ #[doc(hidden)]
+ pub fn enter_reconcile_required(&self, cause: impl Into) -> EngineError {
+ enter_reconcile_required(&self.catalog_commit_lock, &self.background, cause)
+ }
+}
+
+/// Shared one-way terminal transition used by both the externally transferred
+/// logical-writer handle and a durable receipt's private install obligation.
+/// Keeping the lock acquisition here guarantees that neither path can race a
+/// catalogue replacement admitted under [`DurablePublicationGuard`].
+pub(super) fn enter_reconcile_required(
+ catalog_commit_lock: &Mutex<()>,
+ background: &BackgroundError,
+ cause: impl Into,
+) -> EngineError {
+ let cause = cause.into();
+ match catalog_commit_lock.lock() {
+ Ok(_guard) => background.record_reconcile_required_without_wake(cause.clone()),
+ Err(poisoned) => {
+ // Do not recover or clear publication-lock poison. The
+ // PoisonError still owns the mutex guard, so terminal health can
+ // be armed before it is dropped, while every publisher remains
+ // excluded.
+ background.record_reconcile_required_without_wake(cause.clone());
+ drop(poisoned);
+ }
+ }
+ background.wake_waiters();
+ EngineError::WriterReconcileRequired { cause }
+}
+
impl<'a> DurablePublicationGuard<'a> {
pub(super) fn acquire(
lock: &'a Mutex<()>,
@@ -164,6 +225,10 @@ impl BackgroundError {
Arc::new(Self::default())
}
+ pub(super) fn terminal_handle(self: &Arc, catalog_commit_lock: Arc>) -> EngineTerminalHandle {
+ EngineTerminalHandle::new(catalog_commit_lock, Arc::clone(self))
+ }
+
/// Subscribes a worker to failure wake-ups. Held weakly on purpose (see
/// the module doc): a dropped worker simply stops being notified.
pub(super) fn register(&self, waker: Weak) {
@@ -178,12 +243,20 @@ impl BackgroundError {
///
/// The caller must **not** hold any worker's state lock (module doc).
pub(super) fn record(&self, cause: String) {
+ self.record_cause(cause);
+ self.wake_waiters();
+ }
+
+ fn record_cause(&self, cause: String) {
{
let mut slot = self.cause.lock().unwrap_or_else(std::sync::PoisonError::into_inner);
if slot.is_none() {
*slot = Some(cause);
}
}
+ }
+
+ fn wake_waiters(&self) {
// Cause lock released before waking: `wake_on_background_error` takes
// worker state locks, and a worker checks `cause()` under its own
// state lock. Holding both here, in the opposite order, would deadlock.
@@ -212,8 +285,13 @@ impl BackgroundError {
/// Idempotent, and never downgrades: once terminal, always terminal, even
/// if an ordinary failure is recorded afterwards.
pub(super) fn record_reconcile_required(&self, cause: String) {
+ self.record_reconcile_required_without_wake(cause);
+ self.wake_waiters();
+ }
+
+ fn record_reconcile_required_without_wake(&self, cause: String) {
self.reconcile_required.store(true, Ordering::SeqCst);
- self.record(cause);
+ self.record_cause(cause);
}
/// Records `cause` as an ambiguous durable publication and returns the
@@ -295,3 +373,91 @@ impl BackgroundError {
}
}
}
+
+#[cfg(test)]
+mod tests {
+ use std::sync::atomic::{AtomicBool, Ordering};
+ use std::sync::{Arc, Barrier, Mutex};
+
+ use super::{BackgroundError, DurablePublicationGuard};
+ use crate::error::EngineError;
+
+ #[test]
+ fn terminal_handle_linearises_after_an_admitted_publisher_and_refuses_the_next() {
+ let lock = Arc::new(Mutex::new(()));
+ let background = BackgroundError::new();
+ let terminal = background.terminal_handle(Arc::clone(&lock));
+ let publisher_admitted = Arc::new(Barrier::new(2));
+ let release_publisher = Arc::new(Barrier::new(2));
+ let replace_happened = Arc::new(AtomicBool::new(false));
+
+ let publisher = {
+ let lock = Arc::clone(&lock);
+ let background = Arc::clone(&background);
+ let publisher_admitted = Arc::clone(&publisher_admitted);
+ let release_publisher = Arc::clone(&release_publisher);
+ let replace_happened = Arc::clone(&replace_happened);
+ std::thread::spawn(move || {
+ let _guard = DurablePublicationGuard::acquire(&lock, &background, "ADR-070 test publisher")
+ .expect("publisher is admitted while the engine is healthy");
+ publisher_admitted.wait();
+ release_publisher.wait();
+ // Models the publisher's atomic_replace boundary while it
+ // still owns the real ADR-067 authority.
+ replace_happened.store(true, Ordering::SeqCst);
+ })
+ };
+
+ publisher_admitted.wait();
+ let terminal_thread = std::thread::spawn(move || {
+ let error = terminal.enter_reconcile_required("product writer failed");
+ assert!(matches!(error, EngineError::WriterReconcileRequired { .. }));
+ assert!(
+ replace_happened.load(Ordering::SeqCst),
+ "the admitted publisher must finish before terminality linearises"
+ );
+ });
+
+ release_publisher.wait();
+ publisher.join().expect("publisher thread");
+ terminal_thread.join().expect("terminal thread");
+
+ let refused = match DurablePublicationGuard::acquire(&lock, &background, "late publisher") {
+ Ok(_) => panic!("no publisher may be admitted after the terminal point"),
+ Err(error) => error,
+ };
+ assert!(matches!(refused, EngineError::WriterReconcileRequired { .. }));
+ }
+
+ #[test]
+ fn terminal_handle_escalates_a_poisoned_publication_lock_without_recovering_it() {
+ let lock = Arc::new(Mutex::new(()));
+ let background = BackgroundError::new();
+ let terminal = background.terminal_handle(Arc::clone(&lock));
+
+ let unwound = std::panic::catch_unwind(std::panic::AssertUnwindSafe({
+ let lock = Arc::clone(&lock);
+ let background = Arc::clone(&background);
+ move || {
+ let _guard = DurablePublicationGuard::acquire(&lock, &background, "poison witness")
+ .expect("publisher starts healthy");
+ panic!("panic before replacement poisons the publication lock");
+ }
+ }));
+ assert!(unwound.is_err(), "the witness must poison the lock");
+ assert!(lock.is_poisoned(), "control: publication lock is poisoned");
+
+ let error = terminal.enter_reconcile_required("product panic after lock poison");
+ assert!(matches!(error, EngineError::WriterReconcileRequired { .. }));
+ assert!(
+ lock.is_poisoned(),
+ "terminalisation must never clear publication poison"
+ );
+
+ let refused = match DurablePublicationGuard::acquire(&lock, &background, "late publisher") {
+ Ok(_) => panic!("poisoned terminal authority cannot admit a publisher"),
+ Err(error) => error,
+ };
+ assert!(matches!(refused, EngineError::WriterReconcileRequired { .. }));
+ }
+}
diff --git a/crates/basemyai-engine/src/store/engine/io.rs b/crates/basemyai-engine/src/store/engine/io.rs
index fd008c9..878b318 100644
--- a/crates/basemyai-engine/src/store/engine/io.rs
+++ b/crates/basemyai-engine/src/store/engine/io.rs
@@ -60,6 +60,10 @@ pub(super) fn publish_generation_tracked(
if let Err(error) = crate::failpoint_result!("after_generation_rename") {
return DurablePublish::Unknown(error);
}
+ tracker.directory_sync_started();
+ if let Err(error) = crate::failpoint_result!("during_generation_directory_sync") {
+ return DurablePublish::Unknown(error);
+ }
// ENG-DUR-003/004: the generation pointer's rename must be durable
// before any caller (`rotate_key_full`) acts on the strength of it —
// notably the old-generation GC that follows immediately after.
diff --git a/crates/basemyai-engine/src/store/engine/mod.rs b/crates/basemyai-engine/src/store/engine/mod.rs
index e13d6c6..16fa297 100644
--- a/crates/basemyai-engine/src/store/engine/mod.rs
+++ b/crates/basemyai-engine/src/store/engine/mod.rs
@@ -44,6 +44,9 @@ mod rotate;
mod test_support;
mod write;
+#[doc(hidden)]
+pub use background::EngineTerminalHandle;
+
pub use compact::CompactionJob;
pub(crate) use io::gc_old_generation as gc_retired_generation;
@@ -182,6 +185,153 @@ pub struct Batch {
ops: Vec<(Key, Option)>,
}
+#[doc(hidden)]
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub struct SequenceRange {
+ first: SequenceNumber,
+ last: SequenceNumber,
+}
+
+impl SequenceRange {
+ #[must_use]
+ pub fn first(self) -> SequenceNumber {
+ self.first
+ }
+
+ #[must_use]
+ pub fn last(self) -> SequenceNumber {
+ self.last
+ }
+}
+
+#[doc(hidden)]
+pub struct CommitReceipt {
+ sequence_range: SequenceRange,
+ bytes_written: u64,
+ batch: Option,
+ install_token: InstallToken,
+ obligation: ReceiptInstallObligation,
+ payload: Option,
+}
+
+impl CommitReceipt {
+ #[must_use]
+ pub fn sequence_range(&self) -> SequenceRange {
+ self.sequence_range
+ }
+
+ #[must_use]
+ pub fn bytes_written(&self) -> u64 {
+ self.bytes_written
+ }
+}
+
+impl std::fmt::Debug for CommitReceipt {
+ fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+ formatter
+ .debug_struct("CommitReceipt")
+ .field("sequence_range", &self.sequence_range)
+ .field("bytes_written", &self.bytes_written)
+ .field("install_pending", &self.obligation.armed)
+ .finish_non_exhaustive()
+ }
+}
+
+#[derive(Debug)]
+struct InstallToken {
+ sequence_range: SequenceRange,
+ engine_identity: Arc<()>,
+}
+
+struct ReceiptInstallObligation {
+ catalog_commit_lock: Arc>,
+ background: Arc,
+ armed: bool,
+}
+
+impl ReceiptInstallObligation {
+ fn disarm(&mut self) {
+ self.armed = false;
+ }
+}
+
+impl Drop for ReceiptInstallObligation {
+ fn drop(&mut self) {
+ if self.armed {
+ let _ = background::enter_reconcile_required(
+ &self.catalog_commit_lock,
+ &self.background,
+ "durable commit receipt was abandoned before product installation",
+ );
+ }
+ }
+}
+
+#[doc(hidden)]
+#[derive(Debug)]
+pub enum EngineCommitOutcome {
+ Aborted {
+ burned_sequence_range: Option,
+ cause: Option,
+ },
+ Durable(CommitReceipt),
+ OutcomeUnknown {
+ phase: crate::store::wal::WalCommitPhase,
+ cause: EngineError,
+ },
+}
+
+/// Structural publication phase that made a live rotation's durable outcome
+/// impossible to reconcile in the current engine instance (ADR-070 section
+/// 4.1). This is deliberately separate from WAL commit phases: structural
+/// rotations never invent a logical sequence range.
+#[doc(hidden)]
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum StructuralCommitPhase {
+ CatalogReplace,
+ DirectorySync,
+ CryptoGeneration,
+}
+
+/// Ownership verdict for artifacts which a structural publication may have
+/// made authoritative. Recovery, not stale RAM state, decides their lifecycle.
+#[doc(hidden)]
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum StructuralCandidateOwnership {
+ RetainForRecovery,
+}
+
+/// Proof that a live structural operation completed its durable publication
+/// and matching RAM installation. It intentionally carries no WAL sequence.
+#[doc(hidden)]
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub struct StructuralReceipt {
+ _private: (),
+}
+
+impl StructuralReceipt {
+ fn committed() -> Self {
+ Self { _private: () }
+ }
+}
+
+/// Closed engine-level result for a live structural rotation. Legacy rotation
+/// methods flatten this to `Result<()>`; the V2-03 coordinator consumes this
+/// form so ambiguity can never be mistaken for an ordinary retryable error.
+#[doc(hidden)]
+#[derive(Debug)]
+pub enum StructuralCommitOutcome {
+ Aborted {
+ cause: EngineError,
+ },
+ Committed(StructuralReceipt),
+ StructuralReopenRequired {
+ phase: StructuralCommitPhase,
+ candidate_ownership: StructuralCandidateOwnership,
+ cause: EngineError,
+ },
+}
+
impl Batch {
/// Creates an empty batch.
#[must_use]
@@ -501,6 +651,11 @@ pub struct Engine {
/// readers. Writers publish once with Release after an entire batch;
/// readers load once with Acquire at operation entry (ADR-046 I2/I3).
visible_sequence: AtomicU64,
+ /// Runtime-only identity and the at-most-one durable receipt awaiting RAM
+ /// installation. A pending receipt blocks every later logical mutation;
+ /// it is cleared only after the exact engine-bound receipt is consumed.
+ engine_identity: Arc<()>,
+ pending_install: Option,
/// The published, immutable set of live SSTs (ADR-043 §2 amended, J3).
/// Replaced wholesale by [`compact::apply_version_edit`] — never mutated
/// in place (INV-VS-1/2). [`Self::snapshot`] pins it by cloning the
@@ -509,6 +664,10 @@ pub struct Engine {
/// Serialises durable catalogue transactions without holding
/// `view_gate` across filesystem I/O.
catalog_commit_lock: Arc>,
+ /// One-way ADR-070 capability over the same publication lock and sticky
+ /// terminal health. Created at open and later handed to the logical
+ /// writer owner without exposing either authority independently.
+ terminal_handle: Option,
/// Coordinates the `(visible_sequence, current SuperVersion)` capture
/// with batch publication and flush/compaction swaps.
view_gate: Arc>,
@@ -621,6 +780,14 @@ impl Engine {
Arc::clone(&self.published().current)
}
+ /// Transfers the engine-bound terminal capability to its logical writer
+ /// owner. Exactly one caller can take it; the opaque handle has no public
+ /// constructor and is not clonable.
+ #[doc(hidden)]
+ pub fn take_terminal_handle(&mut self) -> Option {
+ self.terminal_handle.take()
+ }
+
#[cfg(test)]
fn catalog_snapshot(&self) -> CatalogState {
self.published().catalog.clone()
@@ -824,7 +991,18 @@ impl Engine {
/// that `close()` must not publish from a RAM state that may already be
/// stale, and it no longer can.
pub fn close(mut self) -> Result<()> {
- let flush = self.flush();
+ // A durable-but-uninstalled receipt makes a healthy flush unsafe: its
+ // WAL bytes must be recovered on reopen, never retired by publishing
+ // the older RAM state. Consuming close still joins both workers.
+ let flush = match self.pending_install {
+ Some(range) => Err(EngineError::WriterReconcileRequired {
+ cause: format!(
+ "cannot close cleanly while durable sequence range {}..={} awaits installation",
+ range.first, range.last
+ ),
+ }),
+ None => self.flush(),
+ };
// Compaction first: an in-flight merge still needs `catalog_commit_lock`
// and a healthy flush pipeline to commit. Stopping the flush worker
// first would fail that commit for a reason unrelated to any fault.
diff --git a/crates/basemyai-engine/src/store/engine/open.rs b/crates/basemyai-engine/src/store/engine/open.rs
index 2ad0a7d..c0e4575 100644
--- a/crates/basemyai-engine/src/store/engine/open.rs
+++ b/crates/basemyai-engine/src/store/engine/open.rs
@@ -372,6 +372,7 @@ impl Engine {
// One sticky failure for both workers (ADR-049 §1), created before
// either so each can register its wake-up on it at spawn.
let background = super::background::BackgroundError::new();
+ let terminal_handle = background.terminal_handle(Arc::clone(&catalog_commit_lock));
let merge_census = super::compact::MergeCensus::new();
let compaction_trigger = super::compaction_worker::CompactionTrigger::new(options.auto_compact_on_flush);
let snapshots = SnapshotRegistry::new();
@@ -421,8 +422,11 @@ impl Engine {
memtable,
last_allocated_sequence: last_sequence,
visible_sequence: AtomicU64::new(last_sequence),
+ engine_identity: Arc::new(()),
+ pending_install: None,
published,
catalog_commit_lock,
+ terminal_handle: Some(terminal_handle),
view_gate,
next_file_id,
next_flush_job_id: 1,
diff --git a/crates/basemyai-engine/src/store/engine/rotate.rs b/crates/basemyai-engine/src/store/engine/rotate.rs
index e306d1a..883ae6c 100644
--- a/crates/basemyai-engine/src/store/engine/rotate.rs
+++ b/crates/basemyai-engine/src/store/engine/rotate.rs
@@ -8,6 +8,7 @@
//! intrinsically one all-or-nothing "new generation" unit, not a sequence
//! of independently reusable phases.
+use std::cell::Cell;
use std::fs;
use std::path::PathBuf;
use std::sync::Arc;
@@ -19,7 +20,7 @@ use crate::fail_point;
use crate::format::catalog::{CatalogState, SstMeta, WalMeta};
use crate::format::generation_meta;
use crate::store::catalog as catalog_io;
-use crate::store::durable::DurablePublish;
+use crate::store::durable::{DurablePublicationTracker, DurablePublish};
use crate::store::memtable::Memtable;
use crate::store::sst_block::BlockSstFile;
use crate::store::version::{GenerationLease, SstHandle, SuperVersion, Version};
@@ -29,7 +30,79 @@ use crate::store::{InternalKey, Value};
use super::background::DurablePublicationGuard;
use super::compact::CaptureContext;
use super::io::{generation_dir, publish_generation_tracked};
-use super::{Engine, PreparedPublishedInstall};
+use super::{
+ Engine, PreparedPublishedInstall, StructuralCandidateOwnership, StructuralCommitOutcome, StructuralCommitPhase,
+ StructuralReceipt,
+};
+
+/// Observes ADR-067's existing publication guard without owning any lock or
+/// health state itself. The only extra state is a local phase label used to
+/// translate `DurablePublish::Unknown` into ADR-070's structural outcome.
+struct StructuralPublicationTracker<'guard, 'lock> {
+ guard: &'guard DurablePublicationGuard<'lock>,
+ progress: &'guard StructuralProgress,
+ pointer_phase: StructuralCommitPhase,
+}
+
+impl<'guard, 'lock> StructuralPublicationTracker<'guard, 'lock> {
+ fn new(
+ guard: &'guard DurablePublicationGuard<'lock>,
+ progress: &'guard StructuralProgress,
+ pointer_phase: StructuralCommitPhase,
+ ) -> Self {
+ Self {
+ guard,
+ progress,
+ pointer_phase,
+ }
+ }
+}
+
+impl DurablePublicationTracker for StructuralPublicationTracker<'_, '_> {
+ fn pointer_replaced(&self) {
+ self.guard.pointer_replaced();
+ self.progress.phase.set(Some(self.pointer_phase));
+ }
+
+ fn directory_sync_started(&self) {
+ self.guard.directory_sync_started();
+ self.progress.phase.set(Some(StructuralCommitPhase::DirectorySync));
+ }
+}
+
+#[derive(Default)]
+struct StructuralProgress {
+ phase: Cell>,
+ committed: Cell,
+}
+
+fn classify_structural(result: Result<()>, progress: &StructuralProgress) -> StructuralCommitOutcome {
+ if progress.committed.get() {
+ return StructuralCommitOutcome::Committed(StructuralReceipt::committed());
+ }
+ match (result, progress.phase.get()) {
+ (Ok(()), None) => StructuralCommitOutcome::Committed(StructuralReceipt::committed()),
+ (Err(cause), Some(phase)) => StructuralCommitOutcome::StructuralReopenRequired {
+ phase,
+ candidate_ownership: StructuralCandidateOwnership::RetainForRecovery,
+ cause,
+ },
+ (Err(cause), None) => StructuralCommitOutcome::Aborted { cause },
+ (Ok(()), Some(_)) => unreachable!("structural progress must be committed after successful installation"),
+ }
+}
+
+fn run_phase_aware(operation: impl FnOnce() -> Result<()>) -> std::thread::Result> {
+ std::panic::catch_unwind(std::panic::AssertUnwindSafe(operation))
+}
+
+fn flatten_structural(outcome: StructuralCommitOutcome) -> Result<()> {
+ match outcome {
+ StructuralCommitOutcome::Committed(_) => Ok(()),
+ StructuralCommitOutcome::Aborted { cause }
+ | StructuralCommitOutcome::StructuralReopenRequired { cause, .. } => Err(cause),
+ }
+}
/// Complete successor-generation RAM image, allocated and validated before
/// `generation.meta` crosses its point of no return.
@@ -87,6 +160,43 @@ impl PreparedFullRotation {
}
impl Engine {
+ fn classify_phase_aware_run(
+ &self,
+ run: std::thread::Result>,
+ progress: &StructuralProgress,
+ ) -> StructuralCommitOutcome {
+ match run {
+ Ok(result) => classify_structural(result, progress),
+ Err(_payload) if progress.committed.get() => {
+ // Publication and RAM installation already completed. Any
+ // later panic belongs to best-effort finalisation and cannot
+ // demote a durable structural commit.
+ StructuralCommitOutcome::Committed(StructuralReceipt::committed())
+ }
+ Err(_payload) if progress.phase.get().is_some() => {
+ // The real ADR-067 guard has already armed terminal health in
+ // Drop while unwinding. This layer only preserves its phase
+ // and candidate-ownership verdict for the coordinator.
+ let cause = match self.background.check_not_terminal() {
+ Err(cause) => cause,
+ Ok(()) => {
+ let invariant = EngineError::CryptoFailure {
+ reason: "structural publication panicked after pointer replacement without terminal health"
+ .to_string(),
+ };
+ self.background.enter_reconcile_required(&invariant)
+ }
+ };
+ StructuralCommitOutcome::StructuralReopenRequired {
+ phase: progress.phase.get().expect("phase checked above"),
+ candidate_ownership: StructuralCandidateOwnership::RetainForRecovery,
+ cause,
+ }
+ }
+ Err(payload) => std::panic::resume_unwind(payload),
+ }
+ }
+
/// Rotates the user key **in place** (ADR-030 §4): the store's DEK is
/// re-wrapped under a KEK derived from `new_key` (fresh salt) and
/// `crypto.meta` is atomically replaced (tmp + fsync + rename). O(1) —
@@ -106,7 +216,30 @@ impl Engine {
/// encryption (nothing to rotate — parity with ADR-007's posture);
/// otherwise I/O errors from the atomic replace.
pub fn rotate_key(&mut self, new_key: &[u8]) -> Result<()> {
- self.rotate_key_with_mode(new_key, crypto::KeyMode::RawKey, crypto::Argon2idProfile::Default)
+ let progress = StructuralProgress::default();
+ let result = self.rotate_key_with_mode(
+ new_key,
+ crypto::KeyMode::RawKey,
+ crypto::Argon2idProfile::Default,
+ &progress,
+ );
+ flatten_structural(classify_structural(result, &progress))
+ }
+
+ /// Phase-aware V2-03 entry point. Hidden from the product API: only the
+ /// live structural coordinator should consume its closed outcome.
+ #[doc(hidden)]
+ pub fn rotate_key_phase_aware(&mut self, new_key: &[u8]) -> StructuralCommitOutcome {
+ let progress = StructuralProgress::default();
+ let run = run_phase_aware(|| {
+ self.rotate_key_with_mode(
+ new_key,
+ crypto::KeyMode::RawKey,
+ crypto::Argon2idProfile::Default,
+ &progress,
+ )
+ });
+ self.classify_phase_aware_run(run, &progress)
}
/// Passphrase counterpart to [`Self::rotate_key`]. The existing DEK is
@@ -123,7 +256,23 @@ impl Engine {
new_passphrase: &[u8],
profile: crypto::Argon2idProfile,
) -> Result<()> {
- self.rotate_key_with_mode(new_passphrase, crypto::KeyMode::Passphrase, profile)
+ let progress = StructuralProgress::default();
+ let result = self.rotate_key_with_mode(new_passphrase, crypto::KeyMode::Passphrase, profile, &progress);
+ flatten_structural(classify_structural(result, &progress))
+ }
+
+ /// Phase-aware passphrase counterpart to [`Self::rotate_key_phase_aware`].
+ #[doc(hidden)]
+ pub fn rotate_passphrase_phase_aware(
+ &mut self,
+ new_passphrase: &[u8],
+ profile: crypto::Argon2idProfile,
+ ) -> StructuralCommitOutcome {
+ let progress = StructuralProgress::default();
+ let run = run_phase_aware(|| {
+ self.rotate_key_with_mode(new_passphrase, crypto::KeyMode::Passphrase, profile, &progress)
+ });
+ self.classify_phase_aware_run(run, &progress)
}
fn rotate_key_with_mode(
@@ -131,6 +280,7 @@ impl Engine {
new_key: &[u8],
mode: crypto::KeyMode,
profile: crypto::Argon2idProfile,
+ progress: &StructuralProgress,
) -> Result<()> {
self.background.check_not_terminal()?;
self.flush_worker.ensure_healthy()?;
@@ -160,11 +310,22 @@ impl Engine {
DurablePublicationGuard::acquire(&commit_lock, &publication_background, "light crypto rotation")?;
#[cfg(any(test, feature = "test-util"))]
drop(publication_waiter);
- match crypto::publish_staged_meta_tracked(&self.dir, &staged, &publication_guard) {
+ let publication = {
+ let tracker = StructuralPublicationTracker::new(
+ &publication_guard,
+ progress,
+ StructuralCommitPhase::CryptoGeneration,
+ );
+ crypto::publish_staged_meta_tracked(&self.dir, &staged, &tracker)
+ };
+ match publication {
DurablePublish::Published(()) => {
// A light rotation re-wraps the same in-memory DEK, so no RAM
// object changes after the pointer is durable.
publication_guard.complete();
+ drop(publication_guard);
+ progress.committed.set(true);
+ fail_point!("after_light_rotation_publish");
Ok(())
}
// The previous wrapping is still the committed one; the old key
@@ -189,7 +350,29 @@ impl Engine {
/// generation. Before pointer publication an error leaves this instance
/// and the active generation unchanged.
pub fn rotate_key_full(&mut self, new_key: &[u8]) -> Result<()> {
- self.rotate_full(new_key, crypto::KeyMode::RawKey, crypto::Argon2idProfile::Default)
+ let progress = StructuralProgress::default();
+ let result = self.rotate_full(
+ new_key,
+ crypto::KeyMode::RawKey,
+ crypto::Argon2idProfile::Default,
+ &progress,
+ );
+ flatten_structural(classify_structural(result, &progress))
+ }
+
+ /// Phase-aware full-generation rotation for the live structural handler.
+ #[doc(hidden)]
+ pub fn rotate_key_full_phase_aware(&mut self, new_key: &[u8]) -> StructuralCommitOutcome {
+ let progress = StructuralProgress::default();
+ let run = run_phase_aware(|| {
+ self.rotate_full(
+ new_key,
+ crypto::KeyMode::RawKey,
+ crypto::Argon2idProfile::Default,
+ &progress,
+ )
+ });
+ self.classify_phase_aware_run(run, &progress)
}
/// Passphrase counterpart to [`Self::rotate_key_full`]. The fresh DEK is
@@ -204,10 +387,30 @@ impl Engine {
new_passphrase: &[u8],
profile: crypto::Argon2idProfile,
) -> Result<()> {
- self.rotate_full(new_passphrase, crypto::KeyMode::Passphrase, profile)
+ let progress = StructuralProgress::default();
+ let result = self.rotate_full(new_passphrase, crypto::KeyMode::Passphrase, profile, &progress);
+ flatten_structural(classify_structural(result, &progress))
}
- fn rotate_full(&mut self, new_key: &[u8], mode: crypto::KeyMode, profile: crypto::Argon2idProfile) -> Result<()> {
+ /// Phase-aware full passphrase rotation for the live structural handler.
+ #[doc(hidden)]
+ pub fn rotate_passphrase_full_phase_aware(
+ &mut self,
+ new_passphrase: &[u8],
+ profile: crypto::Argon2idProfile,
+ ) -> StructuralCommitOutcome {
+ let progress = StructuralProgress::default();
+ let run = run_phase_aware(|| self.rotate_full(new_passphrase, crypto::KeyMode::Passphrase, profile, &progress));
+ self.classify_phase_aware_run(run, &progress)
+ }
+
+ fn rotate_full(
+ &mut self,
+ new_key: &[u8],
+ mode: crypto::KeyMode,
+ profile: crypto::Argon2idProfile,
+ progress: &StructuralProgress,
+ ) -> Result<()> {
// Terminal publication state has precedence over every operation-
// specific validation. In particular, a plaintext engine made
// reconcile-required by a caught seal panic must not mask that state
@@ -449,7 +652,8 @@ impl Engine {
return Err(error);
}
};
- match publish_generation_tracked(&self.root_dir, prepared.generation_id, &commit_guard) {
+ let tracker = StructuralPublicationTracker::new(&commit_guard, progress, StructuralCommitPhase::CatalogReplace);
+ match publish_generation_tracked(&self.root_dir, prepared.generation_id, &tracker) {
DurablePublish::Published(()) => {}
DurablePublish::NotReplaced(error) => {
// The active generation is still the previous one and this
@@ -483,8 +687,13 @@ impl Engine {
// The generation pointer is live. Installation consumes only the
// prebuilt successor image and returns superseded owners for cleanup.
+ // A panic here remains a structural catalogue ambiguity: RAM may be
+ // only partially installed and recovery owns the candidate generation.
+ progress.phase.set(Some(StructuralCommitPhase::CatalogReplace));
+ fail_point!("before_full_rotation_install");
let retired = prepared.install(self);
commit_guard.complete();
+ progress.committed.set(true);
drop(commit_guard);
self.counters.fsync_count += retired.wal.fsync_count();
self.counters.wal_remove_failures += retired.wal.remove_failures();
diff --git a/crates/basemyai-engine/src/store/engine/write.rs b/crates/basemyai-engine/src/store/engine/write.rs
index 2a8b4fa..e456217 100644
--- a/crates/basemyai-engine/src/store/engine/write.rs
+++ b/crates/basemyai-engine/src/store/engine/write.rs
@@ -3,6 +3,7 @@
//! before the memtable is touched, ADR-025), plus the auto-flush trigger
//! and the WAL-record counter bookkeeping every write op shares.
+use std::sync::Arc;
use std::sync::atomic::Ordering;
use crate::error::{EngineError, Result};
@@ -10,12 +11,14 @@ use crate::format::wal::{self, BatchOp, WalOp};
use crate::key::Key;
use crate::store::SequenceNumber;
-use super::{Batch, Engine};
+use super::{Batch, CommitReceipt, Engine, EngineCommitOutcome, InstallToken, SequenceRange};
+use crate::store::wal::{WalCommitPhase, WalUnwindState, WalWriteOutcome};
impl Engine {
/// Inserts or overwrites `key`. Durable once this returns `Ok` — the WAL
/// record is fsynced before the memtable is updated.
pub fn put(&mut self, key: &[u8], value: &[u8]) -> Result<()> {
+ self.ensure_no_pending_install()?;
self.background.check_not_terminal()?;
self.flush_worker.ensure_healthy()?;
self.seal_recovered_tail_before_write()?;
@@ -36,6 +39,7 @@ impl Engine {
/// Deletes `key` (a no-op if it wasn't present). Durable once this
/// returns `Ok`.
pub fn delete(&mut self, key: &[u8]) -> Result<()> {
+ self.ensure_no_pending_install()?;
self.background.check_not_terminal()?;
self.flush_worker.ensure_healthy()?;
self.seal_recovered_tail_before_write()?;
@@ -67,16 +71,65 @@ impl Engine {
/// already relies on for single `put`/`delete` records, just covering
/// the whole batch's bytes instead of one op's.
pub fn apply_batch(&mut self, batch: &Batch) -> Result<()> {
+ match self.commit_batch(batch.clone()) {
+ EngineCommitOutcome::Aborted { cause: None, .. } => Ok(()),
+ EngineCommitOutcome::Aborted { cause: Some(cause), .. } => Err(cause),
+ EngineCommitOutcome::Durable(receipt) => self.install_committed_batch_with(receipt, |_| Ok(())),
+ EngineCommitOutcome::OutcomeUnknown { phase, cause } => match phase {
+ WalCommitPhase::Append => Err(EngineError::WalAppendOutcomeUnknown {
+ cause: cause.to_string(),
+ }),
+ WalCommitPhase::Sync => Err(EngineError::WalSyncOutcomeUnknown {
+ cause: cause.to_string(),
+ }),
+ },
+ }
+ }
+
+ /// ADR-070's closed engine commit primitive. It reserves the exact
+ /// sequence range and establishes WAL durability, but deliberately does
+ /// not install product-visible RAM state; the receipt is the sole token
+ /// accepted by `install_committed_batch`.
+ #[doc(hidden)]
+ pub fn commit_batch(&mut self, batch: Batch) -> EngineCommitOutcome {
+ self.commit_batch_with(batch, ())
+ }
+
+ /// Commits `batch` while carrying an owned, non-persisted installation
+ /// plan through the durability boundary. The payload can only be
+ /// recovered by consuming the exact engine-bound durable receipt in
+ /// [`Self::install_committed_batch`].
+ #[doc(hidden)]
+ pub fn commit_batch_with(&mut self, batch: Batch, payload: T) -> EngineCommitOutcome {
// Terminal writer state has precedence over the empty-batch no-op:
// every mutating API is also an admission surface and must report
// that this engine can only recover by reopening.
- self.background.check_not_terminal()?;
+ if let Err(cause) = self
+ .ensure_no_pending_install()
+ .and_then(|()| self.background.check_not_terminal())
+ {
+ return EngineCommitOutcome::Aborted {
+ burned_sequence_range: None,
+ cause: Some(cause),
+ };
+ }
if batch.is_empty() {
- return Ok(());
+ return EngineCommitOutcome::Aborted {
+ burned_sequence_range: None,
+ cause: None,
+ };
+ }
+ if let Err(cause) = self
+ .flush_worker
+ .ensure_healthy()
+ .and_then(|()| self.seal_recovered_tail_before_write())
+ .and_then(|()| self.preflight_batch(&batch))
+ {
+ return EngineCommitOutcome::Aborted {
+ burned_sequence_range: None,
+ cause: Some(cause),
+ };
}
- self.flush_worker.ensure_healthy()?;
- self.seal_recovered_tail_before_write()?;
- self.preflight_batch(batch)?;
let wal_ops: Vec = batch
.ops
.iter()
@@ -86,30 +139,142 @@ impl Engine {
value: value.clone(),
})
.collect();
- let (base_sequence, last_sequence) = self.reserve_sequences(batch.len())?;
- let written = self.wal.append_batch(base_sequence, &wal_ops)?;
- self.note_wal_record(written);
+ let (base_sequence, last_sequence) = match self.reserve_sequences(batch.len()) {
+ Ok(range) => range,
+ Err(cause) => {
+ return EngineCommitOutcome::Aborted {
+ burned_sequence_range: None,
+ cause: Some(cause),
+ };
+ }
+ };
+ let sequence_range = SequenceRange {
+ first: base_sequence,
+ last: last_sequence,
+ };
+ let background = &self.background;
+ match self.wal.commit_batch(base_sequence, &wal_ops, |state| {
+ let cause = match state {
+ WalUnwindState::Append => "WAL append panicked after write attempt",
+ WalUnwindState::Sync => "WAL sync panicked after append",
+ WalUnwindState::Durable => "WAL commit panicked after durability was confirmed",
+ };
+ background.record_reconcile_required(cause.to_string());
+ }) {
+ WalWriteOutcome::Aborted(cause) => EngineCommitOutcome::Aborted {
+ burned_sequence_range: Some(sequence_range),
+ cause: Some(cause),
+ },
+ WalWriteOutcome::OutcomeUnknown { phase, cause } => {
+ self.background
+ .record_reconcile_required(format!("WAL {phase:?} outcome unknown: {cause}"));
+ EngineCommitOutcome::OutcomeUnknown { phase, cause }
+ }
+ WalWriteOutcome::Durable { bytes_written } => {
+ self.note_wal_record(bytes_written);
+ debug_assert!(
+ self.pending_install.is_none(),
+ "admission rejected a second pending receipt"
+ );
+ self.pending_install = Some(sequence_range);
+ EngineCommitOutcome::Durable(CommitReceipt {
+ sequence_range,
+ bytes_written,
+ batch: Some(batch),
+ install_token: InstallToken {
+ sequence_range,
+ engine_identity: Arc::clone(&self.engine_identity),
+ },
+ obligation: super::ReceiptInstallObligation {
+ catalog_commit_lock: Arc::clone(&self.catalog_commit_lock),
+ background: Arc::clone(&self.background),
+ armed: true,
+ },
+ payload: Some(payload),
+ })
+ }
+ }
+ }
+ #[doc(hidden)]
+ pub fn install_committed_batch_with(
+ &mut self,
+ mut receipt: CommitReceipt,
+ install_product: impl FnOnce(T) -> Result<()>,
+ ) -> Result<()> {
+ let sequence_range = receipt.sequence_range;
+ let identity_matches = Arc::ptr_eq(&self.engine_identity, &receipt.install_token.engine_identity);
+ let pending_matches =
+ self.pending_install == Some(sequence_range) && receipt.install_token.sequence_range == sequence_range;
+ if !identity_matches || !pending_matches {
+ return Err(super::background::enter_reconcile_required(
+ &self.catalog_commit_lock,
+ &self.background,
+ "durable commit receipt belongs to another engine or pending installation",
+ ));
+ }
+ let batch = receipt
+ .batch
+ .take()
+ .expect("an armed durable receipt always owns its batch");
+ let payload = receipt
+ .payload
+ .take()
+ .expect("an armed durable receipt always owns its installation payload");
+ if receipt.bytes_written == 0 || sequence_range.last - sequence_range.first + 1 != batch.len() as u64 {
+ return Err(super::background::enter_reconcile_required(
+ &self.catalog_commit_lock,
+ &self.background,
+ "durable commit receipt failed its internal range validation",
+ ));
+ }
{
let _gate = self.view_gate.lock().unwrap_or_else(std::sync::PoisonError::into_inner);
- for (ordinal, (key, value)) in batch.ops.iter().enumerate() {
- let sequence = base_sequence
+ for (ordinal, (key, value)) in batch.ops.into_iter().enumerate() {
+ let sequence = sequence_range
+ .first
.checked_add(u64::try_from(ordinal).expect("batch length is bounded to 10,000"))
.expect("reserve_sequences validated the complete contiguous range");
match value {
Some(v) => self
.memtable
- .insert_value(key.clone(), sequence, v.clone())
+ .insert_value(key, sequence, v)
.expect("freshly allocated sequence must be unique and nonzero"),
None => self
.memtable
- .insert_tombstone(key.clone(), sequence)
+ .insert_tombstone(key, sequence)
.expect("freshly allocated sequence must be unique and nonzero"),
}
}
- self.visible_sequence.store(last_sequence, Ordering::Release);
+ self.visible_sequence.store(sequence_range.last, Ordering::Release);
+ }
+ // `pending_install` and the receipt obligation deliberately remain
+ // armed while product RAM is installed. An error or unwind drops the
+ // armed receipt, serialising terminalisation with catalogue publish.
+ install_product(payload)?;
+ self.pending_install = None;
+ receipt.obligation.disarm();
+ // Durability and complete product installation already establish the
+ // caller's Committed verdict. Opportunistic seal/flush work may make
+ // the engine sticky for the *next* admission, but must never demote
+ // this completed mutation into a retryable error.
+ if let Err(error) = self.maybe_flush() {
+ self.background
+ .record(format!("post-commit opportunistic flush failed: {error}"));
+ }
+ Ok(())
+ }
+
+ fn ensure_no_pending_install(&self) -> Result<()> {
+ match self.pending_install {
+ Some(range) => Err(EngineError::WriterReconcileRequired {
+ cause: format!(
+ "durable sequence range {}..={} is awaiting mandatory RAM installation",
+ range.first, range.last
+ ),
+ }),
+ None => Ok(()),
}
- self.maybe_flush()
}
/// Seals the mutable memtable when it crosses its threshold and enqueues
@@ -259,6 +424,7 @@ impl Engine {
#[cfg(test)]
mod tests {
use super::*;
+ use crate::failpoint::{self, Action};
use crate::store::engine::test_support::KEY;
fn bounded_options() -> super::super::EngineOptions {
@@ -374,6 +540,323 @@ mod tests {
);
}
+ #[test]
+ fn adr070_commit_outcomes_are_phase_aware_exact_and_terminal_when_ambiguous() {
+ struct ClearFailpoints;
+ impl Drop for ClearFailpoints {
+ fn drop(&mut self) {
+ failpoint::clear_all();
+ }
+ }
+
+ let _clear = ClearFailpoints;
+ failpoint::clear_all();
+
+ // A failure before any write attempt burns the reserved range but
+ // leaves both WAL and writer reusable.
+ let aborted_dir = tempfile::tempdir().expect("tempdir");
+ let mut aborted_engine = Engine::open_with_options(aborted_dir.path(), bounded_options()).expect("open");
+ let mut one = Batch::new();
+ one.put(b"a", b"1");
+ let before = aborted_engine.stats().expect("stats before aborted commit");
+ failpoint::set("before_wal_append_attempt", Action::Error);
+ match aborted_engine.commit_batch(one.clone()) {
+ EngineCommitOutcome::Aborted {
+ burned_sequence_range: Some(range),
+ cause: Some(_),
+ } => assert_eq!(range, SequenceRange { first: 1, last: 1 }),
+ other => panic!("expected pre-write Aborted, got {other:?}"),
+ }
+ failpoint::remove("before_wal_append_attempt");
+ assert_eq!(
+ aborted_engine.stats().expect("stats after aborted commit").wal_bytes,
+ before.wal_bytes,
+ "pre-write Aborted must not touch the WAL"
+ );
+ aborted_engine.apply_batch(&one).expect("writer remains reusable");
+ assert_eq!(
+ aborted_engine.visible_sequence(),
+ 2,
+ "the burned sequence remains a gap"
+ );
+
+ // A normal commit returns the exact range and remains invisible until
+ // its non-forgeable receipt is consumed by installation.
+ let durable_dir = tempfile::tempdir().expect("tempdir");
+ let mut durable_engine = Engine::open_with_options(durable_dir.path(), bounded_options()).expect("open");
+ let mut two = Batch::new();
+ two.put(b"a", b"1");
+ two.put(b"b", b"2");
+ let receipt = match durable_engine.commit_batch(two) {
+ EngineCommitOutcome::Durable(receipt) => receipt,
+ other => panic!("expected Durable receipt, got {other:?}"),
+ };
+ assert_eq!(receipt.sequence_range, SequenceRange { first: 1, last: 2 });
+ assert!(receipt.bytes_written > 0);
+ assert_eq!(durable_engine.visible_sequence(), 0);
+ assert_eq!(durable_engine.get(b"a").expect("not installed"), None);
+ durable_engine
+ .install_committed_batch_with(receipt, |_| Ok(()))
+ .expect("receipt belongs to this engine");
+ assert_eq!(durable_engine.visible_sequence(), 2);
+
+ // An error after write_all is Append-unknown and immediately arms
+ // sticky fail-stop health; no later append is admitted.
+ for (site, expected_phase) in [
+ ("after_wal_append", WalCommitPhase::Append),
+ ("during_wal_sync", WalCommitPhase::Sync),
+ ] {
+ failpoint::clear_all();
+ let dir = tempfile::tempdir().expect("tempdir");
+ let mut engine = Engine::open_with_options(dir.path(), bounded_options()).expect("open");
+ failpoint::set(site, Action::Error);
+ match engine.commit_batch(one.clone()) {
+ EngineCommitOutcome::OutcomeUnknown { phase, .. } => assert_eq!(phase, expected_phase),
+ other => panic!("expected phase-aware OutcomeUnknown at {site}, got {other:?}"),
+ }
+ failpoint::remove(site);
+ let bytes_after_unknown = engine.stats().expect("stats after unknown").wal_bytes;
+ let error = engine
+ .apply_batch(&one)
+ .expect_err("sticky terminal state refuses the next append");
+ assert!(matches!(error, EngineError::WriterReconcileRequired { .. }));
+ assert_eq!(
+ engine.stats().expect("stats after refused retry").wal_bytes,
+ bytes_after_unknown,
+ "terminal writer must not append again"
+ );
+ }
+
+ // Unwinding cannot bypass the returned-outcome path and accidentally
+ // leave a writer healthy after a WAL attempt.
+ for site in ["after_wal_append", "during_wal_sync", "after_wal_fsync"] {
+ failpoint::clear_all();
+ let panic_dir = tempfile::tempdir().expect("tempdir");
+ let mut panic_engine = Engine::open_with_options(panic_dir.path(), bounded_options()).expect("open");
+ failpoint::set(site, Action::Panic);
+ let panic = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
+ let _ = panic_engine.commit_batch(one.clone());
+ }));
+ assert!(panic.is_err(), "injected panic at {site} must unwind");
+ failpoint::remove(site);
+ let bytes_after_panic = panic_engine.stats().expect("stats after panic").wal_bytes;
+ assert!(matches!(
+ panic_engine.apply_batch(&one),
+ Err(EngineError::WriterReconcileRequired { .. })
+ ));
+ assert_eq!(
+ panic_engine
+ .stats()
+ .expect("stats after refused post-panic append")
+ .wal_bytes,
+ bytes_after_panic
+ );
+ }
+
+ // Once sync_all returned success, an error-style probe at the crash
+ // boundary cannot demote the result from Durable.
+ failpoint::clear_all();
+ let confirmed_dir = tempfile::tempdir().expect("tempdir");
+ let mut confirmed = Engine::open_with_options(confirmed_dir.path(), bounded_options()).expect("open");
+ failpoint::set("after_wal_fsync", Action::Error);
+ assert!(matches!(confirmed.commit_batch(one), EngineCommitOutcome::Durable(_)));
+ }
+
+ #[test]
+ fn adr070_pending_receipt_blocks_mutation_until_exact_install() {
+ let dir = tempfile::tempdir().expect("tempdir");
+ let mut engine = Engine::open_with_options(dir.path(), bounded_options()).expect("open");
+ let mut first = Batch::new();
+ first.put(b"one", b"1");
+ let receipt = match engine.commit_batch(first) {
+ EngineCommitOutcome::Durable(receipt) => receipt,
+ other => panic!("expected durable receipt, got {other:?}"),
+ };
+
+ let mut second = Batch::new();
+ second.put(b"two", b"2");
+ assert!(matches!(
+ engine.commit_batch(second.clone()),
+ EngineCommitOutcome::Aborted {
+ cause: Some(EngineError::WriterReconcileRequired { .. }),
+ ..
+ }
+ ));
+ assert!(matches!(
+ engine.put(b"put", b"blocked"),
+ Err(EngineError::WriterReconcileRequired { .. })
+ ));
+ assert!(matches!(
+ engine.delete(b"one"),
+ Err(EngineError::WriterReconcileRequired { .. })
+ ));
+
+ engine
+ .install_committed_batch_with(receipt, |_| Ok(()))
+ .expect("the exact engine consumes its receipt");
+ engine
+ .apply_batch(&second)
+ .expect("next commit is admitted after install");
+ assert_eq!(engine.get(b"one").expect("read first"), Some(b"1".to_vec()));
+ assert_eq!(engine.get(b"two").expect("read second"), Some(b"2".to_vec()));
+ }
+
+ #[test]
+ fn adr070_dropped_receipt_terminalises_its_source_engine() {
+ let dir = tempfile::tempdir().expect("tempdir");
+ let mut engine = Engine::open_with_options(dir.path(), bounded_options()).expect("open");
+ let mut batch = Batch::new();
+ batch.put(b"dur", b"pend");
+ let receipt = match engine.commit_batch(batch) {
+ EngineCommitOutcome::Durable(receipt) => receipt,
+ other => panic!("expected durable receipt, got {other:?}"),
+ };
+
+ drop(receipt);
+ assert!(matches!(
+ engine.put(b"later", b"blocked"),
+ Err(EngineError::WriterReconcileRequired { .. })
+ ));
+ }
+
+ #[test]
+ fn adr070_cross_engine_receipt_terminalises_both_engines() {
+ let source_dir = tempfile::tempdir().expect("source tempdir");
+ let target_dir = tempfile::tempdir().expect("target tempdir");
+ let mut source = Engine::open_with_options(source_dir.path(), bounded_options()).expect("open source");
+ let mut target = Engine::open_with_options(target_dir.path(), bounded_options()).expect("open target");
+ let mut batch = Batch::new();
+ batch.put(b"src", b"dur");
+ let receipt = match source.commit_batch(batch) {
+ EngineCommitOutcome::Durable(receipt) => receipt,
+ other => panic!("expected durable receipt, got {other:?}"),
+ };
+
+ assert!(matches!(
+ target.install_committed_batch_with(receipt, |_| Ok(())),
+ Err(EngineError::WriterReconcileRequired { .. })
+ ));
+ assert!(matches!(
+ target.put(b"target", b"blocked"),
+ Err(EngineError::WriterReconcileRequired { .. })
+ ));
+ assert!(matches!(
+ source.put(b"source", b"blocked"),
+ Err(EngineError::WriterReconcileRequired { .. })
+ ));
+ }
+
+ #[test]
+ fn adr070_close_with_pending_receipt_skips_clean_flush_and_reopen_recovers() {
+ let dir = tempfile::tempdir().expect("tempdir");
+ let mut engine = Engine::open_with_options(dir.path(), bounded_options()).expect("open");
+ let mut batch = Batch::new();
+ batch.put(b"dur", b"play");
+ let receipt = match engine.commit_batch(batch) {
+ EngineCommitOutcome::Durable(receipt) => receipt,
+ other => panic!("expected durable receipt, got {other:?}"),
+ };
+
+ assert!(matches!(
+ engine.close(),
+ Err(EngineError::WriterReconcileRequired { .. })
+ ));
+ drop(receipt);
+
+ let reopened = Engine::open_with_options(dir.path(), bounded_options()).expect("reopen from retained WAL");
+ assert_eq!(reopened.get(b"dur").expect("replayed read"), Some(b"play".to_vec()));
+ }
+
+ #[test]
+ fn adr070_product_install_error_or_panic_keeps_engine_terminal() {
+ for panic in [false, true] {
+ let dir = tempfile::tempdir().expect("tempdir");
+ let mut engine = Engine::open_with_options(dir.path(), bounded_options()).expect("open");
+ let mut batch = Batch::new();
+ batch.put(b"dur", b"plan");
+ let receipt = match engine.commit_batch_with(batch, 7_u8) {
+ EngineCommitOutcome::Durable(receipt) => receipt,
+ other => panic!("expected durable receipt, got {other:?}"),
+ };
+
+ let outcome = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
+ engine.install_committed_batch_with(receipt, |payload| {
+ assert_eq!(payload, 7);
+ assert!(!panic, "injected product RAM install panic");
+ Err(EngineError::InvalidCatalogState {
+ reason: "injected product RAM install error".into(),
+ })
+ })
+ }));
+ if panic {
+ assert!(outcome.is_err(), "the injected install panic must unwind");
+ } else {
+ assert!(matches!(outcome, Ok(Err(EngineError::InvalidCatalogState { .. }))));
+ }
+ assert!(matches!(
+ engine.put(b"next", b"blocked"),
+ Err(EngineError::WriterReconcileRequired { .. })
+ ));
+ }
+ }
+
+ #[test]
+ fn adr070_post_install_seal_failure_preserves_success_and_is_sticky() {
+ struct ClearFailpoints;
+ impl Drop for ClearFailpoints {
+ fn drop(&mut self) {
+ failpoint::clear_all();
+ }
+ }
+
+ let _clear = ClearFailpoints;
+ failpoint::clear_all();
+ let dir = tempfile::tempdir().expect("tempdir");
+ let mut options = bounded_options();
+ options.memtable_flush_threshold = 1;
+ let mut engine = Engine::open_with_options(dir.path(), options).expect("open");
+
+ let mut committed = Batch::new();
+ committed.put(b"one", b"1");
+ let receipt = match engine.commit_batch_with(committed, 7_u8) {
+ EngineCommitOutcome::Durable(receipt) => receipt,
+ other => panic!("expected durable receipt, got {other:?}"),
+ };
+ failpoint::set("before_seal_candidate_wal_fsync", Action::Error);
+ engine
+ .install_committed_batch_with(receipt, |payload| {
+ assert_eq!(payload, 7);
+ Ok(())
+ })
+ .expect("post-install seal failure must not demote a committed product install");
+ failpoint::remove("before_seal_candidate_wal_fsync");
+
+ assert_eq!(engine.get(b"one").expect("committed read"), Some(b"1".to_vec()));
+ let wal_records = engine.stats().expect("stats after committed install").wal_records;
+ assert!(matches!(
+ engine.put(b"two", b"2"),
+ Err(EngineError::BackgroundFlush { .. } | EngineError::WriterReconcileRequired { .. })
+ ));
+ assert_eq!(
+ engine.stats().expect("stats after refused next write").wal_records,
+ wal_records,
+ "the sticky post-commit error must refuse the next append"
+ );
+ assert_eq!(engine.get(b"two").expect("refused key read"), None);
+
+ drop(engine);
+ let reopened = Engine::open_with_options(dir.path(), options).expect("reopen retained WAL");
+ assert_eq!(
+ reopened.get(b"one").expect("replayed committed key"),
+ Some(b"1".to_vec())
+ );
+ assert_eq!(reopened.get(b"two").expect("refused key remains absent"), None);
+ let visible = reopened.scan_prefix(b"one").expect("single logical committed value");
+ assert_eq!(visible.len(), 1);
+ assert_eq!(visible[0].0.as_bytes(), b"one");
+ assert_eq!(visible[0].1, b"1");
+ }
+
#[test]
fn sequence_overflow_is_typed_and_precedes_wal_or_memtable_mutation() {
let dir = tempfile::tempdir().expect("tempdir");
diff --git a/crates/basemyai-engine/src/store/mod.rs b/crates/basemyai-engine/src/store/mod.rs
index d410e69..ad72269 100644
--- a/crates/basemyai-engine/src/store/mod.rs
+++ b/crates/basemyai-engine/src/store/mod.rs
@@ -52,8 +52,9 @@ pub mod writer_runtime;
pub(crate) mod writer_runtime;
pub use engine::{
- Batch, CompactionJob, DEFAULT_BLOCK_CACHE_CAPACITY_BYTES, DEFAULT_BLOCK_SIZE,
- DEFAULT_METADATA_CACHE_CAPACITY_BYTES, Engine, EngineOptions, ScanPage,
+ Batch, CommitReceipt, CompactionJob, DEFAULT_BLOCK_CACHE_CAPACITY_BYTES, DEFAULT_BLOCK_SIZE,
+ DEFAULT_METADATA_CACHE_CAPACITY_BYTES, Engine, EngineCommitOutcome, EngineOptions, EngineTerminalHandle, ScanPage,
+ SequenceRange, StructuralCandidateOwnership, StructuralCommitOutcome, StructuralCommitPhase, StructuralReceipt,
};
// Réservé à N13/R8 (ADR-050 §4.2) — jamais publié. `CommitPolicy` est lu par
// `WriterRuntime::new`, compilé même hors tests ; `GroupCommitOptions` n'est
@@ -69,6 +70,8 @@ pub use repair::{RebuildReport, RepairAction, RepairPlan, plan_repair, rebuild_i
pub use stats::{EngineStats, LatencySummary};
pub use verify::{IntegrityIssue, IssueKind, VerifyMode, VerifyReport, verify_store, verify_store_with_passphrase};
pub use version::{ReadSnapshot, Snapshot};
+#[doc(hidden)]
+pub use wal::WalCommitPhase;
/// A stored value. Kept as a plain alias (not a newtype) since, unlike
/// [`crate::key::Key`], nothing about its ordering or encoding is
diff --git a/crates/basemyai-engine/src/store/wal.rs b/crates/basemyai-engine/src/store/wal.rs
index 251bffc..5a865c9 100644
--- a/crates/basemyai-engine/src/store/wal.rs
+++ b/crates/basemyai-engine/src/store/wal.rs
@@ -70,6 +70,73 @@ pub(crate) struct Wal {
remove_failures: u64,
}
+/// Closed WAL outcome used by ADR-070's internal commit primitive. It keeps
+/// failures before the first attempted write separate from failures after an
+/// append or sync attempt, where absence/durability can no longer be inferred
+/// from a flat `io::Error`.
+pub(crate) enum WalWriteOutcome {
+ Aborted(EngineError),
+ Durable { bytes_written: u64 },
+ OutcomeUnknown { phase: WalCommitPhase, cause: EngineError },
+}
+
+#[doc(hidden)]
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum WalCommitPhase {
+ Append,
+ Sync,
+}
+
+struct WalAmbiguityGuard
+where
+ F: FnOnce(WalUnwindState),
+{
+ state: WalUnwindState,
+ on_ambiguous: Option,
+}
+
+#[derive(Debug, Clone, Copy)]
+pub(crate) enum WalUnwindState {
+ Append,
+ Sync,
+ Durable,
+}
+
+impl WalAmbiguityGuard
+where
+ F: FnOnce(WalUnwindState),
+{
+ fn new(on_ambiguous: F) -> Self {
+ Self {
+ state: WalUnwindState::Append,
+ on_ambiguous: Some(on_ambiguous),
+ }
+ }
+
+ fn enter_sync(&mut self) {
+ self.state = WalUnwindState::Sync;
+ }
+
+ fn confirm_durable(&mut self) {
+ self.state = WalUnwindState::Durable;
+ }
+
+ fn disarm(mut self) {
+ self.on_ambiguous = None;
+ }
+}
+
+impl Drop for WalAmbiguityGuard
+where
+ F: FnOnce(WalUnwindState),
+{
+ fn drop(&mut self) {
+ if let Some(on_ambiguous) = self.on_ambiguous.take() {
+ on_ambiguous(self.state);
+ }
+ }
+}
+
/// Owns a newly-created WAL until its initial fsync succeeds. An error or
/// unwind before that boundary must not leave a reusable canonical file id
/// behind: `create_new` deliberately refuses to overwrite such a path.
@@ -370,6 +437,7 @@ impl Wal {
/// crash can only ever leave either every sub-operation absent (record
/// never made it) or every sub-operation present (record fully synced)
/// — never a partial subset. Returns the on-disk bytes written.
+ #[cfg(test)]
pub(crate) fn append_batch(&mut self, base_sequence: SequenceNumber, ops: &[BatchOp]) -> Result {
if ops.len() > wal::MAX_BATCH_OPS {
return Err(EngineError::WalBatchTooLarge {
@@ -381,6 +449,105 @@ impl Wal {
self.write_record(base_sequence, WalOp::Batch, &[], Some(&payload))
}
+ /// Phase-aware batch append for ADR-070. Unlike the legacy wrapper above,
+ /// this never erases whether a failure happened before or after the WAL
+ /// ambiguity boundary.
+ pub(crate) fn commit_batch(
+ &mut self,
+ base_sequence: SequenceNumber,
+ ops: &[BatchOp],
+ on_ambiguous_unwind: impl FnOnce(WalUnwindState),
+ ) -> WalWriteOutcome {
+ if ops.len() > wal::MAX_BATCH_OPS {
+ return WalWriteOutcome::Aborted(EngineError::WalBatchTooLarge {
+ len: ops.len(),
+ max: wal::MAX_BATCH_OPS,
+ });
+ }
+ let payload = match wal::encode_batch(ops) {
+ Ok(payload) => payload,
+ Err(error) => return WalWriteOutcome::Aborted(error),
+ };
+ self.write_record_phase_aware(base_sequence, WalOp::Batch, &[], Some(&payload), on_ambiguous_unwind)
+ }
+
+ fn write_record_phase_aware(
+ &mut self,
+ base_sequence: SequenceNumber,
+ op: WalOp,
+ key: &[u8],
+ value: Option<&[u8]>,
+ on_ambiguous_unwind: impl FnOnce(WalUnwindState),
+ ) -> WalWriteOutcome {
+ let offset = match self.file.seek(SeekFrom::End(0)) {
+ Ok(offset) => offset,
+ Err(error) => return WalWriteOutcome::Aborted(EngineError::io(self.path.clone(), error)),
+ };
+ let record = match wal::encode_with_sequence(op, offset, base_sequence, key, value) {
+ Ok(record) => record,
+ Err(error) => return WalWriteOutcome::Aborted(error),
+ };
+ let sealed;
+ let on_disk: &[u8] = match &self.crypto {
+ Some(crypto) => {
+ let aad = envelope::wal_envelope_aad_v3(self.store_id, self.wal_epoch, self.file_id, offset);
+ let encrypted = match crypto.seal(&record, &aad) {
+ Ok(encrypted) => encrypted,
+ Err(error) => return WalWriteOutcome::Aborted(error),
+ };
+ sealed = match envelope::encode_wal_envelope(&encrypted.nonce, &encrypted.ciphertext) {
+ Ok(sealed) => sealed,
+ Err(error) => return WalWriteOutcome::Aborted(error),
+ };
+ &sealed
+ }
+ None => &record,
+ };
+
+ if let Err(error) = crate::failpoint_result!("before_wal_append_attempt") {
+ return WalWriteOutcome::Aborted(error);
+ }
+ let mut ambiguity = WalAmbiguityGuard::new(on_ambiguous_unwind);
+ if let Err(error) = self.file.write_all(on_disk) {
+ ambiguity.disarm();
+ return WalWriteOutcome::OutcomeUnknown {
+ phase: WalCommitPhase::Append,
+ cause: EngineError::io(self.path.clone(), error),
+ };
+ }
+ if let Err(error) = crate::failpoint_result!("after_wal_append") {
+ ambiguity.disarm();
+ return WalWriteOutcome::OutcomeUnknown {
+ phase: WalCommitPhase::Append,
+ cause: error,
+ };
+ }
+ ambiguity.enter_sync();
+ if let Err(error) = crate::failpoint_result!("during_wal_sync") {
+ ambiguity.disarm();
+ return WalWriteOutcome::OutcomeUnknown {
+ phase: WalCommitPhase::Sync,
+ cause: error,
+ };
+ }
+ if let Err(error) = self.file.sync_all() {
+ ambiguity.disarm();
+ return WalWriteOutcome::OutcomeUnknown {
+ phase: WalCommitPhase::Sync,
+ cause: EngineError::io(self.path.clone(), error),
+ };
+ }
+ self.fsync_count += 1;
+ ambiguity.confirm_durable();
+ // `Error` after a confirmed fsync cannot revoke durability. Abort and
+ // panic actions still fire at this exact crash-consistency boundary.
+ let _ = crate::failpoint_result!("after_wal_fsync");
+ ambiguity.disarm();
+ WalWriteOutcome::Durable {
+ bytes_written: on_disk.len() as u64,
+ }
+ }
+
/// Explicitly seeks to end-of-file first (see [`Wal::open_for_append`]
/// for why this isn't `O_APPEND`) — the returned position is this
/// record's `record_offset` (ADR-044 §3), known before encoding since it
diff --git a/crates/basemyai-engine/tests/crash/adr066_commit_outcome.rs b/crates/basemyai-engine/tests/crash/adr066_commit_outcome.rs
index b9b6dbf..88669a4 100644
--- a/crates/basemyai-engine/tests/crash/adr066_commit_outcome.rs
+++ b/crates/basemyai-engine/tests/crash/adr066_commit_outcome.rs
@@ -26,7 +26,10 @@ use std::sync::{Mutex, MutexGuard, OnceLock};
use std::thread;
use basemyai_engine::failpoint::{self, Action};
-use basemyai_engine::{Batch, Engine, EngineError, EngineOptions};
+use basemyai_engine::{
+ Batch, Engine, EngineError, EngineOptions, StructuralCandidateOwnership, StructuralCommitOutcome,
+ StructuralCommitPhase,
+};
const KEY: &[u8] = b"adr066 commit outcome passphrase";
const REPLACEMENT_KEY: &[u8] = b"adr066 replacement passphrase";
@@ -81,6 +84,26 @@ fn concurrent_publish_options() -> EngineOptions {
}
}
+fn assert_structural_reopen(outcome: StructuralCommitOutcome, expected_phase: StructuralCommitPhase) -> EngineError {
+ match outcome {
+ StructuralCommitOutcome::StructuralReopenRequired {
+ phase,
+ candidate_ownership,
+ cause,
+ } => {
+ assert_eq!(phase, expected_phase);
+ assert_eq!(
+ candidate_ownership,
+ StructuralCandidateOwnership::RetainForRecovery,
+ "an ambiguous structural publication must transfer candidate lifecycle to reopen"
+ );
+ assert_terminal(&cause, "phase-aware structural outcome");
+ cause
+ }
+ other => panic!("expected StructuralReopenRequired({expected_phase:?}), got {other:?}"),
+ }
+}
+
/// Produces `count` distinct, durable SSTs without allowing automatic
/// compaction. The caller can then exercise the inline compaction commit
/// boundary without racing a background job it did not request.
@@ -472,6 +495,125 @@ fn ambiguous_generation_publication_is_terminal() {
);
}
+#[test]
+fn phase_aware_full_rotation_reports_catalog_replace_and_retains_generation() {
+ let _serial = lock();
+ let _clear = ClearOnDrop;
+ failpoint::clear_all();
+
+ let dir = tempfile::tempdir().expect("tempdir");
+ let mut engine = Engine::open_encrypted_with_options(dir.path(), KEY, manual_options()).expect("open encrypted");
+ engine.put(b"durable", b"before").expect("pre-rotation write");
+ engine.flush().expect("settle before rotation");
+
+ failpoint::set("after_generation_rename", Action::Error);
+ let outcome = engine.rotate_key_full_phase_aware(REPLACEMENT_KEY);
+ failpoint::remove("after_generation_rename");
+
+ let _cause = assert_structural_reopen(outcome, StructuralCommitPhase::CatalogReplace);
+ assert!(
+ dir.path().join("gen-1").is_dir(),
+ "the candidate generation may be authoritative and must remain for recovery"
+ );
+ assert_terminal(
+ &engine
+ .put(b"after", b"must-not-append")
+ .expect_err("the next mutation must be blocked"),
+ "mutation after catalog-replace ambiguity",
+ );
+}
+
+#[test]
+fn phase_aware_full_rotation_reports_directory_sync_and_retains_generation() {
+ let _serial = lock();
+ let _clear = ClearOnDrop;
+ failpoint::clear_all();
+
+ let dir = tempfile::tempdir().expect("tempdir");
+ let mut engine = Engine::open_encrypted_with_options(dir.path(), KEY, manual_options()).expect("open encrypted");
+ engine.put(b"durable", b"before").expect("pre-rotation write");
+ engine.flush().expect("settle before rotation");
+
+ failpoint::set("during_generation_directory_sync", Action::Error);
+ let outcome = engine.rotate_key_full_phase_aware(REPLACEMENT_KEY);
+ failpoint::remove("during_generation_directory_sync");
+
+ let _cause = assert_structural_reopen(outcome, StructuralCommitPhase::DirectorySync);
+ assert!(
+ dir.path().join("gen-1").is_dir(),
+ "directory-sync ambiguity must retain the complete candidate generation"
+ );
+ assert_terminal(
+ &engine
+ .put(b"after", b"must-not-append")
+ .expect_err("the next mutation must be blocked"),
+ "mutation after directory-sync ambiguity",
+ );
+}
+
+#[test]
+fn phase_aware_full_rotation_classifies_panics_at_replace_sync_and_install() {
+ for (site, expected_phase) in [
+ ("after_generation_rename", StructuralCommitPhase::CatalogReplace),
+ ("during_generation_directory_sync", StructuralCommitPhase::DirectorySync),
+ ("before_full_rotation_install", StructuralCommitPhase::CatalogReplace),
+ ] {
+ let _serial = lock();
+ let _clear = ClearOnDrop;
+ failpoint::clear_all();
+
+ let dir = tempfile::tempdir().expect("tempdir");
+ let mut engine =
+ Engine::open_encrypted_with_options(dir.path(), KEY, manual_options()).expect("open encrypted");
+ engine.put(b"durable", b"before").expect("pre-rotation write");
+ engine.flush().expect("settle before rotation");
+
+ failpoint::set(site, Action::Panic);
+ let outcome = engine.rotate_key_full_phase_aware(REPLACEMENT_KEY);
+ failpoint::remove(site);
+
+ let _cause = assert_structural_reopen(outcome, expected_phase);
+ assert!(
+ dir.path().join("gen-1").is_dir(),
+ "panic at {site} must retain the candidate generation for recovery"
+ );
+ assert_terminal(
+ &engine
+ .put(b"after", b"must-not-append")
+ .expect_err("the next mutation must be blocked"),
+ "mutation after structural panic",
+ );
+ }
+}
+
+#[test]
+fn phase_aware_full_rotation_preserves_committed_after_finalisation_panic() {
+ let _serial = lock();
+ let _clear = ClearOnDrop;
+ failpoint::clear_all();
+
+ let dir = tempfile::tempdir().expect("tempdir");
+ let mut engine = Engine::open_encrypted_with_options(dir.path(), KEY, manual_options()).expect("open encrypted");
+ engine.put(b"durable", b"before").expect("pre-rotation write");
+ engine.flush().expect("settle before rotation");
+
+ failpoint::set("after_full_rotation_publish", Action::Panic);
+ let outcome = engine.rotate_key_full_phase_aware(REPLACEMENT_KEY);
+ failpoint::remove("after_full_rotation_publish");
+
+ assert!(
+ matches!(outcome, StructuralCommitOutcome::Committed(_)),
+ "a finalisation panic after guard completion cannot demote the structural commit: {outcome:?}"
+ );
+ engine
+ .put(b"after", b"still-admitted")
+ .expect("the committed engine remains usable after finalisation panic");
+ assert_eq!(
+ engine.get(b"after").expect("read after committed outcome").as_deref(),
+ Some(&b"still-admitted"[..])
+ );
+}
+
/// Same-instance unwind proof for the pointer with the widest blast radius:
/// `generation.meta`. The atomic replacement already selects the staged
/// generation when the panic fires, while every Engine field still describes
@@ -588,6 +730,79 @@ fn ambiguous_crypto_meta_replacement_is_terminal() {
);
}
+#[test]
+fn phase_aware_light_rotation_reports_crypto_generation_and_blocks_next_mutation() {
+ let _serial = lock();
+ let _clear = ClearOnDrop;
+ failpoint::clear_all();
+
+ let dir = tempfile::tempdir().expect("tempdir");
+ let mut engine = Engine::open_encrypted_with_options(dir.path(), KEY, manual_options()).expect("open encrypted");
+ engine.put(b"durable", b"before").expect("pre-rotation write");
+
+ failpoint::set("after_crypto_meta_write", Action::Error);
+ let outcome = engine.rotate_key_phase_aware(REPLACEMENT_KEY);
+ failpoint::remove("after_crypto_meta_write");
+
+ let _cause = assert_structural_reopen(outcome, StructuralCommitPhase::CryptoGeneration);
+ assert!(
+ dir.path().join("crypto.meta").is_file(),
+ "the replaced crypto generation must remain authoritative for reopen"
+ );
+ assert_terminal(
+ &engine
+ .put(b"after", b"must-not-append")
+ .expect_err("the next mutation must be blocked"),
+ "mutation after crypto-generation ambiguity",
+ );
+}
+
+#[test]
+fn phase_aware_light_rotation_classifies_replace_panic_as_crypto_generation() {
+ let _serial = lock();
+ let _clear = ClearOnDrop;
+ failpoint::clear_all();
+
+ let dir = tempfile::tempdir().expect("tempdir");
+ let mut engine = Engine::open_encrypted_with_options(dir.path(), KEY, manual_options()).expect("open encrypted");
+ engine.put(b"durable", b"before").expect("pre-rotation write");
+
+ failpoint::set("after_crypto_meta_write", Action::Panic);
+ let outcome = engine.rotate_key_phase_aware(REPLACEMENT_KEY);
+ failpoint::remove("after_crypto_meta_write");
+
+ let _cause = assert_structural_reopen(outcome, StructuralCommitPhase::CryptoGeneration);
+ assert_terminal(
+ &engine
+ .put(b"after", b"must-not-append")
+ .expect_err("the next mutation must be blocked"),
+ "mutation after crypto-generation panic",
+ );
+}
+
+#[test]
+fn phase_aware_light_rotation_preserves_committed_after_finalisation_panic() {
+ let _serial = lock();
+ let _clear = ClearOnDrop;
+ failpoint::clear_all();
+
+ let dir = tempfile::tempdir().expect("tempdir");
+ let mut engine = Engine::open_encrypted_with_options(dir.path(), KEY, manual_options()).expect("open encrypted");
+
+ failpoint::set("after_light_rotation_publish", Action::Panic);
+ let outcome = engine.rotate_key_phase_aware(REPLACEMENT_KEY);
+ failpoint::remove("after_light_rotation_publish");
+
+ assert!(
+ matches!(outcome, StructuralCommitOutcome::Committed(_)),
+ "a post-complete light-rotation panic cannot demote the commit: {outcome:?}"
+ );
+ engine
+ .put(b"after", b"still-admitted")
+ .expect("publication authority must not remain poisoned after completion");
+ engine.flush().expect("a later catalogue publication remains healthy");
+}
+
/// Catching an unwind after `crypto.meta` was atomically replaced must not
/// resurrect the same Engine as a writer. The in-process DEK is still valid,
/// which makes this case deceptively survivable; terminality is nevertheless
diff --git a/crates/basemyai-mcp/src/error.rs b/crates/basemyai-mcp/src/error.rs
index 0213e7f..3f42b28 100644
--- a/crates/basemyai-mcp/src/error.rs
+++ b/crates/basemyai-mcp/src/error.rs
@@ -4,6 +4,13 @@
use thiserror::Error;
+// JSON-RPC réserve -32000..=-32099 aux erreurs serveur définies par
+// l'application. Ces valeurs figent les catégories ADR-069 qui n'ont pas
+// d'équivalent dans les codes standard JSON-RPC.
+const UNAUTHENTICATED: rmcp::model::ErrorCode = rmcp::model::ErrorCode(-32016);
+const PERMISSION_DENIED: rmcp::model::ErrorCode = rmcp::model::ErrorCode(-32007);
+const FAILED_PRECONDITION: rmcp::model::ErrorCode = rmcp::model::ErrorCode(-32009);
+
/// Erreur du serveur MCP basemyai.
#[derive(Debug, Error)]
#[non_exhaustive]
@@ -94,6 +101,16 @@ impl From for rmcp::ErrorData {
fn memory_error_data(inner: &basemyai::MemoryError, msg: &str) -> rmcp::ErrorData {
use basemyai::MemoryError as M;
match inner {
+ M::InvalidScope(_) => rmcp::ErrorData::invalid_params("invalid scope identifier", None),
+ M::Unauthenticated => rmcp::ErrorData::new(UNAUTHENTICATED, "authentication required", None),
+ M::UnauthorizedScope => {
+ rmcp::ErrorData::new(PERMISSION_DENIED, "operation not authorized for this scope", None)
+ }
+ M::ScopeOperationUnsupported(_) => rmcp::ErrorData::new(
+ FAILED_PRECONDITION,
+ "scope operation not supported in legacy mode",
+ None,
+ ),
M::MissingAgent
| M::UnknownLayer(_)
| M::Extraction(_)
@@ -161,4 +178,31 @@ mod tests {
assert!(!data.message.is_empty());
assert_ne!(data.message, "internal error");
}
+
+ #[test]
+ fn scope_errors_map_to_stable_json_rpc_codes_without_payload_leaks() {
+ let canary = "private-project/secret-agent";
+ let cases = [
+ (
+ basemyai::MemoryError::InvalidScope(canary.to_string()),
+ rmcp::model::ErrorCode::INVALID_PARAMS,
+ ),
+ (basemyai::MemoryError::Unauthenticated, UNAUTHENTICATED),
+ (basemyai::MemoryError::UnauthorizedScope, PERMISSION_DENIED),
+ (
+ basemyai::MemoryError::ScopeOperationUnsupported(canary.to_string()),
+ FAILED_PRECONDITION,
+ ),
+ ];
+
+ for (memory_error, expected_code) in cases {
+ let data: rmcp::ErrorData = McpError::Memory(memory_error).into();
+ assert_eq!(data.code, expected_code);
+ assert_eq!(data.data, None);
+ assert!(
+ !data.message.contains(canary),
+ "MCP error message leaked the canary payload"
+ );
+ }
+ }
}
diff --git a/crates/basemyai-rest/src/http/error.rs b/crates/basemyai-rest/src/http/error.rs
index cac0a53..0af2287 100644
--- a/crates/basemyai-rest/src/http/error.rs
+++ b/crates/basemyai-rest/src/http/error.rs
@@ -183,6 +183,35 @@ impl RestError {
"plaintext store cannot be encrypted in place".to_string(),
None,
),
+ // ADR-069 §5 : taxonomie d'erreurs d'autorisation observables
+ // sans oracle d'existence. `UnauthorizedScope` ne porte aucun
+ // identifiant de scope étranger dans son message — ne jamais
+ // enrichir ce bras avec `e.to_string()` ou un `details`
+ // dérivé de la requête, sous peine de leak cross-scope.
+ M::InvalidScope(_) => (
+ StatusCode::BAD_REQUEST,
+ "invalid_scope",
+ "invalid scope identifier".to_string(),
+ None,
+ ),
+ M::Unauthenticated => (
+ StatusCode::UNAUTHORIZED,
+ "unauthenticated",
+ "authentication required".to_string(),
+ None,
+ ),
+ M::UnauthorizedScope => (
+ StatusCode::FORBIDDEN,
+ "unauthorized_scope",
+ "operation not authorized for this scope".to_string(),
+ None,
+ ),
+ M::ScopeOperationUnsupported(_) => (
+ StatusCode::CONFLICT,
+ "scope_operation_unsupported",
+ "scope operation not supported in legacy mode".to_string(),
+ None,
+ ),
_ => {
tracing::error!(error = %e, "internal error in REST handler");
(
@@ -283,4 +312,43 @@ mod tests {
let with_field = RestError::Validation("importance must be finite".to_string()).with_field("importance");
assert_eq!(with_field.details, Some(serde_json::json!({ "field": "importance" })));
}
+
+ #[test]
+ fn scope_errors_map_to_stable_statuses_and_codes_without_payload_leaks() {
+ let canary = "private-project/secret-agent";
+ let cases = [
+ (
+ MemoryError::InvalidScope(canary.to_string()),
+ StatusCode::BAD_REQUEST,
+ "invalid_scope",
+ ),
+ (
+ MemoryError::Unauthenticated,
+ StatusCode::UNAUTHORIZED,
+ "unauthenticated",
+ ),
+ (
+ MemoryError::UnauthorizedScope,
+ StatusCode::FORBIDDEN,
+ "unauthorized_scope",
+ ),
+ (
+ MemoryError::ScopeOperationUnsupported(canary.to_string()),
+ StatusCode::CONFLICT,
+ "scope_operation_unsupported",
+ ),
+ ];
+
+ for (memory_error, expected_status, expected_code) in cases {
+ let error = RestError::Memory(memory_error);
+ let (status, code, message, details) = error.parts();
+ assert_eq!(status, expected_status);
+ assert_eq!(code, expected_code);
+ assert_eq!(details, None);
+ assert!(
+ !message.contains(canary),
+ "REST error message leaked the canary payload"
+ );
+ }
+ }
}
diff --git a/crates/basemyai/src/error.rs b/crates/basemyai/src/error.rs
index 6928291..b63f132 100644
--- a/crates/basemyai/src/error.rs
+++ b/crates/basemyai/src/error.rs
@@ -111,7 +111,57 @@ pub enum MemoryError {
/// Limite publique acceptee.
max: usize,
},
+
+ /// Identifiant ou scope de requête mal formé (grammaire/format invalide),
+ /// sans lien avec l'existence ou l'autorisation d'un scope donné
+ /// (ADR-069 §5). Reste générique : ne doit pas répéter tel quel un
+ /// identifiant étranger complet si cela pouvait leaker une info
+ /// cross-scope.
+ #[error("invalid scope identifier")]
+ InvalidScope(String),
+
+ /// Aucun credential fourni, ou credential fourni mais non vérifiable
+ /// (ADR-069 §5). Ne porte aucune donnée : la distinction avec
+ /// `UnauthorizedScope` doit rester binaire côté transport (401 vs 403).
+ #[error("authentication required")]
+ Unauthenticated,
+
+ /// Principal authentifié mais sans le grant exact requis pour le scope
+ /// demandé (ADR-069 §5). Message volontairement générique : ne doit
+ /// jamais révéler l'identifiant du scope demandé ni s'il existe
+ /// réellement — un principal autorisé recevant « absent » et un
+ /// principal non autorisé recevant « interdit » est acceptable, mais
+ /// l'inverse (leak d'existence) ne l'est pas.
+ #[error("operation not authorized for this scope")]
+ UnauthorizedScope,
+
+ /// Scope ou opération non encore activé en mode legacy (ADR-069 §5) :
+ /// limitation de fonctionnalité, pas un refus d'autorisation. Le payload
+ /// reste réservé au diagnostic interne : `Display` est volontairement
+ /// constant afin qu'un binding qui stringify l'erreur ne le divulgue pas.
+ #[error("scope operation not supported in legacy mode")]
+ ScopeOperationUnsupported(String),
}
/// Alias de résultat de la couche mémoire.
pub type Result = core::result::Result;
+
+#[cfg(test)]
+mod tests {
+ use super::MemoryError;
+
+ #[test]
+ fn scope_error_display_does_not_expose_diagnostic_payloads() {
+ let canary = "private-project/secret-agent";
+
+ for error in [
+ MemoryError::InvalidScope(canary.to_string()),
+ MemoryError::ScopeOperationUnsupported(canary.to_string()),
+ ] {
+ assert!(
+ !error.to_string().contains(canary),
+ "scope error Display must not expose its diagnostic payload"
+ );
+ }
+ }
+}
diff --git a/crates/basemyai/src/lib.rs b/crates/basemyai/src/lib.rs
index 1eba82e..0e6f900 100644
--- a/crates/basemyai/src/lib.rs
+++ b/crates/basemyai/src/lib.rs
@@ -21,6 +21,7 @@ pub mod maintenance;
mod memory;
pub mod provision;
mod retrieval;
+mod runtime;
pub mod storage;
pub mod temporal;
@@ -46,9 +47,9 @@ pub use maintenance::{
#[cfg(feature = "test-util")]
pub use memory::HashEmbedder;
pub use memory::{
- AgentId, AgentStats, ConversationTurn, ImportReport, MAX_TEXT_LEN, Memory, MemoryEvent, MemoryEventKind,
- MemoryLayer, MemorySubscription, RecallOptions, Record, SOURCE_CONSOLIDATION, SOURCE_IMPORT, SOURCE_USER,
- TrustLevel,
+ AgentId, AgentStats, CallerPrincipal, ConversationTurn, ImportReport, MAX_TEXT_LEN, Memory, MemoryEvent,
+ MemoryEventKind, MemoryId, MemoryLayer, MemoryScope, MemorySubscription, PrincipalIssuer, PrincipalSubject,
+ ProjectId, QualifiedMemoryId, RecallOptions, Record, SOURCE_CONSOLIDATION, SOURCE_IMPORT, SOURCE_USER, TrustLevel,
};
pub use storage::BMAI_FORMAT_VERSION;
diff --git a/crates/basemyai/src/maintenance/epoch.rs b/crates/basemyai/src/maintenance/epoch.rs
new file mode 100644
index 0000000..3d2bfcc
--- /dev/null
+++ b/crates/basemyai/src/maintenance/epoch.rs
@@ -0,0 +1,296 @@
+// SPDX-License-Identifier: BUSL-1.1
+//! Dormant V2-03 maintenance planning preconditions (ADR-070 §9).
+//!
+//! These are value types only: this module deliberately owns no atomic
+//! counter and does not wrap [`crate::storage::MemoryStore`]. The live V1
+//! store cannot yet provide a scope mutation epoch or an exact per-member
+//! version, and incrementing a second counter outside the product writer's
+//! publication gate would create the very second authority ADR-070 forbids.
+//!
+//! At cutover the coordinator owns the epoch and increments it under the
+//! same exclusive product gate as every successful logical publication. A
+//! planner then reads `epoch_before`, performs all of its bounded scans,
+//! reads `epoch_after`, and may emit exactly one [`MaintenanceChunk`] only
+//! when both values match. The handler revalidates that epoch and every
+//! member version before staging a deletion. Until those two engine-bound
+//! seams exist, no live adaptive-forgetting or expired-GC path constructs
+//! these types or claims phantom protection.
+
+use std::collections::HashSet;
+
+/// Runtime-local mutation generation for one exact semantic scope.
+///
+/// It is intentionally neither serializable nor public. Reopen starts a new
+/// runtime-local lineage; this value is not an idempotency key.
+#[derive(Clone, Copy, Debug, Eq, PartialEq)]
+pub(crate) struct ScopeMutationEpoch(u64);
+
+impl ScopeMutationEpoch {
+ pub(crate) const INITIAL: Self = Self(0);
+
+ pub(crate) const fn from_raw(value: u64) -> Self {
+ Self(value)
+ }
+
+ pub(crate) const fn raw(self) -> u64 {
+ self.0
+ }
+
+ pub(crate) fn checked_next(self) -> Option {
+ self.0.checked_add(1).map(Self)
+ }
+}
+
+/// Exact version observed for a member while planning.
+///
+/// The eventual native adapter derives this opaque value from the same
+/// coherent read state used to hydrate the member. It must change whenever a
+/// mutation can invalidate the planner's decision; timestamps or caller ids
+/// are not substitutes.
+#[derive(Clone, Copy, Debug, Eq, PartialEq)]
+pub(crate) struct MaintenanceMemberVersion(u64);
+
+impl MaintenanceMemberVersion {
+ pub(crate) const fn from_raw(value: u64) -> Self {
+ Self(value)
+ }
+}
+
+/// One member and the exact version used by the maintenance decision.
+#[derive(Clone, Debug, Eq, PartialEq)]
+pub(crate) struct VersionedMaintenanceMember {
+ id: String,
+ version: MaintenanceMemberVersion,
+}
+
+impl VersionedMaintenanceMember {
+ pub(crate) fn new(id: String, version: MaintenanceMemberVersion) -> Option {
+ (!id.is_empty()).then_some(Self { id, version })
+ }
+
+ pub(crate) fn id(&self) -> &str {
+ &self.id
+ }
+
+ pub(crate) const fn version(&self) -> MaintenanceMemberVersion {
+ self.version
+ }
+}
+
+/// Closed maintenance classes that currently produce deletion chunks.
+#[derive(Clone, Copy, Debug, Eq, PartialEq)]
+pub(crate) enum MaintenancePlanKind {
+ AdaptiveForgetting,
+ ExpiredGc,
+}
+
+#[derive(Clone, Copy, Debug, Eq, PartialEq)]
+pub(crate) enum MaintenancePlanError {
+ ZeroCapacity,
+ EpochChanged,
+ DuplicateMember,
+ MemberSetChanged,
+ MemberVersionChanged,
+}
+
+/// Bounded builder kept across the planner's scans.
+///
+/// Once full it ignores later victims but the planner must still finish its
+/// scan and call [`Self::finish`] with `epoch_after`. Thus memory remains
+/// `O(max_items)` without weakening phantom detection.
+pub(crate) struct MaintenanceChunkBuilder {
+ kind: MaintenancePlanKind,
+ epoch_before: ScopeMutationEpoch,
+ max_items: usize,
+ members: Vec,
+ selected_ids: HashSet,
+}
+
+impl MaintenanceChunkBuilder {
+ pub(crate) fn new(
+ kind: MaintenancePlanKind,
+ epoch_before: ScopeMutationEpoch,
+ max_items: usize,
+ ) -> Result {
+ if max_items == 0 {
+ return Err(MaintenancePlanError::ZeroCapacity);
+ }
+ Ok(Self {
+ kind,
+ epoch_before,
+ max_items,
+ members: Vec::with_capacity(max_items.min(256)),
+ selected_ids: HashSet::with_capacity(max_items.min(256)),
+ })
+ }
+
+ /// Offers one victim selected from the coherent scan. Returns `true`
+ /// when retained in this chunk and `false` once the one-chunk bound is
+ /// full. Duplicate selected ids are rejected rather than silently
+ /// weakening the read-set.
+ pub(crate) fn offer(&mut self, member: VersionedMaintenanceMember) -> Result {
+ if self.members.len() == self.max_items {
+ return Ok(false);
+ }
+ if !self.selected_ids.insert(member.id.clone()) {
+ return Err(MaintenancePlanError::DuplicateMember);
+ }
+ self.members.push(member);
+ Ok(true)
+ }
+
+ /// Seals at most one chunk only after the planner's final epoch read.
+ pub(crate) fn finish(
+ self,
+ epoch_after: ScopeMutationEpoch,
+ ) -> Result, MaintenancePlanError> {
+ if epoch_after != self.epoch_before {
+ return Err(MaintenancePlanError::EpochChanged);
+ }
+ if self.members.is_empty() {
+ return Ok(None);
+ }
+ Ok(Some(MaintenanceChunk {
+ kind: self.kind,
+ expected_epoch: self.epoch_before,
+ members: self.members,
+ }))
+ }
+}
+
+/// One bounded, runtime-local maintenance intent payload.
+#[derive(Debug, Eq, PartialEq)]
+pub(crate) struct MaintenanceChunk {
+ kind: MaintenancePlanKind,
+ expected_epoch: ScopeMutationEpoch,
+ members: Vec,
+}
+
+impl MaintenanceChunk {
+ pub(crate) const fn kind(&self) -> MaintenancePlanKind {
+ self.kind
+ }
+
+ pub(crate) const fn expected_epoch(&self) -> ScopeMutationEpoch {
+ self.expected_epoch
+ }
+
+ pub(crate) fn members(&self) -> &[VersionedMaintenanceMember] {
+ &self.members
+ }
+
+ /// Revalidates immediately before commit under the coordinator's product
+ /// gate. `current` is the exact, freshly hydrated read-set for the ids in
+ /// this chunk; missing, extra, duplicate or changed members abort it.
+ pub(crate) fn revalidate(
+ &self,
+ current_epoch: ScopeMutationEpoch,
+ current: &[VersionedMaintenanceMember],
+ ) -> Result<(), MaintenancePlanError> {
+ if current_epoch != self.expected_epoch {
+ return Err(MaintenancePlanError::EpochChanged);
+ }
+ if current.len() != self.members.len() {
+ return Err(MaintenancePlanError::MemberSetChanged);
+ }
+ let mut seen = HashSet::with_capacity(current.len());
+ for expected in &self.members {
+ let Some(actual) = current.iter().find(|member| member.id == expected.id) else {
+ return Err(MaintenancePlanError::MemberSetChanged);
+ };
+ if !seen.insert(actual.id.as_str()) {
+ return Err(MaintenancePlanError::MemberSetChanged);
+ }
+ if actual.version != expected.version {
+ return Err(MaintenancePlanError::MemberVersionChanged);
+ }
+ }
+ Ok(())
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ fn member(id: &str, version: u64) -> VersionedMaintenanceMember {
+ VersionedMaintenanceMember::new(id.to_owned(), MaintenanceMemberVersion::from_raw(version))
+ .expect("non-empty test member id")
+ }
+
+ #[test]
+ fn changed_epoch_discards_the_entire_planned_chunk() {
+ let before = ScopeMutationEpoch::from_raw(7);
+ let mut builder =
+ MaintenanceChunkBuilder::new(MaintenancePlanKind::ExpiredGc, before, 2).expect("non-zero bound");
+ assert!(builder.offer(member("a", 1)).expect("first member"));
+
+ assert_eq!(
+ builder.finish(ScopeMutationEpoch::from_raw(8)),
+ Err(MaintenancePlanError::EpochChanged)
+ );
+ }
+
+ #[test]
+ fn planner_emits_only_one_bounded_chunk_but_still_seals_after_scan() {
+ let epoch = ScopeMutationEpoch::INITIAL;
+ let mut builder =
+ MaintenanceChunkBuilder::new(MaintenancePlanKind::AdaptiveForgetting, epoch, 2).expect("non-zero bound");
+ assert!(builder.offer(member("a", 1)).expect("first member"));
+ assert!(builder.offer(member("b", 2)).expect("second member"));
+ assert!(!builder.offer(member("c", 3)).expect("full chunk ignores later victim"));
+
+ let chunk = builder.finish(epoch).expect("stable scan").expect("non-empty chunk");
+ assert_eq!(chunk.kind(), MaintenancePlanKind::AdaptiveForgetting);
+ assert_eq!(chunk.expected_epoch().raw(), 0);
+ assert_eq!(chunk.members(), &[member("a", 1), member("b", 2)]);
+ }
+
+ #[test]
+ fn duplicate_selected_member_is_rejected() {
+ let epoch = ScopeMutationEpoch::INITIAL;
+ let mut builder =
+ MaintenanceChunkBuilder::new(MaintenancePlanKind::ExpiredGc, epoch, 2).expect("non-zero bound");
+ assert!(builder.offer(member("a", 1)).expect("first member"));
+ assert_eq!(
+ builder.offer(member("a", 1)),
+ Err(MaintenancePlanError::DuplicateMember)
+ );
+ }
+
+ #[test]
+ fn handler_revalidation_checks_epoch_set_and_exact_versions() {
+ let epoch = ScopeMutationEpoch::from_raw(11);
+ let mut builder =
+ MaintenanceChunkBuilder::new(MaintenancePlanKind::ExpiredGc, epoch, 2).expect("non-zero bound");
+ assert!(builder.offer(member("a", 4)).expect("first member"));
+ assert!(builder.offer(member("b", 5)).expect("second member"));
+ let chunk = builder.finish(epoch).expect("stable scan").expect("chunk");
+
+ assert_eq!(chunk.revalidate(epoch, &[member("b", 5), member("a", 4)]), Ok(()));
+ assert_eq!(
+ chunk.revalidate(epoch, &[member("a", 4)]),
+ Err(MaintenancePlanError::MemberSetChanged)
+ );
+ assert_eq!(
+ chunk.revalidate(epoch, &[member("a", 9), member("b", 5)]),
+ Err(MaintenancePlanError::MemberVersionChanged)
+ );
+ assert_eq!(
+ chunk.revalidate(ScopeMutationEpoch::from_raw(12), &[member("a", 4), member("b", 5)]),
+ Err(MaintenancePlanError::EpochChanged)
+ );
+ }
+
+ #[test]
+ fn epoch_overflow_is_explicit_and_empty_plan_is_noop() {
+ assert_eq!(ScopeMutationEpoch::from_raw(u64::MAX).checked_next(), None);
+ let epoch = ScopeMutationEpoch::INITIAL;
+ let builder = MaintenanceChunkBuilder::new(MaintenancePlanKind::ExpiredGc, epoch, 1).expect("non-zero bound");
+ assert_eq!(builder.finish(epoch), Ok(None));
+ assert!(matches!(
+ MaintenanceChunkBuilder::new(MaintenancePlanKind::ExpiredGc, epoch, 0),
+ Err(MaintenancePlanError::ZeroCapacity)
+ ));
+ }
+}
diff --git a/crates/basemyai/src/maintenance/mod.rs b/crates/basemyai/src/maintenance/mod.rs
index 65432a6..f913cdc 100644
--- a/crates/basemyai/src/maintenance/mod.rs
+++ b/crates/basemyai/src/maintenance/mod.rs
@@ -20,6 +20,7 @@
//! store partagé injecté par le worker.
pub(crate) mod adaptive_forgetting;
+pub(crate) mod epoch;
pub(crate) mod expired_gc;
pub use adaptive_forgetting::{
diff --git a/crates/basemyai/src/memory/isolation.rs b/crates/basemyai/src/memory/isolation.rs
index 53d0f10..3331822 100644
--- a/crates/basemyai/src/memory/isolation.rs
+++ b/crates/basemyai/src/memory/isolation.rs
@@ -1,18 +1,90 @@
// SPDX-License-Identifier: BUSL-1.1
-//! Isolation multi-agent. Chaque ligne porte un `agent_id` ; **toute** lecture
-//! et écriture sont filtrées par lui **au niveau SQL** (ADR-006). Une fuite
-//! cross-agent est un incident de sécurité, pas un bug fonctionnel.
+//! Isolation sémantique des namespaces mémoire. Les clés legacy natives portent
+//! encore un `agent_id`; toute fuite cross-agent reste un incident de sécurité,
+//! pas un bug fonctionnel. L'isolation physique multi-project arrive au hard
+//! cut V2-08 et n'est pas revendiquée par cette tranche.
+//!
+//! ADR-069 ajoute les types sémantiques canoniques Project/Agent/MemoryScope
+//! (§1) : `ProjectId`, `MemoryId`, `MemoryScope`, `QualifiedMemoryId`,
+//! `PrincipalIssuer`, `PrincipalSubject` et `CallerPrincipal`. Cette tranche
+//! (V2-02A) fige la sémantique et les frontières d'autorité ; elle n'introduit
+//! aucun format physique, aucune isolation vectorielle structurelle et aucun
+//! `ScopeKey` (cf. ADR-069 §8-9).
-/// Identifiant d'agent (tenant logique d'une mémoire).
+/// Grammaire ASCII partagée par `AgentId` et `ProjectId` (ADR-069 §1) :
+/// `[A-Za-z0-9][A-Za-z0-9._-]{0,127}`, 1..=128 octets.
+fn is_valid_ascii_namespace_grammar(s: &str) -> bool {
+ let bytes = s.as_bytes();
+ if bytes.is_empty() || bytes.len() > 128 {
+ return false;
+ }
+ let first = bytes[0];
+ if !first.is_ascii_alphanumeric() {
+ return false;
+ }
+ bytes[1..]
+ .iter()
+ .all(|b| b.is_ascii_alphanumeric() || matches!(b, b'.' | b'_' | b'-'))
+}
+
+/// Validation partagée par `MemoryId`, `PrincipalIssuer` et `PrincipalSubject`
+/// (ADR-069 §1) : UTF-8 non vide dans `1..=max_bytes` octets, sans NUL ni
+/// caractère de contrôle. Aucune normalisation Unicode ni case folding.
+fn is_valid_opaque_utf8(s: &str, max_bytes: usize) -> bool {
+ let len = s.len();
+ if len == 0 || len > max_bytes {
+ return false;
+ }
+ !s.chars().any(|c| c.is_control())
+}
+
+/// Identifiant de projet (namespace produit, ADR-069 §1). Grammaire ASCII
+/// `[A-Za-z0-9][A-Za-z0-9._-]{0,127}`, 1..=128 octets. Égalité byte-exacte et
+/// sensible à la casse (dérivée). Construire un `ProjectId` n'accorde aucun
+/// droit — cf. `MemoryScope` et l'authorizer pour l'autorisation.
+#[derive(Debug, Clone, PartialEq, Eq, Hash)]
+pub struct ProjectId(String);
+
+impl ProjectId {
+ /// Construit un `ProjectId`. `None` si la grammaire ASCII n'est pas
+ /// respectée (vide, > 128 octets, premier caractère non alphanumérique,
+ /// caractère hors `[A-Za-z0-9._-]`).
+ #[must_use]
+ pub fn new(id: impl Into) -> Option {
+ let id = id.into();
+ if is_valid_ascii_namespace_grammar(&id) {
+ Some(Self(id))
+ } else {
+ None
+ }
+ }
+
+ #[must_use]
+ pub fn as_str(&self) -> &str {
+ &self.0
+ }
+}
+
+/// Identifiant d'agent (tenant logique d'une mémoire). Grammaire ASCII
+/// `[A-Za-z0-9][A-Za-z0-9._-]{0,127}`, 1..=128 octets (ADR-069 §1, resserre
+/// l'ancienne validation "non vide"). Égalité byte-exacte et sensible à la
+/// casse (dérivée). Construire un `AgentId` n'accorde aucun droit — le même
+/// `AgentId` dans deux `ProjectId` désigne deux scopes différents.
#[derive(Debug, Clone, PartialEq, Eq, Hash)]
pub struct AgentId(String);
impl AgentId {
- /// Construit un `AgentId`. Vide => `None` (un agent valide est requis).
+ /// Construit un `AgentId`. `None` si la grammaire ASCII n'est pas
+ /// respectée (vide, > 128 octets, premier caractère non alphanumérique,
+ /// caractère hors `[A-Za-z0-9._-]`).
#[must_use]
pub fn new(id: impl Into) -> Option {
let id = id.into();
- if id.is_empty() { None } else { Some(Self(id)) }
+ if is_valid_ascii_namespace_grammar(&id) {
+ Some(Self(id))
+ } else {
+ None
+ }
}
#[must_use]
@@ -20,3 +92,431 @@ impl AgentId {
&self.0
}
}
+
+/// Identifiant de mémoire (ADR-069 §1). UTF-8, 1..=512 octets (mesurés sur la
+/// représentation UTF-8, pas en nombre de caractères), sans NUL ni caractère
+/// de contrôle. Unique seulement dans son `MemoryScope` — toute référence
+/// transverse utilise `QualifiedMemoryId`.
+#[derive(Debug, Clone, PartialEq, Eq, Hash)]
+pub struct MemoryId(String);
+
+impl MemoryId {
+ /// Construit un `MemoryId`. `None` si la chaîne est vide, dépasse 512
+ /// octets, ou contient un NUL/caractère de contrôle.
+ #[must_use]
+ pub fn new(id: impl Into) -> Option {
+ let id = id.into();
+ if is_valid_opaque_utf8(&id, 512) {
+ Some(Self(id))
+ } else {
+ None
+ }
+ }
+
+ #[must_use]
+ pub fn as_str(&self) -> &str {
+ &self.0
+ }
+}
+
+/// Scope exact d'une mémoire (ADR-069 §1). Il n'existe aucun scope global :
+/// `Private(project, agent)` appartient à un agent dans un projet précis (le
+/// même `AgentId` dans deux projets désigne deux scopes différents) ;
+/// `Project(project)` appartient au projet, jamais à un agent implicite.
+///
+/// `#[non_exhaustive]` : de nouvelles variantes de scope pourront être
+/// ajoutées par un futur ADR sans casser les consommateurs existants.
+#[non_exhaustive]
+#[derive(Debug, Clone, PartialEq, Eq, Hash)]
+pub enum MemoryScope {
+ /// Scope privé : appartient à `agent` au sein de `project`.
+ Private { project: ProjectId, agent: AgentId },
+ /// Scope projet : appartient au projet, sans agent implicite.
+ Project { project: ProjectId },
+}
+
+/// Identifiant de mémoire qualifié par son scope (ADR-069 §1). Un
+/// `MemoryId` nu n'est unique que dans son scope ; `QualifiedMemoryId` porte
+/// les deux et sert de référence transverse.
+#[derive(Debug, Clone, PartialEq, Eq, Hash)]
+pub struct QualifiedMemoryId {
+ pub scope: MemoryScope,
+ pub memory_id: MemoryId,
+}
+
+/// Issuer opaque d'un credential authentifié (ADR-069 §1-2). UTF-8, 1..=256
+/// octets, sans NUL ni caractère de contrôle. Comparaison byte-exacte ; le
+/// runtime BaseMyAI ne renormalise jamais issuer/subject après coup — la
+/// normalisation spécifique au protocole (OIDC, etc.) est la responsabilité
+/// de l'adaptateur transport avant construction d'un `CallerPrincipal`.
+#[derive(Debug, Clone, PartialEq, Eq, Hash)]
+pub struct PrincipalIssuer(String);
+
+impl PrincipalIssuer {
+ /// Construit un `PrincipalIssuer`. `None` si la chaîne est vide, dépasse
+ /// 256 octets, ou contient un NUL/caractère de contrôle.
+ #[must_use]
+ pub fn new(id: impl Into) -> Option {
+ let id = id.into();
+ if is_valid_opaque_utf8(&id, 256) {
+ Some(Self(id))
+ } else {
+ None
+ }
+ }
+
+ #[must_use]
+ pub fn as_str(&self) -> &str {
+ &self.0
+ }
+}
+
+/// Subject opaque d'un credential authentifié (ADR-069 §1-2). Mêmes bornes et
+/// discipline de validation que `PrincipalIssuer`.
+#[derive(Debug, Clone, PartialEq, Eq, Hash)]
+pub struct PrincipalSubject(String);
+
+impl PrincipalSubject {
+ /// Construit un `PrincipalSubject`. `None` si la chaîne est vide, dépasse
+ /// 256 octets, ou contient un NUL/caractère de contrôle.
+ #[must_use]
+ pub fn new(id: impl Into) -> Option {
+ let id = id.into();
+ if is_valid_opaque_utf8(&id, 256) {
+ Some(Self(id))
+ } else {
+ None
+ }
+ }
+
+ #[must_use]
+ pub fn as_str(&self) -> &str {
+ &self.0
+ }
+}
+
+/// Identité authentifiée du caller sous la paire opaque `(issuer, subject)`
+/// (ADR-069 §2). Ce n'est ni un namespace, ni une clé persistée, ni un
+/// porteur de grants, ni une preuve cryptographique interprétée par
+/// l'engine.
+///
+/// Champs privés par construction : seul un adaptateur de transport ayant
+/// effectué les vérifications applicables au credential (ou une autorité
+/// opérateur locale de confiance pour stdio/in-process) peut construire un
+/// `CallerPrincipal`. Un body REST, argument MCP ou DTO de binding ne peut
+/// jamais en fournir un directement.
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub struct CallerPrincipal {
+ issuer: PrincipalIssuer,
+ subject: PrincipalSubject,
+}
+
+impl CallerPrincipal {
+ /// Construit un `CallerPrincipal` à partir d'un credential déjà vérifié.
+ ///
+ /// # Invariant appelant
+ ///
+ /// L'appelant de cette fonction doit avoir déjà effectué les
+ /// vérifications applicables au credential (signature, expiration,
+ /// format spécifique au protocole d'authentification, etc.) — ce
+ /// constructeur ne réalise lui-même aucune vérification cryptographique,
+ /// ce n'est pas le rôle de ce type. Ce nom explicite marque la frontière
+ /// de confiance : seul un adaptateur de transport vérifié ou une
+ /// autorité opérateur locale de confiance doit l'appeler (ADR-069 §2).
+ #[must_use]
+ pub fn from_verified_credential(issuer: PrincipalIssuer, subject: PrincipalSubject) -> Self {
+ Self { issuer, subject }
+ }
+
+ #[must_use]
+ pub fn issuer(&self) -> &PrincipalIssuer {
+ &self.issuer
+ }
+
+ #[must_use]
+ pub fn subject(&self) -> &PrincipalSubject {
+ &self.subject
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ // --- AgentId / ProjectId grammar ---
+
+ #[test]
+ fn agent_id_rejects_empty() {
+ assert!(AgentId::new("").is_none());
+ }
+
+ #[test]
+ fn project_id_rejects_empty() {
+ assert!(ProjectId::new("").is_none());
+ }
+
+ #[test]
+ fn agent_id_rejects_non_alphanumeric_first_char() {
+ assert!(AgentId::new("-agent").is_none());
+ assert!(AgentId::new(".agent").is_none());
+ assert!(AgentId::new("_agent").is_none());
+ }
+
+ #[test]
+ fn agent_id_rejects_disallowed_chars() {
+ assert!(AgentId::new("agent x").is_none());
+ assert!(AgentId::new("agent/x").is_none());
+ assert!(AgentId::new("agent@x").is_none());
+ }
+
+ #[test]
+ fn agent_id_accepts_valid_grammar() {
+ assert!(AgentId::new("agent-1").is_some());
+ assert!(AgentId::new("Agent_1.2-3").is_some());
+ assert!(AgentId::new("a").is_some());
+ }
+
+ #[test]
+ fn project_id_accepts_valid_grammar() {
+ assert!(ProjectId::new("project-1").is_some());
+ assert!(ProjectId::new("P1").is_some());
+ }
+
+ #[test]
+ fn namespace_ids_reject_non_ascii_and_are_case_sensitive() {
+ assert!(AgentId::new("agent-é").is_none());
+ assert!(ProjectId::new("équipe").is_none());
+
+ let lower_agent = AgentId::new("agent-a").expect("valid lowercase agent id");
+ let upper_agent = AgentId::new("Agent-a").expect("valid uppercase agent id");
+ assert_ne!(lower_agent, upper_agent);
+
+ let lower_project = ProjectId::new("project-a").expect("valid lowercase project id");
+ let upper_project = ProjectId::new("Project-a").expect("valid uppercase project id");
+ assert_ne!(lower_project, upper_project);
+ }
+
+ #[test]
+ fn agent_id_accepts_exactly_128_bytes() {
+ let id = format!("a{}", "b".repeat(127));
+ assert_eq!(id.len(), 128);
+ assert!(AgentId::new(id).is_some());
+ }
+
+ #[test]
+ fn agent_id_rejects_129_bytes() {
+ let id = format!("a{}", "b".repeat(128));
+ assert_eq!(id.len(), 129);
+ assert!(AgentId::new(id).is_none());
+ }
+
+ #[test]
+ fn project_id_accepts_exactly_128_bytes() {
+ let id = format!("p{}", "q".repeat(127));
+ assert_eq!(id.len(), 128);
+ assert!(ProjectId::new(id).is_some());
+ }
+
+ #[test]
+ fn project_id_rejects_129_bytes() {
+ let id = format!("p{}", "q".repeat(128));
+ assert_eq!(id.len(), 129);
+ assert!(ProjectId::new(id).is_none());
+ }
+
+ // --- MemoryId ---
+
+ #[test]
+ fn memory_id_rejects_nul() {
+ assert!(MemoryId::new("abc\0def").is_none());
+ }
+
+ #[test]
+ fn memory_id_rejects_control_char() {
+ assert!(MemoryId::new("abc\ndef").is_none());
+ assert!(MemoryId::new("abc\tdef").is_none());
+ assert!(MemoryId::new("abc\u{7f}def").is_none());
+ }
+
+ #[test]
+ fn memory_id_accepts_512_bytes() {
+ let id = "x".repeat(512);
+ assert_eq!(id.len(), 512);
+ assert!(MemoryId::new(id).is_some());
+ }
+
+ #[test]
+ fn memory_id_rejects_513_bytes() {
+ let id = "x".repeat(513);
+ assert_eq!(id.len(), 513);
+ assert!(MemoryId::new(id).is_none());
+ }
+
+ #[test]
+ fn memory_id_counts_multibyte_utf8_in_bytes_not_chars() {
+ // 'é' encodes as 2 bytes in UTF-8; 300 of them = 600 bytes > 512.
+ let id = "é".repeat(300);
+ assert_eq!(id.chars().count(), 300);
+ assert!(id.len() > 512);
+ assert!(MemoryId::new(id).is_none());
+
+ // 250 of them = 500 bytes <= 512, should be accepted.
+ let id_ok = "é".repeat(250);
+ assert_eq!(id_ok.len(), 500);
+ assert!(MemoryId::new(id_ok).is_some());
+ }
+
+ #[test]
+ fn memory_id_rejects_empty() {
+ assert!(MemoryId::new("").is_none());
+ }
+
+ // --- MemoryScope ---
+
+ #[test]
+ fn memory_scope_private_differs_by_project() {
+ let agent = AgentId::new("agent-x").expect("valid agent id");
+ let p1 = ProjectId::new("p1").expect("valid project id");
+ let p2 = ProjectId::new("p2").expect("valid project id");
+ let scope1 = MemoryScope::Private {
+ project: p1,
+ agent: agent.clone(),
+ };
+ let scope2 = MemoryScope::Private { project: p2, agent };
+ assert_ne!(scope1, scope2);
+ }
+
+ #[test]
+ fn memory_scope_private_differs_from_project() {
+ let agent = AgentId::new("agent-x").expect("valid agent id");
+ let project = ProjectId::new("p1").expect("valid project id");
+ let private = MemoryScope::Private {
+ project: project.clone(),
+ agent,
+ };
+ let project_scope = MemoryScope::Project { project };
+ assert_ne!(private, project_scope);
+ }
+
+ // --- QualifiedMemoryId ---
+
+ #[test]
+ fn qualified_memory_id_equality_by_scope_and_id() {
+ let project = ProjectId::new("p1").expect("valid project id");
+ let scope = MemoryScope::Project {
+ project: project.clone(),
+ };
+ let memory_id = MemoryId::new("mem-1").expect("valid memory id");
+ let q1 = QualifiedMemoryId {
+ scope: scope.clone(),
+ memory_id: memory_id.clone(),
+ };
+ let q2 = QualifiedMemoryId { scope, memory_id };
+ assert_eq!(q1, q2);
+
+ let other_memory_id = MemoryId::new("mem-2").expect("valid memory id");
+ let q3 = QualifiedMemoryId {
+ scope: MemoryScope::Project { project },
+ memory_id: other_memory_id,
+ };
+ assert_ne!(q1, q3);
+ }
+
+ #[test]
+ fn qualified_memory_id_differs_by_scope() {
+ let memory_id = MemoryId::new("mem-1").expect("valid memory id");
+ let project = ProjectId::new("p1").expect("valid project id");
+ let agent = AgentId::new("agent-x").expect("valid agent id");
+ let q_private = QualifiedMemoryId {
+ scope: MemoryScope::Private {
+ project: project.clone(),
+ agent,
+ },
+ memory_id: memory_id.clone(),
+ };
+ let q_project = QualifiedMemoryId {
+ scope: MemoryScope::Project { project },
+ memory_id,
+ };
+ assert_ne!(q_private, q_project);
+ }
+
+ // --- PrincipalIssuer / PrincipalSubject ---
+
+ #[test]
+ fn principal_issuer_rejects_empty_nul_and_control() {
+ assert!(PrincipalIssuer::new("").is_none());
+ assert!(PrincipalIssuer::new("abc\0def").is_none());
+ assert!(PrincipalIssuer::new("abc\ndef").is_none());
+ }
+
+ #[test]
+ fn principal_issuer_accepts_256_bytes_rejects_257() {
+ let ok = "x".repeat(256);
+ assert!(PrincipalIssuer::new(ok).is_some());
+ let too_long = "x".repeat(257);
+ assert!(PrincipalIssuer::new(too_long).is_none());
+ }
+
+ #[test]
+ fn principal_subject_rejects_empty_nul_and_control() {
+ assert!(PrincipalSubject::new("").is_none());
+ assert!(PrincipalSubject::new("abc\0def").is_none());
+ assert!(PrincipalSubject::new("abc\ndef").is_none());
+ }
+
+ #[test]
+ fn principal_subject_accepts_256_bytes_rejects_257() {
+ let ok = "x".repeat(256);
+ assert!(PrincipalSubject::new(ok).is_some());
+ let too_long = "x".repeat(257);
+ assert!(PrincipalSubject::new(too_long).is_none());
+ }
+
+ #[test]
+ fn principal_bounds_are_measured_in_utf8_bytes() {
+ let exact = "é".repeat(128);
+ assert_eq!(exact.len(), 256);
+ assert!(PrincipalIssuer::new(exact.clone()).is_some());
+ assert!(PrincipalSubject::new(exact).is_some());
+
+ let too_long = "é".repeat(129);
+ assert_eq!(too_long.len(), 258);
+ assert!(PrincipalIssuer::new(too_long.clone()).is_none());
+ assert!(PrincipalSubject::new(too_long).is_none());
+ }
+
+ // --- CallerPrincipal ---
+
+ #[test]
+ fn caller_principal_equal_when_issuer_and_subject_match() {
+ let issuer = PrincipalIssuer::new("issuer-a").expect("valid issuer");
+ let subject = PrincipalSubject::new("subject-a").expect("valid subject");
+ let p1 = CallerPrincipal::from_verified_credential(issuer.clone(), subject.clone());
+ let p2 = CallerPrincipal::from_verified_credential(issuer, subject);
+ assert_eq!(p1, p2);
+ }
+
+ #[test]
+ fn caller_principal_differs_by_issuer_or_subject() {
+ let issuer_a = PrincipalIssuer::new("issuer-a").expect("valid issuer");
+ let issuer_b = PrincipalIssuer::new("issuer-b").expect("valid issuer");
+ let subject = PrincipalSubject::new("subject-a").expect("valid subject");
+ let p1 = CallerPrincipal::from_verified_credential(issuer_a.clone(), subject.clone());
+ let p2 = CallerPrincipal::from_verified_credential(issuer_b, subject);
+ assert_ne!(p1, p2);
+
+ let subject_b = PrincipalSubject::new("subject-b").expect("valid subject");
+ let p3 = CallerPrincipal::from_verified_credential(issuer_a, subject_b);
+ assert_ne!(p1, p3);
+ }
+
+ #[test]
+ fn caller_principal_getters_return_expected_values() {
+ let issuer = PrincipalIssuer::new("issuer-a").expect("valid issuer");
+ let subject = PrincipalSubject::new("subject-a").expect("valid subject");
+ let principal = CallerPrincipal::from_verified_credential(issuer.clone(), subject.clone());
+ assert_eq!(principal.issuer(), &issuer);
+ assert_eq!(principal.subject(), &subject);
+ }
+}
diff --git a/crates/basemyai/src/memory/mod.rs b/crates/basemyai/src/memory/mod.rs
index 86d48e2..152dc16 100644
--- a/crates/basemyai/src/memory/mod.rs
+++ b/crates/basemyai/src/memory/mod.rs
@@ -12,7 +12,9 @@ mod testutil;
mod trust;
pub use event::{MemoryEvent, MemoryEventKind, MemorySubscription};
-pub use isolation::AgentId;
+pub use isolation::{
+ AgentId, CallerPrincipal, MemoryId, MemoryScope, PrincipalIssuer, PrincipalSubject, ProjectId, QualifiedMemoryId,
+};
pub use layer::{AgentStats, MemoryLayer, Record};
pub use porting::ImportReport;
#[cfg(feature = "test-util")]
@@ -1109,6 +1111,32 @@ mod tests {
Arc::new(NativeMemoryStore::open_ephemeral().expect("open ephemeral store"))
}
+ /// Dropping the last writer handle is deliberately non-blocking — the OS
+ /// file lock is released by a detached reaper thread, not synchronously
+ /// by `Drop` itself (see
+ /// `storage::native_store::coordinator::tests::
+ /// dropping_last_writer_handle_is_non_blocking_and_reaper_releases_the_store`).
+ /// A reopen immediately after `drop` can therefore transiently observe
+ /// `StoreLocked` — retry for a bounded window instead of asserting on the
+ /// very first attempt.
+ async fn retry_after_drop(mut attempt: F) -> Result
+ where
+ F: FnMut() -> Fut,
+ Fut: std::future::Future