diff --git a/src/transaction/freemap.rs b/src/transaction/freemap.rs index 4c01f02..ba27d55 100644 --- a/src/transaction/freemap.rs +++ b/src/transaction/freemap.rs @@ -1,8 +1,19 @@ //! transaction::freemap — freemap staging and structural-page recycling: //! the freemap-aware COW allocator (`cow_alloc` / `structural_extend`), -//! `persist_freemap`, orphan reclamation, and the data-page / handle-table -//! COW allocation paths. Split out of `transaction.rs` verbatim; see the -//! parent module for the type and fields. +//! `persist`, orphan reclamation, and the transient-tree trio that the +//! data-page / handle-table / membership COW allocation paths use. +//! +//! The mutable recycle/commit state lives in `FreemapRecycle`, an owned unit +//! held by `TransactionManager` (one field, `freemap`). It bundles the five +//! pieces of structural-recycle + freemap-hint state that move together across +//! begin/commit/rollback. `TransactionManager` reaches the freemap through this +//! type's narrow surface (the trio + the commit-path methods + the lifecycle +//! hooks); see the struct doc below for the recycle model. +//! +//! `cow_alloc` and `structural_extend` stay FREE functions (not methods) so the +//! `#[cfg(test)]` `record_structural_reuse` hook the recycle pin-tests depend on +//! sits at a stable, easily-targeted seam; the `FreemapRecycle` methods wrap +//! them, passing the recycle's own fields by `&mut`. use super::*; @@ -17,7 +28,8 @@ use super::*; /// historical `allocate_data_page` behavior, which this also routes through. /// /// The tree's own COW of the claimed leaf supersedes pages, which the caller -/// drains from `tree.pending_superseded` into `txn_freed_pages` after the call. +/// drains from `tree.pending_superseded` into `structural_superseded` after the +/// call (see `FreemapRecycle::put_tree`). /// /// LAZY-CREATE GUARD: a fresh database has `tree.root == PAGE_ID_NONE` (no tree /// materialized yet). `PAGE_ID_NONE` is `u64::MAX`, NOT the tree's internal @@ -25,7 +37,7 @@ use super::*; /// error rather than reporting "nothing free". We short-circuit that here: a /// None-root tree holds nothing reusable, so we fall straight through to /// `new_page`. The tree is first materialized when a page is *freed* (see -/// persist_freemap), never on the allocation side. +/// `FreemapRecycle::persist`), never on the allocation side. /// /// Pages freed during the CURRENT transaction live in `txn_freed_pages` and are /// NOT in the committed tree until commit, so `allocate_first` can never hand @@ -94,138 +106,191 @@ fn structural_extend(cache: &mut PageCache, structural_reuse: &mut Vec) -> } } -impl TransactionManager { - // --- Watermark-based rollback (ISSUES.md I3 + I7) --- - // - // `PageCache::new_page()` hands out monotonically increasing ids, so - // every page allocated during a transaction has an id strictly greater - // than or equal to the `next_page_id` watermark captured at begin() / - // savepoint() time. `PageCache::truncate(watermark)` drops every cache - // entry AND truncates the file to `watermark` pages, cleanly discarding - // every transaction-allocated page without a per-page tracking list. - // - // This supersedes an earlier per-page `txn_dirty_pages` vector — the - // list was a weaker mechanism (I7 showed it missed intermediate COW - // pages and overflow allocations) and a redundant one once the - // watermark invariant was in place. See memory - // project_chisel_i3_watermark_rollback for the reasoning. - // - // Savepoints capture `cache.next_page_id()` at creation time (see the - // `watermark` field on Savepoint) so `rollback_to(name)` can truncate - // to that specific watermark — discarding every page allocated after - // the savepoint while preserving those allocated before it. +/// Owns the structural-page recycle cluster and the freemap commit/alloc/persist +/// machinery — the crash-durability backbone of the engine. Held by +/// `TransactionManager` as the single `freemap` field. +/// +/// The recycle exists to bound the extend-only freemap's growth (ISSUES.md I18, +/// generalized to the multi-page tree). A freemap mutation (data-alloc-side leaf +/// COW, or persist's frees) must COW the committed freemap pages it touches — it +/// can never overwrite a page the last-durable superblock still references — and +/// each COW supersedes an OLD freemap page. Those old pages ROTATE through a +/// small pool instead of marching the file upward ~1/commit: +/// +/// * superseded THIS transaction -> collected in `structural_superseded`; +/// * promoted at commit -> `pending_structural_frees` (deferred one commit, +/// since the old page is still referenced until the superblock flips); +/// * reused NEXT transaction -> `begin` clones `pending_structural_frees` into +/// `structural_reuse`, which every structural `extend` (freemap COW target) +/// pops from before extending the file. +/// +/// Reusing a DEAD page (vs. a free bit in the tree) preserves the extend-only +/// TERMINATION guarantee: no freemap mutation ever draws structural space from +/// the freemap's own bits. These freemap-page frees are NOT data-reusable (never +/// enter `txn_freed_pages`): a freed freemap page sits at a high id where the +/// lowest-first data allocator would starve it, so routing it back as structural +/// reuse — where demand matches supply at steady state — is what reclaims it. +pub(super) struct FreemapRecycle { + // Best-effort lower bound on the lowest free page id in the committed freemap + // tree, threaded into `FreeMapTree::allocate_first` so a scan starts near the + // answer instead of at id 0. Deliberately NOT transactionally tracked: a + // too-low hint only costs a wasted left-to-right scan, never correctness (the + // scan still returns the true lowest free id), so it needs no begin/rollback + // snapshotting — it PERSISTS across transactions. `allocate_first` advances + // it; a free at a lower id is invisible until the next scan walks back over + // it, which is acceptable slack. Init 0. + hint: u64, + // The dead-freemap-page pool available to reuse as structural COW targets in + // the CURRENT transaction. Seeded from `pending_structural_frees` at `begin`; + // drained by every structural `extend`; the unconsumed remainder is carried + // forward (back into `pending_structural_frees`) at commit. On rollback it is + // cleared (begin re-clones `pending_structural_frees` next time). + structural_reuse: Vec, + // This transaction's freemap-COW supersedes (old freemap pages this txn + // replaced). Accumulated as transient handles drain `tree.pending_superseded` + // here via `put_tree`; promoted to `pending_structural_frees` at commit (the + // one-commit defer). Dropped on rollback (those COWs are truncated above the + // watermark). + structural_superseded: Vec, + // Dead freemap pages carried BETWEEN commits — the committed-baseline recycle. + // `begin` CLONES (not moves) this into `structural_reuse`, so it stays intact + // as the rollback fallback; `commit` overwrites it with this commit's + // superseded + reuse remainder. See the struct doc for the full rotation. + pending_structural_frees: Vec, + // Freemap pages already COW'd/extended by the CURRENT transaction. Because the + // manager rebuilds a transient `FreeMapTree` handle at every allocation site + // (data-page alloc, each HT/membership COW, persist), this set is what lets + // those handles share the "first touch this txn => COW, later touches => + // in-place" discipline: without it every site would re-COW the same freemap + // leaf, turning reclamation into unbounded file growth. Moved into each + // transient handle and read back out (see `take_tree` / `put_tree`). Cleared + // at begin/commit/rollback so the next transaction starts empty — a stale + // entry pointing at a now-committed page would be a CORRECTNESS bug (it would + // suppress a needed COW and mutate a live committed page in place). + session_owned: FxHashSet, +} - /// Snapshot the current `next_page_id` watermark. Cheap — one read - /// through the RefCell. - pub(super) fn cache_watermark(&self) -> u64 { - self.cache.borrow().next_page_id() +impl FreemapRecycle { + /// Fresh recycle for a newly-opened manager: hint at 0, empty pools/set. + pub(super) fn new() -> Self { + FreemapRecycle { + hint: 0, + structural_reuse: Vec::new(), + structural_superseded: Vec::new(), + pending_structural_frees: Vec::new(), + session_owned: FxHashSet::default(), + } } - // --- Freemap-aware page allocation (ISSUES.md R2) --- - // - // `cow_alloc` (free function above) is the shared freemap-aware allocator - // for a fresh page during a transaction: it first tries to reuse an id from - // the committed freemap tree and falls back to extending the file. Data-page - // allocation reaches it through the freemap dance hoisted into - // `SlotPacker::insert`'s caller (`insert_into_data_page` in packing.rs); the - // handle-table / membership COW paths reach it through `ht_insert` below. - // - // Two important scoping rules: - // - // 1. Reuse is disabled when any savepoint is active. A rollback_to - // would need to per-savepoint distinguish dirty entries at - // reused ids from dirty entries at preserved ids, which would - // require an 8 KB freemap snapshot per savepoint and a - // per-savepoint dirty-page list. For v1, the simpler rule is - // "reuse only outside savepoint scopes". Workloads that want - // reuse (e.g. F1 delete_subtree / drop_table) typically don't - // use savepoints at all. + // --- The transient-tree trio --- // - // 2. Pages freed during the CURRENT transaction (in - // `txn_freed_pages`) are NOT reusable within the same - // transaction — their old contents must stay readable via - // `committed_roots` until commit swaps the superblock. This - // is enforced by only merging `txn_freed_pages` into - // `current_freemap` during commit, after the new roots have - // been computed. - // - // Handle-table and membership-index COW pages now share this same - // freemap-aware allocator via `cow_alloc` (each `insert`/`delete` takes an - // `alloc` closure that calls it), so they reuse freed pages before - // extending — that is what bounds their steady-state page count. Overflow - // pages still call `cache.new_page()` directly and always extend, but their - // frees feed the freemap, so a later data- or handle-table allocation can - // reclaim them. Routing overflow through the freemap too would need the - // same allocator-closure plumbing at the overflow module boundary; left as - // a v1 simplification since overflow churn is far smaller than HT churn. + // Each allocation site (data-page alloc, every HT/membership COW, persist, + // and the orphan sweep) materializes a transient `FreeMapTree` from the + // committed roots, threads it through one downstream `insert`/`delete`/`mark`, + // then writes it back. `take_tree` MOVES the session set into the handle so a + // leaf an earlier site already COW'd is treated as in-place-mutable; `put_tree` + // moves it back out and drains the handle's COW supersedes into + // `structural_superseded`. Never drop a handle from `take_tree` without a + // matching `put_tree`, or the session set is lost and later sites re-COW. + /// Build a transient `FreeMapTree` handle from the current freemap roots, - /// MOVING the transaction's `freemap_session_owned` set into it so this - /// handle treats pages an earlier site already COW'd this transaction as - /// in-place-mutable. Pair with `put_freemap_tree`, which moves the (possibly - /// grown) set back out — never drop a handle from `take_` without a matching - /// `put_`, or the session set is lost and later sites re-COW. - pub(super) fn take_freemap_tree(&mut self) -> FreeMapTree { - let mut tree = FreeMapTree::from_roots( - self.current_roots.freemap_page, - self.current_roots.freemap_depth, - ); - tree.session_owned = std::mem::take(&mut self.freemap_session_owned); + /// MOVING `session_owned` into it (so already-COW'd pages are in-place this + /// transaction). Pair with `put_tree`. + pub(super) fn take_tree(&mut self, roots: &Roots) -> FreeMapTree { + let mut tree = FreeMapTree::from_roots(roots.freemap_page, roots.freemap_depth); + tree.session_owned = std::mem::take(&mut self.session_owned); tree } - /// Write a transient handle's grown root/depth back into the current roots, - /// move its session-owned set back into the manager, and drain its - /// COW-superseded freemap pages into `structural_superseded` (the one-commit - /// defer stream — NOT `txn_freed_pages`, since freed freemap pages are - /// recycled as structural reuse, not as data frees). - pub(super) fn put_freemap_tree(&mut self, mut tree: FreeMapTree) { - self.current_roots.freemap_page = tree.root; - self.current_roots.freemap_depth = tree.depth; + /// Write a transient handle's grown root/depth back into `roots`, move its + /// session-owned set back into the recycle, and drain its COW-superseded + /// freemap pages into `structural_superseded` (the one-commit defer stream — + /// NOT `txn_freed_pages`, since freed freemap pages are recycled as structural + /// reuse, not as data frees). + pub(super) fn put_tree(&mut self, roots: &mut Roots, mut tree: FreeMapTree) { + roots.freemap_page = tree.root; + roots.freemap_depth = tree.depth; self.structural_superseded .append(&mut tree.pending_superseded); - self.freemap_session_owned = std::mem::take(&mut tree.session_owned); + self.session_owned = std::mem::take(&mut tree.session_owned); } - /// COW `handle`'s handle-table entry to `entry`, installing the new root - /// and queuing the superseded spine pages for freemap reclamation at - /// commit. Shared by `allocate`, `update`, and `set_client_byte`. + /// Wrap the free `cow_alloc` with the recycle's own hint + structural-reuse + /// pool. The `tree` MUST persist across the whole downstream `insert` (insert + /// may call this several times on the one tree; that shared `session_owned` + /// accumulation is load-bearing), so this takes `&mut tree` rather than + /// owning it. + pub(super) fn cow_alloc_into( + &mut self, + cache: &mut PageCache, + tree: &mut FreeMapTree, + reuse: bool, + ) -> Result { + cow_alloc( + cache, + tree, + &mut self.hint, + &mut self.structural_reuse, + reuse, + ) + } + + // --- Commit-path machinery --- + + /// Mark a single page id free in the working freemap tree, routing every + /// structural COW target through the pooled `structural_extend` (reuse a dead + /// freemap page before extending the file) and lazily materializing the + /// depth-0 root on first use. Lowers `hint` to cover `id` so the next + /// `allocate_first` scan can reach it. /// - /// The superseded pages are appended to `txn_freed_pages` ONLY after the - /// new root is installed in `current_roots`: if the COW fails partway - /// (e.g. `CacheFull`), the local `freed` list is dropped and the still- - /// current old tree keeps all its pages — never freeing a live page. - pub(super) fn ht_insert(&mut self, handle: u64, entry: &HandleEntry) -> Result<()> { - let mut freed: Vec = Vec::new(); - let reuse = self.savepoints.is_empty(); - // Build the freemap-tree handle (with the session set moved in) and - // borrow the hint + structural-reuse pool as locals, all disjoint from - // `self.handle_table`, so the alloc closure (which mutates them) and the - // handle-table insert can both borrow `self` at once. - let mut tree = self.take_freemap_tree(); - let result = { - let hint = &mut self.freemap_hint; - let pool = &mut self.structural_reuse; - let mut cache = self.cache.borrow_mut(); - let mut alloc = |c: &mut PageCache| cow_alloc(c, &mut tree, hint, pool, reuse); - self.handle_table.insert( - &mut cache, - self.current_roots.handle_table_page, - handle, - entry, - &mut alloc, - &mut freed, - ) - }; - // Write back freemap growth (its supersedes go to structural_superseded - // via put_freemap_tree). Done before the `?` so a freemap COW that - // happened before an insert error still returns the session set and - // records the extended root. Handle-table supersedes (`freed`) only land - // in txn_freed_pages after the new root is installed. - self.put_freemap_tree(tree); - let new_root = result?; - self.current_roots.handle_table_page = new_root; - self.txn_freed_pages.append(&mut freed); - Ok(()) + /// The ONE marking path shared by `persist` (this commit's data frees) and + /// `reclaim_orphans` (the defrag orphan-sweep). Both must flow through the + /// same COW + recycle discipline so the structural reuse pool and supersede + /// streams stay consistent; a second marking implementation could silently + /// diverge from the one-commit-defer crash-safety the recycle depends on. + /// Take/put the tree per call: the session-owned set and the reuse pool persist + /// on the recycle across calls, so a multi-id loop still COWs each leaf at most + /// once (the session dedup carries across handles). + pub(super) fn mark_free_committed_path( + &mut self, + cache: &mut PageCache, + roots: &mut Roots, + id: u64, + ) -> Result<()> { + // Take the working handle WITH the transaction's session set so a leaf an + // earlier call (or this commit's data allocations) already COW'd is + // recognized as in-place here, not re-COW'd. + let mut tree = self.take_tree(roots); + // RefCell so the structural-`extend` closure can drain the shared reuse + // pool by `&mut` while the rest of the method still holds `&mut self`. + let structural_reuse = std::cell::RefCell::new(std::mem::take(&mut self.structural_reuse)); + let result = (|| { + let mut extend = + |c: &mut PageCache| structural_extend(c, &mut structural_reuse.borrow_mut()); + + // Lazy materialization: a database that has never freed a page has no + // tree yet (root == PAGE_ID_NONE). Create the depth-0 leaf now, before + // marking, since `mark_free_growing` needs a real root to COW. + // Preserve the session set across the swap. + if tree.root == PAGE_ID_NONE { + let session = std::mem::take(&mut tree.session_owned); + tree = FreeMapTree::create(cache, &mut extend)?; + tree.session_owned.extend(session); + } + tree.mark_free_growing(cache, id, &mut extend) + })(); + // Pull the hint back to cover `id`: the hint advances monotonically via + // `allocate_first`, so a too-high hint would start the next scan above + // `id` and never reuse it. A too-low hint only costs a wasted scan. + // (Mirrors the oracle proptest's `hint = hint.min(id)`.) + self.hint = self.hint.min(id); + // Return the (partly drained) reuse pool and write the tree back even on + // error: its COW supersedes flow to `structural_superseded` via put_tree; + // commit promotes structural_superseded + the leftover reuse pool into + // pending_structural_frees (the one-commit defer). + self.structural_reuse = structural_reuse.into_inner(); + self.put_tree(roots, tree); + result } // Persist the freemap tree at commit time (ISSUES.md R2 / I11 / I18, @@ -237,7 +302,7 @@ impl TransactionManager { // // TWO FREE-STREAMS (the load-bearing distinction the reviewer scrutinizes): // - // * `txn_freed_pages` (DATA frees) — pages freed by this commit's + // * `txn_freed_pages` (DATA frees, passed in) — pages freed by this commit's // data/handle-table/membership COW supersedes. Recorded as FREE in this // commit's new freemap tree, so the NEXT transaction's data/HT // allocations can reuse them. Safe to mark now: the new tree becomes @@ -249,12 +314,8 @@ impl TransactionManager { // supersedes. These are NOT marked free in the tree: a freemap page sits // at a high id where the lowest-first data allocator would starve it, and // marking a freemap page free inside the tree that is recording frees - // could cascade. Instead they ride a separate recycle: superseded this - // commit (`structural_superseded`) -> deferred one commit - // (`pending_structural_frees`, since the old page is still referenced - // until the superblock flips) -> reused as structural COW targets next - // transaction (`structural_reuse`). This makes the freemap pages ROTATE - // among a small set rather than marching the file upward ~1/commit. + // could cascade. Instead they ride the separate recycle described on the + // struct. // // I18 ORDERING preserved by construction. The structural COW never draws a // page from the freemap's own free bits (that would re-COW a leaf and @@ -268,74 +329,24 @@ impl TransactionManager { // (reusing the prior commit's dead leaf id when available, else extend), set // the freed bits, defer the old leaf to the structural recycle. Steady-state // page count matches the pre-tree single-page freemap. - /// Mark a single page id free in the working freemap tree, routing every - /// structural COW target through the pooled `structural_extend` (reuse a dead - /// freemap page before extending the file) and lazily materializing the - /// depth-0 root on first use. Lowers `freemap_hint` to cover `id` so the next - /// `allocate_first` scan can reach it. - /// - /// The ONE marking path shared by `persist_freemap` (this commit's data - /// frees) and `reclaim_freemap_orphans` (the defrag orphan-sweep). Both must - /// flow through the same COW + recycle discipline so the structural reuse pool - /// and supersede streams stay consistent; a second marking implementation - /// could silently diverge from the one-commit-defer crash-safety the recycle - /// depends on. Take/put the tree per call: the session-owned set and the - /// reuse pool persist on the manager across calls, so a multi-id loop still - /// COWs each leaf at most once (the session dedup carries across handles). - fn freemap_mark_free_committed_path(&mut self, id: u64) -> Result<()> { - // Take the working handle WITH the transaction's session set so a leaf an - // earlier call (or this commit's data allocations) already COW'd is - // recognized as in-place here, not re-COW'd. - let mut tree = self.take_freemap_tree(); - // RefCell so the structural-`extend` closure can drain the shared reuse - // pool by `&mut` while the rest of the method still owns `self`. - let structural_reuse = std::cell::RefCell::new(std::mem::take(&mut self.structural_reuse)); - let result = (|| { - let mut cache = self.cache.borrow_mut(); - let mut extend = - |c: &mut PageCache| structural_extend(c, &mut structural_reuse.borrow_mut()); - - // Lazy materialization: a database that has never freed a page has no - // tree yet (root == PAGE_ID_NONE). Create the depth-0 leaf now, before - // marking, since `mark_free_growing` needs a real root to COW. - // Preserve the session set across the swap. - if tree.root == PAGE_ID_NONE { - let session = std::mem::take(&mut tree.session_owned); - tree = FreeMapTree::create(&mut cache, &mut extend)?; - tree.session_owned.extend(session); - } - tree.mark_free_growing(&mut cache, id, &mut extend) - })(); - // Pull the hint back to cover `id`: the hint advances monotonically via - // `allocate_first`, so a too-high hint would start the next scan above - // `id` and never reuse it. A too-low hint only costs a wasted scan. - // (Mirrors the oracle proptest's `hint = hint.min(id)`.) - self.freemap_hint = self.freemap_hint.min(id); - // Return the (partly drained) reuse pool and write the tree back even on - // error: its COW supersedes flow to `structural_superseded` via - // put_freemap_tree; commit promotes structural_superseded + the leftover - // reuse pool into pending_structural_frees (the one-commit defer). - self.structural_reuse = structural_reuse.into_inner(); - self.put_freemap_tree(tree); - result - } - - pub(super) fn persist_freemap(&mut self) -> Result<()> { - // Nothing freed this commit => the committed tree is still exactly right, - // no COW needed. (Structural reuse / supersede streams are only ever - // non-empty when there were frees, so this single check suffices.) - if self.txn_freed_pages.is_empty() { + /// Mark this commit's DATA frees (`txn_freed_pages`) free in a COW of the + /// committed tree, via the shared marking path. Each call take/puts the tree, + /// but the session-owned set persists on the recycle, so a leaf hit by several + /// frees is COW'd once. A no-op when nothing was freed (the recycle/supersede + /// streams are only ever non-empty when there were frees, so the single + /// emptiness check suffices). + pub(super) fn persist( + &mut self, + cache: &mut PageCache, + roots: &mut Roots, + txn_freed_pages: &[u64], + ) -> Result<()> { + if txn_freed_pages.is_empty() { return Ok(()); } - - // Mark this commit's DATA frees free in the new tree via the shared - // marking path. Each call take/puts the tree, but the session-owned set - // persists on the manager, so a leaf hit by several frees is COW'd once. - let freed: Vec = std::mem::take(&mut self.txn_freed_pages); - for id in freed.iter().copied() { - self.freemap_mark_free_committed_path(id)?; + for id in txn_freed_pages.iter().copied() { + self.mark_free_committed_path(cache, roots, id)?; } - self.txn_freed_pages = freed; Ok(()) } @@ -347,13 +358,26 @@ impl TransactionManager { /// supersedes). This sweep walks the live tree to find the reachable set, /// scans the file for freemap-typed pages that are neither reachable nor /// already free, and marks each free — routing the mark through the SAME - /// `freemap_mark_free_committed_path` the commit uses (COW + recycle), so a - /// reclaimed orphan lands in the BITMAP (data-reusable), disjoint from the - /// in-memory recycle pool. Requires an active transaction (called by defrag). + /// `mark_free_committed_path` the commit uses (COW + recycle), so a reclaimed + /// orphan lands in the BITMAP (data-reusable), disjoint from the in-memory + /// recycle pool. Requires an active transaction (called by defrag). /// Returns the count reclaimed. /// - /// THE EXCLUSION SET (get this exactly right): a page in the CURRENT - /// in-memory recycle pool (`structural_reuse` ∪ `structural_superseded` ∪ + /// `savepoint_active` is `!savepoints.is_empty()` from the caller — when true + /// the sweep is a no-op (returns 0). The sweep is the ONLY path that COWs the + /// freemap (draining committed-LIVE pages into the structural streams) while a + /// savepoint is open; ordinary allocation already disables structural reuse + /// under a savepoint. But `rollback_to` rewinds only the roots + cache + /// watermark, NOT the structural streams: a page the sweep drained into + /// `structural_superseded` would survive the rollback, get promoted at commit, + /// and be reused as a COW target in the next transaction while the last-durable + /// superblock still references it — silent durable freemap corruption. + /// Deferring orphan reclamation to a defrag run with no active savepoint avoids + /// the whole interaction, so `rollback_to_inner` correctly needs no + /// structural-stream reset. + /// + /// THE EXCLUSION SET (get this exactly right): a page in the CURRENT in-memory + /// recycle pool (`structural_reuse` ∪ `structural_superseded` ∪ /// `pending_structural_frees`) is LIVE recycling state, NOT an orphan — /// reclaiming it into the bitmap while it is also pool-reusable would /// double-hand-out the page. After a crash the pool is empty, so the @@ -371,24 +395,18 @@ impl TransactionManager { /// fatal on a corrupt LIVE node — only the dead-page scan is softened. The /// scan is O(total_pages) I/O — off the hot path (defrag), bounded, and /// acceptable. - pub(crate) fn reclaim_freemap_orphans(&mut self) -> Result { - // Skip the sweep entirely while a savepoint is active. The sweep is the - // ONLY path that COWs the freemap (draining committed-LIVE pages into the - // structural recycle streams) while a savepoint is open — ordinary - // allocation already disables structural reuse under a savepoint - // (`reuse = self.savepoints.is_empty()`). But `rollback_to` rewinds only - // the roots + cache watermark, NOT the structural streams: a page the - // sweep drained into `structural_superseded` would survive the rollback, - // get promoted at commit, and be reused as a COW target in the next - // transaction while the last-durable superblock still references it — - // silent durable freemap corruption. Deferring orphan reclamation to a - // defrag run with no active savepoint avoids the whole interaction, so - // `rollback_to_inner` correctly needs no structural-stream reset. - if !self.savepoints.is_empty() { + pub(super) fn reclaim_orphans( + &mut self, + cache: &mut PageCache, + roots: &mut Roots, + savepoint_active: bool, + superblock_count: u32, + ) -> Result { + if savepoint_active { return Ok(0); } - let root = self.current_roots.freemap_page; - let depth = self.current_roots.freemap_depth; + let root = roots.freemap_page; + let depth = roots.freemap_depth; if root == PAGE_ID_NONE { return Ok(0); // no tree yet => no freemap pages can be orphaned } @@ -398,64 +416,271 @@ impl TransactionManager { let mut excluded: FxHashSet = FxHashSet::default(); excluded.extend(self.structural_reuse.iter().copied()); excluded.extend(self.structural_superseded.iter().copied()); - // Belt-and-suspenders: `begin()` clones `pending_structural_frees` - // into `structural_reuse`, so every id here is already covered by the - // `structural_reuse` term above. Kept explicitly so the exclusion - // remains correct if `begin()`'s seeding ever changes. + // Belt-and-suspenders: `begin` clones `pending_structural_frees` into + // `structural_reuse`, so every id here is already covered by the + // `structural_reuse` term above. Kept explicitly so the exclusion remains + // correct if `begin`'s seeding ever changes. excluded.extend(self.pending_structural_frees.iter().copied()); - // Collect orphan ids read-only inside a single cache-borrow scope, then - // drop the borrow before marking (the mark path re-borrows the cache). + // Collect orphan ids, then mark them. The original dropped and re-borrowed + // the cache between the collection and mark phases; with `cache` a param we + // hold one continuous borrow — sequential use is equivalent (the mark path + // does not read any cache state the collection left mid-update). let tree = FreeMapTree::from_roots(root, depth); let mut orphans: Vec = Vec::new(); - { - let mut cache = self.cache.borrow_mut(); - // Upper bound: the allocation high-water (`next_page_id`), NOT the - // committed `total_pages`. After a real crash + reopen these are - // equal (open seeds next_page_id from the committed superblock), and - // every orphan — a structural supersede from a committed transaction — - // sits below it. Using next_page_id also covers a page extended - // earlier in THIS session (e.g. the forge-orphan test), which a stale - // committed total_pages would miss. - let total = cache.next_page_id(); - let reachable = tree.reachable_pages(&mut cache)?; - // Pages 0..superblock_count are superblocks; start the scan above them. - for id in self.superblock_count as u64..total { - if reachable.contains(&id) || excluded.contains(&id) { + // Upper bound: the allocation high-water (`next_page_id`), NOT the + // committed `total_pages`. After a real crash + reopen these are equal + // (open seeds next_page_id from the committed superblock), and every orphan + // — a structural supersede from a committed transaction — sits below it. + // Using next_page_id also covers a page extended earlier in THIS session + // (e.g. the forge-orphan test), which a stale committed total_pages would + // miss. + let total = cache.next_page_id(); + let reachable = tree.reachable_pages(cache)?; + // Pages 0..superblock_count are superblocks; start the scan above them. + for id in superblock_count as u64..total { + if reachable.contains(&id) || excluded.contains(&id) { + continue; + } + // Skip a non-reachable page that is GARBAGE/corrupt rather than letting + // it poison the whole maintenance pass (2026-06-22 review: "skip + // unreadable dead pages"). A page not in the live tree cannot be + // confirmed as a freemap orphan if we cannot read its type, and a DEAD + // page's corruption does not affect correctness — so on + // `CorruptPage`/`ChecksumMismatch` we `continue`. We deliberately + // PROPAGATE every other read error (e.g. `IoError`): a real device + // fault should still surface and poison. NOTE: the live-tree walk + // (`reachable_pages` above) still propagates fatal on a corrupt LIVE + // page — only the dead-page scan is softened. + let buf = match cache.get(id) { + Ok(buf) => buf, + Err(ChiselError::CorruptPage { .. } | ChiselError::ChecksumMismatch { .. }) => { continue; } - // Skip a non-reachable page that is GARBAGE/corrupt rather than - // letting it poison the whole maintenance pass (2026-06-22 review - // decision: "skip unreadable dead pages"). A page that is not in - // the live tree cannot be confirmed as a freemap orphan if we - // cannot read its type, and a DEAD page's corruption does not - // affect correctness — so on `CorruptPage`/`ChecksumMismatch` we - // `continue`. We deliberately PROPAGATE every other read error - // (e.g. `IoError`): a real device fault should still surface and - // poison, not be silently swallowed. NOTE: the live-tree walk - // (`reachable_pages` above) still propagates fatal on a corrupt - // LIVE page — only the dead-page scan is softened. - let buf = match cache.get(id) { - Ok(buf) => buf, - Err(ChiselError::CorruptPage { .. } | ChiselError::ChecksumMismatch { .. }) => { - continue; - } - Err(e) => return Err(e), - }; - let ty = buf[0]; - if (ty == crate::page::PageType::FreeMap as u8 - || ty == crate::page::PageType::FreeMapInterior as u8) - && !tree.is_free(&mut cache, id)? - { - orphans.push(id); - } + Err(e) => return Err(e), + }; + let ty = buf[0]; + if (ty == crate::page::PageType::FreeMap as u8 + || ty == crate::page::PageType::FreeMapInterior as u8) + && !tree.is_free(cache, id)? + { + orphans.push(id); } } // Mark each orphan free through the shared committed-marking path (COW + // recycle), landing them in the bitmap as data-reusable space. for id in &orphans { - self.freemap_mark_free_committed_path(*id)?; + self.mark_free_committed_path(cache, roots, *id)?; } Ok(orphans.len() as u64) } + + // --- Lifecycle (operate on own fields; EXACT semantics from lifecycle.rs) --- + + /// At transaction begin: start with an empty session set, and seed the reuse + /// pool from the prior commit's deferred dead freemap pages (those + /// superblock-unreferenced pages are now safe to reuse as this transaction's + /// freemap COW targets). CLONE (not move) `pending_structural_frees` so it + /// stays intact as the rollback fallback — a rolled-back transaction never + /// reached commit, so its structural recycle is exactly the pre-transaction + /// one; `commit` overwrites it on the success path. `structural_superseded` is + /// empty here (only `persist` fills it); clear defensively. `hint` is NOT + /// reset — it persists across transactions (a stale hint only costs a scan). + pub(super) fn begin(&mut self) { + self.session_owned.clear(); + self.structural_reuse = self.pending_structural_frees.clone(); + self.structural_superseded.clear(); + } + + /// At commit: every freemap page COW'd this transaction is now committed + /// (clear the session set). Promote the structural recycle for the next + /// transaction: the pages this commit superseded (`structural_superseded`) + /// become dead the instant the superblock flips, and the reuse-pool remainder + /// (`structural_reuse` ids not consumed as COW targets) is likewise still dead + /// and reusable. Both become next transaction's `pending_structural_frees`. + /// Order: clear pending, then append superseded, then append reuse. + pub(super) fn commit(&mut self) { + self.session_owned.clear(); + self.pending_structural_frees.clear(); + self.pending_structural_frees + .append(&mut self.structural_superseded); + self.pending_structural_frees + .append(&mut self.structural_reuse); + } + + /// At rollback: discard the in-transaction structural working state. + /// `structural_superseded` holds committed-tree freemap pages this aborted + /// transaction COW'd-over; the abort means the committed tree still references + /// them, so they are NOT dead and must never be recycled. `structural_reuse` + /// was the working copy; drop it. The session set: any freemap pages this + /// aborted transaction COW'd sit above the watermark and were just truncated, + /// so their ids must not be treated as in-place-mutable next transaction. + /// `pending_structural_frees` is NOT touched — `begin` CLONED it into + /// `structural_reuse` rather than moving it, so it still holds the + /// pre-transaction dead-freemap-page set (correct: a rolled-back transaction's + /// structural recycle is exactly the pre-transaction one). + pub(super) fn rollback(&mut self) { + self.structural_superseded.clear(); + self.structural_reuse.clear(); + self.session_owned.clear(); + } + + // --- Test-only accessors (the recycle pin-tests read/forge these) --- + + #[cfg(test)] + pub(super) fn structural_reuse(&self) -> &[u64] { + &self.structural_reuse + } + + #[cfg(test)] + pub(super) fn structural_superseded(&self) -> &[u64] { + &self.structural_superseded + } + + #[cfg(test)] + pub(super) fn pending_structural_frees(&self) -> &[u64] { + &self.pending_structural_frees + } + + #[cfg(test)] + pub(super) fn session_owned(&self) -> &FxHashSet { + &self.session_owned + } + + /// Forge state for a pin-test: push a page id into the live reuse pool so the + /// orphan-sweep exclusion test can verify the pool is spared. + #[cfg(test)] + pub(super) fn push_structural_reuse_for_test(&mut self, id: u64) { + self.structural_reuse.push(id); + } +} + +impl TransactionManager { + // --- Watermark-based rollback (ISSUES.md I3 + I7) --- + // + // `PageCache::new_page()` hands out monotonically increasing ids, so + // every page allocated during a transaction has an id strictly greater + // than or equal to the `next_page_id` watermark captured at begin() / + // savepoint() time. `PageCache::truncate(watermark)` drops every cache + // entry AND truncates the file to `watermark` pages, cleanly discarding + // every transaction-allocated page without a per-page tracking list. + // + // This supersedes an earlier per-page `txn_dirty_pages` vector — the + // list was a weaker mechanism (I7 showed it missed intermediate COW + // pages and overflow allocations) and a redundant one once the + // watermark invariant was in place. See memory + // project_chisel_i3_watermark_rollback for the reasoning. + // + // Savepoints capture `cache.next_page_id()` at creation time (see the + // `watermark` field on Savepoint) so `rollback_to(name)` can truncate + // to that specific watermark — discarding every page allocated after + // the savepoint while preserving those allocated before it. + + /// Snapshot the current `next_page_id` watermark. Cheap — one read + /// through the RefCell. + pub(super) fn cache_watermark(&self) -> u64 { + self.cache.borrow().next_page_id() + } + + // --- Freemap-aware page allocation (ISSUES.md R2) --- + // + // `cow_alloc` (free function above) is the shared freemap-aware allocator + // for a fresh page during a transaction: it first tries to reuse an id from + // the committed freemap tree and falls back to extending the file. The six + // allocation call sites reach it through `self.freemap.cow_alloc_into`, + // sandwiched between `self.freemap.take_tree` / `put_tree` (the transient-tree + // trio). Two important scoping rules: + // + // 1. Reuse is disabled when any savepoint is active. A rollback_to + // would need to per-savepoint distinguish dirty entries at + // reused ids from dirty entries at preserved ids, which would + // require an 8 KB freemap snapshot per savepoint and a + // per-savepoint dirty-page list. For v1, the simpler rule is + // "reuse only outside savepoint scopes". Workloads that want + // reuse (e.g. F1 delete_subtree / drop_table) typically don't + // use savepoints at all. + // + // 2. Pages freed during the CURRENT transaction (in + // `txn_freed_pages`) are NOT reusable within the same + // transaction — their old contents must stay readable via + // `committed_roots` until commit swaps the superblock. This + // is enforced by only merging `txn_freed_pages` into the freemap + // tree during commit (via `persist_freemap`), after the new roots + // have been computed. + // + // Handle-table and membership-index COW pages share this same freemap-aware + // allocator (each `insert`/`delete` takes an `alloc` closure that calls it), + // so they reuse freed pages before extending — that is what bounds their + // steady-state page count. Overflow pages still call `cache.new_page()` + // directly and always extend, but their frees feed the freemap, so a later + // data- or handle-table allocation can reclaim them. Routing overflow through + // the freemap too would need the same allocator-closure plumbing at the + // overflow module boundary; left as a v1 simplification since overflow churn + // is far smaller than HT churn. + + /// COW `handle`'s handle-table entry to `entry`, installing the new root + /// and queuing the superseded spine pages for freemap reclamation at + /// commit. Shared by `allocate`, `update`, and `set_client_byte`. + /// + /// The superseded pages are appended to `txn_freed_pages` ONLY after the + /// new root is installed in `current_roots`: if the COW fails partway + /// (e.g. `CacheFull`), the local `freed` list is dropped and the still- + /// current old tree keeps all its pages — never freeing a live page. + pub(super) fn ht_insert(&mut self, handle: u64, entry: &HandleEntry) -> Result<()> { + let mut freed: Vec = Vec::new(); + let reuse = self.savepoints.is_empty(); + // Build the freemap-tree handle (with the session set moved in). The + // alloc closure captures `&mut self.freemap` + the local `tree`, disjoint + // from `self.handle_table`, so both can borrow `self` at once. + let mut tree = self.freemap.take_tree(&self.current_roots); + let result = { + let mut cache = self.cache.borrow_mut(); + let mut alloc = |c: &mut PageCache| self.freemap.cow_alloc_into(c, &mut tree, reuse); + self.handle_table.insert( + &mut cache, + self.current_roots.handle_table_page, + handle, + entry, + &mut alloc, + &mut freed, + ) + }; + // Write back freemap growth (its supersedes go to structural_superseded + // via put_tree). Done before the `?` so a freemap COW that happened before + // an insert error still returns the session set and records the extended + // root. Handle-table supersedes (`freed`) only land in txn_freed_pages + // after the new root is installed. + self.freemap.put_tree(&mut self.current_roots, tree); + let new_root = result?; + self.current_roots.handle_table_page = new_root; + self.txn_freed_pages.append(&mut freed); + Ok(()) + } + + /// Commit-path wrapper: persist this commit's data frees into a COW of the + /// committed freemap tree. `current_roots` (mut), `freemap` (mut), + /// `txn_freed_pages` (immutable read), and the cache borrow are disjoint, so + /// `txn_freed_pages` passes by `&` with no take/restore dance. `commit_inner` + /// calls this BEFORE cache.flush() so the new freemap pages join the same + /// durable write set. + pub(super) fn persist_freemap(&mut self) -> Result<()> { + let mut cache = self.cache.borrow_mut(); + self.freemap + .persist(&mut cache, &mut self.current_roots, &self.txn_freed_pages) + } + + /// Commit-path wrapper: reclaim crash-orphaned freemap pages. Read the + /// savepoint-active flag and superblock count into locals BEFORE borrowing the + /// cache to keep the borrows clean. `pub(crate)` — defrag calls it. + pub(crate) fn reclaim_freemap_orphans(&mut self) -> Result { + let savepoint_active = !self.savepoints.is_empty(); + let superblock_count = self.superblock_count; + let mut cache = self.cache.borrow_mut(); + self.freemap.reclaim_orphans( + &mut cache, + &mut self.current_roots, + savepoint_active, + superblock_count, + ) + } } diff --git a/src/transaction/lifecycle.rs b/src/transaction/lifecycle.rs index 3659e74..6e26056 100644 --- a/src/transaction/lifecycle.rs +++ b/src/transaction/lifecycle.rs @@ -88,24 +88,12 @@ impl TransactionManager { } self.current_roots = self.committed_roots.clone(); // The freemap root+depth ride in current_roots (cloned just above), so - // there is no separate freemap working copy to reset here. The hint is - // untracked (a stale hint only costs a scan), so it is left as-is too. - // The session-owned set is strictly per-transaction: a page COW'd last - // transaction is now committed and must NOT be mutated in place, so start - // empty. (begin already requires no active txn, so it is normally empty, - // but clear defensively.) - self.freemap_session_owned.clear(); - // Seed the structural reuse pool from the prior commit's deferred dead - // freemap pages: those superblock-unreferenced pages are now safe to - // reuse as this transaction's freemap COW targets, so the freemap rotates - // among a bounded set instead of extending. CLONE (not move) so - // `pending_structural_frees` stays intact as the rollback fallback — a - // rolled-back transaction never reached commit, so its structural recycle - // is exactly the pre-transaction one. `commit_inner` overwrites it on the - // success path. `structural_superseded` is empty here (only - // persist_freemap fills it); clear defensively. - self.structural_reuse = self.pending_structural_frees.clone(); - self.structural_superseded.clear(); + // there is no separate freemap working copy to reset here. Reset the + // recycle's per-transaction state: clear the session set and reseed the + // structural reuse pool from the prior commit's deferred dead freemap + // pages (the hint persists across transactions — a stale hint only costs a + // scan). See `FreemapRecycle::begin`. + self.freemap.begin(); // R1: clone the live-slot counts and reset the insert cursor. // The cursor is always None at begin — it only tracks pages // allocated during the current transaction. @@ -337,19 +325,11 @@ impl TransactionManager { // txn_freed_pages were already marked free in the new committed freemap // tree by persist_freemap; clear the vector now that it's done its job. self.txn_freed_pages.clear(); - // Every freemap page COW'd this transaction is now committed; the next - // transaction must COW (not edit in place) any of them it touches. - self.freemap_session_owned.clear(); - // Promote the freemap structural recycle for the next transaction: the - // pages this commit superseded (`structural_superseded`) become dead the - // instant the superblock flips above — and the reuse-pool remainder - // (`structural_reuse` ids not consumed as COW targets) is likewise still - // dead and reusable. Both become next transaction's `pending_structural_frees`. - self.pending_structural_frees.clear(); - self.pending_structural_frees - .append(&mut self.structural_superseded); - self.pending_structural_frees - .append(&mut self.structural_reuse); + // Clear the freemap session set (every page COW'd this transaction is now + // committed) and promote the structural recycle for the next transaction: + // this commit's supersedes + the unconsumed reuse remainder become next + // transaction's `pending_structural_frees`. See `FreemapRecycle::commit`. + self.freemap.commit(); Ok(()) } @@ -426,24 +406,11 @@ impl TransactionManager { // The freemap root+depth were restored by `current_roots = // committed_roots.clone()` above; any dirty freemap pages this // transaction COW'd sit above the watermark and were dropped by the - // truncate. The hint is untracked, so nothing to revert. - // - // `pending_structural_frees` is left intact: begin() CLONED it into - // `structural_reuse` rather than moving it, so it still holds the - // pre-transaction dead-freemap-page set — correct, since a rolled-back - // transaction's structural recycle is exactly the pre-transaction one. - // We DISCARD the in-transaction structural working state: - // * `structural_superseded` holds committed-tree freemap pages this - // aborted transaction COW'd-over; the abort means the committed tree - // still references them, so they are NOT dead and must never be - // recycled. - // * `structural_reuse` was the working copy; drop it. - // * the session-owned set: any freemap pages this aborted transaction - // COW'd sit above the watermark and were just truncated, so their ids - // must not be treated as in-place-mutable next transaction. - self.structural_superseded.clear(); - self.structural_reuse.clear(); - self.freemap_session_owned.clear(); + // truncate. Discard the in-transaction structural working state + // (`structural_superseded` + `structural_reuse` + the session set) while + // leaving `pending_structural_frees` intact as the pre-transaction + // baseline. See `FreemapRecycle::rollback` for the per-stream reasoning. + self.freemap.rollback(); // R1: revert the live-slot counts and drop the insert cursor. self.packer.rollback(); self.active_txn = false; diff --git a/src/transaction/mod.rs b/src/transaction/mod.rs index 3b6e9f1..87bd9bd 100644 --- a/src/transaction/mod.rs +++ b/src/transaction/mod.rs @@ -184,66 +184,14 @@ pub struct TransactionManager { // their old contents must stay readable via `committed_roots` until // commit promotes the new roots. txn_freed_pages: Vec, - // Best-effort lower bound on the lowest free page id in the committed - // freemap tree, threaded into `FreeMapTree::allocate_first` so a scan - // starts near the answer instead of at id 0. Deliberately NOT - // transactionally tracked: a too-low hint only costs a wasted left-to-right - // scan, never correctness (the scan still returns the true lowest free id), - // so it needs no begin/rollback snapshotting. `allocate_first` advances it; - // a free at a lower id is invisible to the hint until the next scan walks - // back over it, which is acceptable slack. Init 0. - freemap_hint: u64, - // Dead freemap pages carried BETWEEN commits, the engine's bounded-growth - // mechanism for the extend-only freemap (ISSUES.md I18, generalized to the - // tree). Lifecycle: - // - // * A freemap mutation (data-alloc-side leaf COW, or persist's frees) must - // COW the committed freemap pages it touches — it can never overwrite a - // page the last-durable superblock still references. Each COW supersedes - // an OLD freemap page. - // * That old page cannot be reused IN THE SAME COMMIT (the commit's new - // freemap root may still reference it until the superblock flips), so it - // is DEFERRED one commit: collected in `structural_superseded` this - // transaction, promoted to `pending_structural_frees` at commit. - // * The NEXT transaction reuses them: `begin()` moves them into - // `structural_reuse`, and every structural `extend` (freemap COW target) - // pops from that pool before extending the file. This is what makes the - // freemap leaf ROTATE among a small set of pages instead of marching the - // file upward ~1 page/commit forever. Reusing a DEAD page (vs. a free bit - // in the tree) keeps the extend-only TERMINATION guarantee — no freemap - // mutation ever draws structural space from the freemap's own bits. - // - // Not data-reusable (never enters `txn_freed_pages`): a freed freemap page - // sits at a high id, and the lowest-first data allocator would starve it, so - // routing it back as structural reuse (where demand matches supply at steady - // state) is what actually reclaims it. - pending_structural_frees: Vec, - // The dead-freemap-page pool available to reuse as structural COW targets in - // the CURRENT transaction. Seeded from `pending_structural_frees` at - // `begin()`; drained by every structural `extend`; the unconsumed remainder - // is carried forward (back into `pending_structural_frees`) at commit. On - // rollback it is moved back wholesale, restoring the pre-transaction - // `pending_structural_frees`. - structural_reuse: Vec, - // This transaction's freemap-COW supersedes (old freemap pages this txn - // replaced). Accumulated as transient handles drain `tree.pending_superseded` - // here via `put_freemap_tree`; promoted to `pending_structural_frees` at - // commit (the one-commit defer). Dropped on rollback (those COWs are - // truncated above the watermark). - structural_superseded: Vec, - // Freemap pages already COW'd/extended by the CURRENT transaction. Because - // the manager rebuilds a transient `FreeMapTree` handle at every allocation - // site (data-page alloc, each HT/membership COW, persist_freemap), this set - // is what lets those handles share the "first touch this txn => COW, later - // touches => in-place" discipline: without it every site would re-COW the - // same freemap leaf, turning reclamation into unbounded file growth. Swapped - // into each transient handle and read back out (see `freemap_tree` helper). - // Cleared at begin (fresh per transaction); also cleared on commit/rollback - // so the next transaction starts empty. A stale entry pointing at a - // now-committed page would be a CORRECTNESS bug (it would suppress a needed - // COW and mutate a live committed page in place), which is exactly why it is - // transaction-scoped, not cross-transaction. - freemap_session_owned: FxHashSet, + // The structural-page recycle cluster and freemap commit/alloc/persist + // machinery — the crash-durability backbone — extracted into its own owned + // unit. No code outside `freemap.rs` touches the inner fields; the rest of + // `TransactionManager` reaches the freemap through `FreemapRecycle`'s narrow + // surface (the transient-tree trio, the commit-path persist/reclaim wrappers + // on this type, and the begin/commit/rollback lifecycle hooks). See the + // `FreemapRecycle` doc in `freemap.rs` for the recycle/rotation model. + freemap: freemap::FreemapRecycle, // R1 slot-packing state (live-slot counts per data page + the insert // cursor), extracted into its own owned unit. No code outside `packing.rs` // touches the inner fields; `TransactionManager` reaches packing through diff --git a/src/transaction/mutate.rs b/src/transaction/mutate.rs index 13f4719..52f2c35 100644 --- a/src/transaction/mutate.rs +++ b/src/transaction/mutate.rs @@ -2,7 +2,6 @@ //! delete_with_tag / delete_many (+ their `_inner` cores). Split out of //! `transaction.rs` verbatim; see the parent module for the type and fields. -use super::freemap::cow_alloc; use super::*; impl TransactionManager { @@ -193,12 +192,10 @@ impl TransactionManager { // InvalidHandle to preserve the public-API behavior. let mut ht_freed: Vec = Vec::new(); let reuse = self.savepoints.is_empty(); - let mut tree = self.take_freemap_tree(); + let mut tree = self.freemap.take_tree(&self.current_roots); let delete_result = { - let hint = &mut self.freemap_hint; - let pool = &mut self.structural_reuse; let mut cache = self.cache.borrow_mut(); - let mut alloc = |c: &mut PageCache| cow_alloc(c, &mut tree, hint, pool, reuse); + let mut alloc = |c: &mut PageCache| self.freemap.cow_alloc_into(c, &mut tree, reuse); self.handle_table.delete( &mut cache, self.current_roots.handle_table_page, @@ -210,7 +207,7 @@ impl TransactionManager { // Install freemap growth (supersedes go to structural_superseded). Done // BEFORE the `?` so a delete that COW'd the freemap leaf yet then errored // still records the extended root and returns the session set. - self.put_freemap_tree(tree); + self.freemap.put_tree(&mut self.current_roots, tree); let (ht_new_root, prev_entry) = delete_result?; let entry = prev_entry.ok_or(ChiselError::InvalidHandle(handle))?; diff --git a/src/transaction/packing.rs b/src/transaction/packing.rs index 8f8bc12..bc99151 100644 --- a/src/transaction/packing.rs +++ b/src/transaction/packing.rs @@ -260,17 +260,14 @@ impl SlotPacker { impl TransactionManager { /// Place a value in a data page and return (page_id, slot_index). /// - /// Thin wrapper over `SlotPacker::insert`. The freemap dance that - /// `allocate_data_page` performs (take/put the transient `FreeMapTree`, - /// touching `current_roots`, `freemap_session_owned`, - /// `structural_superseded`, `freemap_hint`, `structural_reuse`) is hoisted - /// HERE, around the closure, exactly like `ht_insert`: those are - /// `&mut self` operations and cannot run inside a closure that the packer - /// borrow (`&mut self.packer`) would also need. The closure captures only - /// the disjoint freemap field refs plus the local `tree`, and takes `cache` - /// as a param. + /// Thin wrapper over `SlotPacker::insert`. The freemap dance (the + /// transient-tree trio: `freemap.take_tree` / `cow_alloc_into` / `put_tree`) + /// is hoisted HERE, around the closure, exactly like `ht_insert`. The closure + /// captures only `&mut self.freemap` plus the local `tree` — disjoint from + /// `&mut self.packer` — so the packer borrow and the alloc closure both hold + /// `self` at once; `cache` is passed as a param. /// - /// `put_freemap_tree` runs even on the error path — matching the historical + /// `put_tree` runs even on the error path — matching the historical /// `allocate_data_page`, which wrote tree growth back before returning the /// allocation result. The freemap pages were extended (never freed), so on a /// non-fatal failure they are harmless above-watermark scratch, and the @@ -284,17 +281,14 @@ impl TransactionManager { // named separately. let reuse = self.savepoints.is_empty(); let packing_enabled = self.savepoints.is_empty(); - let mut tree = self.take_freemap_tree(); + let mut tree = self.freemap.take_tree(&self.current_roots); let result = { - let hint = &mut self.freemap_hint; - let pool = &mut self.structural_reuse; let mut cache = self.cache.borrow_mut(); - let mut alloc = - |c: &mut PageCache| super::freemap::cow_alloc(c, &mut tree, hint, pool, reuse); + let mut alloc = |c: &mut PageCache| self.freemap.cow_alloc_into(c, &mut tree, reuse); self.packer .insert(&mut cache, &mut alloc, packing_enabled, value) }; - self.put_freemap_tree(tree); + self.freemap.put_tree(&mut self.current_roots, tree); result } diff --git a/src/transaction/recovery.rs b/src/transaction/recovery.rs index b1a9457..fab45eb 100644 --- a/src/transaction/recovery.rs +++ b/src/transaction/recovery.rs @@ -89,11 +89,7 @@ impl TransactionManager { active_txn: false, savepoints: Vec::new(), txn_freed_pages: Vec::new(), - freemap_hint: 0, - pending_structural_frees: Vec::new(), - structural_reuse: Vec::new(), - structural_superseded: Vec::new(), - freemap_session_owned: FxHashSet::default(), + freemap: freemap::FreemapRecycle::new(), // A fresh database has no data pages and no live slots yet. packer: packing::SlotPacker::new(), poisoned: Cell::new(false), @@ -314,11 +310,7 @@ impl TransactionManager { active_txn: false, savepoints: Vec::new(), txn_freed_pages: Vec::new(), - freemap_hint: 0, - pending_structural_frees: Vec::new(), - structural_reuse: Vec::new(), - structural_superseded: Vec::new(), - freemap_session_owned: FxHashSet::default(), + freemap: freemap::FreemapRecycle::new(), packer: packing::SlotPacker::from_committed(committed_live_slots), poisoned: Cell::new(false), #[cfg(test)] diff --git a/src/transaction/staging.rs b/src/transaction/staging.rs index d093549..9815cf5 100644 --- a/src/transaction/staging.rs +++ b/src/transaction/staging.rs @@ -3,7 +3,6 @@ //! roots, the abort-prepare unwind, and the membership-failure injection //! hook. Split out of `transaction.rs` verbatim; see the parent module. -use super::freemap::cow_alloc; use super::*; impl TransactionManager { @@ -45,12 +44,10 @@ impl TransactionManager { freed: &mut Vec, ) -> Result { let reuse = self.savepoints.is_empty(); - let mut tree = self.take_freemap_tree(); + let mut tree = self.freemap.take_tree(&self.current_roots); let result = { - let hint = &mut self.freemap_hint; - let pool = &mut self.structural_reuse; let mut cache = self.cache.borrow_mut(); - let mut alloc = |c: &mut PageCache| cow_alloc(c, &mut tree, hint, pool, reuse); + let mut alloc = |c: &mut PageCache| self.freemap.cow_alloc_into(c, &mut tree, reuse); self.membership_index.insert( &mut cache, self.current_roots.membership_index_page, @@ -64,10 +61,10 @@ impl TransactionManager { // pages were extended (never freed), so a non-fatal failure that discards // `freed`/the candidate root leaves these extra pages as harmless // above-watermark scratch, exactly like the other COW pages on an aborted - // prepare. put_freemap_tree drains the freemap COW supersedes into + // prepare. put_tree drains the freemap COW supersedes into // structural_superseded and returns the session set so the next site in // this transaction stays in-place. - self.put_freemap_tree(tree); + self.freemap.put_tree(&mut self.current_roots, tree); result } @@ -94,12 +91,10 @@ impl TransactionManager { return Err(ChiselError::CacheFull { limit: 0 }); } let reuse = self.savepoints.is_empty(); - let mut tree = self.take_freemap_tree(); + let mut tree = self.freemap.take_tree(&self.current_roots); let result = { - let hint = &mut self.freemap_hint; - let pool = &mut self.structural_reuse; let mut cache = self.cache.borrow_mut(); - let mut alloc = |c: &mut PageCache| cow_alloc(c, &mut tree, hint, pool, reuse); + let mut alloc = |c: &mut PageCache| self.freemap.cow_alloc_into(c, &mut tree, reuse); self.handle_table.insert( &mut cache, self.current_roots.handle_table_page, @@ -112,9 +107,9 @@ impl TransactionManager { // Install freemap growth into roots (and return the session set) so the // NEXT candidate in this allocate (the reverse-map insert) threads the // up-to-date tree and treats already-COW'd freemap pages as in-place. The - // freemap COW supersedes go to structural_superseded via put_freemap_tree. + // freemap COW supersedes go to structural_superseded via put_tree. // See membership_insert_candidate for the abort-safety reasoning. - self.put_freemap_tree(tree); + self.freemap.put_tree(&mut self.current_roots, tree); result } @@ -171,12 +166,10 @@ impl TransactionManager { freed: &mut Vec, ) -> Result<(u64, bool)> { let reuse = self.savepoints.is_empty(); - let mut tree = self.take_freemap_tree(); + let mut tree = self.freemap.take_tree(&self.current_roots); let result = { - let hint = &mut self.freemap_hint; - let pool = &mut self.structural_reuse; let mut cache = self.cache.borrow_mut(); - let mut alloc = |c: &mut PageCache| cow_alloc(c, &mut tree, hint, pool, reuse); + let mut alloc = |c: &mut PageCache| self.freemap.cow_alloc_into(c, &mut tree, reuse); self.membership_index.remove( &mut cache, self.current_roots.membership_index_page, @@ -186,7 +179,7 @@ impl TransactionManager { freed, ) }; - self.put_freemap_tree(tree); + self.freemap.put_tree(&mut self.current_roots, tree); result } diff --git a/src/transaction/tests.rs b/src/transaction/tests.rs index a48e0c8..5d41e26 100644 --- a/src/transaction/tests.rs +++ b/src/transaction/tests.rs @@ -635,8 +635,12 @@ fn structural_recycle_one_commit_defer() { // Capture the promoted recycle the measured transaction inherits, and // drain the warm-up's reuse log so only the measured transaction is seen. - let promoted: std::collections::HashSet = - tm.pending_structural_frees.iter().copied().collect(); + let promoted: std::collections::HashSet = tm + .freemap + .pending_structural_frees() + .iter() + .copied() + .collect(); assert!( !promoted.is_empty(), "precondition: warm-up must leave a non-empty deferred recycle" @@ -661,7 +665,7 @@ fn structural_recycle_one_commit_defer() { // Pages T+1 superseded so far (the body). persist_freemap adds more at // commit; both must stay out of the reuse pops. let superseded_in_t1: std::collections::HashSet = - tm.structural_superseded.iter().copied().collect(); + tm.freemap.structural_superseded().iter().copied().collect(); tm.commit().unwrap(); // Read the log IMMEDIATELY after T+1's commit — before any later @@ -714,7 +718,7 @@ fn structural_recycle_one_commit_defer() { .into_iter() .collect() }; - for id in &tm.pending_structural_frees { + for id in tm.freemap.pending_structural_frees() { assert!( !reachable.contains(id), "one-commit-defer VIOLATION: page {id} is queued for reuse in T+2 but is \ @@ -740,7 +744,7 @@ fn structural_recycle_rollback_resets_pools() { // Establish a non-empty committed recycle so the test exercises a real // restore target, not just emptiness. let survivors = structural_churn(&mut tm, &big, 8, 4); - let committed_recycle: Vec = tm.pending_structural_frees.clone(); + let committed_recycle: Vec = tm.freemap.pending_structural_frees().to_vec(); assert!( !committed_recycle.is_empty(), "precondition: a prior commit must leave a non-empty deferred recycle" @@ -758,7 +762,7 @@ fn structural_recycle_rollback_resets_pools() { // path; assert the test actually dirtied the state it is about to roll // back (else the reset assertions are vacuous). assert!( - !tm.freemap_session_owned.is_empty() || !tm.structural_superseded.is_empty(), + !tm.freemap.session_owned().is_empty() || !tm.freemap.structural_superseded().is_empty(), "precondition: the pre-rollback churn must have mutated freemap session/supersede state" ); @@ -768,11 +772,16 @@ fn structural_recycle_rollback_resets_pools() { // aborted transaction's recycle is exactly the pre-transaction one: the // committed recycle must be intact, and the working pools cleared. assert_eq!( - tm.pending_structural_frees, committed_recycle, + tm.freemap.pending_structural_frees(), + committed_recycle.as_slice(), "rollback must leave the committed deferred recycle intact" ); - let recycle_after: std::collections::HashSet = - tm.pending_structural_frees.iter().copied().collect(); + let recycle_after: std::collections::HashSet = tm + .freemap + .pending_structural_frees() + .iter() + .copied() + .collect(); let committed_set: std::collections::HashSet = committed_recycle.iter().copied().collect(); assert_eq!( recycle_after, committed_set, @@ -780,12 +789,12 @@ fn structural_recycle_rollback_resets_pools() { must equal the committed recycle state" ); assert!( - tm.structural_superseded.is_empty(), + tm.freemap.structural_superseded().is_empty(), "rollback must clear structural_superseded — those committed-tree pages are \ still referenced and must never be recycled" ); assert!( - tm.freemap_session_owned.is_empty(), + tm.freemap.session_owned().is_empty(), "rollback must clear freemap_session_owned — leaking it would suppress a needed \ COW and mutate a live committed page in place next transaction" ); @@ -887,9 +896,9 @@ fn orphan_sweep_skipped_under_savepoint_preserves_committed_freemap() { // The streams the rollback_to does NOT reset must be untouched by the // sweep, or the rollback leaves dangerous residue. assert!( - tm.structural_superseded.is_empty(), + tm.freemap.structural_superseded().is_empty(), "sweep under savepoint leaked into structural_superseded: {:?}", - tm.structural_superseded + tm.freemap.structural_superseded() ); // Roll back to the savepoint (discards the forged page) and commit the @@ -1113,7 +1122,7 @@ fn structural_recycle_no_lost_or_double_free() { .into_iter() .collect() }; - for id in &tm.pending_structural_frees { + for id in tm.freemap.pending_structural_frees() { assert!( !reachable.contains(id), "round {round}: freemap page {id} is in the structural reuse pool AND \ @@ -1130,7 +1139,7 @@ fn structural_recycle_no_lost_or_double_free() { tm.committed_roots.freemap_page, tm.committed_roots.freemap_depth, ); - for id in &tm.pending_structural_frees { + for id in tm.freemap.pending_structural_frees() { assert!( !committed.is_free(&mut cache, *id).unwrap(), "round {round}: structural-reuse page {id} is ALSO marked free in the \ @@ -1210,14 +1219,14 @@ fn reclaim_freemap_orphans_excludes_live_recycle_pool() { let pooled = tm.test_forge_freemap_orphan().unwrap(); tm.begin().unwrap(); - tm.structural_reuse.push(pooled); + tm.freemap.push_structural_reuse_for_test(pooled); let reclaimed = tm.reclaim_freemap_orphans().unwrap(); assert_eq!( reclaimed, 0, "a page in the live recycle pool must NOT be reclaimed as an orphan" ); assert!( - tm.structural_reuse.contains(&pooled), + tm.freemap.structural_reuse().contains(&pooled), "the sweep must leave the live recycle pool untouched" ); tm.rollback().unwrap();