diff --git a/laurus/src/vector/index/flat/searcher.rs b/laurus/src/vector/index/flat/searcher.rs index a2f211bc..606a5172 100644 --- a/laurus/src/vector/index/flat/searcher.rs +++ b/laurus/src/vector/index/flat/searcher.rs @@ -236,7 +236,10 @@ impl VectorIndexSearcher for FlatVectorSearcher { if let Some(ref field_name) = request.field_name { Ok(self.index_reader.doc_ids_for_field(field_name).len() as u64) } else { - Ok(self.index_reader.vector_ids()?.len() as u64) + // Issue #672: `vector_ids()` materializes a String per record + // just to be counted; `vector_count()` is the same number (one + // entry per (doc, field) record) with no allocation. + Ok(self.index_reader.vector_count() as u64) } } } diff --git a/laurus/src/vector/index/hnsw/reader.rs b/laurus/src/vector/index/hnsw/reader.rs index d18f9662..d74884a2 100644 --- a/laurus/src/vector/index/hnsw/reader.rs +++ b/laurus/src/vector/index/hnsw/reader.rs @@ -1090,6 +1090,24 @@ impl HnswIndexReader { &self.vectors } + /// Iterate the interned `(doc_id, field_name)` records without + /// materializing a `String` per record (Issue #672). + /// + /// The trait-level [`VectorIndexReader::vector_ids`] must rehydrate + /// owned `String`s at every call (its signature predates the #633 + /// interning); callers that already hold a concrete + /// `HnswIndexReader` — e.g. the warmup page-fault pass — can borrow + /// the dictionary-backed names instead. + /// + /// # Returns + /// + /// An iterator over `(doc_id, &field_name)` in record order. + pub(crate) fn interned_vector_ids(&self) -> impl Iterator { + self.vector_ids + .iter() + .map(|&(id, fid)| (id, &*self.field_dict[fid as usize])) + } + /// Borrow the optional Stage 2 rerank storage pool. /// /// Returns `Some(_)` only when this reader was loaded against a diff --git a/laurus/src/vector/index/hnsw/searcher.rs b/laurus/src/vector/index/hnsw/searcher.rs index f45ae1fe..c75b60ea 100644 --- a/laurus/src/vector/index/hnsw/searcher.rs +++ b/laurus/src/vector/index/hnsw/searcher.rs @@ -438,7 +438,10 @@ impl VectorIndexSearcher for HnswSearcher { if let Some(ref field_name) = request.field_name { Ok(self.index_reader.doc_ids_for_field(field_name).len() as u64) } else { - Ok(self.index_reader.vector_ids()?.len() as u64) + // Issue #672: `vector_ids()` materializes a String per record + // just to be counted; `vector_count()` is the same number (one + // entry per (doc, field) record) with no allocation. + Ok(self.index_reader.vector_count() as u64) } } @@ -468,10 +471,11 @@ impl VectorIndexSearcher for HnswSearcher { } // Read every stored vector so its backing page is faulted in. The // accumulator (kept live via `black_box`) stops the loop from being - // optimised away as dead code. + // optimised away as dead code. The interned iterator (#672) avoids + // materializing one `String` per record just to name the field. let mut acc = 0u64; - for (doc_id, field) in reader.vector_ids()? { - if let Ok(Some(vector)) = reader.get_vector(doc_id, &field) + for (doc_id, field) in reader.interned_vector_ids() { + if let Ok(Some(vector)) = reader.get_vector(doc_id, field) && let Some(first) = vector.data.first() { acc = acc.wrapping_add(first.to_bits() as u64); diff --git a/laurus/src/vector/index/ivf/searcher.rs b/laurus/src/vector/index/ivf/searcher.rs index 4c660162..3c384084 100644 --- a/laurus/src/vector/index/ivf/searcher.rs +++ b/laurus/src/vector/index/ivf/searcher.rs @@ -330,7 +330,10 @@ impl VectorIndexSearcher for IvfSearcher { if let Some(ref field_name) = request.field_name { Ok(self.index_reader.doc_ids_for_field(field_name).len() as u64) } else { - Ok(self.index_reader.vector_ids()?.len() as u64) + // Issue #672: `vector_ids()` materializes a String per record + // just to be counted; `vector_count()` is the same number (one + // entry per (doc, field) record) with no allocation. + Ok(self.index_reader.vector_count() as u64) } } } diff --git a/laurus/src/vector/index/segment/fanout.rs b/laurus/src/vector/index/segment/fanout.rs index 9e44a851..502dabdc 100644 --- a/laurus/src/vector/index/segment/fanout.rs +++ b/laurus/src/vector/index/segment/fanout.rs @@ -549,21 +549,29 @@ impl VectorIndexSearcher for SegmentFanoutSearcher { // Count distinct live `(doc_id, field)` keys across segments with // the same newest-wins masking as `search`, excluding soft-deleted // docs. + // + // Issue #672: iterate per field through `doc_ids_for_field` (an + // O(1) `Arc` clone on every reader type, #405) instead of + // `vector_ids()`, whose trait-boundary rehydration materializes a + // fresh `String` per record on every call — a full-corpus + // allocation per segment per count query. `field_names()` is + // dictionary-backed and tiny on all three readers. let mut count = 0u64; for (idx, reader) in self.readers.iter().enumerate() { - for (doc_id, field) in reader.vector_ids()? { - if let Some(ref field_name) = request.field_name - && &field != field_name - { - continue; - } - if let Some(bitmap) = &self.bitmap - && bitmap.is_deleted(doc_id) - { - continue; - } - if !self.shadowed(idx, doc_id, &field) { - count += 1; + let fields: Vec = match request.field_name { + Some(ref field_name) => vec![field_name.clone()], + None => reader.field_names()?, + }; + for field in &fields { + for &doc_id in reader.doc_ids_for_field(field).iter() { + if let Some(bitmap) = &self.bitmap + && bitmap.is_deleted(doc_id) + { + continue; + } + if !self.shadowed(idx, doc_id, field) { + count += 1; + } } } } diff --git a/laurus/tests/segment_score_comparability_test.rs b/laurus/tests/segment_score_comparability_test.rs index a9b893c7..dd3a15b5 100644 --- a/laurus/tests/segment_score_comparability_test.rs +++ b/laurus/tests/segment_score_comparability_test.rs @@ -179,3 +179,34 @@ fn min_similarity_applies_to_the_shared_basis_score() { ); } } + +/// Issue #672 regression: `SegmentFanoutSearcher::count` was rewritten +/// from per-segment `vector_ids()` full clones onto the Arc-backed +/// `doc_ids_for_field` path — its semantics must be unchanged: distinct +/// live `(doc_id, field)` keys, counting cross-segment duplicates once +/// (newest-wins), excluding soft-deleted docs, and honoring the field +/// filter. +#[test] +fn count_masks_duplicates_and_deletions() { + use laurus::vector::search::searcher::VectorIndexQueryParams; + + let (index, live) = build_fixture(); + let searcher = index.searcher().unwrap(); + + let count_for = |field: Option<&str>| { + searcher + .count(VectorIndexQuery { + query: grid(0.0), + params: VectorIndexQueryParams::default(), + field_name: field.map(str::to_string), + filter: None, + }) + .unwrap() + }; + + // 30 docs, one deleted; the stale copies of docs 3 and 7 are the + // same (doc, field) keys and must not be double-counted. + assert_eq!(count_for(Some("v")), live.len() as u64); + assert_eq!(count_for(None), live.len() as u64); + assert_eq!(count_for(Some("missing")), 0); +}