From 080bf645a6c48db5ee214aa7854578bf5f15902e Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Mon, 2 Nov 2026 22:15:01 -0800 Subject: [PATCH 01/14] fix(search): admit prose words as identifier subtokens --- .../tracedecay-query/src/retrieval/lexical.rs | 16 +++---- .../src/retrieval/lexical/tests.rs | 37 +++++++++++++--- .../candidate_producers.rs | 43 +++++++++++++++++++ 3 files changed, 82 insertions(+), 14 deletions(-) diff --git a/crates/tracedecay-query/src/retrieval/lexical.rs b/crates/tracedecay-query/src/retrieval/lexical.rs index 45c7c2613f..1e1443c8d2 100644 --- a/crates/tracedecay-query/src/retrieval/lexical.rs +++ b/crates/tracedecay-query/src/retrieval/lexical.rs @@ -158,8 +158,10 @@ pub(super) fn normalize_lexical(value: &str) -> String { /// Whole terms are the same maximal technical tokens the code index extracts /// (`tracedecay_domain::technical_tokens`), so a term the indexer keeps whole /// (`foo-bar`, `a::b`, `p/q.rs`) is queried whole, and subtokens decompose -/// through the same shared grammar. Single-token queries intentionally emit -/// no subtokens so one technical term does not fan out to common subtokens. +/// through the same shared grammar. For multiword prose, even an unsplit +/// word is searched in the Subtoken field so a matching camelCase symbol can +/// be admitted as a candidate. Single-token queries intentionally emit no +/// subtokens so one technical term does not fan out to common subtokens. /// /// Multi-token sanitized input is also retained as a phrase. This gives exact /// diagnostic/error text and natural-language queries a bounded lexical phrase @@ -181,12 +183,10 @@ pub fn lexical_query_parts(query: &str) -> Result = index_tokens .iter() - .flat_map(|token| { - let lowercase = token.to_ascii_lowercase(); - split_subtokens(token) - .into_iter() - .filter(move |part| part != &lowercase) - }) + .flat_map(|token| split_subtokens(token)) .collect(); expected_subtokens.sort(); expected_subtokens.dedup(); @@ -199,6 +194,36 @@ fn query_tokenization_matches_index_tokenization_for_separator_terms() { } } + +#[test] +fn prose_words_can_match_identifier_subtokens_without_broadening_single_term_queries() { + let query = "widget auto tag items forward total"; + let parts = lexical_query_parts(query).expect("prose search"); + assert_eq!( + parts.subtokens, + ["auto", "forward", "items", "tag", "total", "widget"] + .map(str::to_owned) + .to_vec() + ); + assert_eq!(parts.whole_terms.len(), 6); + + let hyphenated = + lexical_query_parts("widget-tool auto-tag query rows per item").expect("hyphenated prose"); + for word in ["widget", "tool", "auto", "tag", "query", "rows", "per", "item"] { + assert!( + hyphenated.subtokens.contains(&word.to_owned()), + "{word} must reach a camelCase subtoken posting" + ); + } + assert!( + lexical_query_parts("fetchWidgetAutoTagItemsForwardTotal") + .expect("single identifier") + .subtokens + .is_empty(), + "single-term queries must retain their narrow admission" + ); +} + fn budget(max_candidates_per_lane: u32) -> RetrievalBudget { RetrievalBudget { max_candidates_per_lane, diff --git a/crates/tracedecay-query/tests/search_quality_suite/candidate_producers.rs b/crates/tracedecay-query/tests/search_quality_suite/candidate_producers.rs index 4cd22cd3ea..b8c4ed5238 100644 --- a/crates/tracedecay-query/tests/search_quality_suite/candidate_producers.rs +++ b/crates/tracedecay-query/tests/search_quality_suite/candidate_producers.rs @@ -6283,6 +6283,49 @@ fn duplicate_whole_terms_do_not_consume_the_global_fuzzy_budget() { ); } + +#[test] +fn prose_query_recovers_camel_case_symbol_from_real_sealed_subtoken_postings() { + let artifact = rust_artifact(&[ + "pub fn fetchWidgetAutoTagItemsForwardTotal() {}\n", + "pub fn widget() {}\n", + "pub fn tag() {}\n", + ]); + let query = "widget auto tag items forward total"; + let parts = tracedecay_query::retrieval::lexical::lexical_query_parts(query) + .expect("parse multiword query"); + let mut request = artifact.request(query, &[], &[], &[], 0, 32); + request.whole_terms = Cow::Owned(parts.whole_terms); + request.subtokens = Cow::Owned(parts.subtokens); + request.phrases = Cow::Owned(parts.phrases); + let results = complete( + artifact + .lane() + .retrieve_lexical(&request) + .expect("sealed lexical search"), + ); + assert!( + candidate_files(&results.candidates).contains(&0), + "the camelCase symbol must not be absent from the bounded page: {results:?}" + ); + assert!( + results.candidates.iter().any(|candidate| { + candidate.file_occurrence_id.as_ref().is_some_and(|file| { + file.as_str() == "file.fixture.0" + && ["widget", "auto", "tag", "items", "forward", "total"] + .iter() + .all(|term| { + results.evidence_by_occurrence[&candidate.source_occurrence_id] + .matched_subtokens + .iter() + .any(|matched| matched == term) + }) + }) + }), + "the long symbol must match every query word in the subtoken field" + ); +} + #[test] fn lexical_artifact_reports_freshness_coverage_and_page_cutoff() { let sources = ["pub fn target() {}\n"; 3]; From 8deba1c19d33258d8003dce70dd3712a5778faf6 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Mon, 2 Nov 2026 22:16:16 -0800 Subject: [PATCH 02/14] test(search): tighten prose subtoken regressions --- crates/tracedecay-query/src/retrieval/lexical/tests.rs | 1 - .../tests/search_quality_suite/candidate_producers.rs | 3 +-- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/crates/tracedecay-query/src/retrieval/lexical/tests.rs b/crates/tracedecay-query/src/retrieval/lexical/tests.rs index d582b3e7b2..b7270a156a 100644 --- a/crates/tracedecay-query/src/retrieval/lexical/tests.rs +++ b/crates/tracedecay-query/src/retrieval/lexical/tests.rs @@ -194,7 +194,6 @@ fn query_tokenization_matches_index_tokenization_for_separator_terms() { } } - #[test] fn prose_words_can_match_identifier_subtokens_without_broadening_single_term_queries() { let query = "widget auto tag items forward total"; diff --git a/crates/tracedecay-query/tests/search_quality_suite/candidate_producers.rs b/crates/tracedecay-query/tests/search_quality_suite/candidate_producers.rs index b8c4ed5238..667496e632 100644 --- a/crates/tracedecay-query/tests/search_quality_suite/candidate_producers.rs +++ b/crates/tracedecay-query/tests/search_quality_suite/candidate_producers.rs @@ -6283,7 +6283,6 @@ fn duplicate_whole_terms_do_not_consume_the_global_fuzzy_budget() { ); } - #[test] fn prose_query_recovers_camel_case_symbol_from_real_sealed_subtoken_postings() { let artifact = rust_artifact(&[ @@ -6318,7 +6317,7 @@ fn prose_query_recovers_camel_case_symbol_from_real_sealed_subtoken_postings() { results.evidence_by_occurrence[&candidate.source_occurrence_id] .matched_subtokens .iter() - .any(|matched| matched == term) + .any(|matched| matched.as_str() == *term) }) }) }), From 1fb0dd349d84e085b686229075d56b160f221cc6 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Mon, 2 Nov 2026 23:52:00 -0800 Subject: [PATCH 03/14] fix(search): deduplicate overlapping candidate source channels --- .../projection/artifact/reader/query.rs | 239 ++++++++++++++++-- 1 file changed, 219 insertions(+), 20 deletions(-) diff --git a/crates/tracedecay-query/src/retrieval/lexical/projection/artifact/reader/query.rs b/crates/tracedecay-query/src/retrieval/lexical/projection/artifact/reader/query.rs index 0ee1e6f516..c79a36d43d 100644 --- a/crates/tracedecay-query/src/retrieval/lexical/projection/artifact/reader/query.rs +++ b/crates/tracedecay-query/src/retrieval/lexical/projection/artifact/reader/query.rs @@ -1169,7 +1169,8 @@ impl<'a> ArtifactQueryV1<'a> { /// The candidate document set for one lexical request: every phrase /// match plus the term sources `admit_candidate_sources` keeps under - /// `MAX_LEXICAL_CANDIDATE_DOCUMENTS_V1`. + /// `MAX_LEXICAL_CANDIDATE_DOCUMENTS_V1`. A term requested through both + /// whole and subtoken channels is charged once for their document union. fn lexical_documents( &self, request: &LexicalLaneRequest<'_>, @@ -1197,31 +1198,16 @@ impl<'a> ArtifactQueryV1<'a> { .iter() .map(|subtoken| normalize_lexical(subtoken)) .collect::>(); - let mut sources = Vec::with_capacity(whole_terms.len() + subtokens.len()); - for term in whole_terms { - if stats.postings.by_term.contains_key(&term) { - sources.push((stats.whole_term_documents(&term), (term, false))); - } - } - for subtoken in subtokens { - if stats.postings.by_term.contains_key(&subtoken) { - sources.push(( - stats.document_frequency(LexicalFieldV1::Subtoken, &subtoken), - (subtoken, true), - )); - } - } + let sources = stats.candidate_sources(whole_terms, subtokens)?; let mut documents = phrase_queries .values() .fold(RoaringBitmap::new(), |union, documents| union | documents); - for (term, subtoken) in admit_candidate_sources(sources, |frequency, (term, _)| { - pruned.push((term.clone(), frequency as u64)); + for source in admit_candidate_sources(sources, |frequency, source| { + pruned.push((source.term.clone(), frequency as u64)); }) { documents |= stats .postings - .documents(&term, |field| { - (field == LexicalFieldV1::Subtoken) == subtoken - }) + .documents(&source.term, |field| source.fields.admits(field)) .map_err(map_query_artifact_error)?; } Ok(documents) @@ -1607,6 +1593,28 @@ struct LexicalStatsCacheV1 { postings: RequestTermPostingsV1, } +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum CandidateSourceFieldsV1 { + Whole, + Subtoken, + All, +} + +impl CandidateSourceFieldsV1 { + fn admits(self, field: LexicalFieldV1) -> bool { + match self { + Self::Whole => field != LexicalFieldV1::Subtoken, + Self::Subtoken => field == LexicalFieldV1::Subtoken, + Self::All => true, + } + } +} + +struct LexicalCandidateSourceV1 { + term: String, + fields: CandidateSourceFieldsV1, +} + fn lexical_terms( prepared: &PreparedLexicalQueryV1<'_>, fuzzy: &FuzzyExpansionsV1, @@ -1663,6 +1671,54 @@ impl LexicalStatsCacheV1 { .map(|(_, frequencies)| frequencies.get(term).copied().unwrap_or_default()) .fold(0usize, usize::saturating_add) } + + /// Charge a normalized term only once when both scoring channels name it. + /// Their actual document union, not the larger individual frequency, is + /// required: the channels may overlap completely or be disjoint. Scoring + /// retains the original field postings and frequencies. Only one transient + /// bitmap is needed here; admitted sources are decoded into the final set. + fn candidate_sources( + &self, + whole_terms: Vec, + subtokens: Vec, + ) -> Result, RetrievalPortError> { + let mut subtokens = subtokens.into_iter().collect::>(); + let mut seen = BTreeSet::new(); + let mut sources = Vec::with_capacity(whole_terms.len() + subtokens.len()); + for term in whole_terms { + if !seen.insert(term.clone()) || !self.postings.by_term.contains_key(&term) { + continue; + } + let (frequency, fields) = if subtokens.remove(&term) { + let documents = self + .postings + .documents(&term, |_| true) + .map_err(map_query_artifact_error)?; + ( + usize::try_from(documents.len()).map_err(contract_error)?, + CandidateSourceFieldsV1::All, + ) + } else { + ( + self.whole_term_documents(&term), + CandidateSourceFieldsV1::Whole, + ) + }; + sources.push((frequency, LexicalCandidateSourceV1 { term, fields })); + } + for term in subtokens { + if self.postings.by_term.contains_key(&term) { + sources.push(( + self.document_frequency(LexicalFieldV1::Subtoken, &term), + LexicalCandidateSourceV1 { + term, + fields: CandidateSourceFieldsV1::Subtoken, + }, + )); + } + } + Ok(sources) + } } struct LexicalDraftV1 { @@ -1968,3 +2024,146 @@ impl EditDistanceScratchV1 { (self.previous[width - 1] <= limit).then_some(self.previous[width - 1]) } } + +#[cfg(test)] +mod candidate_source_tests { + use std::collections::BTreeMap; + use std::ops::Range; + + use roaring::RoaringBitmap; + + use super::super::super::format::PostingListEncoderV1; + use super::{ + CandidateSourceFieldsV1, LexicalStatsCacheV1, RequestTermPostingsV1, TermFieldPostingsV1, + }; + use crate::retrieval::lexical::{LexicalFieldV1, admit_candidate_sources}; + + fn stats(entries: &[(&str, LexicalFieldV1, Range)]) -> LexicalStatsCacheV1 { + let mut stats = LexicalStatsCacheV1 { + field_totals: BTreeMap::new(), + document_frequencies: BTreeMap::new(), + postings: RequestTermPostingsV1::default(), + }; + for (term, field, documents) in entries { + let mut encoder = PostingListEncoderV1::new(true); + for document in documents.clone() { + encoder.push(document, 1).expect("ordered posting"); + } + stats + .document_frequencies + .entry(*field) + .or_default() + .insert((*term).to_owned(), documents.clone().count()); + stats + .postings + .by_term + .entry((*term).to_owned()) + .or_default() + .push(TermFieldPostingsV1 { + field: *field, + postings: encoder.finish().expect("nonempty postings"), + }); + } + stats + } + + fn candidates( + stats: &LexicalStatsCacheV1, + whole: &[&str], + subtokens: &[&str], + ) -> (RoaringBitmap, Vec<(String, usize)>) { + let sources = stats + .candidate_sources( + whole.iter().map(|term| (*term).to_owned()).collect(), + subtokens.iter().map(|term| (*term).to_owned()).collect(), + ) + .expect("candidate sources"); + let mut pruned = Vec::new(); + let admitted = admit_candidate_sources(sources, |frequency, source| { + pruned.push((source.term.clone(), frequency)); + }); + let mut documents = RoaringBitmap::new(); + for source in admitted { + documents |= stats + .postings + .documents(&source.term, |field| source.fields.admits(field)) + .expect("decode admitted postings"); + } + (documents, pruned) + } + + #[test] + fn overlapping_prose_channels_do_not_crowd_out_a_distinct_word() { + let stats = stats(&[ + ("needle", LexicalFieldV1::BodyText, 0..100), + ("needle", LexicalFieldV1::Subtoken, 0..100), + ("target", LexicalFieldV1::BodyText, 100..16_350), + ("target", LexicalFieldV1::Subtoken, 100..16_350), + ]); + let (documents, pruned) = + candidates(&stats, &["needle", "target"], &["needle", "target"]); + assert!(pruned.is_empty(), "{pruned:?}"); + assert_eq!(documents.len(), 16_350); + assert!( + documents.contains(16_349), + "the target-only document must remain eligible" + ); + // Admission grouping must leave both channels available to scoring. + let mut cursors = stats.postings.cursors().unwrap(); + let target_channels = cursors + .iter_mut() + .filter(|cursor| cursor.term == "target") + .map(|cursor| (cursor.field, cursor.frequency_at(16_349).unwrap())) + .collect::>(); + assert_eq!( + target_channels, + vec![ + (LexicalFieldV1::BodyText, Some(1)), + (LexicalFieldV1::Subtoken, Some(1)), + ] + ); + } + + #[test] + fn disjoint_channels_are_charged_for_their_full_union() { + let stats = stats(&[ + ("needle", LexicalFieldV1::BodyText, 0..100), + ("needle", LexicalFieldV1::Subtoken, 100..200), + ("target", LexicalFieldV1::BodyText, 200..16_400), + ("target", LexicalFieldV1::Subtoken, 200..16_400), + ]); + let (documents, pruned) = + candidates(&stats, &["needle", "target"], &["needle", "target"]); + assert_eq!(documents.len(), 200); + assert_eq!(pruned, vec![("target".to_owned(), 16_200)]); + assert!(!documents.contains(16_399)); + } + + #[test] + fn repeated_sources_are_deduplicated_without_widening_requested_fields() { + let stats = stats(&[ + ("needle", LexicalFieldV1::BodyText, 0..100), + ("needle", LexicalFieldV1::Subtoken, 100..200), + ("part", LexicalFieldV1::Subtoken, 200..210), + ]); + let sources = stats + .candidate_sources( + vec!["needle".to_owned(), "needle".to_owned()], + vec!["part".to_owned(), "part".to_owned()], + ) + .unwrap(); + assert_eq!(sources.len(), 2); + assert_eq!(sources[0].0, 100); + assert_eq!(sources[0].1.fields, CandidateSourceFieldsV1::Whole); + assert_eq!(sources[1].0, 10); + assert_eq!(sources[1].1.fields, CandidateSourceFieldsV1::Subtoken); + let (documents, pruned) = candidates(&stats, &["needle", "needle"], &["part", "part"]); + assert!(pruned.is_empty()); + assert_eq!(documents.len(), 110); + assert!( + !documents.contains(100), + "unrequested subtoken channel stays excluded" + ); + assert!(documents.contains(209)); + } +} From bebc834973b138f5446f63ed6c7c3cc618aefd77 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Tue, 3 Nov 2026 02:23:28 -0800 Subject: [PATCH 04/14] style(search): format lexical candidate regressions --- .../retrieval/lexical/projection/artifact/reader/query.rs | 6 ++---- crates/tracedecay-query/src/retrieval/lexical/tests.rs | 4 +++- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/crates/tracedecay-query/src/retrieval/lexical/projection/artifact/reader/query.rs b/crates/tracedecay-query/src/retrieval/lexical/projection/artifact/reader/query.rs index c79a36d43d..398bf554a2 100644 --- a/crates/tracedecay-query/src/retrieval/lexical/projection/artifact/reader/query.rs +++ b/crates/tracedecay-query/src/retrieval/lexical/projection/artifact/reader/query.rs @@ -2100,8 +2100,7 @@ mod candidate_source_tests { ("target", LexicalFieldV1::BodyText, 100..16_350), ("target", LexicalFieldV1::Subtoken, 100..16_350), ]); - let (documents, pruned) = - candidates(&stats, &["needle", "target"], &["needle", "target"]); + let (documents, pruned) = candidates(&stats, &["needle", "target"], &["needle", "target"]); assert!(pruned.is_empty(), "{pruned:?}"); assert_eq!(documents.len(), 16_350); assert!( @@ -2132,8 +2131,7 @@ mod candidate_source_tests { ("target", LexicalFieldV1::BodyText, 200..16_400), ("target", LexicalFieldV1::Subtoken, 200..16_400), ]); - let (documents, pruned) = - candidates(&stats, &["needle", "target"], &["needle", "target"]); + let (documents, pruned) = candidates(&stats, &["needle", "target"], &["needle", "target"]); assert_eq!(documents.len(), 200); assert_eq!(pruned, vec![("target".to_owned(), 16_200)]); assert!(!documents.contains(16_399)); diff --git a/crates/tracedecay-query/src/retrieval/lexical/tests.rs b/crates/tracedecay-query/src/retrieval/lexical/tests.rs index b7270a156a..c8cfba3aeb 100644 --- a/crates/tracedecay-query/src/retrieval/lexical/tests.rs +++ b/crates/tracedecay-query/src/retrieval/lexical/tests.rs @@ -208,7 +208,9 @@ fn prose_words_can_match_identifier_subtokens_without_broadening_single_term_que let hyphenated = lexical_query_parts("widget-tool auto-tag query rows per item").expect("hyphenated prose"); - for word in ["widget", "tool", "auto", "tag", "query", "rows", "per", "item"] { + for word in [ + "widget", "tool", "auto", "tag", "query", "rows", "per", "item", + ] { assert!( hyphenated.subtokens.contains(&word.to_owned()), "{word} must reach a camelCase subtoken posting" From e7bf43619a46f9a3fd00c41c20c64ad99b66a570 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Tue, 3 Nov 2026 03:34:01 -0800 Subject: [PATCH 05/14] fix(search): admit unique lexical candidates without eager scans Keep whole and subtoken channels independently selectable and charge only new document IDs under the shared candidate cap. Prune provably oversized postings from stored frequencies before decoding and check cancellation within bounded posting batches. Cover rare whole-word recall, overlap, common-source pruning, and mid-stream cancellation. --- .../tracedecay-query/src/retrieval/lexical.rs | 63 ++-- .../projection/artifact/reader/query.rs | 270 ++++++++++++++---- .../src/retrieval/lexical/tests.rs | 88 ++++-- 3 files changed, 314 insertions(+), 107 deletions(-) diff --git a/crates/tracedecay-query/src/retrieval/lexical.rs b/crates/tracedecay-query/src/retrieval/lexical.rs index 1e1443c8d2..4f4375cd98 100644 --- a/crates/tracedecay-query/src/retrieval/lexical.rs +++ b/crates/tracedecay-query/src/retrieval/lexical.rs @@ -10,6 +10,7 @@ use std::borrow::Cow; use std::collections::{BTreeMap, BTreeSet}; +use roaring::RoaringBitmap; use serde::{Deserialize, Serialize}; use tracedecay_domain::{ CodeGenerationId, CompactCandidate, ComponentRevision, CursorPayloadDigest, @@ -78,45 +79,53 @@ pub const MAX_LEXICAL_FIELD_FILTERS_V1: usize = 9; /// proximity term. pub const MAX_LEXICAL_QUERY_TERM_BYTES_V1: usize = 512; -/// Summed document-frequency budget for lexical term-source admission. Every -/// candidate is decoded from its row and scored, so the union of the request's -/// term sources, not the winner cap, decides the lane's transient allocation -/// and wall time: unbounded, a natural-language task whose terms include -/// common words hydrated ~74k rows of a 472k-chunk corpus per read (~0.95 GB -/// decoded, 5.7 s) and missed the context deadline. Term sources are admitted -/// in ascending document-frequency order until their summed frequencies would -/// exceed this bound; the most selective source is always admitted so a -/// single common-term query still answers. Phrase sources are admitted separately. +/// Unique-document budget for lexical term-source admission. Every candidate +/// is decoded and scored, so source selection determines transient allocation +/// and wall time. Previously a natural-language query hydrated ~74k rows of +/// a 472k-chunk corpus (~0.95 GB decoded, 5.7 s), missing the deadline. +/// Sources are considered in ascending stored field-frequency order. Only +/// new document IDs charge the budget; the first nonempty source is always +/// admitted so a single common-term query still answers. Phrase sources +/// are admitted separately. /// This is a recall/latency policy, not a hard document or allocation ceiling: -/// documents matching only pruned terms cannot rank. The initial 16,384 value +/// documents matching only pruned sources cannot rank. The 16,384 value /// retains the measured policy (~100 MiB decode churn at ~6.6 KiB per row); /// changing the reader cache must not change candidate eligibility. pub const MAX_LEXICAL_CANDIDATE_DOCUMENTS_V1: usize = 16_384; -/// Admit `(document_frequency, source)` pairs rarest-first while the summed -/// frequency stays within [`MAX_LEXICAL_CANDIDATE_DOCUMENTS_V1`]; the rarest -/// nonempty source is always admitted. Ties keep request order so admission -/// is deterministic. Sources past the bound still weigh admitted candidates -/// through scoring; a document matching only those sources is never hydrated. +/// Visit posting sources rarest-first, counting unique documents rather than +/// charging overlapping whole/subtoken postings twice. The reader supplies a +/// bounded delta; only the rarest nonempty source may exceed the cap. pub(crate) fn admit_candidate_sources( mut sources: Vec<(usize, S)>, + mut new_documents: impl FnMut( + &S, + &RoaringBitmap, + usize, + ) -> Result, RetrievalPortError>, mut on_pruned: impl FnMut(usize, &S), -) -> Vec { +) -> Result { sources.retain(|(frequency, _)| *frequency > 0); sources.sort_by_key(|(frequency, _)| *frequency); - let total = sources.len(); - let mut admitted_documents = 0usize; - let mut admitted = Vec::with_capacity(total); - for (ordinal, (frequency, source)) in sources.into_iter().enumerate() { - let next = admitted_documents.saturating_add(frequency); - if ordinal > 0 && next > MAX_LEXICAL_CANDIDATE_DOCUMENTS_V1 { - on_pruned(frequency, &source); - continue; + let mut admitted = RoaringBitmap::new(); + let mut admitted_sources = 0usize; + for (frequency, source) in sources { + let remaining = if admitted_sources == 0 { + usize::MAX + } else { + MAX_LEXICAL_CANDIDATE_DOCUMENTS_V1.saturating_sub( + usize::try_from(admitted.len()).unwrap_or(usize::MAX), + ) + }; + match new_documents(&source, &admitted, remaining)? { + Some(delta) if delta.len() <= remaining as u64 => { + admitted |= delta; + admitted_sources += 1; + } + _ => on_pruned(frequency, &source), } - admitted_documents = next; - admitted.push(source); } - admitted + Ok(admitted) } fn candidate_admission_outcome( diff --git a/crates/tracedecay-query/src/retrieval/lexical/projection/artifact/reader/query.rs b/crates/tracedecay-query/src/retrieval/lexical/projection/artifact/reader/query.rs index 398bf554a2..234b99a935 100644 --- a/crates/tracedecay-query/src/retrieval/lexical/projection/artifact/reader/query.rs +++ b/crates/tracedecay-query/src/retrieval/lexical/projection/artifact/reader/query.rs @@ -23,7 +23,8 @@ use super::{map_query_artifact_error, map_query_sql_error}; use crate::retrieval::exact::{ExactAdmissionAuthority, ExactLaneEvidence, ExactLaneRequest}; use crate::retrieval::lexical::{ LexicalFieldFilterV1, LexicalFieldV1, LexicalLaneEvidence, LexicalLaneRequest, - MAX_FUZZY_TERM_EXPANSIONS_V1, MAX_LEXICAL_QUERY_TERM_BYTES_V1, admit_candidate_sources, + MAX_FUZZY_TERM_EXPANSIONS_V1, MAX_LEXICAL_CANDIDATE_DOCUMENTS_V1, + MAX_LEXICAL_QUERY_TERM_BYTES_V1, admit_candidate_sources, candidate_admission_outcome, field_admitted, }; use crate::retrieval::ports::{ @@ -236,6 +237,41 @@ impl RequestTermPostingsV1 { } Ok(documents) } + + /// Decode only the documents a source adds to the existing candidate set. + /// Stop as soon as it cannot fit; check cancellation during long postings + /// rather than waiting for the next row-hydration checkpoint. + fn new_documents_bounded( + &self, + term: &str, + admit: impl Fn(LexicalFieldV1) -> bool, + existing: &RoaringBitmap, + remaining: usize, + control: &dyn RetrievalExecutionControl, + ) -> Result, RetrievalPortError> { + let mut added = RoaringBitmap::new(); + let mut visited = 0usize; + for list in self.by_term.get(term).into_iter().flatten() { + if !admit(list.field) { + continue; + } + for posting in PostingListDecoderV1::new(&list.postings, true) { + if visited.is_multiple_of(RETRIEVAL_CANDIDATE_BATCH_SIZE) { + retrieval_checkpoint(control)?; + } + visited += 1; + let (document, _) = posting.map_err(map_query_artifact_error)?; + if !existing.contains(document) { + added.insert(document); + if added.len() > remaining as u64 { + return Ok(None); + } + } + } + } + retrieval_checkpoint(control)?; + Ok(Some(added)) + } } struct PostingCursorV1<'a> { @@ -1198,19 +1234,36 @@ impl<'a> ArtifactQueryV1<'a> { .iter() .map(|subtoken| normalize_lexical(subtoken)) .collect::>(); - let sources = stats.candidate_sources(whole_terms, subtokens)?; - let mut documents = phrase_queries + let sources = stats.candidate_sources(whole_terms, subtokens); + let term_documents = admit_candidate_sources( + sources, + |source, existing, remaining| { + // A source with more independently guaranteed documents than + // the entire cap cannot fit after another source was admitted. + // This avoids decoding very common postings merely to prune. + if remaining != usize::MAX + && source.minimum_documents > MAX_LEXICAL_CANDIDATE_DOCUMENTS_V1 + { + return Ok(None); + } + stats.postings.new_documents_bounded( + &source.term, + |field| source.fields.admits(field), + existing, + remaining, + request.control, + ) + }, + |frequency, source| { + if !pruned.iter().any(|(term, _)| term == &source.term) { + pruned.push((source.term.clone(), frequency as u64)); + } + }, + )?; + let phrase_documents = phrase_queries .values() .fold(RoaringBitmap::new(), |union, documents| union | documents); - for source in admit_candidate_sources(sources, |frequency, source| { - pruned.push((source.term.clone(), frequency as u64)); - }) { - documents |= stats - .postings - .documents(&source.term, |field| source.fields.admits(field)) - .map_err(map_query_artifact_error)?; - } - Ok(documents) + Ok(phrase_documents | term_documents) } fn exact_documents( @@ -1597,7 +1650,6 @@ struct LexicalStatsCacheV1 { enum CandidateSourceFieldsV1 { Whole, Subtoken, - All, } impl CandidateSourceFieldsV1 { @@ -1605,7 +1657,6 @@ impl CandidateSourceFieldsV1 { match self { Self::Whole => field != LexicalFieldV1::Subtoken, Self::Subtoken => field == LexicalFieldV1::Subtoken, - Self::All => true, } } } @@ -1613,6 +1664,7 @@ impl CandidateSourceFieldsV1 { struct LexicalCandidateSourceV1 { term: String, fields: CandidateSourceFieldsV1, + minimum_documents: usize, } fn lexical_terms( @@ -1672,53 +1724,59 @@ impl LexicalStatsCacheV1 { .fold(0usize, usize::saturating_add) } - /// Charge a normalized term only once when both scoring channels name it. - /// Their actual document union, not the larger individual frequency, is - /// required: the channels may overlap completely or be disjoint. Scoring - /// retains the original field postings and frequencies. Only one transient - /// bitmap is needed here; admitted sources are decoded into the final set. + /// Keep whole-word and subtoken channels independently eligible. Their + /// stored field frequencies order sources without eager bitmap decoding; + /// admission later accounts for exact new document IDs across channels. fn candidate_sources( &self, whole_terms: Vec, subtokens: Vec, - ) -> Result, RetrievalPortError> { - let mut subtokens = subtokens.into_iter().collect::>(); + ) -> Vec<(usize, LexicalCandidateSourceV1)> { let mut seen = BTreeSet::new(); let mut sources = Vec::with_capacity(whole_terms.len() + subtokens.len()); for term in whole_terms { if !seen.insert(term.clone()) || !self.postings.by_term.contains_key(&term) { continue; } - let (frequency, fields) = if subtokens.remove(&term) { - let documents = self - .postings - .documents(&term, |_| true) - .map_err(map_query_artifact_error)?; - ( - usize::try_from(documents.len()).map_err(contract_error)?, - CandidateSourceFieldsV1::All, - ) - } else { - ( - self.whole_term_documents(&term), - CandidateSourceFieldsV1::Whole, - ) - }; - sources.push((frequency, LexicalCandidateSourceV1 { term, fields })); + let frequency = self.whole_term_documents(&term); + if frequency == 0 { + continue; + } + let minimum_documents = self + .document_frequencies + .iter() + .filter(|(field, _)| **field != LexicalFieldV1::Subtoken) + .map(|(_, terms)| terms.get(&term).copied().unwrap_or_default()) + .max() + .unwrap_or_default(); + sources.push(( + frequency, + LexicalCandidateSourceV1 { + term, + fields: CandidateSourceFieldsV1::Whole, + minimum_documents, + }, + )); } - for term in subtokens { - if self.postings.by_term.contains_key(&term) { + for term in subtokens.into_iter().collect::>() { + if !self.postings.by_term.contains_key(&term) { + continue; + } + let frequency = self.document_frequency(LexicalFieldV1::Subtoken, &term); + if frequency > 0 { sources.push(( - self.document_frequency(LexicalFieldV1::Subtoken, &term), + frequency, LexicalCandidateSourceV1 { term, fields: CandidateSourceFieldsV1::Subtoken, + minimum_documents: frequency, }, )); } } - Ok(sources) + sources } + } struct LexicalDraftV1 { @@ -2067,29 +2125,53 @@ mod candidate_source_tests { stats } + struct ActiveControl; + + impl crate::retrieval::ports::RetrievalExecutionControl for ActiveControl { + fn is_cancelled(&self) -> bool { + false + } + + fn elapsed_micros(&self) -> u64 { + 0 + } + } + fn candidates( stats: &LexicalStatsCacheV1, whole: &[&str], subtokens: &[&str], ) -> (RoaringBitmap, Vec<(String, usize)>) { - let sources = stats - .candidate_sources( - whole.iter().map(|term| (*term).to_owned()).collect(), - subtokens.iter().map(|term| (*term).to_owned()).collect(), - ) - .expect("candidate sources"); + let sources = stats.candidate_sources( + whole.iter().map(|term| (*term).to_owned()).collect(), + subtokens.iter().map(|term| (*term).to_owned()).collect(), + ); let mut pruned = Vec::new(); - let admitted = admit_candidate_sources(sources, |frequency, source| { - pruned.push((source.term.clone(), frequency)); - }); - let mut documents = RoaringBitmap::new(); - for source in admitted { - documents |= stats - .postings - .documents(&source.term, |field| source.fields.admits(field)) - .expect("decode admitted postings"); - } - (documents, pruned) + let admitted = admit_candidate_sources( + sources, + |source, existing, remaining| { + if remaining != usize::MAX + && source.minimum_documents + > crate::retrieval::lexical::MAX_LEXICAL_CANDIDATE_DOCUMENTS_V1 + { + return Ok(None); + } + stats.postings.new_documents_bounded( + &source.term, + |field| source.fields.admits(field), + existing, + remaining, + &ActiveControl, + ) + }, + |frequency, source| { + if !pruned.iter().any(|(term, _)| term == &source.term) { + pruned.push((source.term.clone(), frequency)); + } + }, + ) + .expect("candidate admission"); + (admitted, pruned) } #[test] @@ -2137,6 +2219,75 @@ mod candidate_source_tests { assert!(!documents.contains(16_399)); } + #[test] + fn rare_whole_word_is_not_lost_behind_a_common_subtoken() { + let stats = stats(&[ + ("cache", LexicalFieldV1::BodyText, 0..1), + ("cache", LexicalFieldV1::Subtoken, 1..20_001), + ("token", LexicalFieldV1::BodyText, 20_001..20_101), + ("token", LexicalFieldV1::Subtoken, 20_001..20_101), + ]); + let (documents, pruned) = candidates(&stats, &["cache", "token"], &["cache", "token"]); + assert_eq!(documents.len(), 101); + assert!(documents.contains(0), "rare whole-word cache must survive"); + assert!(documents.contains(20_100)); + assert_eq!(pruned, vec![("cache".to_owned(), 20_000)]); + } + + #[test] + fn rejected_common_source_is_not_decoded_before_admission() { + let mut stats = stats(&[ + ("needle", LexicalFieldV1::BodyText, 0..1), + ("get", LexicalFieldV1::Subtoken, 1..30_001), + ]); + // The stored frequency alone proves this source cannot fit. If it + // were eagerly decoded, this intentionally invalid payload would fail. + stats.postings.by_term.get_mut("get").unwrap()[0].postings = vec![255]; + let (documents, pruned) = candidates(&stats, &["needle"], &["get"]); + assert_eq!(documents.len(), 1); + assert!(documents.contains(0)); + assert_eq!(pruned, vec![("get".to_owned(), 30_000)]); + } + + #[test] + fn posting_admission_checks_cancellation_during_decode() { + use std::sync::atomic::{AtomicUsize, Ordering}; + + struct CancelAfterBatch(AtomicUsize); + + impl crate::retrieval::ports::RetrievalExecutionControl for CancelAfterBatch { + fn is_cancelled(&self) -> bool { + self.0.fetch_add(1, Ordering::SeqCst) > 0 + } + + fn elapsed_micros(&self) -> u64 { + 0 + } + } + + let stats = stats(&[("needle", LexicalFieldV1::BodyText, 0..1_024)]); + let control = CancelAfterBatch(AtomicUsize::new(0)); + let error = stats + .postings + .new_documents_bounded( + "needle", + |_| true, + &RoaringBitmap::new(), + usize::MAX, + &control, + ) + .expect_err("cancelled during the posting stream"); + assert!(matches!( + error, + crate::retrieval::ports::RetrievalPortError::Cancelled + )); + assert_eq!( + control.0.load(Ordering::SeqCst), + 2, + "the request was interrupted at the second 128-posting checkpoint" + ); + } + #[test] fn repeated_sources_are_deduplicated_without_widening_requested_fields() { let stats = stats(&[ @@ -2148,8 +2299,7 @@ mod candidate_source_tests { .candidate_sources( vec!["needle".to_owned(), "needle".to_owned()], vec!["part".to_owned(), "part".to_owned()], - ) - .unwrap(); + ); assert_eq!(sources.len(), 2); assert_eq!(sources[0].0, 100); assert_eq!(sources[0].1.fields, CandidateSourceFieldsV1::Whole); diff --git a/crates/tracedecay-query/src/retrieval/lexical/tests.rs b/crates/tracedecay-query/src/retrieval/lexical/tests.rs index c8cfba3aeb..71059cbb15 100644 --- a/crates/tracedecay-query/src/retrieval/lexical/tests.rs +++ b/crates/tracedecay-query/src/retrieval/lexical/tests.rs @@ -68,11 +68,41 @@ where T::try_from(format!("sha256:{}", byte.to_string().repeat(64))).expect("valid fixture digest") } +fn synthetic_admission( + sources: Vec<(usize, &'static str)>, + ranges: &[(&str, std::ops::Range)], + pruned: &mut Vec<(String, u64)>, +) -> roaring::RoaringBitmap { + admit_candidate_sources( + sources, + |term, existing, remaining| { + let range = ranges + .iter() + .find(|(name, _)| name == term) + .expect("source") + .1 + .clone(); + let mut delta = roaring::RoaringBitmap::new(); + for document in range { + if !existing.contains(document) { + delta.insert(document); + } + if delta.len() > remaining as u64 { + return Ok(None); + } + } + Ok(Some(delta)) + }, + |frequency, term| pruned.push(((*term).to_owned(), frequency as u64)), + ) + .expect("admission") +} + #[test] fn candidate_sources_admit_rarest_first_within_the_document_budget() { let budget = MAX_LEXICAL_CANDIDATE_DOCUMENTS_V1; let mut pruned = Vec::new(); - let admitted = admit_candidate_sources( + let documents = synthetic_admission( vec![ (budget * 3, "the"), (10, "pendingtoolkind"), @@ -80,20 +110,24 @@ fn candidate_sources_admit_rarest_first_within_the_document_budget() { (budget * 2, "tool"), (200, "kind"), ], - |frequency, term| pruned.push((term.to_string(), frequency as u64)), - ); - assert_eq!( - admitted, - ["pendingtoolkind", "kind"], - "`pending` would push the admitted total past the budget, and every later \ - (more common) source is pruned with it" + &[ + ("pendingtoolkind", 0..10), + ("kind", 10..210), + ("pending", 210..(210 + (budget - 100) as u32)), + ("tool", 30_000..(30_000 + (budget * 2) as u32)), + ("the", 70_000..(70_000 + (budget * 3) as u32)), + ], + &mut pruned, ); + assert_eq!(documents.len(), 210); + assert!(documents.contains(209)); + assert!(!documents.contains(210)); assert_eq!( pruned, [ ("pending".to_owned(), (budget - 100) as u64), ("tool".to_owned(), (budget * 2) as u64), - ("the".to_owned(), (budget * 3) as u64) + ("the".to_owned(), (budget * 3) as u64), ] ); let outcome = super::candidate_admission_outcome( @@ -113,24 +147,38 @@ fn candidate_sources_admit_rarest_first_within_the_document_budget() { #[test] fn candidate_sources_ignore_empty_sources_before_admitting_a_common_term() { let budget = MAX_LEXICAL_CANDIDATE_DOCUMENTS_V1; - assert_eq!( - admit_candidate_sources( - vec![(0, "missing"), (budget + 1, "common"), (0, "absent")], - |_, _| {} - ), - ["common"], - "empty sources must not consume the first nonempty source exception" + let documents = synthetic_admission( + vec![(0, "missing"), (budget + 1, "common"), (0, "absent")], + &[("common", 0..(budget + 1) as u32)], + &mut Vec::new(), + ); + assert_eq!(documents.len(), (budget + 1) as u64); + assert!( + synthetic_admission( + vec![(0, "missing"), (0, "absent")], + &[], + &mut Vec::new(), + ) + .is_empty() ); - assert!(admit_candidate_sources(vec![(0, "missing"), (0, "absent")], |_, _| {}).is_empty()); } #[test] fn candidate_sources_keep_request_order_across_equal_frequencies() { let half = MAX_LEXICAL_CANDIDATE_DOCUMENTS_V1 / 2; - assert_eq!( - admit_candidate_sources(vec![(half, "b"), (half, "a"), (half, "c")], |_, _| {}), - ["b", "a"] + let mut pruned = Vec::new(); + let documents = synthetic_admission( + vec![(half, "b"), (half, "a"), (half, "c")], + &[ + ("b", 0..half as u32), + ("a", half as u32..(half * 2) as u32), + ("c", (half * 2) as u32..(half * 3) as u32), + ], + &mut pruned, ); + assert_eq!(documents.len(), (half * 2) as u64); + assert!(!documents.contains((half * 2) as u32)); + assert_eq!(pruned, vec![("c".to_owned(), half as u64)]); } #[test] From 70148243f4dd7dcb89bfcdb0c85bef7d57f46438 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Tue, 3 Nov 2026 03:41:21 -0800 Subject: [PATCH 06/14] fix(search): preserve complete overlapping candidate evidence Do not prune a common source when the first oversized source may already cover all of its documents. Share one lower-bound guard between runtime admission and tests, and cover complete overlap and truly disjoint oversized channels. Keep test imports at module scope. --- .../projection/artifact/reader/query.rs | 56 +++++++++++++++---- 1 file changed, 44 insertions(+), 12 deletions(-) diff --git a/crates/tracedecay-query/src/retrieval/lexical/projection/artifact/reader/query.rs b/crates/tracedecay-query/src/retrieval/lexical/projection/artifact/reader/query.rs index 234b99a935..ee92222362 100644 --- a/crates/tracedecay-query/src/retrieval/lexical/projection/artifact/reader/query.rs +++ b/crates/tracedecay-query/src/retrieval/lexical/projection/artifact/reader/query.rs @@ -1238,12 +1238,7 @@ impl<'a> ArtifactQueryV1<'a> { let term_documents = admit_candidate_sources( sources, |source, existing, remaining| { - // A source with more independently guaranteed documents than - // the entire cap cannot fit after another source was admitted. - // This avoids decoding very common postings merely to prune. - if remaining != usize::MAX - && source.minimum_documents > MAX_LEXICAL_CANDIDATE_DOCUMENTS_V1 - { + if source.cannot_fit(existing, remaining) { return Ok(None); } stats.postings.new_documents_bounded( @@ -1667,6 +1662,19 @@ struct LexicalCandidateSourceV1 { minimum_documents: usize, } +impl LexicalCandidateSourceV1 { + /// A source cannot fit even if every already-admitted document overlaps. + /// When the first source exceeds the cap, its existing documents may + /// entirely cover a second large channel, so its size alone cannot prune. + fn cannot_fit(&self, existing: &RoaringBitmap, remaining: usize) -> bool { + remaining != usize::MAX + && self.minimum_documents + > usize::try_from(existing.len()) + .unwrap_or(usize::MAX) + .saturating_add(remaining) + } +} + fn lexical_terms( prepared: &PreparedLexicalQueryV1<'_>, fuzzy: &FuzzyExpansionsV1, @@ -2087,6 +2095,7 @@ impl EditDistanceScratchV1 { mod candidate_source_tests { use std::collections::BTreeMap; use std::ops::Range; + use std::sync::atomic::{AtomicUsize, Ordering}; use roaring::RoaringBitmap; @@ -2150,10 +2159,7 @@ mod candidate_source_tests { let admitted = admit_candidate_sources( sources, |source, existing, remaining| { - if remaining != usize::MAX - && source.minimum_documents - > crate::retrieval::lexical::MAX_LEXICAL_CANDIDATE_DOCUMENTS_V1 - { + if source.cannot_fit(existing, remaining) { return Ok(None); } stats.postings.new_documents_bounded( @@ -2219,6 +2225,34 @@ mod candidate_source_tests { assert!(!documents.contains(16_399)); } + #[test] + fn fully_overlapping_oversized_channels_do_not_claim_partial_evidence() { + let stats = stats(&[ + ("cache", LexicalFieldV1::BodyText, 0..20_000), + ("cache", LexicalFieldV1::Subtoken, 0..20_000), + ]); + let (documents, pruned) = candidates(&stats, &["cache"], &["cache"]); + assert_eq!(documents.len(), 20_000); + assert!(documents.contains(19_999)); + assert!( + pruned.is_empty(), + "the second channel adds no documents despite exceeding the cap: {pruned:?}" + ); + } + + #[test] + fn oversized_disjoint_channels_still_report_omitted_candidates() { + let stats = stats(&[ + ("cache", LexicalFieldV1::BodyText, 0..20_000), + ("cache", LexicalFieldV1::Subtoken, 20_000..40_000), + ]); + let (documents, pruned) = candidates(&stats, &["cache"], &["cache"]); + assert_eq!(documents.len(), 20_000); + assert!(documents.contains(19_999)); + assert!(!documents.contains(20_000)); + assert_eq!(pruned, vec![("cache".to_owned(), 20_000)]); + } + #[test] fn rare_whole_word_is_not_lost_behind_a_common_subtoken() { let stats = stats(&[ @@ -2251,8 +2285,6 @@ mod candidate_source_tests { #[test] fn posting_admission_checks_cancellation_during_decode() { - use std::sync::atomic::{AtomicUsize, Ordering}; - struct CancelAfterBatch(AtomicUsize); impl crate::retrieval::ports::RetrievalExecutionControl for CancelAfterBatch { From b4ae1682e7f1bb06bc0ea2f91896ab62ffa0f965 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Tue, 3 Nov 2026 03:44:36 -0800 Subject: [PATCH 07/14] fix(search): remove unused candidate cap import The shared admission helper owns the cap after source admission was consolidated; avoid a stale query-reader import. --- .../src/retrieval/lexical/projection/artifact/reader/query.rs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/crates/tracedecay-query/src/retrieval/lexical/projection/artifact/reader/query.rs b/crates/tracedecay-query/src/retrieval/lexical/projection/artifact/reader/query.rs index ee92222362..faecff1238 100644 --- a/crates/tracedecay-query/src/retrieval/lexical/projection/artifact/reader/query.rs +++ b/crates/tracedecay-query/src/retrieval/lexical/projection/artifact/reader/query.rs @@ -23,8 +23,7 @@ use super::{map_query_artifact_error, map_query_sql_error}; use crate::retrieval::exact::{ExactAdmissionAuthority, ExactLaneEvidence, ExactLaneRequest}; use crate::retrieval::lexical::{ LexicalFieldFilterV1, LexicalFieldV1, LexicalLaneEvidence, LexicalLaneRequest, - MAX_FUZZY_TERM_EXPANSIONS_V1, MAX_LEXICAL_CANDIDATE_DOCUMENTS_V1, - MAX_LEXICAL_QUERY_TERM_BYTES_V1, admit_candidate_sources, + MAX_FUZZY_TERM_EXPANSIONS_V1, MAX_LEXICAL_QUERY_TERM_BYTES_V1, admit_candidate_sources, candidate_admission_outcome, field_admitted, }; use crate::retrieval::ports::{ From fa6ce120ea1e06e38924a0ec863121833f6e59b9 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Tue, 3 Nov 2026 06:22:29 -0800 Subject: [PATCH 08/14] style(search): match Bazel rustfmt output for candidate regressions Apply formatting from the failing hosted rustfmt_check_2024 gate. Preserve admission behavior, budgets and all assertions. --- crates/tracedecay-query/src/retrieval/lexical.rs | 5 ++--- .../lexical/projection/artifact/reader/query.rs | 10 ++++------ crates/tracedecay-query/src/retrieval/lexical/tests.rs | 7 +------ 3 files changed, 7 insertions(+), 15 deletions(-) diff --git a/crates/tracedecay-query/src/retrieval/lexical.rs b/crates/tracedecay-query/src/retrieval/lexical.rs index 4f4375cd98..1a08b08a5a 100644 --- a/crates/tracedecay-query/src/retrieval/lexical.rs +++ b/crates/tracedecay-query/src/retrieval/lexical.rs @@ -113,9 +113,8 @@ pub(crate) fn admit_candidate_sources( let remaining = if admitted_sources == 0 { usize::MAX } else { - MAX_LEXICAL_CANDIDATE_DOCUMENTS_V1.saturating_sub( - usize::try_from(admitted.len()).unwrap_or(usize::MAX), - ) + MAX_LEXICAL_CANDIDATE_DOCUMENTS_V1 + .saturating_sub(usize::try_from(admitted.len()).unwrap_or(usize::MAX)) }; match new_documents(&source, &admitted, remaining)? { Some(delta) if delta.len() <= remaining as u64 => { diff --git a/crates/tracedecay-query/src/retrieval/lexical/projection/artifact/reader/query.rs b/crates/tracedecay-query/src/retrieval/lexical/projection/artifact/reader/query.rs index faecff1238..f4cab58639 100644 --- a/crates/tracedecay-query/src/retrieval/lexical/projection/artifact/reader/query.rs +++ b/crates/tracedecay-query/src/retrieval/lexical/projection/artifact/reader/query.rs @@ -1783,7 +1783,6 @@ impl LexicalStatsCacheV1 { } sources } - } struct LexicalDraftV1 { @@ -2326,11 +2325,10 @@ mod candidate_source_tests { ("needle", LexicalFieldV1::Subtoken, 100..200), ("part", LexicalFieldV1::Subtoken, 200..210), ]); - let sources = stats - .candidate_sources( - vec!["needle".to_owned(), "needle".to_owned()], - vec!["part".to_owned(), "part".to_owned()], - ); + let sources = stats.candidate_sources( + vec!["needle".to_owned(), "needle".to_owned()], + vec!["part".to_owned(), "part".to_owned()], + ); assert_eq!(sources.len(), 2); assert_eq!(sources[0].0, 100); assert_eq!(sources[0].1.fields, CandidateSourceFieldsV1::Whole); diff --git a/crates/tracedecay-query/src/retrieval/lexical/tests.rs b/crates/tracedecay-query/src/retrieval/lexical/tests.rs index 71059cbb15..160fbc9576 100644 --- a/crates/tracedecay-query/src/retrieval/lexical/tests.rs +++ b/crates/tracedecay-query/src/retrieval/lexical/tests.rs @@ -154,12 +154,7 @@ fn candidate_sources_ignore_empty_sources_before_admitting_a_common_term() { ); assert_eq!(documents.len(), (budget + 1) as u64); assert!( - synthetic_admission( - vec![(0, "missing"), (0, "absent")], - &[], - &mut Vec::new(), - ) - .is_empty() + synthetic_admission(vec![(0, "missing"), (0, "absent")], &[], &mut Vec::new(),).is_empty() ); } From 7f9947864b142cc924d0219b1301e561b6d8cc7a Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Tue, 3 Nov 2026 10:27:02 -0800 Subject: [PATCH 09/14] test(search): re-pin recovered lexical train ranking The exact-head Linux search evaluation finds train-020's digest-mismatch anchor at rank 5 after the candidate union fix. Three other train misses remain, and protected recall remains 8/8 without forbidden hits. Pin the measured ordered train fallback receipt and explicitly guard the recovered query. Preserve the validation receipt, corpus/workload identity, and all resource and conceptual-miss acceptance checks. --- crates/tracedecay-search-eval/src/report_tests.rs | 12 +++++++++++- .../query-lexical-graph-workload-v1.json | 2 +- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/crates/tracedecay-search-eval/src/report_tests.rs b/crates/tracedecay-search-eval/src/report_tests.rs index b40af42ef7..ccf170b398 100644 --- a/crates/tracedecay-search-eval/src/report_tests.rs +++ b/crates/tracedecay-search-eval/src/report_tests.rs @@ -75,7 +75,6 @@ fn assert_baseline_report_retains_raw_fallback_current_and_exact_ten_x_samples( [ "train-016", "train-019", - "train-020", "train-026", "validation-015", "validation-017", @@ -85,6 +84,17 @@ fn assert_baseline_report_retains_raw_fallback_current_and_exact_ten_x_samples( "validation-027", ] ); + let recovered = report + .profiles + .iter() + .flat_map(|profile| profile.queries.iter()) + .find(|query| query.query_id == "train-020") + .expect("train-020 remains in the pinned workload"); + assert_eq!( + recovered.status, + crate::DirectEvaluationStatusV1::Pass, + "the digest-mismatch query must remain reachable after subtoken admission" + ); assert!(report.profiles.iter().all(|profile| { profile.quality.protected_recall_at_10.denominator > 0 && profile.quality.protected_recall_at_10.numerator diff --git a/tests/fixtures/search_quality/query-lexical-graph-workload-v1.json b/tests/fixtures/search_quality/query-lexical-graph-workload-v1.json index b7e73cd319..11266b93bb 100644 --- a/tests/fixtures/search_quality/query-lexical-graph-workload-v1.json +++ b/tests/fixtures/search_quality/query-lexical-graph-workload-v1.json @@ -133,7 +133,7 @@ } ], "expected_query_fallback_digests": { - "train": "sha256:7f52a7213e6caab00e6a2bfca4a24ccdfe3f37fcd2312fd86da34807a397b846", + "train": "sha256:bba75d4e95661bc4d34e95a4a57facc80fb62f68cf944c07b59b13532be509d6", "validation": "sha256:f8f6e5973a2c07c2e3dc46ec270c75e819f6dc333030f8acabdf1580311577eb" }, "profile_matrix": [ From a1bf7de732d661e2fcfe088e91151d1ab4b1ddbe Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Tue, 3 Nov 2026 13:21:25 -0800 Subject: [PATCH 10/14] test(index): wait for serving seat in lexical recovery CI Use the validated shared fixture repair from #4731 after this branch's native Linux tests failed in the ignored-dependency publication race. The fixture now relies only on its canonical fresh serving-generation wait instead of missing a completed publication broadcast. Keep the stronger readiness assertion, its deadline and every admission regression. The matching source change passed native Linux in #4731; this branch's separate search quality regressions and full CI remain unresolved. --- .../ignored_dependency_admission_tests.rs | 28 ++----------------- 1 file changed, 3 insertions(+), 25 deletions(-) diff --git a/crates/tracedecay-code-index-runtime/src/project_reads/ignored_dependency_admission_tests.rs b/crates/tracedecay-code-index-runtime/src/project_reads/ignored_dependency_admission_tests.rs index 35ab833191..9135cf4f90 100644 --- a/crates/tracedecay-code-index-runtime/src/project_reads/ignored_dependency_admission_tests.rs +++ b/crates/tracedecay-code-index-runtime/src/project_reads/ignored_dependency_admission_tests.rs @@ -21,7 +21,6 @@ use tracedecay_tool_catalog::{CapabilityId, UseCaseId}; use crate::code_index_scheduler::{CodeIndexSchedulerRegistryV1, LatestCompleteCodeIndexV1}; use crate::project_reads::project_code_index_ignored_dependency_admission_port; -use tracedecay_runtime_core::path_safety::canonical_existing_identity; const PROJECT_ID: &str = "project.project-open-ignored-dependency"; @@ -70,7 +69,6 @@ export function generationAnchor() { return 1; } .mount_worktree(project_id(), root.path(), store.path().to_path_buf()) .await .expect("mount code-index scheduler"); - wait_for_initial_generation(®istry, root.path()).await; let baseline = latest(®istry, root.path()).await; let generation = baseline.generation(); let snapshot = generation.snapshot(); @@ -242,31 +240,11 @@ fn request_context(scope: ResolvedScope, suffix: &str) -> RequestContext { .expect("request context") } -async fn wait_for_initial_generation(registry: &CodeIndexSchedulerRegistryV1, project_root: &Path) { - if registry.latest_generation_id(project_root).await.is_some() { - return; - } - let canonical_root = canonical_existing_identity(project_root).expect("canonical fixture root"); - let mut publications = registry.subscribe_generation_publications(); - tokio::time::timeout(Duration::from_secs(5), async { - loop { - let event = publications.recv().await.expect("generation publication"); - if event.project_root == canonical_root { - break; - } - } - }) - .await - .expect("initial generation publication"); -} - /// Wait for the complete serving generation, not merely its publication. /// -/// The generation-publication broadcast and `latest_generation_id` both fire -/// on the reconcile pass, ahead of the bind and the serving swap that fill -/// `serving_generation`. Reading the complete arm the instant -/// `wait_for_initial_generation` returns therefore raced the seat and reported -/// a missing generation for a store that had one. +/// Publication and `latest_generation_id` precede the serving swap. Wait +/// for the actual fresh serving state before using imports as admission +/// evidence; no earlier broadcast establishes readiness. async fn latest( registry: &CodeIndexSchedulerRegistryV1, project_root: &Path, From bed2aa2e528dcde05a6bea6a90f2d66f4f56d729 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Tue, 3 Nov 2026 14:42:01 -0800 Subject: [PATCH 11/14] fix(search): stop prose words double-scoring as subtokens Multiword prose also searched each plain word as a subtoken, and subtoken postings split every chunk token, so a word the text already held whole scored twice. That reordered ordinary prose results and lost validation-018 and validation-019. Prose words now earn subtoken credit only when the row does not hold them whole and at least two of them occur only inside identifiers. This restores both validation queries and also recovers validation-017. train-020 returns to the expected-miss list: a trace showed its recovery came from stop-word fragments, so the assertion guarding it is removed. Both query-fallback receipts are re-pinned from the measured rankings. Co-Authored-By: Claude Opus 5.5 --- .../src/retrieval/lexical/projection.rs | 113 ++++++++++++------ .../candidate_producers.rs | 60 ++++++++++ .../src/report_tests.rs | 13 +- .../query-lexical-graph-workload-v1.json | 4 +- 4 files changed, 137 insertions(+), 53 deletions(-) diff --git a/crates/tracedecay-query/src/retrieval/lexical/projection.rs b/crates/tracedecay-query/src/retrieval/lexical/projection.rs index d70f5457d5..a9810db8c8 100644 --- a/crates/tracedecay-query/src/retrieval/lexical/projection.rs +++ b/crates/tracedecay-query/src/retrieval/lexical/projection.rs @@ -1052,50 +1052,85 @@ fn score_lexical_row( if !score_name_fields && is_identifier_name_field(field) { continue; } - if field != LexicalFieldV1::Subtoken { - for (query_term, normalized) in &prepared.whole_terms { - let exact_tf = term_frequency(field, normalized); - if exact_tf > 0 { - add_score( - &mut field_scores, - field, - bm25(field, exact_tf, document_frequency(field, normalized)), - ); - matched_whole_terms.insert((*query_term).to_owned()); - collect_term_kinds(exact_terms, normalized, &mut matched_kinds); - } - if let Some(expansions) = fuzzy.by_query.get(*query_term) { - for expansion in expansions { - let fuzzy_tf = term_frequency(field, expansion); - if fuzzy_tf == 0 { - continue; - } - let score = bm25(field, fuzzy_tf, document_frequency(field, expansion)) - .saturating_mul(FUZZY_SCORE_MILLIS) - / 1_000; - add_score(&mut field_scores, field, score); - matched_whole_terms.insert((*query_term).to_owned()); - spelling_variants.insert(LexicalSpellingVariantV1 { - query: (*query_term).to_owned(), - alternative: expansion.clone(), - }); - typo_recovery_applied = true; - collect_term_kinds(exact_terms, expansion, &mut matched_kinds); + if field == LexicalFieldV1::Subtoken { + continue; + } + for (query_term, normalized) in &prepared.whole_terms { + let exact_tf = term_frequency(field, normalized); + if exact_tf > 0 { + add_score( + &mut field_scores, + field, + bm25(field, exact_tf, document_frequency(field, normalized)), + ); + matched_whole_terms.insert((*query_term).to_owned()); + collect_term_kinds(exact_terms, normalized, &mut matched_kinds); + } + if let Some(expansions) = fuzzy.by_query.get(*query_term) { + for expansion in expansions { + let fuzzy_tf = term_frequency(field, expansion); + if fuzzy_tf == 0 { + continue; } + let score = bm25(field, fuzzy_tf, document_frequency(field, expansion)) + .saturating_mul(FUZZY_SCORE_MILLIS) + / 1_000; + add_score(&mut field_scores, field, score); + matched_whole_terms.insert((*query_term).to_owned()); + spelling_variants.insert(LexicalSpellingVariantV1 { + query: (*query_term).to_owned(), + alternative: expansion.clone(), + }); + typo_recovery_applied = true; + collect_term_kinds(exact_terms, expansion, &mut matched_kinds); } } - } else { - for (subtoken, normalized) in &prepared.subtokens { - let tf = term_frequency(field, normalized); - if tf > 0 { - add_score( - &mut field_scores, - field, - bm25(field, tf, document_frequency(field, normalized)), - ); - matched_subtokens.insert((*subtoken).to_owned()); + } + } + if field_lengths.contains_key(&LexicalFieldV1::Subtoken) { + // Multiword prose also searches each plain word as a subtoken so it + // can reach a camelCase identifier built from those words. Subtoken + // postings split every chunk token, so a word the text holds whole + // is there too; crediting it again double-counts ordinary prose. + // A single word inside an unrelated identifier (`is` in `is_empty`) + // is incidental, so prose words earn credit only when at least two + // of them occur in this row solely inside identifiers. + let mut scored = Vec::with_capacity(prepared.subtokens.len()); + let mut embedded_prose_words = 0usize; + for (subtoken, normalized) in &prepared.subtokens { + let tf = term_frequency(LexicalFieldV1::Subtoken, normalized); + if tf == 0 { + continue; + } + let prose_word = prepared + .whole_terms + .iter() + .any(|(_, whole)| whole == normalized); + if prose_word { + let held_whole = [LexicalFieldV1::BodyText, LexicalFieldV1::PreambleText] + .into_iter() + .any(|text| term_frequency(text, normalized) > 0); + if held_whole { + continue; } + embedded_prose_words += 1; + } + scored.push((*subtoken, normalized.as_str(), tf, prose_word)); + } + for (subtoken, normalized, tf, prose_word) in scored { + if prose_word && embedded_prose_words < 2 { + continue; } + add_score( + &mut field_scores, + LexicalFieldV1::Subtoken, + bm25( + LexicalFieldV1::Subtoken, + tf, + document_frequency(LexicalFieldV1::Subtoken, normalized), + ), + ); + matched_subtokens.insert(subtoken.to_owned()); } } for (phrase, normalized) in &prepared.phrases { diff --git a/crates/tracedecay-query/tests/search_quality_suite/candidate_producers.rs b/crates/tracedecay-query/tests/search_quality_suite/candidate_producers.rs index 667496e632..17d97eff8e 100644 --- a/crates/tracedecay-query/tests/search_quality_suite/candidate_producers.rs +++ b/crates/tracedecay-query/tests/search_quality_suite/candidate_producers.rs @@ -6325,6 +6325,66 @@ fn prose_query_recovers_camel_case_symbol_from_real_sealed_subtoken_postings() { ); } +#[test] +fn prose_words_the_text_holds_whole_earn_no_second_subtoken_score() { + // The first row states the words; the second only hides them inside its + // name. Prose subtoken admission exists to reach the second row, and must + // not give the first row a second score for words it already matched. + let artifact = rust_artifact(&[ + "/// Reports when the digest does not match its payload.\npub fn validate() {}\n", + "pub fn digestPayloadMismatch() {}\n", + ]); + let query = "digest payload"; + let parts = tracedecay_query::retrieval::lexical::lexical_query_parts(query) + .expect("parse multiword query"); + let mut request = artifact.request(query, &[], &[], &[], 0, 32); + request.whole_terms = Cow::Owned(parts.whole_terms); + request.subtokens = Cow::Owned(parts.subtokens); + request.phrases = Cow::Owned(parts.phrases); + let results = complete( + artifact + .lane() + .retrieve_lexical(&request) + .expect("sealed lexical search"), + ); + let in_file = |ordinal: usize| { + results + .candidates + .iter() + .filter(move |candidate| { + candidate + .file_occurrence_id + .as_ref() + .is_some_and(|file| file.as_str() == format!("file.fixture.{ordinal}")) + }) + .map(|candidate| &results.evidence_by_occurrence[&candidate.source_occurrence_id]) + .collect::>() + }; + let stated = in_file(0); + assert!( + !stated.is_empty() + && stated.iter().all(|evidence| { + evidence.matched_subtokens.is_empty() + && evidence + .field_scores_micros + .iter() + .all(|(field, _)| *field != LexicalFieldV1::Subtoken) + }), + "words the text holds whole must not score again as subtokens: {results:?}" + ); + assert!( + in_file(1).iter().any(|evidence| { + ["digest", "payload"].iter().all(|term| { + evidence + .matched_subtokens + .iter() + .any(|matched| matched == term) + }) + }), + "the camelCase symbol must still match both words as subtokens: {results:?}" + ); +} + #[test] fn lexical_artifact_reports_freshness_coverage_and_page_cutoff() { let sources = ["pub fn target() {}\n"; 3]; diff --git a/crates/tracedecay-search-eval/src/report_tests.rs b/crates/tracedecay-search-eval/src/report_tests.rs index ccf170b398..03c67a0738 100644 --- a/crates/tracedecay-search-eval/src/report_tests.rs +++ b/crates/tracedecay-search-eval/src/report_tests.rs @@ -75,26 +75,15 @@ fn assert_baseline_report_retains_raw_fallback_current_and_exact_ten_x_samples( [ "train-016", "train-019", + "train-020", "train-026", "validation-015", - "validation-017", "validation-023", "validation-024", "validation-025", "validation-027", ] ); - let recovered = report - .profiles - .iter() - .flat_map(|profile| profile.queries.iter()) - .find(|query| query.query_id == "train-020") - .expect("train-020 remains in the pinned workload"); - assert_eq!( - recovered.status, - crate::DirectEvaluationStatusV1::Pass, - "the digest-mismatch query must remain reachable after subtoken admission" - ); assert!(report.profiles.iter().all(|profile| { profile.quality.protected_recall_at_10.denominator > 0 && profile.quality.protected_recall_at_10.numerator diff --git a/tests/fixtures/search_quality/query-lexical-graph-workload-v1.json b/tests/fixtures/search_quality/query-lexical-graph-workload-v1.json index 11266b93bb..b2ec012fc6 100644 --- a/tests/fixtures/search_quality/query-lexical-graph-workload-v1.json +++ b/tests/fixtures/search_quality/query-lexical-graph-workload-v1.json @@ -133,8 +133,8 @@ } ], "expected_query_fallback_digests": { - "train": "sha256:bba75d4e95661bc4d34e95a4a57facc80fb62f68cf944c07b59b13532be509d6", - "validation": "sha256:f8f6e5973a2c07c2e3dc46ec270c75e819f6dc333030f8acabdf1580311577eb" + "train": "sha256:b179db2dd27e33dca852d791b899f9ce530daac8213a5128beb804b4bfab305b", + "validation": "sha256:fdc9412c0c3d75649e18191c5538ef71372d9ba300ef13e15c6b1005140dcee3" }, "profile_matrix": [ { From 4b73fa174260f296987a492716a35c68384fd3e1 Mon Sep 17 00:00:00 2001 From: ScriptedAlchemy Date: Tue, 3 Nov 2026 14:57:10 -0800 Subject: [PATCH 12/14] fix(search): preserve prose recall without duplicate lexical credit --- .../tracedecay-query/src/retrieval/lexical.rs | 19 ++-- .../src/retrieval/lexical/projection.rs | 69 ++++++------- .../src/retrieval/lexical/tests.rs | 20 +++- .../candidate_producers.rs | 99 ++++++++++++++++--- .../src/report_tests.rs | 14 ++- .../query-lexical-graph-workload-v1.json | 4 +- 6 files changed, 159 insertions(+), 66 deletions(-) diff --git a/crates/tracedecay-query/src/retrieval/lexical.rs b/crates/tracedecay-query/src/retrieval/lexical.rs index 1a08b08a5a..292aad5b2d 100644 --- a/crates/tracedecay-query/src/retrieval/lexical.rs +++ b/crates/tracedecay-query/src/retrieval/lexical.rs @@ -166,10 +166,10 @@ pub(super) fn normalize_lexical(value: &str) -> String { /// Whole terms are the same maximal technical tokens the code index extracts /// (`tracedecay_domain::technical_tokens`), so a term the indexer keeps whole /// (`foo-bar`, `a::b`, `p/q.rs`) is queried whole, and subtokens decompose -/// through the same shared grammar. For multiword prose, even an unsplit -/// word is searched in the Subtoken field so a matching camelCase symbol can -/// be admitted as a candidate. Single-token queries intentionally emit no -/// subtokens so one technical term does not fan out to common subtokens. +/// through the same shared grammar. For multiword prose, unsplit content +/// words use the symbol route's policy before searching the Subtoken field, +/// so camelCase symbols can be admitted without expanding stopwords. +/// Single-token queries emit no subtokens, preserving their narrow admission. /// /// Multi-token sanitized input is also retained as a phrase. This gives exact /// diagnostic/error text and natural-language queries a bounded lexical phrase @@ -191,10 +191,13 @@ pub fn lexical_query_parts(query: &str) -> Result LexicalRowScoreV1 { let mut field_scores = BTreeMap::new(); + let mut whole_term_scores = BTreeMap::<&str, u64>::new(); let mut matched_whole_terms = BTreeSet::new(); let mut matched_subtokens = BTreeSet::new(); let mut matched_phrases = BTreeSet::new(); @@ -1058,11 +1059,18 @@ fn score_lexical_row( for (query_term, normalized) in &prepared.whole_terms { let exact_tf = term_frequency(field, normalized); if exact_tf > 0 { - add_score( - &mut field_scores, - field, - bm25(field, exact_tf, document_frequency(field, normalized)), - ); + let score = bm25(field, exact_tf, document_frequency(field, normalized)); + add_score(&mut field_scores, field, score); + let weighted_score = + if field == LexicalFieldV1::Documentation && prepared.whole_terms.len() != 1 { + score.saturating_mul(DOCUMENTATION_NL_SCORE_MILLIS) / 1_000 + } else { + score + }; + whole_term_scores + .entry(normalized.as_str()) + .and_modify(|current| *current = (*current).max(weighted_score)) + .or_insert(weighted_score); matched_whole_terms.insert((*query_term).to_owned()); collect_term_kinds(exact_terms, normalized, &mut matched_kinds); } @@ -1088,49 +1096,28 @@ fn score_lexical_row( } } if field_lengths.contains_key(&LexicalFieldV1::Subtoken) { - // Multiword prose also searches each plain word as a subtoken so it - // can reach a camelCase identifier built from those words. Subtoken - // postings split every chunk token, so a word the text holds whole - // is there too; crediting it again double-counts ordinary prose. - // A single word inside an unrelated identifier (`is` in `is_empty`) - // is incidental, so prose words earn credit only when at least two - // of them occur in this row solely inside identifiers. - let mut scored = Vec::with_capacity(prepared.subtokens.len()); - let mut embedded_prose_words = 0usize; for (subtoken, normalized) in &prepared.subtokens { let tf = term_frequency(LexicalFieldV1::Subtoken, normalized); if tf == 0 { continue; } - let prose_word = prepared - .whole_terms - .iter() - .any(|(_, whole)| whole == normalized); - if prose_word { - let held_whole = [LexicalFieldV1::BodyText, LexicalFieldV1::PreambleText] - .into_iter() - .any(|text| term_frequency(text, normalized) > 0); - if held_whole { - continue; - } - embedded_prose_words += 1; - } - scored.push((*subtoken, normalized.as_str(), tf, prose_word)); - } - for (subtoken, normalized, tf, prose_word) in scored { - if prose_word && embedded_prose_words < 2 { - continue; - } - add_score( - &mut field_scores, + // Whole-word and subtoken postings can represent the same + // evidence. Add only the stronger representation's excess. + let score = bm25( LexicalFieldV1::Subtoken, - bm25( - LexicalFieldV1::Subtoken, - tf, - document_frequency(LexicalFieldV1::Subtoken, normalized), - ), + tf, + document_frequency(LexicalFieldV1::Subtoken, normalized), + ) + .saturating_sub( + whole_term_scores + .get(normalized.as_str()) + .copied() + .unwrap_or_default(), ); - matched_subtokens.insert(subtoken.to_owned()); + if score > 0 { + add_score(&mut field_scores, LexicalFieldV1::Subtoken, score); + matched_subtokens.insert((*subtoken).to_owned()); + } } } for (phrase, normalized) in &prepared.phrases { diff --git a/crates/tracedecay-query/src/retrieval/lexical/tests.rs b/crates/tracedecay-query/src/retrieval/lexical/tests.rs index 160fbc9576..b74310511e 100644 --- a/crates/tracedecay-query/src/retrieval/lexical/tests.rs +++ b/crates/tracedecay-query/src/retrieval/lexical/tests.rs @@ -203,7 +203,7 @@ fn shared_query_parser_retains_phrase_and_identifier_subtokens() { /// query-side subtokens equal the extraction subtokens. #[test] fn query_tokenization_matches_index_tokenization_for_separator_terms() { - let text = "find foo-bar a::b p/q.rs --flag E0308 TS1234 ERR_X"; + let text = "foo-bar a::b p/q.rs --flag E0308 TS1234 ERR_X"; let parts = lexical_query_parts(text).expect("lexical query parts"); let index_tokens: Vec = technical_tokens(text) @@ -268,6 +268,24 @@ fn prose_words_can_match_identifier_subtokens_without_broadening_single_term_que ); } +#[test] +fn prose_subtoken_expansion_uses_the_existing_content_word_policy() { + let query = "how do i find the widget auto tag"; + let parts = lexical_query_parts(query).expect("prose search"); + assert_eq!( + parts.subtokens, + ["auto", "tag", "widget"].map(str::to_owned) + ); + for word in ["how", "do", "i", "find", "the"] { + assert!(parts.whole_terms.iter().any(|term| term == word)); + } + assert_eq!(parts.phrases, [query]); + + let technical = lexical_query_parts("TypeOf Widget").expect("technical query"); + assert!(technical.subtokens.iter().any(|term| term == "type")); + assert!(technical.subtokens.iter().any(|term| term == "of")); +} + fn budget(max_candidates_per_lane: u32) -> RetrievalBudget { RetrievalBudget { max_candidates_per_lane, diff --git a/crates/tracedecay-query/tests/search_quality_suite/candidate_producers.rs b/crates/tracedecay-query/tests/search_quality_suite/candidate_producers.rs index 17d97eff8e..fdbfcbf7ef 100644 --- a/crates/tracedecay-query/tests/search_quality_suite/candidate_producers.rs +++ b/crates/tracedecay-query/tests/search_quality_suite/candidate_producers.rs @@ -6070,6 +6070,59 @@ fn fielded_bm25_keeps_whole_identifiers_and_subtokens_distinct() { })); } +#[test] +fn prose_subtokens_add_identifier_matches_without_rescoring_whole_words() { + let artifact = rust_artifact(&[ + "pub fn reserveStock() {}\n", + "pub fn reserve() { let stock = 1; }\n", + ]); + let lane = artifact.lane(); + let whole = complete( + lane.retrieve_lexical(&artifact.request( + "reserve stock", + &["reserve", "stock"], + &[], + &[], + 0, + 8, + )) + .expect("whole-word retrieval"), + ); + let expanded = complete( + lane.retrieve_lexical(&artifact.request( + "reserve stock", + &["reserve", "stock"], + &["reserve", "stock"], + &[], + 0, + 8, + )) + .expect("expanded retrieval"), + ); + assert!(candidate_files(&whole.candidates).contains(&1)); + assert_eq!( + candidate_files(&expanded.candidates), + BTreeSet::from([0, 1]) + ); + for candidate in &whole.candidates { + if candidate + .file_occurrence_id + .as_ref() + .map(|file| file.as_str()) + != Some("file.fixture.1") + { + continue; + } + let occurrence = &candidate.source_occurrence_id; + let evidence = &whole.evidence_by_occurrence[occurrence]; + assert_eq!( + expanded.evidence_by_occurrence[occurrence].field_scores_micros, + evidence.field_scores_micros, + "an ordinary word already scored in its source field is not new subtoken evidence" + ); + } +} + #[test] fn lexical_phrase_and_bounded_fuzzy_recovery_are_deterministic() { let artifact = rust_artifact(&[ @@ -6326,10 +6379,10 @@ fn prose_query_recovers_camel_case_symbol_from_real_sealed_subtoken_postings() { } #[test] -fn prose_words_the_text_holds_whole_earn_no_second_subtoken_score() { +fn prose_words_do_not_add_two_full_scores_for_the_same_evidence() { // The first row states the words; the second only hides them inside its // name. Prose subtoken admission exists to reach the second row, and must - // not give the first row a second score for words it already matched. + // not add two full scores for words the first row already matched. let artifact = rust_artifact(&[ "/// Reports when the digest does not match its payload.\npub fn validate() {}\n", "pub fn digestPayloadMismatch() {}\n", @@ -6347,6 +6400,18 @@ fn prose_words_the_text_holds_whole_earn_no_second_subtoken_score() { .retrieve_lexical(&request) .expect("sealed lexical search"), ); + let whole = complete( + artifact + .lane() + .retrieve_lexical(&artifact.request(query, &["digest", "payload"], &[], &[], 0, 32)) + .expect("whole-word control"), + ); + let subtoken = complete( + artifact + .lane() + .retrieve_lexical(&artifact.request(query, &[], &["digest", "payload"], &[], 0, 32)) + .expect("subtoken control"), + ); let in_file = |ordinal: usize| { results .candidates @@ -6361,17 +6426,25 @@ fn prose_words_the_text_holds_whole_earn_no_second_subtoken_score() { .collect::>() }; let stated = in_file(0); - assert!( - !stated.is_empty() - && stated.iter().all(|evidence| { - evidence.matched_subtokens.is_empty() - && evidence - .field_scores_micros - .iter() - .all(|(field, _)| *field != LexicalFieldV1::Subtoken) - }), - "words the text holds whole must not score again as subtokens: {results:?}" - ); + assert!(!stated.is_empty()); + let total = |evidence: &LexicalLaneEvidence| { + evidence + .field_scores_micros + .iter() + .map(|(_, score)| score) + .sum::() + }; + for evidence in stated { + let occurrence = &evidence.binding.source_occurrence; + let whole_score = total(&whole.evidence_by_occurrence[occurrence]); + let subtoken_score = total(&subtoken.evidence_by_occurrence[occurrence]); + assert!(whole_score > 0 && subtoken_score > 0); + assert!(total(evidence) >= whole_score.max(subtoken_score)); + assert!( + total(evidence) < whole_score + subtoken_score, + "overlapping representations must not add both full scores: {evidence:?}" + ); + } assert!( in_file(1).iter().any(|evidence| { ["digest", "payload"].iter().all(|term| { diff --git a/crates/tracedecay-search-eval/src/report_tests.rs b/crates/tracedecay-search-eval/src/report_tests.rs index 03c67a0738..5d87916362 100644 --- a/crates/tracedecay-search-eval/src/report_tests.rs +++ b/crates/tracedecay-search-eval/src/report_tests.rs @@ -75,7 +75,6 @@ fn assert_baseline_report_retains_raw_fallback_current_and_exact_ten_x_samples( [ "train-016", "train-019", - "train-020", "train-026", "validation-015", "validation-023", @@ -84,6 +83,19 @@ fn assert_baseline_report_retains_raw_fallback_current_and_exact_ten_x_samples( "validation-027", ] ); + for query_id in ["train-020", "validation-018", "validation-019"] { + let query = report + .profiles + .iter() + .flat_map(|profile| &profile.queries) + .find(|query| query.query_id == query_id) + .expect("ranking regression query is present"); + assert_eq!( + query.status, + crate::DirectEvaluationStatusV1::Pass, + "{query_id} must retain useful top-ten retrieval" + ); + } assert!(report.profiles.iter().all(|profile| { profile.quality.protected_recall_at_10.denominator > 0 && profile.quality.protected_recall_at_10.numerator diff --git a/tests/fixtures/search_quality/query-lexical-graph-workload-v1.json b/tests/fixtures/search_quality/query-lexical-graph-workload-v1.json index b2ec012fc6..a3f50a061a 100644 --- a/tests/fixtures/search_quality/query-lexical-graph-workload-v1.json +++ b/tests/fixtures/search_quality/query-lexical-graph-workload-v1.json @@ -133,8 +133,8 @@ } ], "expected_query_fallback_digests": { - "train": "sha256:b179db2dd27e33dca852d791b899f9ce530daac8213a5128beb804b4bfab305b", - "validation": "sha256:fdc9412c0c3d75649e18191c5538ef71372d9ba300ef13e15c6b1005140dcee3" + "train": "sha256:469266a120472b3203ad5a4d520b1a4e539e4c44ae6f5cf9babe1fb64ddfc1a3", + "validation": "sha256:94e8499d1c6237a7c7cb2f5008f34edb17e76cb6a9b1144b79851595f569fd09" }, "profile_matrix": [ { From 0d0f2a840cc441144e0b6daef2f2e934f0c8c3d6 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Tue, 3 Nov 2026 15:26:54 -0800 Subject: [PATCH 13/14] fix(search): preserve subtoken-only retrieval under field filters Only subtract overlapping whole-term evidence from a subtoken score when the whole-word field is admitted by the request. A Subtoken whitelist or a BodyText exclusion must not erase otherwise matching prose rows. Cover both filtered routes against real sealed lexical postings. Keep the existing unfiltered ranking and the recovered camelCase search regressions. Rust Bazel validation is required on this new branch head. --- .../src/retrieval/lexical/projection.rs | 6 ++- .../candidate_producers.rs | 52 +++++++++++++++++++ 2 files changed, 57 insertions(+), 1 deletion(-) diff --git a/crates/tracedecay-query/src/retrieval/lexical/projection.rs b/crates/tracedecay-query/src/retrieval/lexical/projection.rs index ddc191c28a..96b4425291 100644 --- a/crates/tracedecay-query/src/retrieval/lexical/projection.rs +++ b/crates/tracedecay-query/src/retrieval/lexical/projection.rs @@ -1053,7 +1053,11 @@ fn score_lexical_row( if !score_name_fields && is_identifier_name_field(field) { continue; } - if field == LexicalFieldV1::Subtoken { + // Only an admitted whole-word field can cover subtoken evidence. + // Otherwise a Subtoken-only request loses rows before lane filtering. + if field == LexicalFieldV1::Subtoken + || !super::field_admitted(field_filters, field) + { continue; } for (query_term, normalized) in &prepared.whole_terms { diff --git a/crates/tracedecay-query/tests/search_quality_suite/candidate_producers.rs b/crates/tracedecay-query/tests/search_quality_suite/candidate_producers.rs index fdbfcbf7ef..f186edc00e 100644 --- a/crates/tracedecay-query/tests/search_quality_suite/candidate_producers.rs +++ b/crates/tracedecay-query/tests/search_quality_suite/candidate_producers.rs @@ -6123,6 +6123,58 @@ fn prose_subtokens_add_identifier_matches_without_rescoring_whole_words() { } } +#[test] +fn filtered_subtoken_search_keeps_prose_evidence_when_whole_fields_are_excluded() { + let artifact = rust_artifact(&[ + "pub fn process() { let digest = payload; }\n", + "pub fn digestPayloadMismatch() {}\n", + ]); + for filter in [ + LexicalFieldFilterV1 { + field: LexicalFieldV1::Subtoken, + include: true, + }, + LexicalFieldFilterV1 { + field: LexicalFieldV1::BodyText, + include: false, + }, + ] { + let mut request = artifact.request( + "digest payload", + &["digest", "payload"], + &["digest", "payload"], + &[], + 0, + 8, + ); + request.field_filters = Cow::Owned(vec![filter]); + let results = complete( + artifact + .lane() + .retrieve_lexical(&request) + .expect("filtered subtoken search"), + ); + assert!( + results.candidates.iter().any(|candidate| { + candidate.file_occurrence_id.as_ref().is_some_and(|file| { + file.as_str() == "file.fixture.0" + && results + .evidence_by_occurrence + .get(&candidate.source_occurrence_id) + .is_some_and(|evidence| { + evidence.field_scores_micros.iter().any(|(field, score)| { + *field == LexicalFieldV1::Subtoken && *score > 0 + }) && ["digest", "payload"].iter().all(|word| { + evidence.matched_subtokens.iter().any(|term| term == word) + }) + }) + }) + }), + "excluded whole fields must not zero out admitted subtokens: {results:?}" + ); + } +} + #[test] fn lexical_phrase_and_bounded_fuzzy_recovery_are_deterministic() { let artifact = rust_artifact(&[ From 213f2e1efa3ec6605c572f69850a69c6d87969a8 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Tue, 3 Nov 2026 16:22:31 -0800 Subject: [PATCH 14/14] style(search): rustfmt the lexical field filter check Co-Authored-By: Claude Opus 5.5 --- crates/tracedecay-query/src/retrieval/lexical/projection.rs | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/crates/tracedecay-query/src/retrieval/lexical/projection.rs b/crates/tracedecay-query/src/retrieval/lexical/projection.rs index 96b4425291..084403e6e8 100644 --- a/crates/tracedecay-query/src/retrieval/lexical/projection.rs +++ b/crates/tracedecay-query/src/retrieval/lexical/projection.rs @@ -1055,9 +1055,7 @@ fn score_lexical_row( } // Only an admitted whole-word field can cover subtoken evidence. // Otherwise a Subtoken-only request loses rows before lane filtering. - if field == LexicalFieldV1::Subtoken - || !super::field_admitted(field_filters, field) - { + if field == LexicalFieldV1::Subtoken || !super::field_admitted(field_filters, field) { continue; } for (query_term, normalized) in &prepared.whole_terms {