diff --git a/crates/tracedecay-code-index-runtime/src/project_reads/ignored_dependency_admission_tests.rs b/crates/tracedecay-code-index-runtime/src/project_reads/ignored_dependency_admission_tests.rs index 35ab833191..9135cf4f90 100644 --- a/crates/tracedecay-code-index-runtime/src/project_reads/ignored_dependency_admission_tests.rs +++ b/crates/tracedecay-code-index-runtime/src/project_reads/ignored_dependency_admission_tests.rs @@ -21,7 +21,6 @@ use tracedecay_tool_catalog::{CapabilityId, UseCaseId}; use crate::code_index_scheduler::{CodeIndexSchedulerRegistryV1, LatestCompleteCodeIndexV1}; use crate::project_reads::project_code_index_ignored_dependency_admission_port; -use tracedecay_runtime_core::path_safety::canonical_existing_identity; const PROJECT_ID: &str = "project.project-open-ignored-dependency"; @@ -70,7 +69,6 @@ export function generationAnchor() { return 1; } .mount_worktree(project_id(), root.path(), store.path().to_path_buf()) .await .expect("mount code-index scheduler"); - wait_for_initial_generation(®istry, root.path()).await; let baseline = latest(®istry, root.path()).await; let generation = baseline.generation(); let snapshot = generation.snapshot(); @@ -242,31 +240,11 @@ fn request_context(scope: ResolvedScope, suffix: &str) -> RequestContext { .expect("request context") } -async fn wait_for_initial_generation(registry: &CodeIndexSchedulerRegistryV1, project_root: &Path) { - if registry.latest_generation_id(project_root).await.is_some() { - return; - } - let canonical_root = canonical_existing_identity(project_root).expect("canonical fixture root"); - let mut publications = registry.subscribe_generation_publications(); - tokio::time::timeout(Duration::from_secs(5), async { - loop { - let event = publications.recv().await.expect("generation publication"); - if event.project_root == canonical_root { - break; - } - } - }) - .await - .expect("initial generation publication"); -} - /// Wait for the complete serving generation, not merely its publication. /// -/// The generation-publication broadcast and `latest_generation_id` both fire -/// on the reconcile pass, ahead of the bind and the serving swap that fill -/// `serving_generation`. Reading the complete arm the instant -/// `wait_for_initial_generation` returns therefore raced the seat and reported -/// a missing generation for a store that had one. +/// Publication and `latest_generation_id` precede the serving swap. Wait +/// for the actual fresh serving state before using imports as admission +/// evidence; no earlier broadcast establishes readiness. async fn latest( registry: &CodeIndexSchedulerRegistryV1, project_root: &Path, diff --git a/crates/tracedecay-query/src/retrieval/lexical.rs b/crates/tracedecay-query/src/retrieval/lexical.rs index 45c7c2613f..292aad5b2d 100644 --- a/crates/tracedecay-query/src/retrieval/lexical.rs +++ b/crates/tracedecay-query/src/retrieval/lexical.rs @@ -10,6 +10,7 @@ use std::borrow::Cow; use std::collections::{BTreeMap, BTreeSet}; +use roaring::RoaringBitmap; use serde::{Deserialize, Serialize}; use tracedecay_domain::{ CodeGenerationId, CompactCandidate, ComponentRevision, CursorPayloadDigest, @@ -78,45 +79,52 @@ pub const MAX_LEXICAL_FIELD_FILTERS_V1: usize = 9; /// proximity term. pub const MAX_LEXICAL_QUERY_TERM_BYTES_V1: usize = 512; -/// Summed document-frequency budget for lexical term-source admission. Every -/// candidate is decoded from its row and scored, so the union of the request's -/// term sources, not the winner cap, decides the lane's transient allocation -/// and wall time: unbounded, a natural-language task whose terms include -/// common words hydrated ~74k rows of a 472k-chunk corpus per read (~0.95 GB -/// decoded, 5.7 s) and missed the context deadline. Term sources are admitted -/// in ascending document-frequency order until their summed frequencies would -/// exceed this bound; the most selective source is always admitted so a -/// single common-term query still answers. Phrase sources are admitted separately. +/// Unique-document budget for lexical term-source admission. Every candidate +/// is decoded and scored, so source selection determines transient allocation +/// and wall time. Previously a natural-language query hydrated ~74k rows of +/// a 472k-chunk corpus (~0.95 GB decoded, 5.7 s), missing the deadline. +/// Sources are considered in ascending stored field-frequency order. Only +/// new document IDs charge the budget; the first nonempty source is always +/// admitted so a single common-term query still answers. Phrase sources +/// are admitted separately. /// This is a recall/latency policy, not a hard document or allocation ceiling: -/// documents matching only pruned terms cannot rank. The initial 16,384 value +/// documents matching only pruned sources cannot rank. The 16,384 value /// retains the measured policy (~100 MiB decode churn at ~6.6 KiB per row); /// changing the reader cache must not change candidate eligibility. pub const MAX_LEXICAL_CANDIDATE_DOCUMENTS_V1: usize = 16_384; -/// Admit `(document_frequency, source)` pairs rarest-first while the summed -/// frequency stays within [`MAX_LEXICAL_CANDIDATE_DOCUMENTS_V1`]; the rarest -/// nonempty source is always admitted. Ties keep request order so admission -/// is deterministic. Sources past the bound still weigh admitted candidates -/// through scoring; a document matching only those sources is never hydrated. +/// Visit posting sources rarest-first, counting unique documents rather than +/// charging overlapping whole/subtoken postings twice. The reader supplies a +/// bounded delta; only the rarest nonempty source may exceed the cap. pub(crate) fn admit_candidate_sources( mut sources: Vec<(usize, S)>, + mut new_documents: impl FnMut( + &S, + &RoaringBitmap, + usize, + ) -> Result, RetrievalPortError>, mut on_pruned: impl FnMut(usize, &S), -) -> Vec { +) -> Result { sources.retain(|(frequency, _)| *frequency > 0); sources.sort_by_key(|(frequency, _)| *frequency); - let total = sources.len(); - let mut admitted_documents = 0usize; - let mut admitted = Vec::with_capacity(total); - for (ordinal, (frequency, source)) in sources.into_iter().enumerate() { - let next = admitted_documents.saturating_add(frequency); - if ordinal > 0 && next > MAX_LEXICAL_CANDIDATE_DOCUMENTS_V1 { - on_pruned(frequency, &source); - continue; + let mut admitted = RoaringBitmap::new(); + let mut admitted_sources = 0usize; + for (frequency, source) in sources { + let remaining = if admitted_sources == 0 { + usize::MAX + } else { + MAX_LEXICAL_CANDIDATE_DOCUMENTS_V1 + .saturating_sub(usize::try_from(admitted.len()).unwrap_or(usize::MAX)) + }; + match new_documents(&source, &admitted, remaining)? { + Some(delta) if delta.len() <= remaining as u64 => { + admitted |= delta; + admitted_sources += 1; + } + _ => on_pruned(frequency, &source), } - admitted_documents = next; - admitted.push(source); } - admitted + Ok(admitted) } fn candidate_admission_outcome( @@ -158,8 +166,10 @@ pub(super) fn normalize_lexical(value: &str) -> String { /// Whole terms are the same maximal technical tokens the code index extracts /// (`tracedecay_domain::technical_tokens`), so a term the indexer keeps whole /// (`foo-bar`, `a::b`, `p/q.rs`) is queried whole, and subtokens decompose -/// through the same shared grammar. Single-token queries intentionally emit -/// no subtokens so one technical term does not fan out to common subtokens. +/// through the same shared grammar. For multiword prose, unsplit content +/// words use the symbol route's policy before searching the Subtoken field, +/// so camelCase symbols can be admitted without expanding stopwords. +/// Single-token queries emit no subtokens, preserving their narrow admission. /// /// Multi-token sanitized input is also retained as a phrase. This gives exact /// diagnostic/error text and natural-language queries a bounded lexical phrase @@ -182,10 +192,11 @@ pub fn lexical_query_parts(query: &str) -> Result LexicalRowScoreV1 { let mut field_scores = BTreeMap::new(); + let mut whole_term_scores = BTreeMap::<&str, u64>::new(); let mut matched_whole_terms = BTreeSet::new(); let mut matched_subtokens = BTreeSet::new(); let mut matched_phrases = BTreeSet::new(); @@ -1052,49 +1053,72 @@ fn score_lexical_row( if !score_name_fields && is_identifier_name_field(field) { continue; } - if field != LexicalFieldV1::Subtoken { - for (query_term, normalized) in &prepared.whole_terms { - let exact_tf = term_frequency(field, normalized); - if exact_tf > 0 { - add_score( - &mut field_scores, - field, - bm25(field, exact_tf, document_frequency(field, normalized)), - ); - matched_whole_terms.insert((*query_term).to_owned()); - collect_term_kinds(exact_terms, normalized, &mut matched_kinds); - } - if let Some(expansions) = fuzzy.by_query.get(*query_term) { - for expansion in expansions { - let fuzzy_tf = term_frequency(field, expansion); - if fuzzy_tf == 0 { - continue; - } - let score = bm25(field, fuzzy_tf, document_frequency(field, expansion)) - .saturating_mul(FUZZY_SCORE_MILLIS) - / 1_000; - add_score(&mut field_scores, field, score); - matched_whole_terms.insert((*query_term).to_owned()); - spelling_variants.insert(LexicalSpellingVariantV1 { - query: (*query_term).to_owned(), - alternative: expansion.clone(), - }); - typo_recovery_applied = true; - collect_term_kinds(exact_terms, expansion, &mut matched_kinds); + // Only an admitted whole-word field can cover subtoken evidence. + // Otherwise a Subtoken-only request loses rows before lane filtering. + if field == LexicalFieldV1::Subtoken || !super::field_admitted(field_filters, field) { + continue; + } + for (query_term, normalized) in &prepared.whole_terms { + let exact_tf = term_frequency(field, normalized); + if exact_tf > 0 { + let score = bm25(field, exact_tf, document_frequency(field, normalized)); + add_score(&mut field_scores, field, score); + let weighted_score = + if field == LexicalFieldV1::Documentation && prepared.whole_terms.len() != 1 { + score.saturating_mul(DOCUMENTATION_NL_SCORE_MILLIS) / 1_000 + } else { + score + }; + whole_term_scores + .entry(normalized.as_str()) + .and_modify(|current| *current = (*current).max(weighted_score)) + .or_insert(weighted_score); + matched_whole_terms.insert((*query_term).to_owned()); + collect_term_kinds(exact_terms, normalized, &mut matched_kinds); + } + if let Some(expansions) = fuzzy.by_query.get(*query_term) { + for expansion in expansions { + let fuzzy_tf = term_frequency(field, expansion); + if fuzzy_tf == 0 { + continue; } + let score = bm25(field, fuzzy_tf, document_frequency(field, expansion)) + .saturating_mul(FUZZY_SCORE_MILLIS) + / 1_000; + add_score(&mut field_scores, field, score); + matched_whole_terms.insert((*query_term).to_owned()); + spelling_variants.insert(LexicalSpellingVariantV1 { + query: (*query_term).to_owned(), + alternative: expansion.clone(), + }); + typo_recovery_applied = true; + collect_term_kinds(exact_terms, expansion, &mut matched_kinds); } } - } else { - for (subtoken, normalized) in &prepared.subtokens { - let tf = term_frequency(field, normalized); - if tf > 0 { - add_score( - &mut field_scores, - field, - bm25(field, tf, document_frequency(field, normalized)), - ); - matched_subtokens.insert((*subtoken).to_owned()); - } + } + } + if field_lengths.contains_key(&LexicalFieldV1::Subtoken) { + for (subtoken, normalized) in &prepared.subtokens { + let tf = term_frequency(LexicalFieldV1::Subtoken, normalized); + if tf == 0 { + continue; + } + // Whole-word and subtoken postings can represent the same + // evidence. Add only the stronger representation's excess. + let score = bm25( + LexicalFieldV1::Subtoken, + tf, + document_frequency(LexicalFieldV1::Subtoken, normalized), + ) + .saturating_sub( + whole_term_scores + .get(normalized.as_str()) + .copied() + .unwrap_or_default(), + ); + if score > 0 { + add_score(&mut field_scores, LexicalFieldV1::Subtoken, score); + matched_subtokens.insert((*subtoken).to_owned()); } } } diff --git a/crates/tracedecay-query/src/retrieval/lexical/projection/artifact/reader/query.rs b/crates/tracedecay-query/src/retrieval/lexical/projection/artifact/reader/query.rs index 0ee1e6f516..f4cab58639 100644 --- a/crates/tracedecay-query/src/retrieval/lexical/projection/artifact/reader/query.rs +++ b/crates/tracedecay-query/src/retrieval/lexical/projection/artifact/reader/query.rs @@ -236,6 +236,41 @@ impl RequestTermPostingsV1 { } Ok(documents) } + + /// Decode only the documents a source adds to the existing candidate set. + /// Stop as soon as it cannot fit; check cancellation during long postings + /// rather than waiting for the next row-hydration checkpoint. + fn new_documents_bounded( + &self, + term: &str, + admit: impl Fn(LexicalFieldV1) -> bool, + existing: &RoaringBitmap, + remaining: usize, + control: &dyn RetrievalExecutionControl, + ) -> Result, RetrievalPortError> { + let mut added = RoaringBitmap::new(); + let mut visited = 0usize; + for list in self.by_term.get(term).into_iter().flatten() { + if !admit(list.field) { + continue; + } + for posting in PostingListDecoderV1::new(&list.postings, true) { + if visited.is_multiple_of(RETRIEVAL_CANDIDATE_BATCH_SIZE) { + retrieval_checkpoint(control)?; + } + visited += 1; + let (document, _) = posting.map_err(map_query_artifact_error)?; + if !existing.contains(document) { + added.insert(document); + if added.len() > remaining as u64 { + return Ok(None); + } + } + } + } + retrieval_checkpoint(control)?; + Ok(Some(added)) + } } struct PostingCursorV1<'a> { @@ -1169,7 +1204,8 @@ impl<'a> ArtifactQueryV1<'a> { /// The candidate document set for one lexical request: every phrase /// match plus the term sources `admit_candidate_sources` keeps under - /// `MAX_LEXICAL_CANDIDATE_DOCUMENTS_V1`. + /// `MAX_LEXICAL_CANDIDATE_DOCUMENTS_V1`. A term requested through both + /// whole and subtoken channels is charged once for their document union. fn lexical_documents( &self, request: &LexicalLaneRequest<'_>, @@ -1197,34 +1233,31 @@ impl<'a> ArtifactQueryV1<'a> { .iter() .map(|subtoken| normalize_lexical(subtoken)) .collect::>(); - let mut sources = Vec::with_capacity(whole_terms.len() + subtokens.len()); - for term in whole_terms { - if stats.postings.by_term.contains_key(&term) { - sources.push((stats.whole_term_documents(&term), (term, false))); - } - } - for subtoken in subtokens { - if stats.postings.by_term.contains_key(&subtoken) { - sources.push(( - stats.document_frequency(LexicalFieldV1::Subtoken, &subtoken), - (subtoken, true), - )); - } - } - let mut documents = phrase_queries + let sources = stats.candidate_sources(whole_terms, subtokens); + let term_documents = admit_candidate_sources( + sources, + |source, existing, remaining| { + if source.cannot_fit(existing, remaining) { + return Ok(None); + } + stats.postings.new_documents_bounded( + &source.term, + |field| source.fields.admits(field), + existing, + remaining, + request.control, + ) + }, + |frequency, source| { + if !pruned.iter().any(|(term, _)| term == &source.term) { + pruned.push((source.term.clone(), frequency as u64)); + } + }, + )?; + let phrase_documents = phrase_queries .values() .fold(RoaringBitmap::new(), |union, documents| union | documents); - for (term, subtoken) in admit_candidate_sources(sources, |frequency, (term, _)| { - pruned.push((term.clone(), frequency as u64)); - }) { - documents |= stats - .postings - .documents(&term, |field| { - (field == LexicalFieldV1::Subtoken) == subtoken - }) - .map_err(map_query_artifact_error)?; - } - Ok(documents) + Ok(phrase_documents | term_documents) } fn exact_documents( @@ -1607,6 +1640,40 @@ struct LexicalStatsCacheV1 { postings: RequestTermPostingsV1, } +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum CandidateSourceFieldsV1 { + Whole, + Subtoken, +} + +impl CandidateSourceFieldsV1 { + fn admits(self, field: LexicalFieldV1) -> bool { + match self { + Self::Whole => field != LexicalFieldV1::Subtoken, + Self::Subtoken => field == LexicalFieldV1::Subtoken, + } + } +} + +struct LexicalCandidateSourceV1 { + term: String, + fields: CandidateSourceFieldsV1, + minimum_documents: usize, +} + +impl LexicalCandidateSourceV1 { + /// A source cannot fit even if every already-admitted document overlaps. + /// When the first source exceeds the cap, its existing documents may + /// entirely cover a second large channel, so its size alone cannot prune. + fn cannot_fit(&self, existing: &RoaringBitmap, remaining: usize) -> bool { + remaining != usize::MAX + && self.minimum_documents + > usize::try_from(existing.len()) + .unwrap_or(usize::MAX) + .saturating_add(remaining) + } +} + fn lexical_terms( prepared: &PreparedLexicalQueryV1<'_>, fuzzy: &FuzzyExpansionsV1, @@ -1663,6 +1730,59 @@ impl LexicalStatsCacheV1 { .map(|(_, frequencies)| frequencies.get(term).copied().unwrap_or_default()) .fold(0usize, usize::saturating_add) } + + /// Keep whole-word and subtoken channels independently eligible. Their + /// stored field frequencies order sources without eager bitmap decoding; + /// admission later accounts for exact new document IDs across channels. + fn candidate_sources( + &self, + whole_terms: Vec, + subtokens: Vec, + ) -> Vec<(usize, LexicalCandidateSourceV1)> { + let mut seen = BTreeSet::new(); + let mut sources = Vec::with_capacity(whole_terms.len() + subtokens.len()); + for term in whole_terms { + if !seen.insert(term.clone()) || !self.postings.by_term.contains_key(&term) { + continue; + } + let frequency = self.whole_term_documents(&term); + if frequency == 0 { + continue; + } + let minimum_documents = self + .document_frequencies + .iter() + .filter(|(field, _)| **field != LexicalFieldV1::Subtoken) + .map(|(_, terms)| terms.get(&term).copied().unwrap_or_default()) + .max() + .unwrap_or_default(); + sources.push(( + frequency, + LexicalCandidateSourceV1 { + term, + fields: CandidateSourceFieldsV1::Whole, + minimum_documents, + }, + )); + } + for term in subtokens.into_iter().collect::>() { + if !self.postings.by_term.contains_key(&term) { + continue; + } + let frequency = self.document_frequency(LexicalFieldV1::Subtoken, &term); + if frequency > 0 { + sources.push(( + frequency, + LexicalCandidateSourceV1 { + term, + fields: CandidateSourceFieldsV1::Subtoken, + minimum_documents: frequency, + }, + )); + } + } + sources + } } struct LexicalDraftV1 { @@ -1968,3 +2088,259 @@ impl EditDistanceScratchV1 { (self.previous[width - 1] <= limit).then_some(self.previous[width - 1]) } } + +#[cfg(test)] +mod candidate_source_tests { + use std::collections::BTreeMap; + use std::ops::Range; + use std::sync::atomic::{AtomicUsize, Ordering}; + + use roaring::RoaringBitmap; + + use super::super::super::format::PostingListEncoderV1; + use super::{ + CandidateSourceFieldsV1, LexicalStatsCacheV1, RequestTermPostingsV1, TermFieldPostingsV1, + }; + use crate::retrieval::lexical::{LexicalFieldV1, admit_candidate_sources}; + + fn stats(entries: &[(&str, LexicalFieldV1, Range)]) -> LexicalStatsCacheV1 { + let mut stats = LexicalStatsCacheV1 { + field_totals: BTreeMap::new(), + document_frequencies: BTreeMap::new(), + postings: RequestTermPostingsV1::default(), + }; + for (term, field, documents) in entries { + let mut encoder = PostingListEncoderV1::new(true); + for document in documents.clone() { + encoder.push(document, 1).expect("ordered posting"); + } + stats + .document_frequencies + .entry(*field) + .or_default() + .insert((*term).to_owned(), documents.clone().count()); + stats + .postings + .by_term + .entry((*term).to_owned()) + .or_default() + .push(TermFieldPostingsV1 { + field: *field, + postings: encoder.finish().expect("nonempty postings"), + }); + } + stats + } + + struct ActiveControl; + + impl crate::retrieval::ports::RetrievalExecutionControl for ActiveControl { + fn is_cancelled(&self) -> bool { + false + } + + fn elapsed_micros(&self) -> u64 { + 0 + } + } + + fn candidates( + stats: &LexicalStatsCacheV1, + whole: &[&str], + subtokens: &[&str], + ) -> (RoaringBitmap, Vec<(String, usize)>) { + let sources = stats.candidate_sources( + whole.iter().map(|term| (*term).to_owned()).collect(), + subtokens.iter().map(|term| (*term).to_owned()).collect(), + ); + let mut pruned = Vec::new(); + let admitted = admit_candidate_sources( + sources, + |source, existing, remaining| { + if source.cannot_fit(existing, remaining) { + return Ok(None); + } + stats.postings.new_documents_bounded( + &source.term, + |field| source.fields.admits(field), + existing, + remaining, + &ActiveControl, + ) + }, + |frequency, source| { + if !pruned.iter().any(|(term, _)| term == &source.term) { + pruned.push((source.term.clone(), frequency)); + } + }, + ) + .expect("candidate admission"); + (admitted, pruned) + } + + #[test] + fn overlapping_prose_channels_do_not_crowd_out_a_distinct_word() { + let stats = stats(&[ + ("needle", LexicalFieldV1::BodyText, 0..100), + ("needle", LexicalFieldV1::Subtoken, 0..100), + ("target", LexicalFieldV1::BodyText, 100..16_350), + ("target", LexicalFieldV1::Subtoken, 100..16_350), + ]); + let (documents, pruned) = candidates(&stats, &["needle", "target"], &["needle", "target"]); + assert!(pruned.is_empty(), "{pruned:?}"); + assert_eq!(documents.len(), 16_350); + assert!( + documents.contains(16_349), + "the target-only document must remain eligible" + ); + // Admission grouping must leave both channels available to scoring. + let mut cursors = stats.postings.cursors().unwrap(); + let target_channels = cursors + .iter_mut() + .filter(|cursor| cursor.term == "target") + .map(|cursor| (cursor.field, cursor.frequency_at(16_349).unwrap())) + .collect::>(); + assert_eq!( + target_channels, + vec![ + (LexicalFieldV1::BodyText, Some(1)), + (LexicalFieldV1::Subtoken, Some(1)), + ] + ); + } + + #[test] + fn disjoint_channels_are_charged_for_their_full_union() { + let stats = stats(&[ + ("needle", LexicalFieldV1::BodyText, 0..100), + ("needle", LexicalFieldV1::Subtoken, 100..200), + ("target", LexicalFieldV1::BodyText, 200..16_400), + ("target", LexicalFieldV1::Subtoken, 200..16_400), + ]); + let (documents, pruned) = candidates(&stats, &["needle", "target"], &["needle", "target"]); + assert_eq!(documents.len(), 200); + assert_eq!(pruned, vec![("target".to_owned(), 16_200)]); + assert!(!documents.contains(16_399)); + } + + #[test] + fn fully_overlapping_oversized_channels_do_not_claim_partial_evidence() { + let stats = stats(&[ + ("cache", LexicalFieldV1::BodyText, 0..20_000), + ("cache", LexicalFieldV1::Subtoken, 0..20_000), + ]); + let (documents, pruned) = candidates(&stats, &["cache"], &["cache"]); + assert_eq!(documents.len(), 20_000); + assert!(documents.contains(19_999)); + assert!( + pruned.is_empty(), + "the second channel adds no documents despite exceeding the cap: {pruned:?}" + ); + } + + #[test] + fn oversized_disjoint_channels_still_report_omitted_candidates() { + let stats = stats(&[ + ("cache", LexicalFieldV1::BodyText, 0..20_000), + ("cache", LexicalFieldV1::Subtoken, 20_000..40_000), + ]); + let (documents, pruned) = candidates(&stats, &["cache"], &["cache"]); + assert_eq!(documents.len(), 20_000); + assert!(documents.contains(19_999)); + assert!(!documents.contains(20_000)); + assert_eq!(pruned, vec![("cache".to_owned(), 20_000)]); + } + + #[test] + fn rare_whole_word_is_not_lost_behind_a_common_subtoken() { + let stats = stats(&[ + ("cache", LexicalFieldV1::BodyText, 0..1), + ("cache", LexicalFieldV1::Subtoken, 1..20_001), + ("token", LexicalFieldV1::BodyText, 20_001..20_101), + ("token", LexicalFieldV1::Subtoken, 20_001..20_101), + ]); + let (documents, pruned) = candidates(&stats, &["cache", "token"], &["cache", "token"]); + assert_eq!(documents.len(), 101); + assert!(documents.contains(0), "rare whole-word cache must survive"); + assert!(documents.contains(20_100)); + assert_eq!(pruned, vec![("cache".to_owned(), 20_000)]); + } + + #[test] + fn rejected_common_source_is_not_decoded_before_admission() { + let mut stats = stats(&[ + ("needle", LexicalFieldV1::BodyText, 0..1), + ("get", LexicalFieldV1::Subtoken, 1..30_001), + ]); + // The stored frequency alone proves this source cannot fit. If it + // were eagerly decoded, this intentionally invalid payload would fail. + stats.postings.by_term.get_mut("get").unwrap()[0].postings = vec![255]; + let (documents, pruned) = candidates(&stats, &["needle"], &["get"]); + assert_eq!(documents.len(), 1); + assert!(documents.contains(0)); + assert_eq!(pruned, vec![("get".to_owned(), 30_000)]); + } + + #[test] + fn posting_admission_checks_cancellation_during_decode() { + struct CancelAfterBatch(AtomicUsize); + + impl crate::retrieval::ports::RetrievalExecutionControl for CancelAfterBatch { + fn is_cancelled(&self) -> bool { + self.0.fetch_add(1, Ordering::SeqCst) > 0 + } + + fn elapsed_micros(&self) -> u64 { + 0 + } + } + + let stats = stats(&[("needle", LexicalFieldV1::BodyText, 0..1_024)]); + let control = CancelAfterBatch(AtomicUsize::new(0)); + let error = stats + .postings + .new_documents_bounded( + "needle", + |_| true, + &RoaringBitmap::new(), + usize::MAX, + &control, + ) + .expect_err("cancelled during the posting stream"); + assert!(matches!( + error, + crate::retrieval::ports::RetrievalPortError::Cancelled + )); + assert_eq!( + control.0.load(Ordering::SeqCst), + 2, + "the request was interrupted at the second 128-posting checkpoint" + ); + } + + #[test] + fn repeated_sources_are_deduplicated_without_widening_requested_fields() { + let stats = stats(&[ + ("needle", LexicalFieldV1::BodyText, 0..100), + ("needle", LexicalFieldV1::Subtoken, 100..200), + ("part", LexicalFieldV1::Subtoken, 200..210), + ]); + let sources = stats.candidate_sources( + vec!["needle".to_owned(), "needle".to_owned()], + vec!["part".to_owned(), "part".to_owned()], + ); + assert_eq!(sources.len(), 2); + assert_eq!(sources[0].0, 100); + assert_eq!(sources[0].1.fields, CandidateSourceFieldsV1::Whole); + assert_eq!(sources[1].0, 10); + assert_eq!(sources[1].1.fields, CandidateSourceFieldsV1::Subtoken); + let (documents, pruned) = candidates(&stats, &["needle", "needle"], &["part", "part"]); + assert!(pruned.is_empty()); + assert_eq!(documents.len(), 110); + assert!( + !documents.contains(100), + "unrequested subtoken channel stays excluded" + ); + assert!(documents.contains(209)); + } +} diff --git a/crates/tracedecay-query/src/retrieval/lexical/tests.rs b/crates/tracedecay-query/src/retrieval/lexical/tests.rs index 69ef708f8d..b74310511e 100644 --- a/crates/tracedecay-query/src/retrieval/lexical/tests.rs +++ b/crates/tracedecay-query/src/retrieval/lexical/tests.rs @@ -68,11 +68,41 @@ where T::try_from(format!("sha256:{}", byte.to_string().repeat(64))).expect("valid fixture digest") } +fn synthetic_admission( + sources: Vec<(usize, &'static str)>, + ranges: &[(&str, std::ops::Range)], + pruned: &mut Vec<(String, u64)>, +) -> roaring::RoaringBitmap { + admit_candidate_sources( + sources, + |term, existing, remaining| { + let range = ranges + .iter() + .find(|(name, _)| name == term) + .expect("source") + .1 + .clone(); + let mut delta = roaring::RoaringBitmap::new(); + for document in range { + if !existing.contains(document) { + delta.insert(document); + } + if delta.len() > remaining as u64 { + return Ok(None); + } + } + Ok(Some(delta)) + }, + |frequency, term| pruned.push(((*term).to_owned(), frequency as u64)), + ) + .expect("admission") +} + #[test] fn candidate_sources_admit_rarest_first_within_the_document_budget() { let budget = MAX_LEXICAL_CANDIDATE_DOCUMENTS_V1; let mut pruned = Vec::new(); - let admitted = admit_candidate_sources( + let documents = synthetic_admission( vec![ (budget * 3, "the"), (10, "pendingtoolkind"), @@ -80,20 +110,24 @@ fn candidate_sources_admit_rarest_first_within_the_document_budget() { (budget * 2, "tool"), (200, "kind"), ], - |frequency, term| pruned.push((term.to_string(), frequency as u64)), - ); - assert_eq!( - admitted, - ["pendingtoolkind", "kind"], - "`pending` would push the admitted total past the budget, and every later \ - (more common) source is pruned with it" + &[ + ("pendingtoolkind", 0..10), + ("kind", 10..210), + ("pending", 210..(210 + (budget - 100) as u32)), + ("tool", 30_000..(30_000 + (budget * 2) as u32)), + ("the", 70_000..(70_000 + (budget * 3) as u32)), + ], + &mut pruned, ); + assert_eq!(documents.len(), 210); + assert!(documents.contains(209)); + assert!(!documents.contains(210)); assert_eq!( pruned, [ ("pending".to_owned(), (budget - 100) as u64), ("tool".to_owned(), (budget * 2) as u64), - ("the".to_owned(), (budget * 3) as u64) + ("the".to_owned(), (budget * 3) as u64), ] ); let outcome = super::candidate_admission_outcome( @@ -113,24 +147,33 @@ fn candidate_sources_admit_rarest_first_within_the_document_budget() { #[test] fn candidate_sources_ignore_empty_sources_before_admitting_a_common_term() { let budget = MAX_LEXICAL_CANDIDATE_DOCUMENTS_V1; - assert_eq!( - admit_candidate_sources( - vec![(0, "missing"), (budget + 1, "common"), (0, "absent")], - |_, _| {} - ), - ["common"], - "empty sources must not consume the first nonempty source exception" + let documents = synthetic_admission( + vec![(0, "missing"), (budget + 1, "common"), (0, "absent")], + &[("common", 0..(budget + 1) as u32)], + &mut Vec::new(), + ); + assert_eq!(documents.len(), (budget + 1) as u64); + assert!( + synthetic_admission(vec![(0, "missing"), (0, "absent")], &[], &mut Vec::new(),).is_empty() ); - assert!(admit_candidate_sources(vec![(0, "missing"), (0, "absent")], |_, _| {}).is_empty()); } #[test] fn candidate_sources_keep_request_order_across_equal_frequencies() { let half = MAX_LEXICAL_CANDIDATE_DOCUMENTS_V1 / 2; - assert_eq!( - admit_candidate_sources(vec![(half, "b"), (half, "a"), (half, "c")], |_, _| {}), - ["b", "a"] + let mut pruned = Vec::new(); + let documents = synthetic_admission( + vec![(half, "b"), (half, "a"), (half, "c")], + &[ + ("b", 0..half as u32), + ("a", half as u32..(half * 2) as u32), + ("c", (half * 2) as u32..(half * 3) as u32), + ], + &mut pruned, ); + assert_eq!(documents.len(), (half * 2) as u64); + assert!(!documents.contains((half * 2) as u32)); + assert_eq!(pruned, vec![("c".to_owned(), half as u64)]); } #[test] @@ -160,7 +203,7 @@ fn shared_query_parser_retains_phrase_and_identifier_subtokens() { /// query-side subtokens equal the extraction subtokens. #[test] fn query_tokenization_matches_index_tokenization_for_separator_terms() { - let text = "find foo-bar a::b p/q.rs --flag E0308 TS1234 ERR_X"; + let text = "foo-bar a::b p/q.rs --flag E0308 TS1234 ERR_X"; let parts = lexical_query_parts(text).expect("lexical query parts"); let index_tokens: Vec = technical_tokens(text) @@ -181,12 +224,7 @@ fn query_tokenization_matches_index_tokenization_for_separator_terms() { let mut expected_subtokens: Vec = index_tokens .iter() - .flat_map(|token| { - let lowercase = token.to_ascii_lowercase(); - split_subtokens(token) - .into_iter() - .filter(move |part| part != &lowercase) - }) + .flat_map(|token| split_subtokens(token)) .collect(); expected_subtokens.sort(); expected_subtokens.dedup(); @@ -199,6 +237,55 @@ fn query_tokenization_matches_index_tokenization_for_separator_terms() { } } +#[test] +fn prose_words_can_match_identifier_subtokens_without_broadening_single_term_queries() { + let query = "widget auto tag items forward total"; + let parts = lexical_query_parts(query).expect("prose search"); + assert_eq!( + parts.subtokens, + ["auto", "forward", "items", "tag", "total", "widget"] + .map(str::to_owned) + .to_vec() + ); + assert_eq!(parts.whole_terms.len(), 6); + + let hyphenated = + lexical_query_parts("widget-tool auto-tag query rows per item").expect("hyphenated prose"); + for word in [ + "widget", "tool", "auto", "tag", "query", "rows", "per", "item", + ] { + assert!( + hyphenated.subtokens.contains(&word.to_owned()), + "{word} must reach a camelCase subtoken posting" + ); + } + assert!( + lexical_query_parts("fetchWidgetAutoTagItemsForwardTotal") + .expect("single identifier") + .subtokens + .is_empty(), + "single-term queries must retain their narrow admission" + ); +} + +#[test] +fn prose_subtoken_expansion_uses_the_existing_content_word_policy() { + let query = "how do i find the widget auto tag"; + let parts = lexical_query_parts(query).expect("prose search"); + assert_eq!( + parts.subtokens, + ["auto", "tag", "widget"].map(str::to_owned) + ); + for word in ["how", "do", "i", "find", "the"] { + assert!(parts.whole_terms.iter().any(|term| term == word)); + } + assert_eq!(parts.phrases, [query]); + + let technical = lexical_query_parts("TypeOf Widget").expect("technical query"); + assert!(technical.subtokens.iter().any(|term| term == "type")); + assert!(technical.subtokens.iter().any(|term| term == "of")); +} + fn budget(max_candidates_per_lane: u32) -> RetrievalBudget { RetrievalBudget { max_candidates_per_lane, diff --git a/crates/tracedecay-query/tests/search_quality_suite/candidate_producers.rs b/crates/tracedecay-query/tests/search_quality_suite/candidate_producers.rs index 4cd22cd3ea..f186edc00e 100644 --- a/crates/tracedecay-query/tests/search_quality_suite/candidate_producers.rs +++ b/crates/tracedecay-query/tests/search_quality_suite/candidate_producers.rs @@ -6070,6 +6070,111 @@ fn fielded_bm25_keeps_whole_identifiers_and_subtokens_distinct() { })); } +#[test] +fn prose_subtokens_add_identifier_matches_without_rescoring_whole_words() { + let artifact = rust_artifact(&[ + "pub fn reserveStock() {}\n", + "pub fn reserve() { let stock = 1; }\n", + ]); + let lane = artifact.lane(); + let whole = complete( + lane.retrieve_lexical(&artifact.request( + "reserve stock", + &["reserve", "stock"], + &[], + &[], + 0, + 8, + )) + .expect("whole-word retrieval"), + ); + let expanded = complete( + lane.retrieve_lexical(&artifact.request( + "reserve stock", + &["reserve", "stock"], + &["reserve", "stock"], + &[], + 0, + 8, + )) + .expect("expanded retrieval"), + ); + assert!(candidate_files(&whole.candidates).contains(&1)); + assert_eq!( + candidate_files(&expanded.candidates), + BTreeSet::from([0, 1]) + ); + for candidate in &whole.candidates { + if candidate + .file_occurrence_id + .as_ref() + .map(|file| file.as_str()) + != Some("file.fixture.1") + { + continue; + } + let occurrence = &candidate.source_occurrence_id; + let evidence = &whole.evidence_by_occurrence[occurrence]; + assert_eq!( + expanded.evidence_by_occurrence[occurrence].field_scores_micros, + evidence.field_scores_micros, + "an ordinary word already scored in its source field is not new subtoken evidence" + ); + } +} + +#[test] +fn filtered_subtoken_search_keeps_prose_evidence_when_whole_fields_are_excluded() { + let artifact = rust_artifact(&[ + "pub fn process() { let digest = payload; }\n", + "pub fn digestPayloadMismatch() {}\n", + ]); + for filter in [ + LexicalFieldFilterV1 { + field: LexicalFieldV1::Subtoken, + include: true, + }, + LexicalFieldFilterV1 { + field: LexicalFieldV1::BodyText, + include: false, + }, + ] { + let mut request = artifact.request( + "digest payload", + &["digest", "payload"], + &["digest", "payload"], + &[], + 0, + 8, + ); + request.field_filters = Cow::Owned(vec![filter]); + let results = complete( + artifact + .lane() + .retrieve_lexical(&request) + .expect("filtered subtoken search"), + ); + assert!( + results.candidates.iter().any(|candidate| { + candidate.file_occurrence_id.as_ref().is_some_and(|file| { + file.as_str() == "file.fixture.0" + && results + .evidence_by_occurrence + .get(&candidate.source_occurrence_id) + .is_some_and(|evidence| { + evidence.field_scores_micros.iter().any(|(field, score)| { + *field == LexicalFieldV1::Subtoken && *score > 0 + }) && ["digest", "payload"].iter().all(|word| { + evidence.matched_subtokens.iter().any(|term| term == word) + }) + }) + }) + }), + "excluded whole fields must not zero out admitted subtokens: {results:?}" + ); + } +} + #[test] fn lexical_phrase_and_bounded_fuzzy_recovery_are_deterministic() { let artifact = rust_artifact(&[ @@ -6283,6 +6388,128 @@ fn duplicate_whole_terms_do_not_consume_the_global_fuzzy_budget() { ); } +#[test] +fn prose_query_recovers_camel_case_symbol_from_real_sealed_subtoken_postings() { + let artifact = rust_artifact(&[ + "pub fn fetchWidgetAutoTagItemsForwardTotal() {}\n", + "pub fn widget() {}\n", + "pub fn tag() {}\n", + ]); + let query = "widget auto tag items forward total"; + let parts = tracedecay_query::retrieval::lexical::lexical_query_parts(query) + .expect("parse multiword query"); + let mut request = artifact.request(query, &[], &[], &[], 0, 32); + request.whole_terms = Cow::Owned(parts.whole_terms); + request.subtokens = Cow::Owned(parts.subtokens); + request.phrases = Cow::Owned(parts.phrases); + let results = complete( + artifact + .lane() + .retrieve_lexical(&request) + .expect("sealed lexical search"), + ); + assert!( + candidate_files(&results.candidates).contains(&0), + "the camelCase symbol must not be absent from the bounded page: {results:?}" + ); + assert!( + results.candidates.iter().any(|candidate| { + candidate.file_occurrence_id.as_ref().is_some_and(|file| { + file.as_str() == "file.fixture.0" + && ["widget", "auto", "tag", "items", "forward", "total"] + .iter() + .all(|term| { + results.evidence_by_occurrence[&candidate.source_occurrence_id] + .matched_subtokens + .iter() + .any(|matched| matched.as_str() == *term) + }) + }) + }), + "the long symbol must match every query word in the subtoken field" + ); +} + +#[test] +fn prose_words_do_not_add_two_full_scores_for_the_same_evidence() { + // The first row states the words; the second only hides them inside its + // name. Prose subtoken admission exists to reach the second row, and must + // not add two full scores for words the first row already matched. + let artifact = rust_artifact(&[ + "/// Reports when the digest does not match its payload.\npub fn validate() {}\n", + "pub fn digestPayloadMismatch() {}\n", + ]); + let query = "digest payload"; + let parts = tracedecay_query::retrieval::lexical::lexical_query_parts(query) + .expect("parse multiword query"); + let mut request = artifact.request(query, &[], &[], &[], 0, 32); + request.whole_terms = Cow::Owned(parts.whole_terms); + request.subtokens = Cow::Owned(parts.subtokens); + request.phrases = Cow::Owned(parts.phrases); + let results = complete( + artifact + .lane() + .retrieve_lexical(&request) + .expect("sealed lexical search"), + ); + let whole = complete( + artifact + .lane() + .retrieve_lexical(&artifact.request(query, &["digest", "payload"], &[], &[], 0, 32)) + .expect("whole-word control"), + ); + let subtoken = complete( + artifact + .lane() + .retrieve_lexical(&artifact.request(query, &[], &["digest", "payload"], &[], 0, 32)) + .expect("subtoken control"), + ); + let in_file = |ordinal: usize| { + results + .candidates + .iter() + .filter(move |candidate| { + candidate + .file_occurrence_id + .as_ref() + .is_some_and(|file| file.as_str() == format!("file.fixture.{ordinal}")) + }) + .map(|candidate| &results.evidence_by_occurrence[&candidate.source_occurrence_id]) + .collect::>() + }; + let stated = in_file(0); + assert!(!stated.is_empty()); + let total = |evidence: &LexicalLaneEvidence| { + evidence + .field_scores_micros + .iter() + .map(|(_, score)| score) + .sum::() + }; + for evidence in stated { + let occurrence = &evidence.binding.source_occurrence; + let whole_score = total(&whole.evidence_by_occurrence[occurrence]); + let subtoken_score = total(&subtoken.evidence_by_occurrence[occurrence]); + assert!(whole_score > 0 && subtoken_score > 0); + assert!(total(evidence) >= whole_score.max(subtoken_score)); + assert!( + total(evidence) < whole_score + subtoken_score, + "overlapping representations must not add both full scores: {evidence:?}" + ); + } + assert!( + in_file(1).iter().any(|evidence| { + ["digest", "payload"].iter().all(|term| { + evidence + .matched_subtokens + .iter() + .any(|matched| matched == term) + }) + }), + "the camelCase symbol must still match both words as subtokens: {results:?}" + ); +} + #[test] fn lexical_artifact_reports_freshness_coverage_and_page_cutoff() { let sources = ["pub fn target() {}\n"; 3]; diff --git a/crates/tracedecay-search-eval/src/report_tests.rs b/crates/tracedecay-search-eval/src/report_tests.rs index b40af42ef7..5d87916362 100644 --- a/crates/tracedecay-search-eval/src/report_tests.rs +++ b/crates/tracedecay-search-eval/src/report_tests.rs @@ -75,16 +75,27 @@ fn assert_baseline_report_retains_raw_fallback_current_and_exact_ten_x_samples( [ "train-016", "train-019", - "train-020", "train-026", "validation-015", - "validation-017", "validation-023", "validation-024", "validation-025", "validation-027", ] ); + for query_id in ["train-020", "validation-018", "validation-019"] { + let query = report + .profiles + .iter() + .flat_map(|profile| &profile.queries) + .find(|query| query.query_id == query_id) + .expect("ranking regression query is present"); + assert_eq!( + query.status, + crate::DirectEvaluationStatusV1::Pass, + "{query_id} must retain useful top-ten retrieval" + ); + } assert!(report.profiles.iter().all(|profile| { profile.quality.protected_recall_at_10.denominator > 0 && profile.quality.protected_recall_at_10.numerator diff --git a/tests/fixtures/search_quality/query-lexical-graph-workload-v1.json b/tests/fixtures/search_quality/query-lexical-graph-workload-v1.json index b7e73cd319..a3f50a061a 100644 --- a/tests/fixtures/search_quality/query-lexical-graph-workload-v1.json +++ b/tests/fixtures/search_quality/query-lexical-graph-workload-v1.json @@ -133,8 +133,8 @@ } ], "expected_query_fallback_digests": { - "train": "sha256:7f52a7213e6caab00e6a2bfca4a24ccdfe3f37fcd2312fd86da34807a397b846", - "validation": "sha256:f8f6e5973a2c07c2e3dc46ec270c75e819f6dc333030f8acabdf1580311577eb" + "train": "sha256:469266a120472b3203ad5a4d520b1a4e539e4c44ae6f5cf9babe1fb64ddfc1a3", + "validation": "sha256:94e8499d1c6237a7c7cb2f5008f34edb17e76cb6a9b1144b79851595f569fd09" }, "profile_matrix": [ {