fix(search): recover camelCase symbols from prose words

crates/tracedecay-code-index-runtime/src/project_reads/ignored_dependency_admission_tests.rs

@@ -21,7 +21,6 @@ use tracedecay_tool_catalog::{CapabilityId, UseCaseId};
 
 use crate::code_index_scheduler::{CodeIndexSchedulerRegistryV1, LatestCompleteCodeIndexV1};
 use crate::project_reads::project_code_index_ignored_dependency_admission_port;
-use tracedecay_runtime_core::path_safety::canonical_existing_identity;
 
 const PROJECT_ID: &str = "project.project-open-ignored-dependency";
 
@@ -70,7 +69,6 @@ export function generationAnchor() { return 1; }
             .mount_worktree(project_id(), root.path(), store.path().to_path_buf())
             .await
             .expect("mount code-index scheduler");
-        wait_for_initial_generation(&registry, root.path()).await;
         let baseline = latest(&registry, root.path()).await;
         let generation = baseline.generation();
         let snapshot = generation.snapshot();
@@ -242,31 +240,11 @@ fn request_context(scope: ResolvedScope, suffix: &str) -> RequestContext {
     .expect("request context")
 }
 
-async fn wait_for_initial_generation(registry: &CodeIndexSchedulerRegistryV1, project_root: &Path) {
-    if registry.latest_generation_id(project_root).await.is_some() {
-        return;
-    }
-    let canonical_root = canonical_existing_identity(project_root).expect("canonical fixture root");
-    let mut publications = registry.subscribe_generation_publications();
-    tokio::time::timeout(Duration::from_secs(5), async {
-        loop {
-            let event = publications.recv().await.expect("generation publication");
-            if event.project_root == canonical_root {
-                break;
-            }
-        }
-    })
-    .await
-    .expect("initial generation publication");
-}
-
 /// Wait for the complete serving generation, not merely its publication.
 ///
-/// The generation-publication broadcast and `latest_generation_id` both fire
-/// on the reconcile pass, ahead of the bind and the serving swap that fill
-/// `serving_generation`. Reading the complete arm the instant
-/// `wait_for_initial_generation` returns therefore raced the seat and reported
-/// a missing generation for a store that had one.
+/// Publication and `latest_generation_id` precede the serving swap. Wait
+/// for the actual fresh serving state before using imports as admission
+/// evidence; no earlier broadcast establishes readiness.
 async fn latest(
     registry: &CodeIndexSchedulerRegistryV1,
     project_root: &Path,

crates/tracedecay-query/src/retrieval/lexical.rs

@@ -10,6 +10,7 @@
 use std::borrow::Cow;
 use std::collections::{BTreeMap, BTreeSet};
 
+use roaring::RoaringBitmap;
 use serde::{Deserialize, Serialize};
 use tracedecay_domain::{
     CodeGenerationId, CompactCandidate, ComponentRevision, CursorPayloadDigest,
@@ -78,45 +79,52 @@ pub const MAX_LEXICAL_FIELD_FILTERS_V1: usize = 9;
 /// proximity term.
 pub const MAX_LEXICAL_QUERY_TERM_BYTES_V1: usize = 512;
 
-/// Summed document-frequency budget for lexical term-source admission. Every
-/// candidate is decoded from its row and scored, so the union of the request's
-/// term sources, not the winner cap, decides the lane's transient allocation
-/// and wall time: unbounded, a natural-language task whose terms include
-/// common words hydrated ~74k rows of a 472k-chunk corpus per read (~0.95 GB
-/// decoded, 5.7 s) and missed the context deadline. Term sources are admitted
-/// in ascending document-frequency order until their summed frequencies would
-/// exceed this bound; the most selective source is always admitted so a
-/// single common-term query still answers. Phrase sources are admitted separately.
+/// Unique-document budget for lexical term-source admission. Every candidate
+/// is decoded and scored, so source selection determines transient allocation
+/// and wall time. Previously a natural-language query hydrated ~74k rows of
+/// a 472k-chunk corpus (~0.95 GB decoded, 5.7 s), missing the deadline.
+/// Sources are considered in ascending stored field-frequency order. Only
+/// new document IDs charge the budget; the first nonempty source is always
+/// admitted so a single common-term query still answers. Phrase sources
+/// are admitted separately.
 /// This is a recall/latency policy, not a hard document or allocation ceiling:
-/// documents matching only pruned terms cannot rank. The initial 16,384 value
+/// documents matching only pruned sources cannot rank. The 16,384 value
 /// retains the measured policy (~100 MiB decode churn at ~6.6 KiB per row);
 /// changing the reader cache must not change candidate eligibility.
 pub const MAX_LEXICAL_CANDIDATE_DOCUMENTS_V1: usize = 16_384;
 
-/// Admit `(document_frequency, source)` pairs rarest-first while the summed
-/// frequency stays within [`MAX_LEXICAL_CANDIDATE_DOCUMENTS_V1`]; the rarest
-/// nonempty source is always admitted. Ties keep request order so admission
-/// is deterministic. Sources past the bound still weigh admitted candidates
-/// through scoring; a document matching only those sources is never hydrated.
+/// Visit posting sources rarest-first, counting unique documents rather than
+/// charging overlapping whole/subtoken postings twice. The reader supplies a
+/// bounded delta; only the rarest nonempty source may exceed the cap.
 pub(crate) fn admit_candidate_sources<S>(
     mut sources: Vec<(usize, S)>,
+    mut new_documents: impl FnMut(
+        &S,
+        &RoaringBitmap,
+        usize,
+    ) -> Result<Option<RoaringBitmap>, RetrievalPortError>,
     mut on_pruned: impl FnMut(usize, &S),
-) -> Vec<S> {
+) -> Result<RoaringBitmap, RetrievalPortError> {
     sources.retain(|(frequency, _)| *frequency > 0);
     sources.sort_by_key(|(frequency, _)| *frequency);
-    let total = sources.len();
-    let mut admitted_documents = 0usize;
-    let mut admitted = Vec::with_capacity(total);
-    for (ordinal, (frequency, source)) in sources.into_iter().enumerate() {
-        let next = admitted_documents.saturating_add(frequency);
-        if ordinal > 0 && next > MAX_LEXICAL_CANDIDATE_DOCUMENTS_V1 {
-            on_pruned(frequency, &source);
-            continue;
+    let mut admitted = RoaringBitmap::new();
+    let mut admitted_sources = 0usize;
+    for (frequency, source) in sources {
+        let remaining = if admitted_sources == 0 {
+            usize::MAX
+        } else {
+            MAX_LEXICAL_CANDIDATE_DOCUMENTS_V1
+                .saturating_sub(usize::try_from(admitted.len()).unwrap_or(usize::MAX))
+        };
+        match new_documents(&source, &admitted, remaining)? {
+            Some(delta) if delta.len() <= remaining as u64 => {
+                admitted |= delta;
+                admitted_sources += 1;
+            }
+            _ => on_pruned(frequency, &source),
         }
-        admitted_documents = next;
-        admitted.push(source);
     }
-    admitted
+    Ok(admitted)
 }
 
 fn candidate_admission_outcome<E>(
@@ -158,8 +166,10 @@ pub(super) fn normalize_lexical(value: &str) -> String {
 /// Whole terms are the same maximal technical tokens the code index extracts
 /// (`tracedecay_domain::technical_tokens`), so a term the indexer keeps whole
 /// (`foo-bar`, `a::b`, `p/q.rs`) is queried whole, and subtokens decompose
-/// through the same shared grammar. Single-token queries intentionally emit
-/// no subtokens so one technical term does not fan out to common subtokens.
+/// through the same shared grammar. For multiword prose, unsplit content
+/// words use the symbol route's policy before searching the Subtoken field,
+/// so camelCase symbols can be admitted without expanding stopwords.
+/// Single-token queries emit no subtokens, preserving their narrow admission.
 ///
 /// Multi-token sanitized input is also retained as a phrase. This gives exact
 /// diagnostic/error text and natural-language queries a bounded lexical phrase
@@ -182,10 +192,11 @@ pub fn lexical_query_parts(query: &str) -> Result<LexicalQueryPartsV1, Retrieval
         whole_terms.push(token.to_owned());
         if split_identifiers {
             let lowercase = token.to_ascii_lowercase();
+            let expand_unsplit = !preferred_symbol_tokens(token).is_empty();
             subtokens.extend(
                 split_subtokens(token)
                     .into_iter()
-                    .filter(|part| part != &lowercase),
+                    .filter(|part| part != &lowercase || expand_unsplit),
             );
         }
     }

crates/tracedecay-query/src/retrieval/lexical/projection.rs

@@ -1025,6 +1025,7 @@ fn score_lexical_row(
     echo_penalty: bool,
 ) -> LexicalRowScoreV1 {
     let mut field_scores = BTreeMap::new();
+    let mut whole_term_scores = BTreeMap::<&str, u64>::new();
     let mut matched_whole_terms = BTreeSet::new();
     let mut matched_subtokens = BTreeSet::new();
     let mut matched_phrases = BTreeSet::new();
@@ -1052,49 +1053,72 @@ fn score_lexical_row(
         if !score_name_fields && is_identifier_name_field(field) {
             continue;
         }
-        if field != LexicalFieldV1::Subtoken {
-            for (query_term, normalized) in &prepared.whole_terms {
-                let exact_tf = term_frequency(field, normalized);
-                if exact_tf > 0 {
-                    add_score(
-                        &mut field_scores,
-                        field,
-                        bm25(field, exact_tf, document_frequency(field, normalized)),
-                    );
-                    matched_whole_terms.insert((*query_term).to_owned());
-                    collect_term_kinds(exact_terms, normalized, &mut matched_kinds);
-                }
-                if let Some(expansions) = fuzzy.by_query.get(*query_term) {
-                    for expansion in expansions {
-                        let fuzzy_tf = term_frequency(field, expansion);
-                        if fuzzy_tf == 0 {
-                            continue;
-                        }
-                        let score = bm25(field, fuzzy_tf, document_frequency(field, expansion))
-                            .saturating_mul(FUZZY_SCORE_MILLIS)
-                            / 1_000;
-                        add_score(&mut field_scores, field, score);
-                        matched_whole_terms.insert((*query_term).to_owned());
-                        spelling_variants.insert(LexicalSpellingVariantV1 {
-                            query: (*query_term).to_owned(),
-                            alternative: expansion.clone(),
-                        });
-                        typo_recovery_applied = true;
-                        collect_term_kinds(exact_terms, expansion, &mut matched_kinds);
+        // Only an admitted whole-word field can cover subtoken evidence.
+        // Otherwise a Subtoken-only request loses rows before lane filtering.
+        if field == LexicalFieldV1::Subtoken || !super::field_admitted(field_filters, field) {
+            continue;
+        }
+        for (query_term, normalized) in &prepared.whole_terms {
+            let exact_tf = term_frequency(field, normalized);
+            if exact_tf > 0 {
+                let score = bm25(field, exact_tf, document_frequency(field, normalized));
+                add_score(&mut field_scores, field, score);
+                let weighted_score =
+                    if field == LexicalFieldV1::Documentation && prepared.whole_terms.len() != 1 {
+                        score.saturating_mul(DOCUMENTATION_NL_SCORE_MILLIS) / 1_000
+                    } else {
+                        score
+                    };
+                whole_term_scores
+                    .entry(normalized.as_str())
+                    .and_modify(|current| *current = (*current).max(weighted_score))
+                    .or_insert(weighted_score);
+                matched_whole_terms.insert((*query_term).to_owned());
+                collect_term_kinds(exact_terms, normalized, &mut matched_kinds);
+            }
+            if let Some(expansions) = fuzzy.by_query.get(*query_term) {
+                for expansion in expansions {
+                    let fuzzy_tf = term_frequency(field, expansion);
+                    if fuzzy_tf == 0 {
+                        continue;
                     }
+                    let score = bm25(field, fuzzy_tf, document_frequency(field, expansion))
+                        .saturating_mul(FUZZY_SCORE_MILLIS)
+                        / 1_000;
+                    add_score(&mut field_scores, field, score);
+                    matched_whole_terms.insert((*query_term).to_owned());
+                    spelling_variants.insert(LexicalSpellingVariantV1 {
+                        query: (*query_term).to_owned(),
+                        alternative: expansion.clone(),
+                    });
+                    typo_recovery_applied = true;
+                    collect_term_kinds(exact_terms, expansion, &mut matched_kinds);
                 }
             }
-        } else {
-            for (subtoken, normalized) in &prepared.subtokens {
-                let tf = term_frequency(field, normalized);
-                if tf > 0 {
-                    add_score(
-                        &mut field_scores,
-                        field,
-                        bm25(field, tf, document_frequency(field, normalized)),
-                    );
-                    matched_subtokens.insert((*subtoken).to_owned());
-                }
+        }
+    }
+    if field_lengths.contains_key(&LexicalFieldV1::Subtoken) {
+        for (subtoken, normalized) in &prepared.subtokens {
+            let tf = term_frequency(LexicalFieldV1::Subtoken, normalized);
+            if tf == 0 {
+                continue;
+            }
+            // Whole-word and subtoken postings can represent the same
+            // evidence. Add only the stronger representation's excess.
+            let score = bm25(
+                LexicalFieldV1::Subtoken,
+                tf,
+                document_frequency(LexicalFieldV1::Subtoken, normalized),
+            )
+            .saturating_sub(
+                whole_term_scores
+                    .get(normalized.as_str())
+                    .copied()
+                    .unwrap_or_default(),
+            );
+            if score > 0 {
+                add_score(&mut field_scores, LexicalFieldV1::Subtoken, score);
+                matched_subtokens.insert((*subtoken).to_owned());
             }
         }
     }