From a5206f6dd87cf76a2b96df2d7c53c45017aeeb7f Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Thu, 17 Sep 2026 09:35:21 +0000 Subject: [PATCH 01/14] fix(code-index): stabilize Fresh reads and dirty retained seats Dashboard freshness treated a bare pending wake as refresh_in_flight, so Fresh flipped to Verifying between settled status samples. Dirty remount seating required an already-decoded generation, so a held decode barrier returned None instead of a pointer Noop. Keep quiet undecoded seats from claiming currency while allowing dirty pointer Noops without joining the decode flight, and wait for owner quiescence before declaring ready. Co-authored-by: Zack Jackson --- .../src/code_index_scheduler/reconcile.rs | 52 +++++++++++-------- .../registry/serving_reads.rs | 15 +++--- .../src/code_index_scheduler/tests/mod.rs | 20 ++++++- 3 files changed, 55 insertions(+), 32 deletions(-) diff --git a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/reconcile.rs b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/reconcile.rs index f6b29a1565..0c339b4b7e 100644 --- a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/reconcile.rs +++ b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/reconcile.rs @@ -1520,32 +1520,34 @@ impl CodeIndexWorktreeSchedulerV1 { }; // Equal metadata is not currency. A quiet Noop is only honest when // this pass decoded the generation and re-derived every sealed digest. - // Otherwise return None so the caller runs the content proof now; - // a later wake is not a substitute, because this empty-slot seat is - // the first branch and would keep swallowing the pass. + // A dirty remount (hints, config drift, or a moved frontier) must still + // seat from the durable pointer without joining the publication decode + // barrier — activation may already own that flight. + let frontier_sweep = self.retained_frontier_stat_sweep(&pointer); let content_matches = decoded.as_ref().is_some_and(|generation| { - self.retained_frontier_stat_sweep(&pointer) - .is_some_and(|sweep| { - sweep.content_matches( - &self.project_root, - &SourceContentManifestV1::for_snapshot(generation.snapshot()), - &self.shutting_down, - ) - }) + frontier_sweep.as_ref().is_some_and(|sweep| { + sweep.content_matches( + &self.project_root, + &SourceContentManifestV1::for_snapshot(generation.snapshot()), + &self.shutting_down, + ) + }) }); - if !retained_empty_seat_settles_source(decoded.is_some(), content_matches) { - return Ok(None); - } - let Some(generation) = decoded else { - return Ok(None); - }; let dirty = { let hints = self .hints .lock() .unwrap_or_else(std::sync::PoisonError::into_inner); hints.overflow || !hints.paths.is_empty() - } || configuration_changed; + } || configuration_changed + || frontier_sweep.is_none() + || (decoded.is_some() && !content_matches); + if !dirty && !retained_empty_seat_settles_source(decoded.is_some(), content_matches) { + // Quiet + undecoded (or unproven) must not settle: a later wake is + // not a substitute, because this empty-slot seat is the first + // branch and would keep swallowing the content-proof pass. + return Ok(None); + } if dirty { self.request_background_reconcile(); } @@ -1553,8 +1555,13 @@ impl CodeIndexWorktreeSchedulerV1 { // `load_active_shared` here parked remount on the publication // barrier while activation owned it, so the seated event never // published and the dirty successor extract never started. - self.adopt_ignored_source_roster(&generation); - let snapshot_content_identity = generation.snapshot().content_identity.clone(); + let snapshot_content_identity = if let Some(generation) = decoded { + self.adopt_ignored_source_roster(&generation); + generation.snapshot().content_identity.clone() + } else { + ContentDigest::new(pointer.snapshot_content_identity.clone()) + .map_err(|error| CodeIndexSchedulerErrorV1::Identity(error.to_string()))? + }; self.latest_content_identity = Some(snapshot_content_identity.clone()); Ok(Some(CodeIndexReconcileOutcomeV1::Noop( CodeIndexNoopEvidenceV1 { @@ -3787,8 +3794,9 @@ fn changed_paths_between_trees( /// A quiet empty-slot seat may end the pass only when the generation was /// decoded and its sealed file digests still match the bytes on disk. /// -/// Equal stat metadata with no decoded generation is not currency. Callers -/// that treat `false` as a settled Noop will skip the content proof. +/// Equal stat metadata with no decoded generation is not currency. Dirty +/// remount seating bypasses this gate and may emit a Noop from the durable +/// pointer without joining the publication decode barrier. pub(crate) fn retained_empty_seat_settles_source( generation_decoded: bool, content_matches: bool, diff --git a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/registry/serving_reads.rs b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/registry/serving_reads.rs index 1d41d5560e..a98a5f7613 100644 --- a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/registry/serving_reads.rs +++ b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/registry/serving_reads.rs @@ -277,7 +277,6 @@ impl CodeIndexSchedulerRegistryV1 { generation_recovery, build_progress, hints, - pending_wake, source_freshness, graph_activation_enabled, ) = { @@ -295,7 +294,6 @@ impl CodeIndexSchedulerRegistryV1 { Arc::clone(&worktree.generation_recovery), Arc::clone(&worktree.build_progress), Arc::clone(&worktree.hints), - Arc::clone(&worktree.pending_wake), worktree.source_freshness.clone(), worktree.graph_activation.policy().is_enabled(), ) @@ -318,13 +316,12 @@ impl CodeIndexSchedulerRegistryV1 { } progress }); - let refresh_in_flight = reconcile_in_progress.load(Ordering::Acquire) != 0 - || pending_wake - .state - .lock() - .unwrap_or_else(std::sync::PoisonError::into_inner) - .micros - != 0; + // `Verifying` / `Refreshing` name an executing source proof or + // rebuild pass. A bare pending wake is only a scheduled follow-up; + // counting it here flipped Fresh→Verifying between consecutive + // status reads after a settled seat (registry publication feeds). + // The pass guard (`reconcile_in_progress`) is the durable signal. + let refresh_in_flight = reconcile_in_progress.load(Ordering::Acquire) != 0; let source_change_pending = source_freshness.source_change_pending(); let parked = convergence_park .read() diff --git a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/tests/mod.rs b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/tests/mod.rs index 845dfa9f10..a8cf5e9564 100644 --- a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/tests/mod.rs +++ b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/tests/mod.rs @@ -1509,7 +1509,25 @@ async fn wait_for_dashboard_ready(registry: &CodeIndexSchedulerRegistryV1, path: && freshness.coverage == tracedecay_contracts::code_index_freshness::CodeIndexFreshnessCoverageV1::Complete }); if ready { - break; + // Fresh is projected whenever refresh_in_flight is briefly + // false between owner passes. Join the seating pass and + // re-sample so ready is not a trough before Verifying. + wait_for_quiescent_owner_pass(registry, path).await; + let still_ready = registry + .dashboard_freshness(path) + .await + .is_some_and(|freshness| { + freshness.staleness_state + == Some( + tracedecay_contracts::code_index_freshness::CodeIndexStalenessStateV1::Fresh, + ) + && freshness.coverage + == tracedecay_contracts::code_index_freshness::CodeIndexFreshnessCoverageV1::Complete + }); + if still_ready && !registry.reconcile_in_progress_for_test(path).await { + break; + } + continue; } tokio::time::sleep(Duration::from_millis(2)).await; } From 7bdc33d3cb1646cc3fc96809485a80398ef5db38 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Thu, 17 Sep 2026 09:37:40 +0000 Subject: [PATCH 02/14] fix(transport): clear LCM, proxy, codex, partial, rebuild reds Treat RefreshWorkerMissing as absence on core mounts, keep the host EOF watch sender alive while draining in-flight proxy responses, surface backtick Codex remediation when the host CLI is missing, bind the fact-commit barrier to expect_content so foreign writes cannot steal PartialEffect induction, and keep graph-rebuild search pages small enough to stay untruncated. Co-authored-by: Zack Jackson --- .../src/agents/codex.rs | 17 +++++++ crates/tracedecay-cli/src/agent_cmd.rs | 19 ++++++-- .../src/fact_store/commit_barrier.rs | 30 ++++++++++--- .../src/fact_store/crud/queries.rs | 10 ++++- .../src/fact_store/envelope.rs | 34 +++++++++++++- .../src/fact_store/mod.rs | 4 +- .../src/session_retrieval/lcm.rs | 32 ++++++++++--- crates/tracedecay/src/daemon/core_proxy.rs | 13 ++++-- .../graph_rebuild_status_test.rs | 5 ++- .../typed_terminal_restart_acceptance.rs | 45 ++++++++++++++++++- .../transport_boundaries.rs | 14 +++--- 11 files changed, 193 insertions(+), 30 deletions(-) diff --git a/crates/tracedecay-agent-hosts/src/agents/codex.rs b/crates/tracedecay-agent-hosts/src/agents/codex.rs index 5a3740c85d..6d29c4388e 100644 --- a/crates/tracedecay-agent-hosts/src/agents/codex.rs +++ b/crates/tracedecay-agent-hosts/src/agents/codex.rs @@ -89,6 +89,23 @@ impl AgentIntegration for CodexIntegration { ctx: &InstallContext, ) -> Result { install_codex_plugin(&ctx.home, &ctx.tracedecay_bin)?; + // Core apply drives `codex plugin add` when the host CLI is present. + // When it is not, stop with the same backtick remediation preflight + // uses so operators (and lifecycle tests) can activate natively. + if plugin_registry::require_codex_plugin_cli().is_err() { + let marketplace_name = codex_exact_personal_marketplace_name(&ctx.home) + .ok() + .flatten() + .unwrap_or_else(|| codex_cached_marketplace_name(&ctx.home)); + return Ok(NonInteractiveInstallOutcome::DeferredUserAction( + DeferredUserAction { + remediation: format!( + "Codex activates plugins through its native cache. Run `codex plugin add tracedecay@{marketplace_name}` after TraceDecay stages the source package." + ), + staged_paths: Vec::new(), + }, + )); + } Ok(NonInteractiveInstallOutcome::Ready) } diff --git a/crates/tracedecay-cli/src/agent_cmd.rs b/crates/tracedecay-cli/src/agent_cmd.rs index e8107c66f1..082805f642 100644 --- a/crates/tracedecay-cli/src/agent_cmd.rs +++ b/crates/tracedecay-cli/src/agent_cmd.rs @@ -1175,9 +1175,22 @@ fn host_bundle_error_for_agent( == tracedecay_agent_hosts::agents::host_bundle::HostBundleError::UnsupportedCapability { return tracedecay_domain::errors::TraceDecayError::Config { - message: "Codex plugin activation could not be completed through `codex plugin add`. \ - Confirm the `codex` CLI is on PATH and retry; hook trust still requires \ - `/hooks` inside Codex after a successful add." + message: "Codex activates plugins through its native cache. Run `codex plugin add \ + tracedecay@personal` after TraceDecay stages the source package. Confirm \ + the `codex` CLI is on PATH and retry; hook trust still requires `/hooks` \ + inside Codex after a successful add." + .to_string(), + }; + } + if matches!( + &error, + tracedecay_agent_hosts::agents::host_bundle::HostBundleError::HostCliUnavailable { .. } + ) && agent_id == "codex" + { + return tracedecay_domain::errors::TraceDecayError::Config { + message: "Codex activates plugins through its native cache. Run `codex plugin add \ + tracedecay@personal` after TraceDecay stages the source package. Install \ + the `codex` CLI or add it to PATH, then retry." .to_string(), }; } diff --git a/crates/tracedecay-session-memory/src/fact_store/commit_barrier.rs b/crates/tracedecay-session-memory/src/fact_store/commit_barrier.rs index 9a660a7c7a..b15cde24ad 100644 --- a/crates/tracedecay-session-memory/src/fact_store/commit_barrier.rs +++ b/crates/tracedecay-session-memory/src/fact_store/commit_barrier.rs @@ -34,12 +34,15 @@ const RELEASE_POLL: Duration = Duration::from_millis(10); /// Parks after a durable fact commit when the harness armed the barrier. /// /// A no-op unless [`BARRIER_DIR_ENV`] names a directory holding an `armed` -/// file. Renaming `armed` to `claimed` is the claim: a concurrent fact commit -/// cannot also consume this one-shot barrier, so exactly the mutation the test -/// is watching is the one that parks. Arrival is published as `arrived`; the -/// park ends when `release` appears, or when the bounded wait expires so a -/// failed test cannot strand a live daemon. -pub(super) async fn wait_after_durable_fact_commit() { +/// file. When that directory also holds `expect_content`, only a commit whose +/// `committed_content` matches that file byte-for-byte (trimmed) may claim the +/// barrier — so a foreign project-open or curator write cannot steal the park +/// from the mutation the journey is watching. Renaming `armed` to `claimed` is +/// the claim: a concurrent matching fact commit cannot also consume this +/// one-shot barrier. Arrival is published as `arrived`; the park ends when +/// `release` appears, or when the bounded wait expires so a failed test cannot +/// strand a live daemon. +pub(super) async fn wait_after_durable_fact_commit(committed_content: Option<&str>) { let Some(root) = std::env::var_os(BARRIER_DIR_ENV) else { return; }; @@ -48,6 +51,21 @@ pub(super) async fn wait_after_durable_fact_commit() { if !matches!(armed.try_exists(), Ok(true)) { return; } + let expect = root.join("expect_content"); + if matches!(expect.try_exists(), Ok(true)) { + let expected = match std::fs::read_to_string(&expect) { + Ok(value) => value, + Err(_) => return, + }; + let expected = expected.trim(); + if expected.is_empty() { + return; + } + match committed_content { + Some(actual) if actual.trim() == expected => {} + _ => return, + } + } if std::fs::rename(&armed, root.join("claimed")).is_err() { return; } diff --git a/crates/tracedecay-session-memory/src/fact_store/crud/queries.rs b/crates/tracedecay-session-memory/src/fact_store/crud/queries.rs index 4ddd07de40..73afbc2b7e 100644 --- a/crates/tracedecay-session-memory/src/fact_store/crud/queries.rs +++ b/crates/tracedecay-session-memory/src/fact_store/crud/queries.rs @@ -1025,7 +1025,15 @@ impl DatabaseFactStore<'_> { // settled yet. Acceptance harnesses park exactly here to make a // budget that expires after the commit point reproducible. #[cfg(feature = "test-transport")] - crate::fact_store::commit_barrier::wait_after_durable_fact_commit().await; + { + let barrier_content = batch + .assertion() + .map(|assertion| assertion.payload().content().to_owned()); + crate::fact_store::commit_barrier::wait_after_durable_fact_commit( + barrier_content.as_deref(), + ) + .await; + } } else { transaction .rollback() diff --git a/crates/tracedecay-session-memory/src/fact_store/envelope.rs b/crates/tracedecay-session-memory/src/fact_store/envelope.rs index 08377b9dd7..ea7433d855 100644 --- a/crates/tracedecay-session-memory/src/fact_store/envelope.rs +++ b/crates/tracedecay-session-memory/src/fact_store/envelope.rs @@ -212,6 +212,30 @@ impl DatabaseFactStore<'_> { -> Pin> + Send + 'tx>> + Send + 'static, + ) -> FactStoreResult { + self.project_memory_write_with_barrier_content( + write_control, + None, + graph_source_changed, + work, + ) + .await + } + + /// Like [`Self::project_memory_write`], but names the fact content the + /// test-transport commit barrier may match against `expect_content`. + #[hotpath::skip] + pub(super) async fn project_memory_write_with_barrier_content( + &self, + write_control: &FactWriteControl, + barrier_content: Option, + graph_source_changed: impl FnOnce(&T) -> bool + Send + 'static, + work: impl for<'tx> FnOnce( + &'tx Transaction<'_>, + ) + -> Pin> + Send + 'tx>> + + Send + + 'static, ) -> FactStoreResult { let db = (*self.db).clone(); let write_control = write_control.clone(); @@ -219,7 +243,9 @@ impl DatabaseFactStore<'_> { // can still deny the write before its commit-start transition; after // that transition the bounded transaction commit runs to completion. tokio::spawn(async move { - let result = execute_project_memory_write(db.clone(), write_control, work).await; + let result = + execute_project_memory_write(db.clone(), write_control, barrier_content, work) + .await; if result.as_ref().is_ok_and(graph_source_changed) { super::graph::publish_project_memory_graph_after_write(db).await; } @@ -233,6 +259,7 @@ impl DatabaseFactStore<'_> { async fn execute_project_memory_write( db: Database, write_control: FactWriteControl, + barrier_content: Option, work: impl for<'tx> FnOnce( &'tx Transaction<'_>, ) -> Pin> + Send + 'tx>> @@ -274,7 +301,10 @@ async fn execute_project_memory_write( // yet. Acceptance harnesses park exactly here to make a budget that // expires after the commit point reproducible. #[cfg(feature = "test-transport")] - crate::fact_store::commit_barrier::wait_after_durable_fact_commit().await; + crate::fact_store::commit_barrier::wait_after_durable_fact_commit( + barrier_content.as_deref(), + ) + .await; Ok(value) } Err(error) => match transaction.rollback().await { diff --git a/crates/tracedecay-session-memory/src/fact_store/mod.rs b/crates/tracedecay-session-memory/src/fact_store/mod.rs index f724da87cd..2ce760ac8b 100644 --- a/crates/tracedecay-session-memory/src/fact_store/mod.rs +++ b/crates/tracedecay-session-memory/src/fact_store/mod.rs @@ -503,8 +503,10 @@ impl ProjectMemoryFactStore for DatabaseFactStore<'_> { request: ProjectMemoryFactAddCommandV1, write_control: &FactWriteControl, ) -> FactStoreResult { - self.project_memory_write( + let barrier_content = Some(request.content().to_owned()); + self.project_memory_write_with_barrier_content( write_control, + barrier_content, |outcome: &ProjectMemoryFactAddOutcomeV1| { outcome.commit_receipt().is_some() && !outcome.commit_replayed() }, diff --git a/crates/tracedecay-session-runtime/src/session_retrieval/lcm.rs b/crates/tracedecay-session-runtime/src/session_retrieval/lcm.rs index 855d5310e7..53cb997df1 100644 --- a/crates/tracedecay-session-runtime/src/session_retrieval/lcm.rs +++ b/crates/tracedecay-session-runtime/src/session_retrieval/lcm.rs @@ -169,6 +169,27 @@ impl DaemonSessionRetrievalService { /// projection is pending — so the worker's own serving state, with its /// backlog and blocker, is the answer instead. Once the worker is current /// the temporal outcome stands: nothing is going to project the session. + /// + /// A missing refresh worker is not convergence: core/direct mounts never + /// attach one, and zero rows there are evidence of absence, not a pending + /// catch-up. Only historical-convergence staleness remaps absence. + fn historically_converging_unavailable(&self) -> Option { + let unavailable = self.refresh_not_current()?; + match unavailable.reason { + SessionRetrievalUnavailableReason::HistoricalConvergence + | SessionRetrievalUnavailableReason::HistoricalRetry + | SessionRetrievalUnavailableReason::HistoricalBlocked => Some(unavailable), + SessionRetrievalUnavailableReason::ServiceNotConfigured + | SessionRetrievalUnavailableReason::RefreshWorkerMissing + | SessionRetrievalUnavailableReason::RefreshWorkerRecovering + | SessionRetrievalUnavailableReason::RefreshWorkerStalled + | SessionRetrievalUnavailableReason::RefreshWorkerStopped + | SessionRetrievalUnavailableReason::TemporalStoreUnavailable + | SessionRetrievalUnavailableReason::TemporalStoreReadFailed + | SessionRetrievalUnavailableReason::HydrationUnavailable => None, + } + } + fn converging_projection_unavailable( &self, unavailable: &SessionRetrievalUnavailable, @@ -176,7 +197,7 @@ impl DaemonSessionRetrievalService { if unavailable.reason != SessionRetrievalUnavailableReason::TemporalStoreUnavailable { return None; } - self.refresh_not_current() + self.historically_converging_unavailable() } #[hotpath::measure(label = "daemon.session_retrieval.lcm_describe", future = true)] @@ -302,11 +323,12 @@ impl DaemonSessionRetrievalService { (Some(result), retrieval) } // Zero temporal rows for the session is only evidence of absence - // once the refresh worker is current; while it is still - // converging history the honest answer is that state, not a - // complete description at generation zero. + // once history is not still converging. A missing worker (core / + // direct mounts) is not convergence — treat CompleteZero as + // absence there. While historical catch-up is in flight, surface + // that state instead of a complete description at generation zero. SessionRetrievalOutcome::CompleteZero { .. } - if direct.is_none() && self.refresh_not_current().is_some() => + if direct.is_none() && self.historically_converging_unavailable().is_some() => { return describe_retrieval_outcome( outcome, diff --git a/crates/tracedecay/src/daemon/core_proxy.rs b/crates/tracedecay/src/daemon/core_proxy.rs index fa9502d1c2..cda1573ed3 100644 --- a/crates/tracedecay/src/daemon/core_proxy.rs +++ b/crates/tracedecay/src/daemon/core_proxy.rs @@ -160,8 +160,13 @@ pub(crate) async fn proxy_transport_to_daemon_with_drain_bound( let (mut reader, mut writer) = transport.split(); let (input_tx, mut input_rx) = tokio::sync::mpsc::unbounded_channel(); let (eof_tx, mut eof_rx) = tokio::sync::watch::channel(false); + // Keep a Sender alive for the whole proxy lifetime. `read_host` only + // marks EOF; if it owned the sole Sender, dropping it on host close would + // make later `eof.changed()` calls fail as "monitor closed" and abort + // before the in-flight daemon response could be drained to the host. + let eof_signal = eof_tx.clone(); - let read_host = async { + let read_host = async move { loop { match reader.read_line().await { Ok(Some(line)) => { @@ -170,7 +175,7 @@ pub(crate) async fn proxy_transport_to_daemon_with_drain_bound( } } Ok(None) => { - let _ = eof_tx.send(true); + let _ = eof_signal.send(true); return Ok(()); } Err(error) => return Err(error.into()), @@ -186,7 +191,9 @@ pub(crate) async fn proxy_transport_to_daemon_with_drain_bound( &mut writer, drain_bound, ); - tokio::try_join!(read_host, proxy)?; + let result = tokio::try_join!(read_host, proxy); + drop(eof_tx); + result?; Ok(()) } diff --git a/crates/tracedecay/tests/transport_acceptance_suite/graph_rebuild_status_test.rs b/crates/tracedecay/tests/transport_acceptance_suite/graph_rebuild_status_test.rs index 51d23de6ed..5b02449a74 100644 --- a/crates/tracedecay/tests/transport_acceptance_suite/graph_rebuild_status_test.rs +++ b/crates/tracedecay/tests/transport_acceptance_suite/graph_rebuild_status_test.rs @@ -116,11 +116,14 @@ async fn search( project: &Path, query: &str, ) -> Value { + // Keep the page tiny: a generation-scale refresh batch otherwise returns + // multi-dozen-KiB candidate bodies that MCP truncates into a handle, and + // the wait helpers never see top-level `results` / `code_generation`. tool( harness, project, "tracedecay_search", - json!({"query": query, "limit": 20, "format": "json"}), + json!({"query": query, "limit": 3, "format": "json"}), ) .await } diff --git a/crates/tracedecay/tests/transport_acceptance_suite/typed_terminal_restart_acceptance.rs b/crates/tracedecay/tests/transport_acceptance_suite/typed_terminal_restart_acceptance.rs index 76c5468628..8609233c82 100644 --- a/crates/tracedecay/tests/transport_acceptance_suite/typed_terminal_restart_acceptance.rs +++ b/crates/tracedecay/tests/transport_acceptance_suite/typed_terminal_restart_acceptance.rs @@ -175,13 +175,23 @@ fn add_fact_settling_after_its_deadline( barrier_dir: &Path, content: &str, ) -> Value { + // One-shot markers: clear a previous park so a leftover `arrived` cannot + // make this helper release before the CLI request reaches the boundary. + for marker in ["armed", "claimed", "arrived", "release", "expect_content"] { + let _ = std::fs::remove_file(barrier_dir.join(marker)); + } + // Bind the barrier to this fact's content so a foreign write (project-open + // side effect, curator, graph publish) cannot claim `arrived` while the + // CLI add races past an unarmed commit boundary. + std::fs::write(barrier_dir.join("expect_content"), content.as_bytes()) + .expect("pin the fact commit barrier to the parked content"); std::fs::write(barrier_dir.join("armed"), b"armed\n").expect("arm the fact commit barrier"); let mut command = tool_command( home, project, "tracedecay_fact_store_add", - &json!({ "content": content, "category": "general" }), + &json!({ "content": content, "category": "general", "format": "json" }), ); command .env( @@ -226,8 +236,18 @@ fn add_fact_settling_after_its_deadline( // test spawned it, so `spawn + deadline` can still be earlier than the real // expiry. Arrival is strictly after that clock started, so holding a full // deadline plus a margin beyond arrival always outlives it. + // + // Keep asserting the CLI is still blocked: an early success here means the + // commit path skipped the barrier (or a foreign claim wrote `arrived`). let release_at = arrived_at + PARTIAL_EFFECT_DEADLINE + Duration::from_secs(1); while Instant::now() < release_at { + assert!( + child + .try_wait() + .expect("inspect the parked fact_store add") + .is_none(), + "the fact_store add settled before its request deadline could expire at the commit barrier" + ); std::thread::sleep(Duration::from_millis(20)); } std::fs::write(barrier_dir.join("release"), b"release\n") @@ -236,7 +256,28 @@ fn add_fact_settling_after_its_deadline( let output = child .wait_with_output() .expect("collect the parked fact_store add"); - parse_tool_output("tracedecay_fact_store_add", &output) + let raw = parse_tool_output("tracedecay_fact_store_add", &output); + // Compatibility MCP results nest the typed envelope in content text when + // `format: json` is set; also accept a top-level problem field. + cli_problem_envelope(&raw, "CLI fact_store_add partial effect") +} + +/// Normalizes the CLI `tracedecay tool --json` MCP payload to `{ "problem": ... }`. +fn cli_problem_envelope(payload: &Value, context: &str) -> Value { + let typed = typed_envelope(payload); + for candidate in [ + payload.clone(), + typed.clone(), + typed["value"].clone(), + typed["data"].clone(), + typed["outcome"]["value"].clone(), + payload["value"].clone(), + ] { + if candidate["problem"].is_object() { + return json!({ "problem": candidate["problem"].clone() }); + } + } + panic!("{context}: no typed problem envelope in the payload: {payload}") } fn assert_partial_effect_committed_receipt(payload: &Value, context: &str) { diff --git a/crates/tracedecay/tests/transport_acceptance_suite/typed_terminal_restart_acceptance/transport_boundaries.rs b/crates/tracedecay/tests/transport_acceptance_suite/typed_terminal_restart_acceptance/transport_boundaries.rs index 68c8ec6474..8ec4ecbda1 100644 --- a/crates/tracedecay/tests/transport_acceptance_suite/typed_terminal_restart_acceptance/transport_boundaries.rs +++ b/crates/tracedecay/tests/transport_acceptance_suite/typed_terminal_restart_acceptance/transport_boundaries.rs @@ -333,16 +333,18 @@ fn problem_envelope(payload: &Value, context: &str) -> Value { /// starts its deadline clock itself: arrival is strictly after that clock /// started, so holding a full budget plus a margin beyond arrival always /// outlives it. -fn park_at_commit_barrier(barrier_dir: &Path, request: F) -> T +fn park_at_commit_barrier(barrier_dir: &Path, content: &str, request: F) -> T where F: FnOnce() -> T + Send + 'static, T: Send + 'static, { // The barrier is one-shot per directory: clear a previous park's markers // so the same daemon process can serve more than one parked request. - for marker in ["armed", "claimed", "arrived", "release"] { + for marker in ["armed", "claimed", "arrived", "release", "expect_content"] { let _ = std::fs::remove_file(barrier_dir.join(marker)); } + std::fs::write(barrier_dir.join("expect_content"), content.as_bytes()) + .expect("pin the fact commit barrier to the parked content"); std::fs::write(barrier_dir.join("armed"), b"armed\n").expect("arm the fact commit barrier"); let started = Instant::now(); @@ -425,7 +427,7 @@ fn partial_effect_survives_http_mcp_and_rust_sdk_across_restart() { token: mount.token.clone(), }; let http_identity = identity.clone(); - let (http_status, http_body) = park_at_commit_barrier(&barrier_path, move || { + let (http_status, http_body) = park_at_commit_barrier(&barrier_path, HTTP_MARKER, move || { post_application( &http_mount_for_request, &http_identity, @@ -447,7 +449,7 @@ fn partial_effect_survives_http_mcp_and_rust_sdk_across_restart() { // the `_meta` object on `tools/call`. let mcp_home = home_path.clone(); let mcp_project = project_path.clone(); - let mcp_response = park_at_commit_barrier(&barrier_path, move || { + let mcp_response = park_at_commit_barrier(&barrier_path, MCP_MARKER, move || { mcp_tool_call( &mcp_home, &mcp_project, @@ -469,7 +471,7 @@ fn partial_effect_survives_http_mcp_and_rust_sdk_across_restart() { token: mount.token.clone(), }; let sdk_identity = identity.clone(); - let sdk_error = park_at_commit_barrier(&barrier_path, move || { + let sdk_error = park_at_commit_barrier(&barrier_path, SDK_MARKER, move || { let client = sdk_client(&sdk_mount, &sdk_identity); let request = serde_json::from_value(fact_add_body(SDK_MARKER)).expect("canonical fact-add request"); @@ -546,7 +548,7 @@ fn partial_effect_survives_http_mcp_and_rust_sdk_across_restart() { token: mount.token.clone(), }; let restart_identity = identity.clone(); - let (_, restart_body) = park_at_commit_barrier(&barrier_path, move || { + let (_, restart_body) = park_at_commit_barrier(&barrier_path, POST_RESTART_MARKER, move || { post_application( &restart_mount, &restart_identity, From f40f81f9ea2bf7936712addc1829295dcc25996f Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Thu, 17 Sep 2026 09:39:34 +0000 Subject: [PATCH 03/14] fix(daemon): decode partitioned generation in fan-out wait The restored-generation wait used monolithic decode_sealed against the daemon's partitioned manifests, so readiness never observed an on-disk generation and timed out after 180s. Co-authored-by: Zack Jackson --- .../advanced_workflow_journey/task_session.rs | 89 ++++++++++++++++--- 1 file changed, 79 insertions(+), 10 deletions(-) diff --git a/crates/tracedecay/tests/daemon_suite/advanced_workflow_journey/task_session.rs b/crates/tracedecay/tests/daemon_suite/advanced_workflow_journey/task_session.rs index 1b4f6e0a42..2eeb7c2387 100644 --- a/crates/tracedecay/tests/daemon_suite/advanced_workflow_journey/task_session.rs +++ b/crates/tracedecay/tests/daemon_suite/advanced_workflow_journey/task_session.rs @@ -7,6 +7,9 @@ use std::process::{Child, Stdio}; use std::time::{Duration, Instant}; use serde_json::{Value, json}; +use tracedecay_code_index::production::{ + CodeIndexPublishedGenerationV1, SealedGenerationSegmentReadV1, +}; use tracedecay_code_index_retention::code_index_generations::{ DurablePublicationPointerV1, scoped_code_index_store_root, }; @@ -18,6 +21,7 @@ use tracedecay_contracts::{ }; use tracedecay_domain::{ ProjectId, RetrieverKind, TaskId, TemporalModeV1, UtcMicros, WorkAttemptIdentityV1, + sha256_hex_suffix, }; use tracedecay_sdk::client::Client; use tracedecay_sdk::operations::WorkRetrieveEvidence; @@ -313,12 +317,49 @@ fn wait_for_code_generation(home: &Path, project: &Path) { while read_active_code_generation(home, project).is_none() { assert!( Instant::now() < deadline, - "timed out waiting for the restored code generation" + "timed out waiting for the restored code generation: {}", + code_generation_wait_diagnostics(home, project) ); std::thread::sleep(Duration::from_millis(250)); } } +fn code_generation_wait_diagnostics(home: &Path, project: &Path) -> String { + let profile = home.join(".tracedecay"); + let Ok(layout) = tracedecay_runtime_core::storage::resolve_layout(project, &profile) else { + return format!( + "layout unresolved for project {} under {}", + project.display(), + profile.display() + ); + }; + let scope = scoped_code_index_store_root(&layout.data_root.join("code-index-v1"), project); + let pointer_path = scope.join("active-code-generation-v1.json"); + if !pointer_path.is_file() { + return format!("missing active pointer at {}", pointer_path.display()); + } + match std::fs::read(&pointer_path) + .ok() + .and_then(|bytes| serde_json::from_slice::(&bytes).ok()) + { + Some(pointer) => { + let generation_path = scope + .join("code-generations-v1") + .join(&pointer.generation_file); + format!( + "pointer={} generation_file_present={} segments_root_present={}", + pointer_path.display(), + generation_path.is_file(), + scope.join("code-generation-segments-v1").is_dir() + ) + } + None => format!( + "active pointer present but undecodable at {}", + pointer_path.display() + ), + } +} + /// The core query authority mounts after the first sealed generation is /// seated, on a deferred owner. Poll the typed SDK until TaskSession evidence /// hydrates rather than asserting on the mount's timing. @@ -348,7 +389,7 @@ fn wait_for_task_session_available(client: &Client, scope: &TaskSessionEvidenceS fn read_active_code_generation( home: &Path, project: &Path, -) -> Option { +) -> Option { let layout = tracedecay_runtime_core::storage::resolve_layout(project, &home.join(".tracedecay")) .ok()?; @@ -357,15 +398,43 @@ fn read_active_code_generation( &std::fs::read(scope.join("active-code-generation-v1.json")).ok()?, ) .ok()?; - tracedecay_code_index::production::CodeIndexPublishedGenerationV1::decode_sealed( - &std::fs::read( - scope - .join("code-generations-v1") - .join(pointer.generation_file), - ) - .ok()?, + let sealed = std::fs::read( + scope + .join("code-generations-v1") + .join(pointer.generation_file), ) - .ok() + .ok()?; + // The daemon publishes partitioned manifests; monolithic `decode_sealed` + // rejects them, which made this wait time out with the generation on disk. + let segments_root = scope.join("code-generation-segments-v1"); + CodeIndexPublishedGenerationV1::decode_partitioned_sealed(&sealed, |request, buffer| { + let (digest, size_bytes, offset, length) = match request { + SealedGenerationSegmentReadV1::Whole { digest, size_bytes } => { + (digest, size_bytes, 0, size_bytes) + } + SealedGenerationSegmentReadV1::Range { + digest, + size_bytes, + offset, + length, + } => (digest, size_bytes, offset, length), + }; + let digest_hex = + sha256_hex_suffix(digest.as_str()).expect("sealed segment digest is sha256"); + let segment = std::fs::read(segments_root.join(format!("segment-{digest_hex}.json"))) + .expect("sealed generation segment"); + assert_eq!( + segment.len() as u64, + size_bytes, + "segment size matches manifest" + ); + let start = usize::try_from(offset).expect("segment offset"); + let end = start + usize::try_from(length).expect("segment length"); + buffer.clear(); + buffer.extend_from_slice(&segment[start..end]); + Ok(()) + }) + .ok()? } /// The exact Work evidence scope one TaskSession availability sweep reads: From 3fad093d52cf005073dd189034368935cac7eaab Mon Sep 17 00:00:00 2001 From: ScriptedAlchemy Date: Thu, 17 Sep 2026 10:44:21 -0700 Subject: [PATCH 04/14] style: format merged master updates --- crates/tracedecay-code-index/src/parallelism.rs | 4 ++-- .../src/session_retrieval/tests.rs | 5 +++-- crates/tracedecay/src/mcp/server/routing.rs | 14 ++++++-------- 3 files changed, 11 insertions(+), 12 deletions(-) diff --git a/crates/tracedecay-code-index/src/parallelism.rs b/crates/tracedecay-code-index/src/parallelism.rs index 5bd3604e27..8f8c51698e 100644 --- a/crates/tracedecay-code-index/src/parallelism.rs +++ b/crates/tracedecay-code-index/src/parallelism.rs @@ -685,8 +685,8 @@ mod tests { #[test] fn install_runs_the_caller_on_an_admitted_rayon_worker() { - let on_worker = install(|| rayon::current_thread_index().is_some()) - .expect("code-index worker pool"); + let on_worker = + install(|| rayon::current_thread_index().is_some()).expect("code-index worker pool"); assert!( on_worker, "direct-seal encode only parallelizes when the caller is already a rayon worker" diff --git a/crates/tracedecay-session-runtime/src/session_retrieval/tests.rs b/crates/tracedecay-session-runtime/src/session_retrieval/tests.rs index e790cda7e6..421551c090 100644 --- a/crates/tracedecay-session-runtime/src/session_retrieval/tests.rs +++ b/crates/tracedecay-session-runtime/src/session_retrieval/tests.rs @@ -1188,8 +1188,9 @@ async fn describe_without_a_refresh_worker_does_not_pretend_history_is_convergin DaemonSessionRetrievalService::new_without_refresh_worker(harness.registered.clone(), root) .expect("registered retrieval service"); let context = admitted_lookup_context(scope); - let cancellation = tracedecay_contracts::CancellationSignal::active("cancellation.session-lookup") - .expect("cancellation"); + let cancellation = + tracedecay_contracts::CancellationSignal::active("cancellation.session-lookup") + .expect("cancellation"); let command = LcmDescribeServiceCommand::new( "codex", SessionId::new("session.describe.worker-missing").expect("session identity"), diff --git a/crates/tracedecay/src/mcp/server/routing.rs b/crates/tracedecay/src/mcp/server/routing.rs index fc2a40b200..7a734ce0bb 100644 --- a/crates/tracedecay/src/mcp/server/routing.rs +++ b/crates/tracedecay/src/mcp/server/routing.rs @@ -217,13 +217,9 @@ async fn resolve_initialize_roots_project_route( } let discovery = repository_discovery(); for root in roots { - let route = resolve_private_project_route_within( - &root, - registry_db, - resolver.clone(), - &discovery, - ) - .await; + let route = + resolve_private_project_route_within(&root, registry_db, resolver.clone(), &discovery) + .await; if !matches!( &route, WorkspaceProjectRoute::Failed(ProjectRouteFailure { @@ -376,7 +372,9 @@ async fn resolve_initialize_roots_project_path( } let registry_db = registry_db?; for root in roots { - match resolve_initialize_root_project_path(&root, registry_db, &repository_discovery()).await { + match resolve_initialize_root_project_path(&root, registry_db, &repository_discovery()) + .await + { Ok(Some(project_path)) => return Some(project_path), Ok(None) => {} Err(_) => return None, From e221b911d32e725f94915d2f4f5f958decd84012 Mon Sep 17 00:00:00 2001 From: ScriptedAlchemy Date: Thu, 17 Sep 2026 10:56:40 -0700 Subject: [PATCH 05/14] fix(query): cancel candidate work at bounded batch checkpoints --- .../src/code_index_executor.rs | 15 ++- .../src/code_index_scheduler/queries.rs | 2 + .../src/code_index_scheduler/query_runtime.rs | 1 + .../src/code_index_scheduler/serving.rs | 17 ++- .../serving/family_report.rs | 44 ++++++-- .../tests/search_permit_release.rs | 47 +++++--- .../src/code_index_scheduler/tests/serving.rs | 1 + .../src/bin/tracedecay_search_bench.rs | 1 + .../tracedecay-query/src/retrieval/exact.rs | 18 ++- .../src/retrieval/exact/tests.rs | 15 ++- .../tracedecay-query/src/retrieval/lexical.rs | 33 +++--- .../lexical/projection/artifact/reader.rs | 103 ++++++++++++++---- .../artifact/reader/family_report.rs | 9 +- .../retrieval/lexical/projection/in_memory.rs | 17 ++- .../tracedecay-query/src/retrieval/ports.rs | 14 +++ .../tests/retrieval_contract_spine.rs | 14 ++- .../candidate_producers.rs | 84 +++++++++++++- .../tests/search_quality_suite/single_root.rs | 1 + .../src/candidate_output.rs | 1 + 19 files changed, 349 insertions(+), 88 deletions(-) diff --git a/crates/tracedecay-code-index-runtime/src/code_index_executor.rs b/crates/tracedecay-code-index-runtime/src/code_index_executor.rs index 46427f5be0..3dd88a35e2 100644 --- a/crates/tracedecay-code-index-runtime/src/code_index_executor.rs +++ b/crates/tracedecay-code-index-runtime/src/code_index_executor.rs @@ -1515,16 +1515,23 @@ where ); } }; + let read_control = Arc::clone(&control); let read = tokio::task::spawn_blocking(move || { let _permit = permit; - owners.similar(&request, control.as_ref()) + owners.similar(&request, read_control.as_ref()) }) .await; + if let Some(reason) = control.request_termination() { + return unavailable(reason); + } match read { Ok(Ok(Some(result))) => { code_search::CodeIndexSimilarOutcomeV1::Complete(Box::new(result)) } Ok(Ok(None)) => code_search::CodeIndexSimilarOutcomeV1::NotFound, + Ok(Err(RetrievalPortError::Cancelled)) => { + unavailable(code_search::CodeIndexSearchUnavailableReasonV1::Cancelled) + } Ok(Err(_)) | Err(_) => { unavailable(code_search::CodeIndexSearchUnavailableReasonV1::Internal) } @@ -1701,11 +1708,15 @@ where ); } }; + let read_control = Arc::clone(&control); let read = tokio::task::spawn_blocking(move || { let _permit = permit; - owners.redundancy(&request, control.as_ref()) + owners.redundancy(&request, read_control.as_ref()) }) .await; + if let Some(reason) = control.request_termination() { + return unavailable(reason); + } match read { Ok(Ok(outcome)) => Ok(outcome), Ok(Err( diff --git a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/queries.rs b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/queries.rs index 25cabb4f5f..2a4a1f4fb4 100644 --- a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/queries.rs +++ b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/queries.rs @@ -2635,12 +2635,14 @@ impl CallableCodeQueryPort for CodeIndexSchedulerRegistryV1 { ) .unwrap_or_else(|_| panic!("static exact rule revision")), ); + let exact_control = CallableRetrievalExecutionControl::for_request(context.request); let lane_request = ExactLaneRequest { literals: authority.parse_literals(&query_view, base), generation: served_generation.clone(), budget: base.budget, base: base.clone(), query_view: &query_view, + control: exact_control.as_ref(), }; let Ok(owners) = latest.production_query_owners_with_budget(&base.budget) else { return unavailable(finished_at); diff --git a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/query_runtime.rs b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/query_runtime.rs index f804c3f2fe..25773ac116 100644 --- a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/query_runtime.rs +++ b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/query_runtime.rs @@ -743,6 +743,7 @@ where generation: generation.clone(), literals: parser.parse_literals(query_view, request), budget: request.budget, + control: graph_control.as_ref(), }) })?; let route_plan = LexicalRoutePlanV1::plan(query_view.as_str(), &input.lexical_routing)?; diff --git a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/serving.rs b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/serving.rs index 4b7ea6f784..07d9a81730 100644 --- a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/serving.rs +++ b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/serving.rs @@ -84,7 +84,7 @@ use crate::{ LexicalLaneEvidence, LexicalLaneRequest, LexicalLaneRetriever, code_lexical_artifact_build_memory_budget_for, }, - ports::RetrievalPortError, + ports::{RETRIEVAL_CANDIDATE_BATCH_SIZE, RetrievalPortError}, }, }; @@ -92,7 +92,7 @@ use super::{DaemonCodeIndexPublicationStoreV1, ProfiledStdMutex, queries}; /// Page bounds for streaming one sealed generation into the durable lexical /// text artifact. One page is one bounded unit of background build progress. -pub(super) const TEXT_ARTIFACT_PAGE_CHUNKS_V1: usize = 128; +pub(super) const TEXT_ARTIFACT_PAGE_CHUNKS_V1: usize = RETRIEVAL_CANDIDATE_BATCH_SIZE; const TEXT_ARTIFACT_PAGE_BYTES_V1: usize = 4 * 1024 * 1024; const CLONE_SUCCESSOR_MEMORY_BUDGET_BYTES_V1: usize = 128 * 1024 * 1024; const TEXT_ARTIFACT_BASE_BATCH_PAGES_V1: usize = 64; @@ -659,6 +659,7 @@ impl ProductionCodeIndexQueryOwnersV1 { Option, RetrievalPortError, > { + checkpoint_text_artifact_control(control)?; let source = match &request.target { tracedecay_query::code_search::CodeIndexSimilarTargetV1::SymbolOccurrence( occurrence, @@ -667,7 +668,7 @@ impl ProductionCodeIndexQueryOwnersV1 { self.hydration.clone_body_by_source_range(path, *span) } } - .map_err(|error| RetrievalPortError::AuthorityUnavailable(error.to_string()))?; + .map_err(family_report::clone_artifact_error)?; let Some(source) = source else { return Ok(None); }; @@ -709,6 +710,7 @@ impl ProductionCodeIndexQueryOwnersV1 { ); } } + checkpoint_text_artifact_control(control)?; Ok(Some( tracedecay_query::code_search::CodeIndexSimilarCompletedV1 { source, @@ -737,11 +739,15 @@ impl ProductionCodeIndexQueryOwnersV1 { let mut cursor = start_cursor.cloned(); let mut complete = false; while members.len() < limit { + checkpoint_text_artifact_control(control)?; let page = self .hydration .clone_exact_page(&source.occurrence, key, cursor.as_ref(), limit, control) - .map_err(|error| RetrievalPortError::AuthorityUnavailable(error.to_string()))?; - for member in page.members { + .map_err(family_report::clone_artifact_error)?; + for (ordinal, member) in page.members.into_iter().enumerate() { + if ordinal.is_multiple_of(RETRIEVAL_CANDIDATE_BATCH_SIZE) { + checkpoint_text_artifact_control(control)?; + } if member.occurrence.symbol_occurrence_id == source.occurrence.symbol_occurrence_id { continue; @@ -758,6 +764,7 @@ impl ProductionCodeIndexQueryOwnersV1 { break; } } + checkpoint_text_artifact_control(control)?; match page.next_cursor { Some(next) if members.len() < limit => cursor = Some(next), Some(next) => return Ok((members, false, Some(next))), diff --git a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/serving/family_report.rs b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/serving/family_report.rs index 2717fa9769..ebf6112a70 100644 --- a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/serving/family_report.rs +++ b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/serving/family_report.rs @@ -3,7 +3,7 @@ use std::path::{Component, Path}; use tracedecay_code_index::clones::{ CloneExactKeyV1, CloneNormalizationClassV1, CodeIndexCloneBodyV1, }; -use tracedecay_code_index::production::CodeIndexExecutionControlV1; +use tracedecay_code_index::production::{CodeIndexExecutionControlV1, CodeIndexInterruptionV1}; use tracedecay_contracts::retrieval::{ RedundancyCoverageV1, RedundancyFamilyV1, RedundancyPartialReasonV1, RedundancyRankingV1, RedundancyResultV1, SimilarFamilyV1, SimilarMatchClassV1, SimilarOccurrenceV1, @@ -14,8 +14,8 @@ use tracedecay_query::retrieval::lexical::{ CloneArtifactCursorV1, CloneExactArtifactMemberV1, CodeLexicalArtifactErrorV1, }; -use super::ProductionCodeIndexQueryOwnersV1; -use crate::query::retrieval::ports::RetrievalPortError; +use super::{ProductionCodeIndexQueryOwnersV1, checkpoint_text_artifact_control}; +use crate::query::retrieval::ports::{RETRIEVAL_CANDIDATE_BATCH_SIZE, RetrievalPortError}; impl ProductionCodeIndexQueryOwnersV1 { pub(crate) fn redundancy( @@ -23,6 +23,7 @@ impl ProductionCodeIndexQueryOwnersV1 { request: &CodeIndexRedundancyQueryV1, control: &dyn CodeIndexExecutionControlV1, ) -> Result { + checkpoint_text_artifact_control(control)?; let family_page_limit = request.family_limit.min(request.work_limit / 3).max(1); let pull_request_scope_digest = match &request.scope { CodeIndexRedundancyScopeV1::PullRequest { @@ -63,7 +64,7 @@ impl ProductionCodeIndexQueryOwnersV1 { family_page_limit, control, ) - .map_err(redundancy_artifact_error)?; + .map_err(clone_artifact_error)?; let page_continuation = page.next_cursor.clone(); let mut families = Vec::with_capacity(page.families.len()); let mut examined_families = 0usize; @@ -72,6 +73,7 @@ impl ProductionCodeIndexQueryOwnersV1 { let mut work_exhausted = false; let mut report_continuation = request.cursor.clone(); for candidate in page.families { + checkpoint_text_artifact_control(control)?; if work_spent.saturating_add(3) > request.work_limit { work_exhausted = true; break; @@ -81,7 +83,7 @@ impl ProductionCodeIndexQueryOwnersV1 { let source = self .hydration .clone_body(&candidate.representative) - .map_err(|error| RetrievalPortError::AuthorityUnavailable(error.to_string()))? + .map_err(clone_artifact_error)? .ok_or_else(|| { RetrievalPortError::AuthorityUnavailable( "clone family representative is unavailable".to_owned(), @@ -152,6 +154,7 @@ impl ProductionCodeIndexQueryOwnersV1 { break; } } + checkpoint_text_artifact_control(control)?; let source_generation = self.hydration.metadata().generation.clone(); let (coverage, next_cursor) = redundancy_coverage( work_exhausted, @@ -184,6 +187,7 @@ impl ProductionCodeIndexQueryOwnersV1 { let mut cursor = None; let mut work_spent = 0usize; loop { + checkpoint_text_artifact_control(control)?; if members.len() >= result_limit { return Ok(RedundancyMemberReadV1 { members, @@ -214,9 +218,12 @@ impl ProductionCodeIndexQueryOwnersV1 { page_limit, control, ) - .map_err(|error| RetrievalPortError::AuthorityUnavailable(error.to_string()))?; + .map_err(clone_artifact_error)?; work_spent = work_spent.saturating_add(page.members.len()); - for member in page.members { + for (ordinal, member) in page.members.into_iter().enumerate() { + if ordinal.is_multiple_of(RETRIEVAL_CANDIDATE_BATCH_SIZE) { + checkpoint_text_artifact_control(control)?; + } if report_path_matches(&member.occurrence.path, path, include_generated_paths) && tracedecay_code_index::clones::verify_exact_clone_payload( &source.payload, @@ -227,6 +234,7 @@ impl ProductionCodeIndexQueryOwnersV1 { members.push(member); } } + checkpoint_text_artifact_control(control)?; match page.next_cursor { Some(next) => cursor = Some(next), None => { @@ -243,8 +251,14 @@ impl ProductionCodeIndexQueryOwnersV1 { } } -fn redundancy_artifact_error(error: CodeLexicalArtifactErrorV1) -> RetrievalPortError { +pub(super) fn clone_artifact_error(error: CodeLexicalArtifactErrorV1) -> RetrievalPortError { match error { + CodeLexicalArtifactErrorV1::Interrupted(CodeIndexInterruptionV1::Cancelled) => { + RetrievalPortError::Cancelled + } + CodeLexicalArtifactErrorV1::Interrupted(CodeIndexInterruptionV1::DeadlineExceeded) => { + RetrievalPortError::BudgetExceeded + } CodeLexicalArtifactErrorV1::Contract(message) if message == "clone family cursor does not match its artifact or request" => { @@ -333,7 +347,19 @@ fn redundancy_coverage( #[cfg(test)] mod tests { - use super::is_generated_path; + use super::{clone_artifact_error, is_generated_path}; + use tracedecay_code_index::production::CodeIndexInterruptionV1; + use tracedecay_query::retrieval::{RetrievalPortError, lexical::CodeLexicalArtifactErrorV1}; + + #[test] + fn clone_page_interruption_preserves_cancellation() { + assert_eq!( + clone_artifact_error(CodeLexicalArtifactErrorV1::Interrupted( + CodeIndexInterruptionV1::Cancelled, + )), + RetrievalPortError::Cancelled, + ); + } #[test] fn generated_member_labels_follow_the_canonical_path_policy() { diff --git a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/tests/search_permit_release.rs b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/tests/search_permit_release.rs index 8efc139f08..da81470c0f 100644 --- a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/tests/search_permit_release.rs +++ b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/tests/search_permit_release.rs @@ -2,7 +2,7 @@ //! not the blocking worker's natural completion. //! //! `MAX_CONCURRENT_CODE_INDEX_SEARCHES` is one, so a request that its caller -//! has already abandoned must release the permit as soon as its lexical scan +//! has already abandoned must release the permit as soon as its candidate scan //! observes the request control — otherwise every following search fails //! `search_capacity_unavailable` until the abandoned scan hydrates the rest of //! its candidate corpus. @@ -60,9 +60,9 @@ impl CodeIndexMcpReadGrantV1 for FixtureGrant { /// /// The executor's request control consults `route_is_registered` on every /// `is_cancelled` check. Calls from the test's runtime thread (the pre-permit -/// check and the executor's settlement poll) answer immediately; the first -/// call from any other thread is the scan itself, running under -/// `spawn_blocking` with the execution permit already held. That call reports +/// check and the executor's settlement poll) answer immediately; calls from +/// other threads are the scan itself, running under +/// `spawn_blocking` with the execution permit already held. The selected checkpoint reports /// the pause to the test and blocks until the test resumes it, so the test /// can observe the permit-held state and cancel the request at a point that /// is deterministic rather than timing-dependent. @@ -71,6 +71,7 @@ struct PausingAdmission { authority: CodeIndexSearchAuthorityV1, runtime_thread: ThreadId, scan_checkpoints: Arc, + pause_at: usize, scan_paused: Arc, resume: Arc>>, } @@ -82,7 +83,7 @@ impl CodeIndexMcpReadAdmissionV1 for PausingAdmission { if std::thread::current().id() == self.runtime_thread { return true; } - if self.scan_checkpoints.fetch_add(1, Ordering::SeqCst) == 0 { + if self.scan_checkpoints.fetch_add(1, Ordering::SeqCst) == self.pause_at { self.scan_paused.notify_one(); self.resume .lock() @@ -171,19 +172,30 @@ fn unavailable_reason( } } -/// Pause a lexical scan at its first control checkpoint after the permit is +/// Pause a candidate scan at its first control checkpoint after the permit is /// acquired, prove the permit is held (a concurrent search is refused with /// `search_capacity_unavailable`), cancel the paused request, and prove that /// resuming it unwinds with the typed cancellation reason, performs no further /// row checkpoints, and hands the permit to the next request, which is /// admitted and completes normally. #[tokio::test] -async fn cancelled_lexical_scan_releases_the_search_permit_to_the_next_request() { - let sources = (0..16) +async fn cancelled_scan_releases_the_search_permit_to_the_next_request() { + cancelled_scan_releases_permit("alpha", 0, 16).await; +} + +#[tokio::test] +async fn cancelled_exact_batch_releases_the_search_permit_to_the_next_request() { + // Entry into the exact lane and artifact reader, then the first batch. + // The fourth observation is the next batch boundary, after 128 candidates. + cancelled_scan_releases_permit(r#""return value""#, 3, 384).await; +} + +async fn cancelled_scan_releases_permit(query: &str, pause_at: usize, source_count: usize) { + let sources = (0..source_count) .map(|ordinal| { ( - format!("src/alpha_{ordinal:02}.rs"), - format!("pub fn alpha_{ordinal:02}() -> u32 {{ {ordinal} }}\n"), + format!("src/alpha_{ordinal:03}.rs"), + format!("pub fn alpha_{ordinal:03}() -> u32 {{ let value = {ordinal}; return value; }}\n"), ) }) .collect::>(); @@ -206,6 +218,7 @@ async fn cancelled_lexical_scan_releases_the_search_permit_to_the_next_request() }, runtime_thread: std::thread::current().id(), scan_checkpoints: Arc::new(AtomicUsize::new(0)), + pause_at, scan_paused: Arc::new(tokio::sync::Notify::new()), resume: Arc::new(StdMutex::new(resume_rx)), }; @@ -218,16 +231,15 @@ async fn cancelled_lexical_scan_releases_the_search_permit_to_the_next_request() let cancellation = CancellationSignal::active("cancellation.search-permit.fixture").expect("cancellation"); - let abandoned = tokio::spawn(executor(search_request( - fixture.path(), - Some(cancellation.clone()), - ))); + let mut request = search_request(fixture.path(), Some(cancellation.clone())); + request.query = query.to_owned(); + let abandoned = tokio::spawn(executor(request)); // Deterministic rendezvous: the scan itself reports when it reaches its - // first checkpoint holding the permit. The timeout only bounds a failure + // selected checkpoint holding the permit. The timeout only bounds a failure // in which the scan never consults the control while it holds the permit. tokio::time::timeout(Duration::from_mins(1), admission.scan_paused.notified()) .await - .expect("the lexical scan must consult the request control while holding the permit"); + .expect("the candidate scan must consult the request control while holding the permit"); let refused = executor(search_request(fixture.path(), None)).await; assert_eq!( @@ -251,7 +263,8 @@ async fn cancelled_lexical_scan_releases_the_search_permit_to_the_next_request() ); let checkpoints = admission.scan_checkpoints.load(Ordering::SeqCst); assert_eq!( - checkpoints, 1, + checkpoints, + pause_at + 1, "the resumed scan observes cancellation at the checkpoint it paused in and \ hydrates nothing further" ); diff --git a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/tests/serving.rs b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/tests/serving.rs index b208dafd74..1efb57f459 100644 --- a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/tests/serving.rs +++ b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/tests/serving.rs @@ -400,6 +400,7 @@ fn production_text_serving_builds_publishes_and_reopens_the_artifact_head() { ); let exact = owners .retrieve_exact(&ExactLaneRequest { + control: &ReadyRetrievalControlV1, literals: authority.parse_literals(&query_view, &base), generation: generation.clone(), budget: base.budget, diff --git a/crates/tracedecay-query/src/bin/tracedecay_search_bench.rs b/crates/tracedecay-query/src/bin/tracedecay_search_bench.rs index be34e3450e..74a7f91665 100644 --- a/crates/tracedecay-query/src/bin/tracedecay_search_bench.rs +++ b/crates/tracedecay-query/src/bin/tracedecay_search_bench.rs @@ -940,6 +940,7 @@ where let exact_started = Instant::now(); let exact_outcome = exact_lane .retrieve_exact(&ExactLaneRequest { + control: &ActiveControl, base: request.clone(), query_view, generation: generation.clone(), diff --git a/crates/tracedecay-query/src/retrieval/exact.rs b/crates/tracedecay-query/src/retrieval/exact.rs index 14ab326e79..1b0e3309be 100644 --- a/crates/tracedecay-query/src/retrieval/exact.rs +++ b/crates/tracedecay-query/src/retrieval/exact.rs @@ -22,8 +22,9 @@ use tracedecay_domain::{ use super::ports::{ CodeCandidateBindingV1, ExactTermPostingReadPort, LaneBoundEvidence, LaneEvidenceRejections, - RetrievalPortError, candidate_checkpoint_prefix, checkpoint_digest, contract_error, - lane_bound_evidence, lane_candidate_cap, + RETRIEVAL_CANDIDATE_BATCH_SIZE, RetrievalExecutionControl, RetrievalPortError, + candidate_checkpoint_prefix, checkpoint_digest, contract_error, lane_bound_evidence, + lane_candidate_cap, retrieval_checkpoint, }; /// Wording the exact lane uses when a port-emitted batch fails the shared @@ -36,7 +37,6 @@ const EXACT_REJECTIONS: LaneEvidenceRejections = LaneEvidenceRejections { /// Exact technical literals are parsed under a versioned exact-admission /// specification before any lane executes. -#[derive(Debug, PartialEq, Eq)] pub struct ExactLaneRequest<'a> { pub base: RetrievalRequest, pub query_view: &'a EphemeralSanitizedQueryViewV1, @@ -45,6 +45,8 @@ pub struct ExactLaneRequest<'a> { /// admission validator. The lane never re-derives exact status. pub literals: Vec, pub budget: RetrievalBudget, + /// The same live request authority used by the other retrieval lanes. + pub control: &'a dyn RetrievalExecutionControl, } impl ExactLaneRequest<'_> { @@ -631,7 +633,10 @@ where // every candidate still compares against the verified minted proof. let mut verified_proofs: BTreeMap<(ExactFieldV1, Vec, Vec), ExactAdmissionProof> = BTreeMap::new(); - for candidate in &batch.candidates { + for (ordinal, candidate) in batch.candidates.iter().enumerate() { + if ordinal.is_multiple_of(RETRIEVAL_CANDIDATE_BATCH_SIZE) { + retrieval_checkpoint(request.control)?; + } let evidence = lane_bound_evidence( batch, candidate, @@ -742,6 +747,9 @@ where let mut candidates = Vec::with_capacity(admitted.len()); let mut evidence_by_occurrence = BTreeMap::new(); for (ordinal, (mut candidate, evidence)) in admitted.into_iter().enumerate() { + if ordinal.is_multiple_of(RETRIEVAL_CANDIDATE_BATCH_SIZE) { + retrieval_checkpoint(request.control)?; + } candidate.ordinal_rank = ordinal as u32; candidate.raw_score = FixedPointScore( (evidence.matched_literals.len() as u64) @@ -750,6 +758,7 @@ where evidence_by_occurrence.insert(candidate.source_occurrence_id.clone(), evidence); candidates.push(candidate); } + retrieval_checkpoint(request.control)?; let checkpoint_digest = exact_checkpoint_digest(&request.generation, &candidates)?; let rebuilt = RetrieverBatch { candidates, @@ -783,6 +792,7 @@ where request: &ExactLaneRequest<'_>, ) -> Result>, RetrievalPortError> { request.validate()?; + retrieval_checkpoint(request.control)?; self.enforce_request_literals(request)?; let outcome = match self.postings.read_exact_postings(request) { Ok(outcome) => outcome, diff --git a/crates/tracedecay-query/src/retrieval/exact/tests.rs b/crates/tracedecay-query/src/retrieval/exact/tests.rs index 580cac7aca..230fca18d8 100644 --- a/crates/tracedecay-query/src/retrieval/exact/tests.rs +++ b/crates/tracedecay-query/src/retrieval/exact/tests.rs @@ -21,9 +21,21 @@ use super::{ ExactLaneRequest, ExactLaneRetriever, ExactLiteralV1, }; use crate::retrieval::ports::{ - CodeCandidateBindingV1, CodeOccurrenceRefV1, ExactTermPostingReadPort, RetrievalPortError, + CodeCandidateBindingV1, CodeOccurrenceRefV1, ExactTermPostingReadPort, + RetrievalExecutionControl, RetrievalPortError, }; +struct ActiveControl; + +impl RetrievalExecutionControl for ActiveControl { + fn is_cancelled(&self) -> bool { + false + } + fn elapsed_micros(&self) -> u64 { + 0 + } +} + fn id(value: &str) -> T where T: TryFrom, @@ -179,6 +191,7 @@ fn exact_request( .expect("query sanitizes"), )); ExactLaneRequest { + control: &ActiveControl, literals: authority.parse_literals(query_view, &base), base, query_view, diff --git a/crates/tracedecay-query/src/retrieval/lexical.rs b/crates/tracedecay-query/src/retrieval/lexical.rs index 7c3be3e56e..133282f3e0 100644 --- a/crates/tracedecay-query/src/retrieval/lexical.rs +++ b/crates/tracedecay-query/src/retrieval/lexical.rs @@ -20,8 +20,9 @@ use tracedecay_domain::{ use super::ports::{ CodeCandidateBindingV1, LaneBoundEvidence, LaneEvidenceRejections, LexicalPostingReadPort, - RetrievalExecutionControl, RetrievalPortError, candidate_checkpoint_prefix, checkpoint_digest, - contract_error, lane_bound_evidence, lane_candidate_cap, + RETRIEVAL_CANDIDATE_BATCH_SIZE, RetrievalExecutionControl, RetrievalPortError, + candidate_checkpoint_prefix, checkpoint_digest, contract_error, lane_bound_evidence, + lane_candidate_cap, retrieval_checkpoint, }; mod projection; @@ -328,7 +329,7 @@ pub struct LexicalLaneRequest<'a> { pub score_domain: ScoreDomainId, pub budget: RetrievalBudget, /// The live request authority the lane consults between bounded units of - /// row work ([`lexical_checkpoint`]). The candidate-source bound keeps one + /// row work. The candidate-source bound keeps one /// request's hydration finite, but a caller that has already settled — /// cancelled, past its deadline, or revoked — must not keep the shared /// search execution permit occupied while the remaining rows decode and @@ -337,21 +338,6 @@ pub struct LexicalLaneRequest<'a> { pub control: &'a dyn RetrievalExecutionControl, } -/// The lexical lane's cooperative cancellation checkpoint. -/// -/// Called before each candidate row is decoded and scored, and between the -/// scan's phases, so cancellation performs at most one further row visit -/// after the signal. An uncancelled request never observes it, which keeps -/// candidate order, evidence, and coverage identical to an unchecked scan. -pub(crate) fn lexical_checkpoint( - control: &dyn RetrievalExecutionControl, -) -> Result<(), RetrievalPortError> { - if control.is_cancelled() { - return Err(RetrievalPortError::Cancelled); - } - Ok(()) -} - /// Per-occurrence lexical-lane evidence with its field score breakdown. /// /// Each channel reports its raw score, rank, normalized feature, and fusion @@ -587,7 +573,10 @@ where let mut admitted: Vec<(CompactCandidate, LexicalLaneEvidence, FixedPointScore)> = Vec::with_capacity(batch.candidates.len()); let mut excluded = 0_u64; - for candidate in &batch.candidates { + for (ordinal, candidate) in batch.candidates.iter().enumerate() { + if ordinal.is_multiple_of(RETRIEVAL_CANDIDATE_BATCH_SIZE) { + retrieval_checkpoint(request.control)?; + } let evidence = lane_bound_evidence( batch, candidate, @@ -642,6 +631,9 @@ where let mut candidates = Vec::with_capacity(admitted.len()); let mut evidence_by_occurrence = BTreeMap::new(); for (ordinal, (mut candidate, evidence, raw_score)) in admitted.into_iter().enumerate() { + if ordinal.is_multiple_of(RETRIEVAL_CANDIDATE_BATCH_SIZE) { + retrieval_checkpoint(request.control)?; + } candidate.ordinal_rank = ordinal as u32; candidate.raw_score = raw_score; evidence_by_occurrence.insert(candidate.source_occurrence_id.clone(), evidence); @@ -655,6 +647,7 @@ where let eligible = batch.coverage.eligible.max(seen).saturating_sub(excluded); let capped = batch.coverage.capped.saturating_add(truncated as u64); let exhausted = truncated == 0 && batch.coverage.capped == 0; + retrieval_checkpoint(request.control)?; let checkpoint_digest = lexical_checkpoint_digest(&request.generation, &candidates)?; let rebuilt = RetrieverBatch { candidates, @@ -687,7 +680,7 @@ where request: &LexicalLaneRequest<'_>, ) -> Result>, RetrievalPortError> { request.validate()?; - lexical_checkpoint(request.control)?; + retrieval_checkpoint(request.control)?; let outcome = match self.postings.read_lexical_postings(request) { Ok(outcome) => outcome, // A missing lexical authority rejects the request as a typed diff --git a/crates/tracedecay-query/src/retrieval/lexical/projection/artifact/reader.rs b/crates/tracedecay-query/src/retrieval/lexical/projection/artifact/reader.rs index 197e103a08..cfc1aadd0e 100644 --- a/crates/tracedecay-query/src/retrieval/lexical/projection/artifact/reader.rs +++ b/crates/tracedecay-query/src/retrieval/lexical/projection/artifact/reader.rs @@ -59,7 +59,8 @@ use crate::retrieval::exact::{ExactAdmissionAuthority, ExactLaneEvidence, ExactL use crate::retrieval::ports::RetrievalExecutionControl; use crate::retrieval::ports::{ CodeCandidateBindingV1, CodeOccurrenceRefV1, ExactTermPostingReadPort, LexicalPostingReadPort, - RetrievalPortError, contract_error, lane_candidate_cap, + RETRIEVAL_CANDIDATE_BATCH_SIZE, RetrievalPortError, contract_error, lane_candidate_cap, + retrieval_checkpoint, }; use super::super::{ @@ -72,7 +73,7 @@ use super::super::{ use crate::retrieval::lexical::{ LexicalFieldFilterV1, LexicalFieldV1, LexicalLaneEvidence, LexicalLaneRequest, LexicalSpellingVariantV1, MAX_FUZZY_TERM_EXPANSIONS_V1, MAX_LEXICAL_QUERY_TERM_BYTES_V1, - admit_candidate_sources, candidate_admission_outcome, field_admitted, lexical_checkpoint, + admit_candidate_sources, candidate_admission_outcome, field_admitted, }; impl LexicalFieldTextV1 for ArtifactRowV1 { @@ -777,9 +778,12 @@ impl CodeLexicalArtifactReaderV1 { .map_err(sqlite_error)?; let mut members = Vec::with_capacity(fetch); while let Some(row) = rows.next().map_err(sqlite_error)? { - checkpoint(control)?; + if members.len().is_multiple_of(RETRIEVAL_CANDIDATE_BATCH_SIZE) { + checkpoint(control)?; + } members.push(self.verified_clone_member(authority, key, row)?); } + checkpoint(control)?; let next_cursor = (members.len() > limit) .then(|| { members.get(limit - 1).map(|member| CloneArtifactCursorV1 { @@ -1475,6 +1479,7 @@ fn rewrite_union_query_parameters( fn visit_document_ids( connection: &Connection, query: &DocumentQueryV1, + control: &dyn RetrievalExecutionControl, mut visitor: impl FnMut(u32) -> Result<(), RetrievalPortError>, ) -> Result<(), RetrievalPortError> { hotpath::measure_block!("query.stream.visit_documents", { @@ -1493,10 +1498,14 @@ fn visit_document_ids( .map_err(map_query_sql_error)?; let mut visited = 0u64; while let Some(row) = rows.next().map_err(map_query_sql_error)? { + if visited.is_multiple_of(RETRIEVAL_CANDIDATE_BATCH_SIZE as u64) { + retrieval_checkpoint(control)?; + } let document = row.get::<_, i64>(0).map_err(map_query_sql_error)?; visitor(u32::try_from(document).map_err(contract_error)?)?; visited += 1; } + retrieval_checkpoint(control)?; hotpath::gauge!("query.stream.rows_total").inc(visited); Ok(()) }) @@ -1506,9 +1515,8 @@ fn visit_document_ids( /// one SQLite statement. The correlated posting lookup seeks the maintained /// document index; it never emits the row BLOB once per matching term. /// -/// `control` is consulted before every row leaves SQLite, so a cancelled or -/// expired request stops after the row already stepped instead of decoding -/// and scoring the rest of its admitted candidate set. +/// Request authority is consulted before each page-sized batch and at stream +/// completion, bounding abandoned work without a route lookup per candidate. fn visit_lexical_rows( connection: &Connection, documents: &DocumentQueryV1, @@ -1623,7 +1631,9 @@ fn visit_lexical_rows( .map_err(map_query_sql_error)?; let mut visited = 0u64; while let Some(row) = rows.next().map_err(map_query_sql_error)? { - lexical_checkpoint(control)?; + if visited.is_multiple_of(RETRIEVAL_CANDIDATE_BATCH_SIZE as u64) { + retrieval_checkpoint(control)?; + } let document = u32::try_from(row.get::<_, i64>(0).map_err(map_query_sql_error)?) .map_err(contract_error)?; let chunk_id: String = row.get(1).map_err(map_query_sql_error)?; @@ -1665,6 +1675,7 @@ fn visit_lexical_rows( visited = visited.saturating_add(1); } drop(rows); + retrieval_checkpoint(control)?; metrics.observe_statement(&statement)?; metrics.rows(visited); Ok(()) @@ -2090,7 +2101,7 @@ impl<'a> ArtifactQueryV1<'a> { let prepared = PreparedLexicalQueryV1::new(request); let terms = lexical_terms(&prepared, &fuzzy); let stats = self.lexical_stats(&terms)?; - lexical_checkpoint(control)?; + retrieval_checkpoint(control)?; let mut phrase_queries = BTreeMap::new(); for (_, normalized) in &prepared.phrases { let query = ngram_document_query( @@ -2178,6 +2189,9 @@ impl<'a> ArtifactQueryV1<'a> { let mut candidates = Vec::with_capacity(selected.len()); let mut evidence_by_occurrence = BTreeMap::new(); for (ordinal, entry) in selected.into_iter().enumerate() { + if ordinal.is_multiple_of(RETRIEVAL_CANDIDATE_BATCH_SIZE) { + retrieval_checkpoint(control)?; + } let RankedLexicalEntryV1 { key: (_, _, _), score, @@ -2206,6 +2220,7 @@ impl<'a> ArtifactQueryV1<'a> { evidence_by_occurrence.insert(candidate.source_occurrence_id.clone(), evidence); candidates.push(candidate); } + retrieval_checkpoint(control)?; Ok(candidate_admission_outcome( capped_batch( self.document_count, @@ -2224,6 +2239,7 @@ impl<'a> ArtifactQueryV1<'a> { request: &ExactLaneRequest, authority: &A, ) -> Result>, RetrievalPortError> { + retrieval_checkpoint(request.control)?; let documents = self.exact_documents(request)?; // Same bounded selection as the lexical lane: keys mirror the exact // lane's canonical order (admitted literal count, then occurrence), @@ -2239,7 +2255,7 @@ impl<'a> ArtifactQueryV1<'a> { let mut eligible = 0u64; let mut ranked = BinaryHeap::new(); let mut proofs = LiteralProofCacheV1::new(request.literals.len()); - self.visit_documents(&documents, |document| { + self.visit_documents(&documents, request.control, |document| { let row = self.row(document)?; let (matched_literals, matched_kinds) = exact_matches_artifact(&row, request); if matched_literals.is_empty() { @@ -2268,11 +2284,15 @@ impl<'a> ArtifactQueryV1<'a> { ); Ok(()) })?; + retrieval_checkpoint(request.control)?; let selected = ranked.into_sorted_vec(); let truncated = eligible - selected.len() as u64; let mut candidates = Vec::with_capacity(selected.len()); let mut evidence_by_occurrence = BTreeMap::new(); for (ordinal, entry) in selected.into_iter().enumerate() { + if ordinal.is_multiple_of(RETRIEVAL_CANDIDATE_BATCH_SIZE) { + retrieval_checkpoint(request.control)?; + } let RankedExactEntryV1 { key: (_, _, document), admitted_ordinal, @@ -2302,6 +2322,7 @@ impl<'a> ArtifactQueryV1<'a> { evidence_by_occurrence.insert(candidate.source_occurrence_id.clone(), evidence); candidates.push(candidate); } + retrieval_checkpoint(request.control)?; Ok(RetrieverOutcome::Complete(capped_batch( self.document_count, eligible, @@ -2492,10 +2513,11 @@ impl<'a> ArtifactQueryV1<'a> { fn visit_documents( &self, query: &DocumentQueryV1, + control: &dyn RetrievalExecutionControl, visitor: impl FnMut(u32) -> Result<(), RetrievalPortError>, ) -> Result<(), RetrievalPortError> { self.metrics.probe(); - visit_document_ids(self.connection, query, visitor) + visit_document_ids(self.connection, query, control, visitor) } #[hotpath::measure(label = "query.lane.fuzzy.expand")] @@ -3778,7 +3800,7 @@ mod tests { fn streamed_documents(connection: &Connection, query: &DocumentQueryV1) -> Vec { let mut documents = Vec::new(); - visit_document_ids(connection, query, |document| { + visit_document_ids(connection, query, &AlwaysActiveControl, |document| { documents.push(document); Ok(()) }) @@ -4386,18 +4408,45 @@ mod tests { (connection, field) } - /// Cancellation reaches the row stream between rows: a request cancelled - /// after `k` consultations decodes exactly `k - 1` rows, unwinds with the - /// typed cancellation error, and never visits the remaining candidates. + #[test] + fn lexical_candidate_batches_bound_cancellation_probes() { + let (connection, field) = lexical_row_stream_fixture(256); + let documents = DocumentQueryV1::term(field, "alpha".to_owned()); + let terms = BTreeSet::from(["alpha".to_owned()]); + let control = CancelAtObservation::new(usize::MAX); + let mut visited = 0; + visit_lexical_rows( + &connection, + &documents, + &terms, + &ArtifactQueryMetricsV1::default(), + LexicalArtifactLayoutV1::V10, + &control, + |_, _, _, _| { + visited += 1; + Ok(()) + }, + ) + .expect("active request visits its candidates"); + assert_eq!(visited, 256); + assert!( + control.observations() <= 5, + "request authority must be consulted per batch, not per candidate: {} probes", + control.observations() + ); + } + + /// A request cancelled between batches unwinds before decoding the next + /// page's candidates, with a typed error rather than partial success. /// The same stream under an active control visits every row, so the /// checkpoint changes nothing for an uncancelled request. #[test] fn lexical_row_stream_unwinds_at_the_first_checkpoint_after_cancellation() { - let (connection, field) = lexical_row_stream_fixture(256); + let (connection, field) = lexical_row_stream_fixture(512); let documents = DocumentQueryV1::term(field, "alpha".to_owned()); let terms = BTreeSet::from(["alpha".to_owned()]); - let control = CancelAtObservation::new(8); + let control = CancelAtObservation::new(2); let mut visited = 0usize; let error = visit_lexical_rows( &connection, @@ -4414,12 +4463,12 @@ mod tests { .expect_err("a cancelled request must not stream to completion"); assert_eq!(error, RetrievalPortError::Cancelled); assert_eq!( - visited, 7, + visited, 128, "every row before the cancelling checkpoint is visited and none after it" ); assert_eq!( control.observations(), - 8, + 2, "the stream stops consulting the control once it reports cancellation" ); @@ -4437,7 +4486,23 @@ mod tests { }, ) .expect("an uncancelled request streams every candidate row"); - assert_eq!(complete, 256); + assert_eq!(complete, 512); + } + + #[test] + fn exact_document_stream_cancels_before_the_next_candidate_batch() { + let (connection, field) = lexical_row_stream_fixture(512); + let documents = DocumentQueryV1::term(field, "alpha".to_owned()); + let control = CancelAtObservation::new(2); + let mut visited = Vec::new(); + let error = visit_document_ids(&connection, &documents, &control, |document| { + visited.push(document); + Ok(()) + }) + .expect_err("the next candidate batch must not start"); + assert_eq!(error, RetrievalPortError::Cancelled); + assert_eq!(visited, (0..128).collect::>()); + assert_eq!(control.observations(), 2); } #[test] diff --git a/crates/tracedecay-query/src/retrieval/lexical/projection/artifact/reader/family_report.rs b/crates/tracedecay-query/src/retrieval/lexical/projection/artifact/reader/family_report.rs index f534af3a23..8ce8024331 100644 --- a/crates/tracedecay-query/src/retrieval/lexical/projection/artifact/reader/family_report.rs +++ b/crates/tracedecay-query/src/retrieval/lexical/projection/artifact/reader/family_report.rs @@ -12,6 +12,7 @@ use super::{CodeLexicalArtifactReaderV1, MAX_CLONE_EXACT_PAGE_MEMBERS_V1}; use crate::retrieval::lexical::projection::artifact::{ CodeLexicalArtifactErrorV1, checkpoint, sqlite_error, }; +use crate::retrieval::ports::RETRIEVAL_CANDIDATE_BATCH_SIZE; #[derive(Clone, Debug, PartialEq, Eq, serde::Deserialize, serde::Serialize)] struct CloneFamilyCursorV1 { @@ -199,7 +200,12 @@ impl CodeLexicalArtifactReaderV1 { .map_err(sqlite_error)?; let mut families = Vec::with_capacity(fetch); while let Some(row) = rows.next().map_err(sqlite_error)? { - checkpoint(control)?; + if families + .len() + .is_multiple_of(RETRIEVAL_CANDIDATE_BATCH_SIZE) + { + checkpoint(control)?; + } let class = match row.get::<_, i64>(0).map_err(sqlite_error)? { 1 => CloneNormalizationClassV1::Conservative, 2 => CloneNormalizationClassV1::Rename, @@ -246,6 +252,7 @@ impl CodeLexicalArtifactReaderV1 { continuation, }); } + checkpoint(control)?; let next_cursor = (families.len() > limit) .then(|| { families diff --git a/crates/tracedecay-query/src/retrieval/lexical/projection/in_memory.rs b/crates/tracedecay-query/src/retrieval/lexical/projection/in_memory.rs index 16d58ece5f..4bd651d1a3 100644 --- a/crates/tracedecay-query/src/retrieval/lexical/projection/in_memory.rs +++ b/crates/tracedecay-query/src/retrieval/lexical/projection/in_memory.rs @@ -21,7 +21,6 @@ use tracedecay_domain::{ use super::super::{ LexicalFieldV1, LexicalLaneEvidence, LexicalLaneRequest, LexicalSpellingVariantV1, MAX_FUZZY_TERM_EXPANSIONS_V1, admit_candidate_sources, candidate_admission_outcome, - lexical_checkpoint, }; use super::{ CodeLexicalProjectionMetadataV1, ECHO_SCORE_MILLIS, ExactMatchRowViewV1, FUZZY_SCORE_MILLIS, @@ -35,7 +34,7 @@ use super::{ use crate::retrieval::exact::{ExactAdmissionAuthority, ExactLaneEvidence, ExactLaneRequest}; use crate::retrieval::ports::{ CodeCandidateBindingV1, CodeOccurrenceRefV1, ExactTermPostingReadPort, LexicalPostingReadPort, - RetrievalPortError, contract_error, + RETRIEVAL_CANDIDATE_BATCH_SIZE, RetrievalPortError, contract_error, retrieval_checkpoint, }; mod postings; @@ -850,8 +849,10 @@ impl CodeLexicalProjectionAdapterV1 { .lexical_documents(request, &fuzzy, &phrase_candidates, &mut pruned); let mut pairs = Vec::new(); let mut excluded = self.rows.len() as u64 - documents.len(); - for document in documents { - lexical_checkpoint(request.control)?; + for (ordinal, document) in documents.into_iter().enumerate() { + if ordinal.is_multiple_of(RETRIEVAL_CANDIDATE_BATCH_SIZE) { + retrieval_checkpoint(request.control)?; + } let row = &self.rows[document as usize]; let score = self.score_row( document, @@ -896,6 +897,7 @@ impl CodeLexicalProjectionAdapterV1 { evidence_by_occurrence.insert(candidate.source_occurrence_id.clone(), evidence); candidates.push(candidate); } + retrieval_checkpoint(request.control)?; hotpath::gauge!("query.lane.lexical.candidates").set(candidates.len()); hotpath::gauge!("query.lane.lexical.examined").set(self.rows.len()); Ok(candidate_admission_outcome( @@ -1258,6 +1260,7 @@ where &self, request: &ExactLaneRequest, ) -> Result>, RetrievalPortError> { + retrieval_checkpoint(request.control)?; self.projection.validate_generation(&request.generation)?; if let Some(outcome) = self.projection.stale_outcome() { return Ok(outcome); @@ -1266,7 +1269,10 @@ where let mut pairs = Vec::new(); let mut excluded = self.projection.rows.len() as u64 - documents.len(); let mut proofs = LiteralProofCacheV1::new(request.literals.len()); - for document in documents { + for (ordinal, document) in documents.into_iter().enumerate() { + if ordinal.is_multiple_of(RETRIEVAL_CANDIDATE_BATCH_SIZE) { + retrieval_checkpoint(request.control)?; + } let row = &self.projection.rows[document as usize]; let (matched_literals, matched_kinds) = exact_matches(row.exact_match_view(), request); if matched_literals.is_empty() { @@ -1310,6 +1316,7 @@ where evidence_by_occurrence.insert(candidate.source_occurrence_id.clone(), evidence); candidates.push(candidate); } + retrieval_checkpoint(request.control)?; Ok(RetrieverOutcome::Complete(RetrieverBatch { coverage: RetrieverCoverage { examined: self.projection.rows.len() as u64, diff --git a/crates/tracedecay-query/src/retrieval/ports.rs b/crates/tracedecay-query/src/retrieval/ports.rs index d9e2129e86..2bebe71960 100644 --- a/crates/tracedecay-query/src/retrieval/ports.rs +++ b/crates/tracedecay-query/src/retrieval/ports.rs @@ -29,6 +29,20 @@ pub trait RetrievalExecutionControl: Send + Sync { fn elapsed_micros(&self) -> u64; } +/// One sealed lexical artifact page's worth of candidate work. Readers keep +/// streaming rows, but consult request authority before starting the next page. +pub const RETRIEVAL_CANDIDATE_BATCH_SIZE: usize = 128; + +pub(crate) fn retrieval_checkpoint( + control: &dyn RetrievalExecutionControl, +) -> Result<(), RetrievalPortError> { + if control.is_cancelled() { + Err(RetrievalPortError::Cancelled) + } else { + Ok(()) + } +} + /// Incompatible indexes or models never trigger silent fallback. #[derive(Clone, Debug, Error, PartialEq, Eq)] pub enum RetrievalPortError { diff --git a/crates/tracedecay-query/tests/retrieval_contract_spine.rs b/crates/tracedecay-query/tests/retrieval_contract_spine.rs index 30724bee0b..9774e25bb9 100644 --- a/crates/tracedecay-query/tests/retrieval_contract_spine.rs +++ b/crates/tracedecay-query/tests/retrieval_contract_spine.rs @@ -11,9 +11,20 @@ use tracedecay_domain::{ }; use tracedecay_query::retrieval::exact::{ExactLaneEvidence, ExactLaneRequest, ExactLiteralV1}; use tracedecay_query::retrieval::ports::{ - CodeCandidateBindingV1, CodeOccurrenceRefV1, RetrievalPortError, + CodeCandidateBindingV1, CodeOccurrenceRefV1, RetrievalExecutionControl, RetrievalPortError, }; +struct ActiveControl; + +impl RetrievalExecutionControl for ActiveControl { + fn is_cancelled(&self) -> bool { + false + } + fn elapsed_micros(&self) -> u64 { + 0 + } +} + fn id(value: &str) -> T where T: TryFrom, @@ -85,6 +96,7 @@ fn request_and_proof() -> (ExactLaneRequest<'static>, ExactAdmissionProof) { )); ( ExactLaneRequest { + control: &ActiveControl, base, query_view, generation: CodeGenerationId::new("generation.contract").unwrap(), diff --git a/crates/tracedecay-query/tests/search_quality_suite/candidate_producers.rs b/crates/tracedecay-query/tests/search_quality_suite/candidate_producers.rs index f2de165799..dd07a39b61 100644 --- a/crates/tracedecay-query/tests/search_quality_suite/candidate_producers.rs +++ b/crates/tracedecay-query/tests/search_quality_suite/candidate_producers.rs @@ -464,7 +464,7 @@ fn generation_backed_projection( fn real_lexical_source_fixture_with_files(file_count: usize) -> RealLexicalSourceFixture { assert!(file_count >= 1, "fixture needs at least one file"); let identity_source = b"import type { Widget } from \"widget-kit\";\nexport function render(value: Widget) { return value; }\n"; - let sources = (0..file_count) + let mut sources = (0..file_count) .map(|ordinal| { if ordinal == 0 { ( @@ -473,8 +473,7 @@ fn real_lexical_source_fixture_with_files(file_count: usize) -> RealLexicalSourc identity_source.to_vec(), ) } else { - // Zero-padded ordinals keep lexicographic file order equal to - // generation order, which snapshot intake requires. + // Keep the original small-fixture identities stable. ( format!("file.artifact.{ordinal:02}"), format!("src/artifact_{ordinal:02}.ts"), @@ -485,7 +484,8 @@ fn real_lexical_source_fixture_with_files(file_count: usize) -> RealLexicalSourc ) } }) - .collect(); + .collect::>(); + sources.sort_by(|left, right| left.1.cmp(&right.1)); real_lexical_source_fixture_from_sources(sources) } @@ -5759,6 +5759,7 @@ fn artifact_exact_reader_prefers_admitted_matches_over_denied_best() { let base = base_request(exact_query, 1); let exact_query_view = query_view(exact_query); let exact_request = ExactLaneRequest { + control: &ACTIVE_CONTROL, literals: authority.parse_literals(&exact_query_view, &base), base, query_view: &exact_query_view, @@ -5788,6 +5789,79 @@ fn artifact_exact_reader_prefers_admitted_matches_over_denied_best() { ); } +#[test] +fn exact_candidate_scan_stops_before_the_next_batch_after_cancellation() { + struct CancelDuringScan { + checks: AtomicUsize, + } + impl RetrievalExecutionControl for CancelDuringScan { + fn is_cancelled(&self) -> bool { + self.checks.fetch_add(1, Ordering::SeqCst) >= 2 + } + fn elapsed_micros(&self) -> u64 { + 0 + } + } + + let fixture = real_lexical_source_fixture_with_files(256); + let metadata = fixture.metadata.clone(); + let generation = metadata.generation.clone(); + let build_control = ArtifactControl { cancelled: false }; + let (pages, _) = drain_verified_pages(&fixture, 128); + let directory = tempfile::tempdir().expect("artifact tempdir"); + let path = directory.path().join("cancel-exact.sqlite"); + let mut builder = + CodeLexicalArtifactBuilderV1::create(&path, metadata).expect("create real artifact"); + for page in &pages { + builder + .append_page(page, &build_control) + .expect("append page"); + } + let mut source = fixture.open_source(128); + let verified = builder + .rebuild_and_finalize(&mut source, &build_control) + .expect("finalize artifact"); + let reader = CodeLexicalArtifactReaderV1::open_with_control( + &path, + &verified, + CODE_LEXICAL_ARTIFACT_QUERY_CACHE_BUDGET_BYTES_V1, + &build_control, + ) + .expect("open artifact"); + let authority = CentralExactAdmissionAuthorityV1::new(id("exact-rules.v1")); + let query = r#""return value""#; + let base = base_request(query, 8); + let view = query_view(query); + let control = CancelDuringScan { + checks: AtomicUsize::new(0), + }; + let mut request = ExactLaneRequest { + literals: authority.parse_literals(&view, &base), + base, + query_view: &view, + generation, + budget: budget(8), + control: &control, + }; + let exact = reader.exact_adapter(authority); + assert_eq!( + exact.read_exact_postings(&request), + Err(RetrievalPortError::Cancelled), + "the exact artifact scan must consult the carried request control between candidate batches", + ); + request.control = &ACTIVE_CONTROL; + let first = complete(exact.read_exact_postings(&request).expect("active read")); + assert!( + first.coverage.eligible > 128, + "fixture must span candidate batches" + ); + assert_eq!(first.candidates.len(), 8); + assert_eq!( + first, + complete(exact.read_exact_postings(&request).expect("repeat read")) + ); +} + #[test] fn disk_artifact_ledger_charges_stay_page_local_across_corpus_scaling() { let control = ArtifactControl { cancelled: false }; @@ -5955,6 +6029,7 @@ fn disk_artifact_reader_selects_bounded_top_k_with_lane_tie_order_and_coverage() let base = base_request(exact_query, 7); let exact_query_view = query_view(exact_query); let exact_request = ExactLaneRequest { + control: &ACTIVE_CONTROL, literals: authority.parse_literals(&exact_query_view, &base), base, query_view: &exact_query_view, @@ -6214,6 +6289,7 @@ fn exact_projection_emits_only_authority_minted_proofs() { let base = base_request(query, 16); let query_view = query_view(query); let request = ExactLaneRequest { + control: &ACTIVE_CONTROL, literals: authority.parse_literals(&query_view, &base), base, query_view: &query_view, diff --git a/crates/tracedecay-query/tests/search_quality_suite/single_root.rs b/crates/tracedecay-query/tests/search_quality_suite/single_root.rs index 1df60b937d..9e87387a20 100644 --- a/crates/tracedecay-query/tests/search_quality_suite/single_root.rs +++ b/crates/tracedecay-query/tests/search_quality_suite/single_root.rs @@ -323,6 +323,7 @@ fn fixture(disposition: GraphDisposition) -> SingleRootFixture { CentralExactAdmissionAuthorityV1::new(id::("exact-rules.v1")); let exact_query_view = query_view("--release"); let exact_request = ExactLaneRequest { + control: &FixtureRetrievalExecutionControl, literals: authority.parse_literals(&exact_query_view, &request), base: request.clone(), query_view: &exact_query_view, diff --git a/crates/tracedecay-search-eval/src/candidate_output.rs b/crates/tracedecay-search-eval/src/candidate_output.rs index a389cca978..ba0fccd162 100644 --- a/crates/tracedecay-search-eval/src/candidate_output.rs +++ b/crates/tracedecay-search-eval/src/candidate_output.rs @@ -748,6 +748,7 @@ fn compose_production_query( let budget = retrieval_budget(); let exact_request = ExactLaneRequest { + control: &ActiveControl, base: request.clone(), query_view: &query_view, generation: generation_id.clone(), From 2aff0f6f6f702497bc4e5b073fe153b246632234 Mon Sep 17 00:00:00 2001 From: ScriptedAlchemy Date: Thu, 17 Sep 2026 11:12:24 -0700 Subject: [PATCH 06/14] fix(query): preserve callable and rebuild cancellation --- .../src/code_index_scheduler/queries.rs | 245 ++++++++++++------ .../src/code_index_scheduler/tests/mod.rs | 14 +- .../retrieval/lexical/projection/in_memory.rs | 8 + .../candidate_producers.rs | 67 +++++ 4 files changed, 253 insertions(+), 81 deletions(-) diff --git a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/queries.rs b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/queries.rs index 2a4a1f4fb4..be5c9e3f54 100644 --- a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/queries.rs +++ b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/queries.rs @@ -61,7 +61,7 @@ use tracedecay_query::retrieval::lexical::{ LexicalFieldFilterV1, LexicalFieldV1, LexicalLaneRequest, lexical_query_parts, }; use tracedecay_query::retrieval::ports::{ - CodeCandidateBindingV1, CodeOccurrenceRefV1, RetrievalExecutionControl, + CodeCandidateBindingV1, CodeOccurrenceRefV1, RetrievalExecutionControl, RetrievalPortError, }; use tracedecay_query::retrieval::{ AdmittedGenerationContextV1, NativeCodeOccurrenceV1, NativeExactRecordV1, NativeGraphRecordV1, @@ -2597,6 +2597,73 @@ fn application_budget_usage(usage: RetrievalBudgetUsage) -> OperationBudgetUsage type PortFuture<'a, T> = Pin>> + Send + 'a>>; +fn execute_prepared_exact_query( + prepared: &PreparedTextCallableQueryV1, + context: &RetrievalPortContext<'_>, + request: &ExactOccurrenceRequest, + query_binding_digest: ManifestDigest, +) -> RetrievalPortOutcome> { + let latest = &prepared.latest; + let served_generation = latest.metadata().manifest().generation_id.clone(); + let finished_at = query_finished_at(); + let base = prepared.query.request(); + let Ok(query_view) = tracedecay_domain::EphemeralSanitizedQueryViewV1::sanitize( + request.literal.clone(), + callable_query_sanitizer_revision(), + callable_query_normalization_revision(), + ) else { + return unavailable(finished_at); + }; + let authority = CentralExactAdmissionAuthorityV1::new( + ExactAdmissionRuleRevision::new(tracedecay_query::retrieval::QUERY_EXACT_RULE_REVISION_V1) + .unwrap_or_else(|_| panic!("static exact rule revision")), + ); + let exact_control = CallableRetrievalExecutionControl::for_request(context.request); + let lane_request = ExactLaneRequest { + literals: authority.parse_literals(&query_view, base), + generation: served_generation.clone(), + budget: base.budget, + base: base.clone(), + query_view: &query_view, + control: exact_control.as_ref(), + }; + let Ok(owners) = latest.production_query_owners_with_budget(&base.budget) else { + return unavailable(finished_at); + }; + let records = TextArtifactNativeRecordReadPortV1 { + generation: served_generation.clone(), + owners: std::sync::Arc::clone(&owners), + }; + let Ok(native_context) = + AdmittedGenerationContextV1::admit(served_generation.clone(), &records) + else { + return unavailable_for_generation(finished_at, served_generation); + }; + let outcome = match owners.retrieve_exact(&lane_request) { + Ok(outcome) => { + let Ok(outcome) = + native_context.exact(outcome, &request.literal, request.kind, |path| { + path_is_in_code_query_scope(path, &request.scope) + }) + else { + return unavailable(finished_at); + }; + outcome + } + Err(RetrievalPortError::Cancelled) => NativeLaneOutcomeV1::Cancelled, + Err(_) => return unavailable(finished_at), + }; + finish_native_lane_query( + prepared, + context, + "code_exact_occurrence", + query_binding_digest, + &request.meta.page, + outcome, + application_exact_record, + ) +} + impl CallableCodeQueryPort for CodeIndexSchedulerRegistryV1 { fn exact_occurrence<'a>( &'a self, @@ -2618,66 +2685,7 @@ impl CallableCodeQueryPort for CodeIndexSchedulerRegistryV1 { &request.meta.order, ) ); - let latest = &prepared.latest; - let served_generation = latest.metadata().manifest().generation_id.clone(); - let finished_at = query_finished_at(); - let base = prepared.query.request(); - let Ok(query_view) = tracedecay_domain::EphemeralSanitizedQueryViewV1::sanitize( - request.literal.clone(), - callable_query_sanitizer_revision(), - callable_query_normalization_revision(), - ) else { - return unavailable(finished_at); - }; - let authority = CentralExactAdmissionAuthorityV1::new( - ExactAdmissionRuleRevision::new( - tracedecay_query::retrieval::QUERY_EXACT_RULE_REVISION_V1, - ) - .unwrap_or_else(|_| panic!("static exact rule revision")), - ); - let exact_control = CallableRetrievalExecutionControl::for_request(context.request); - let lane_request = ExactLaneRequest { - literals: authority.parse_literals(&query_view, base), - generation: served_generation.clone(), - budget: base.budget, - base: base.clone(), - query_view: &query_view, - control: exact_control.as_ref(), - }; - let Ok(owners) = latest.production_query_owners_with_budget(&base.budget) else { - return unavailable(finished_at); - }; - let records = TextArtifactNativeRecordReadPortV1 { - generation: served_generation.clone(), - owners: std::sync::Arc::clone(&owners), - }; - let Ok(native_context) = - AdmittedGenerationContextV1::admit(served_generation.clone(), &records) - else { - return unavailable_for_generation(finished_at, served_generation); - }; - let outcome = owners.retrieve_exact(&lane_request); - match outcome { - Ok(outcome) => { - let Ok(outcome) = - native_context.exact(outcome, &request.literal, request.kind, |path| { - path_is_in_code_query_scope(path, &request.scope) - }) - else { - return unavailable(finished_at); - }; - finish_native_lane_query( - &prepared, - &context, - "code_exact_occurrence", - query_binding_digest, - &request.meta.page, - outcome, - application_exact_record, - ) - } - Err(_) => unavailable(finished_at), - } + execute_prepared_exact_query(&prepared, &context, request, query_binding_digest) }) } @@ -2766,26 +2774,27 @@ impl CallableCodeQueryPort for CodeIndexSchedulerRegistryV1 { else { return unavailable_for_generation(finished_at, served_generation); }; - let outcome = owners.retrieve_lexical(&lane_request); - match outcome { + let outcome = match owners.retrieve_lexical(&lane_request) { Ok(outcome) => { let Ok(outcome) = native_context.lexical(outcome, |path| { path_is_in_code_query_scope(path, &request.scope) }) else { return unavailable(finished_at); }; - finish_native_lane_query( - &prepared, - &context, - "code_phrase_search", - query_binding_digest, - &request.meta.page, - outcome, - application_lexical_record, - ) + outcome } - Err(_) => unavailable(finished_at), - } + Err(RetrievalPortError::Cancelled) => NativeLaneOutcomeV1::Cancelled, + Err(_) => return unavailable(finished_at), + }; + finish_native_lane_query( + &prepared, + &context, + "code_phrase_search", + query_binding_digest, + &request.meta.page, + outcome, + application_lexical_record, + ) }) } @@ -3902,6 +3911,94 @@ fn navigation_symbol_query<'a>( #[cfg(test)] mod tests { use super::*; + use crate::code_index_scheduler::tests::{ + GitFixture, application_context, mounted_core_query_worktree, query_meta, + wait_for_queryable_text_generation, + }; + use tracedecay_contracts::{ + CallableCodeOperationKind, CancellationContext, CodeQueryScope, callable_code_operation, + }; + + #[tokio::test] + async fn callable_exact_read_preserves_cancellation_after_generation_admission() { + let fixture = GitFixture::new(&[("src/lib.rs", "pub fn cancellation_target() {}\n")]); + let store = tempfile::tempdir().expect("isolated store"); + let (registry, scope) = mounted_core_query_worktree(&fixture, &store).await; + let latest = wait_for_queryable_text_generation(®istry, fixture.path()).await; + let generation = latest.metadata().manifest().generation_id.clone(); + let operation = callable_code_operation(CallableCodeOperationKind::ExactOccurrence) + .expect("exact operation"); + let context = application_context(&operation, scope.repository_id, scope.worktree_id); + let request = ExactOccurrenceRequest::new( + "cancellation_target", + None, + CodeQueryScope::new(generation.clone(), None).expect("exact generation scope"), + query_meta(), + ) + .expect("exact request"); + let port_context = RetrievalPortContext { + request: &context, + operation: &operation, + }; + let binding = canonical_sha256(&( + "code_exact_occurrence", + &request.literal, + &request.kind, + &request.scope, + &request.meta.projection, + &request.meta.order, + )) + .expect("request binding"); + let prepared = registry + .prepare_text_callable_query( + &port_context, + &generation, + &request.meta.page, + request.meta.temporal, + "code_exact_occurrence", + binding.clone(), + ) + .await + .expect("real mounted artifact admission"); + let active = registry.exact_occurrence(port_context, &request).await; + assert!( + matches!(active, RetrievalPortOutcome::Completed(_)), + "{active:?}" + ); + + let cancelled = context.with_cancellation( + CancellationContext::cancelled("cancel.callable-exact", query_finished_at()) + .expect("cancelled request context"), + ); + let outcome = execute_prepared_exact_query( + &prepared, + &RetrievalPortContext { + request: &cancelled, + operation: &operation, + }, + &request, + binding, + ); + let RetrievalPortOutcome::Cancelled(evidence) = outcome else { + panic!("a cancelled exact read must retain its typed outcome: {outcome:?}"); + }; + assert_eq!(evidence.temporal.source_generation, Some(generation)); + assert!(evidence.payload.is_none()); + assert!( + evidence + .omissions + .iter() + .any(|omission| omission.reason == OmissionReason::Cancelled) + ); + assert_eq!( + evidence + .cancellation + .expect("cancellation observation") + .stage, + CancellationStage::DuringRead + ); + registry.shutdown().await; + } #[test] fn generation_resolution_wait_reserves_outer_settlement_margin() { diff --git a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/tests/mod.rs b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/tests/mod.rs index a8cf5e9564..abf3684028 100644 --- a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/tests/mod.rs +++ b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/tests/mod.rs @@ -69,7 +69,7 @@ fn canonical_temp_root() -> std::path::PathBuf { canonical_root_identity(&std::env::temp_dir()) } -struct GitFixture { +pub(super) struct GitFixture { root: TempDir, } @@ -81,7 +81,7 @@ const RETAINED_REVISION_0: &[(&str, &str)] = &[("src/lib.rs", "pub fn retained_revision() -> usize { 0 }\n")]; impl GitFixture { - fn new(files: &[(&str, &str)]) -> Self { + pub(super) fn new(files: &[(&str, &str)]) -> Self { if files == ALPHA_LIB_V1 { return Self::from_template(alpha_lib_v1_template()); } @@ -130,7 +130,7 @@ impl GitFixture { Self { root } } - fn path(&self) -> &Path { + pub(super) fn path(&self) -> &Path { self.root.path() } @@ -619,7 +619,7 @@ impl RetrievalExecutionControl for ReadyRetrievalControlV1 { } } -fn application_context( +pub(super) fn application_context( operation: &tracedecay_contracts::ApplicationOperation, repository: RepositoryId, worktree: WorktreeId, @@ -655,7 +655,7 @@ fn application_context( .expect("request context") } -fn query_meta() -> RetrievalRequestMeta { +pub(super) fn query_meta() -> RetrievalRequestMeta { RetrievalRequestMeta::current( PageRequest::first(16).expect("page"), ResultProjection::Evidence, @@ -1014,7 +1014,7 @@ fn core_search_request(query: &str) -> super::query_runtime::QuerySearchExecutio /// Mount one worktree, publish an initial generation, and mount the core /// query authority for its exact scope. -async fn mounted_core_query_worktree( +pub(super) async fn mounted_core_query_worktree( fixture: &GitFixture, store: &TempDir, ) -> (CodeIndexSchedulerRegistryV1, ResolvedScope) { @@ -1543,7 +1543,7 @@ async fn wait_for_dashboard_ready(registry: &CodeIndexSchedulerRegistryV1, path: /// through the text owner, so that slot is the typed receipt this wait joins. /// The text lane publishes the per-worktree serving-generation watch, so /// [`wait_until_serving_seat`] blocks on that signal rather than sampling. -async fn wait_for_queryable_text_generation( +pub(super) async fn wait_for_queryable_text_generation( registry: &CodeIndexSchedulerRegistryV1, path: &Path, ) -> super::LatestCodeTextGenerationV1 { diff --git a/crates/tracedecay-query/src/retrieval/lexical/projection/in_memory.rs b/crates/tracedecay-query/src/retrieval/lexical/projection/in_memory.rs index 4bd651d1a3..cdb752f968 100644 --- a/crates/tracedecay-query/src/retrieval/lexical/projection/in_memory.rs +++ b/crates/tracedecay-query/src/retrieval/lexical/projection/in_memory.rs @@ -885,6 +885,7 @@ impl CodeLexicalProjectionAdapterV1 { }; pairs.push((candidate, evidence)); } + retrieval_checkpoint(request.control)?; pairs.sort_by(|left, right| { left.0 .source_occurrence_id @@ -893,6 +894,9 @@ impl CodeLexicalProjectionAdapterV1 { let mut candidates = Vec::with_capacity(pairs.len()); let mut evidence_by_occurrence = BTreeMap::new(); for (ordinal, (mut candidate, evidence)) in pairs.into_iter().enumerate() { + if ordinal.is_multiple_of(RETRIEVAL_CANDIDATE_BATCH_SIZE) { + retrieval_checkpoint(request.control)?; + } candidate.ordinal_rank = ordinal as u32; evidence_by_occurrence.insert(candidate.source_occurrence_id.clone(), evidence); candidates.push(candidate); @@ -1304,6 +1308,7 @@ where }; pairs.push((candidate, evidence)); } + retrieval_checkpoint(request.control)?; pairs.sort_by(|left, right| { left.0 .source_occurrence_id @@ -1312,6 +1317,9 @@ where let mut candidates = Vec::with_capacity(pairs.len()); let mut evidence_by_occurrence = BTreeMap::new(); for (ordinal, (mut candidate, evidence)) in pairs.into_iter().enumerate() { + if ordinal.is_multiple_of(RETRIEVAL_CANDIDATE_BATCH_SIZE) { + retrieval_checkpoint(request.control)?; + } candidate.ordinal_rank = ordinal as u32; evidence_by_occurrence.insert(candidate.source_occurrence_id.clone(), evidence); candidates.push(candidate); diff --git a/crates/tracedecay-query/tests/search_quality_suite/candidate_producers.rs b/crates/tracedecay-query/tests/search_quality_suite/candidate_producers.rs index dd07a39b61..f41e9056e6 100644 --- a/crates/tracedecay-query/tests/search_quality_suite/candidate_producers.rs +++ b/crates/tracedecay-query/tests/search_quality_suite/candidate_producers.rs @@ -5862,6 +5862,73 @@ fn exact_candidate_scan_stops_before_the_next_batch_after_cancellation() { ); } +#[test] +fn in_memory_rebuilds_observe_cancellation_at_phase_and_batch_boundaries() { + let fixture = real_lexical_source_fixture_with_files(256); + let (pages, _) = drain_verified_pages(&fixture, 128); + let projection = CodeLexicalProjectionAdapterV1::new_admitted( + fixture.metadata.clone(), + pages + .iter() + .flat_map(|page| page.chunks().iter().cloned()) + .collect::>(), + page_symbol_displays(&pages), + ) + .expect("real admitted in-memory projection"); + + // Empty rebuilds must still consult the phase boundary. For the wide + // fixture, cancellation occurs after enough observations to enter a later + // rebuild batch; entry and final checks alone cannot trigger it. + for (term, cancel_at, has_matches) in [("absentzzxyz", 2, false), ("widget", 10, true)] { + let control = CancelAtObservation::new(cancel_at); + let mut request = lexical_request(term, &[term], &[], &[], 0, 1024); + request.generation = fixture.metadata.generation.clone(); + let baseline = complete( + projection + .read_lexical_postings(&request) + .expect("active lexical read"), + ); + assert_eq!(baseline.candidates.len() > 128, has_matches); + request.control = &control; + assert_eq!( + projection.read_lexical_postings(&request), + Err(RetrievalPortError::Cancelled) + ); + assert_eq!(control.observations(), cancel_at); + } + + let authority = CentralExactAdmissionAuthorityV1::new(id("exact-rules.v1")); + let exact = projection.exact_adapter(authority.clone()); + for (query, cancel_at, has_matches) in [ + (r#""absentzzxyz""#, 3, false), + (r#""return value""#, 9, true), + ] { + let control = CancelAtObservation::new(cancel_at); + let view = query_view(query); + let base = base_request(query, 1024); + let mut request = ExactLaneRequest { + literals: authority.parse_literals(&view, &base), + base, + query_view: &view, + generation: fixture.metadata.generation.clone(), + budget: budget(1024), + control: &ACTIVE_CONTROL, + }; + let baseline = complete( + exact + .read_exact_postings(&request) + .expect("active exact read"), + ); + assert_eq!(baseline.candidates.len() > 128, has_matches); + request.control = &control; + assert_eq!( + exact.read_exact_postings(&request), + Err(RetrievalPortError::Cancelled) + ); + assert_eq!(control.observations(), cancel_at); + } +} + #[test] fn disk_artifact_ledger_charges_stay_page_local_across_corpus_scaling() { let control = ArtifactControl { cancelled: false }; From 11ab3edc20a37f98937cf7fc267483778c332e14 Mon Sep 17 00:00:00 2001 From: ScriptedAlchemy Date: Thu, 17 Sep 2026 11:15:55 -0700 Subject: [PATCH 07/14] refactor(mcp): simplify routing and ledger paths --- crates/tracedecay/src/mcp/server/ledger.rs | 88 +++++++++++---------- crates/tracedecay/src/mcp/server/routing.rs | 69 ++++++++-------- 2 files changed, 84 insertions(+), 73 deletions(-) diff --git a/crates/tracedecay/src/mcp/server/ledger.rs b/crates/tracedecay/src/mcp/server/ledger.rs index 42c73405ae..ab2425e444 100644 --- a/crates/tracedecay/src/mcp/server/ledger.rs +++ b/crates/tracedecay/src/mcp/server/ledger.rs @@ -9,6 +9,51 @@ use tracedecay_global_db::RegisteredGlobalDb; /// is *finite* — a wedged recorder task can never hang the caller (tests, /// shutdown drains) indefinitely as the previous unbounded loop allowed. const LEDGER_SETTLE_TIMEOUT: Duration = Duration::from_secs(10); +const MAX_SPAN_IDENTIFIER_BYTES: usize = 256; + +fn bounded_span_identifier(value: Option<&str>) -> Option { + value + .filter(|value| { + !value.is_empty() + && value.len() <= MAX_SPAN_IDENTIFIER_BYTES + && !value.chars().any(char::is_control) + }) + .map(str::to_string) +} + +fn derive_hook_span_git_context( + cwd: &Path, + project_root: PathBuf, + active_project_root: &Path, +) -> Option<(String, Option)> { + let deadline = tracedecay_runtime_core::cancellation::MonotonicDeadline::at( + std::time::Instant::now() + std::time::Duration::from_secs(2), + ); + let cancellation = tracedecay_runtime_core::cancellation::CancellationToken::new(); + let worktree_raw = + match tracedecay_runtime_core::git_discovery::discover_repository_identity_with_control( + cwd, + deadline, + &cancellation, + ) { + tracedecay_runtime_core::git_discovery::GitRepositoryIdentityOutcome::Resolved( + identity, + ) => identity.worktree_root, + tracedecay_runtime_core::git_discovery::GitRepositoryIdentityOutcome::NotRepository => { + project_root + } + tracedecay_runtime_core::git_discovery::GitRepositoryIdentityOutcome::Unknown(_) => { + return None; + } + }; + let worktree_raw = + hook_events::authorize_add_branch_at_root(&worktree_raw, active_project_root).ok()?; + let worktree = git_correlation::normalize_worktree(&worktree_raw.to_string_lossy()); + let branch = bounded_span_identifier( + tracedecay_runtime_core::branch::current_branch(&worktree_raw).as_deref(), + ); + Some((worktree, branch)) +} fn configuration_authority_unavailable(detail: impl std::fmt::Display) -> TraceDecayError { TraceDecayError::Config { @@ -431,22 +476,11 @@ impl McpServer { event: &hook_events::HookEvent, selected: &crate::mcp::project_route::ResolvedProjectRoute, ) { - const MAX_SPAN_IDENTIFIER_BYTES: usize = 256; - let Some(route) = event.route.as_ref() else { return; }; - let bounded_identifier = |value: Option<&str>| { - value - .filter(|value| { - !value.is_empty() - && value.len() <= MAX_SPAN_IDENTIFIER_BYTES - && !value.chars().any(char::is_control) - }) - .map(str::to_string) - }; - let Some(session_id) = bounded_identifier(route.session_id.as_deref()) + let Some(session_id) = bounded_span_identifier(route.session_id.as_deref()) .and_then(|value| tracedecay_privacy::protect_sensitive_structural_id(&value).ok()) else { return; @@ -461,7 +495,7 @@ impl McpServer { let Some(db) = self.project_session_db.clone() else { return; }; - let thread_id = bounded_identifier(route.thread_id.as_deref()) + let thread_id = bounded_span_identifier(route.thread_id.as_deref()) .and_then(|value| tracedecay_privacy::protect_sensitive_structural_id(&value).ok()); let ts = crate::project::current_timestamp(); // Session-only pre-debounce: the full key needs branch/worktree, which @@ -498,33 +532,7 @@ impl McpServer { // spawn git, so it runs on the blocking pool, off the // notification hot path. let derived = tokio::task::spawn_blocking(move || { - let deadline = tracedecay_runtime_core::cancellation::MonotonicDeadline::at( - std::time::Instant::now() + std::time::Duration::from_secs(2), - ); - let cancellation = tracedecay_runtime_core::cancellation::CancellationToken::new(); - let worktree_raw = match tracedecay_runtime_core::git_discovery::discover_repository_identity_with_control( - &cwd, - deadline, - &cancellation, - ) { - tracedecay_runtime_core::git_discovery::GitRepositoryIdentityOutcome::Resolved( - identity, - ) => identity.worktree_root, - tracedecay_runtime_core::git_discovery::GitRepositoryIdentityOutcome::NotRepository => { - project_root - } - tracedecay_runtime_core::git_discovery::GitRepositoryIdentityOutcome::Unknown(_) => { - return None; - } - }; - let worktree_raw = - hook_events::authorize_add_branch_at_root(&worktree_raw, &active_project_root) - .ok()?; - let worktree = git_correlation::normalize_worktree(&worktree_raw.to_string_lossy()); - let branch = bounded_identifier( - tracedecay_runtime_core::branch::current_branch(&worktree_raw).as_deref(), - ); - Some((worktree, branch)) + derive_hook_span_git_context(&cwd, project_root, &active_project_root) }) .await; let Ok(Some((worktree, branch))) = derived else { diff --git a/crates/tracedecay/src/mcp/server/routing.rs b/crates/tracedecay/src/mcp/server/routing.rs index 7a734ce0bb..f5f439e84b 100644 --- a/crates/tracedecay/src/mcp/server/routing.rs +++ b/crates/tracedecay/src/mcp/server/routing.rs @@ -251,6 +251,38 @@ pub(crate) async fn resolve_private_project_route( .await } +async fn resolve_private_selected_path( + requested_path: &Path, + registry_db: &RegisteredGlobalDb, + discovery: &RepositoryDiscovery, +) -> Result { + match resolve_initialize_root_project_path(requested_path, registry_db, discovery).await { + Ok(Some(path)) => Ok(path), + Ok(None) => Err(ProjectRouteFailure { + kind: ProjectRouteFailureKind::NotFound, + detail: format!( + "workspace {} did not resolve to a registered project", + requested_path.display() + ), + }), + Err(InitializeRootResolutionError::AmbiguousIdentity) => Err(ProjectRouteFailure { + kind: ProjectRouteFailureKind::Ambiguous, + detail: format!( + "workspace {} matches multiple registered projects", + requested_path.display() + ), + }), + Err(InitializeRootResolutionError::AuthorityUnavailable) => Err(ProjectRouteFailure { + kind: ProjectRouteFailureKind::Unavailable, + detail: "private project route authority is unavailable".to_owned(), + }), + Err(InitializeRootResolutionError::Discovery(reason)) => Err(ProjectRouteFailure { + kind: ProjectRouteFailureKind::Unavailable, + detail: format!("repository discovery {reason}"), + }), + } +} + async fn resolve_private_project_route_within( requested_path: &Path, registry_db: Option<&RegisteredGlobalDb>, @@ -264,38 +296,9 @@ async fn resolve_private_project_route_within( }); }; let selected_path = - match resolve_initialize_root_project_path(requested_path, registry_db, discovery).await { - Ok(Some(path)) => path, - Ok(None) => { - return WorkspaceProjectRoute::Failed(ProjectRouteFailure { - kind: ProjectRouteFailureKind::NotFound, - detail: format!( - "workspace {} did not resolve to a registered project", - requested_path.display() - ), - }); - } - Err(InitializeRootResolutionError::AmbiguousIdentity) => { - return WorkspaceProjectRoute::Failed(ProjectRouteFailure { - kind: ProjectRouteFailureKind::Ambiguous, - detail: format!( - "workspace {} matches multiple registered projects", - requested_path.display() - ), - }); - } - Err(InitializeRootResolutionError::AuthorityUnavailable) => { - return WorkspaceProjectRoute::Failed(ProjectRouteFailure { - kind: ProjectRouteFailureKind::Unavailable, - detail: "private project route authority is unavailable".to_owned(), - }); - } - Err(InitializeRootResolutionError::Discovery(reason)) => { - return WorkspaceProjectRoute::Failed(ProjectRouteFailure { - kind: ProjectRouteFailureKind::Unavailable, - detail: format!("repository discovery {reason}"), - }); - } + match resolve_private_selected_path(requested_path, registry_db, discovery).await { + Ok(path) => path, + Err(failure) => return WorkspaceProjectRoute::Failed(failure), }; let context = match registry_db .project_registry_context_by_alias(&selected_path) @@ -415,7 +418,7 @@ async fn resolve_initialize_root_project_path( .await { Ok(Some(context)) => { - candidates.push((identity.worktree_root, context.project.project_id)) + candidates.push((identity.worktree_root, context.project.project_id)); } Ok(None) => {} Err(_) => return Err(InitializeRootResolutionError::AuthorityUnavailable), From 417831a1742f85d63d22892ca543094c3c611757 Mon Sep 17 00:00:00 2001 From: ScriptedAlchemy Date: Thu, 17 Sep 2026 11:31:08 -0700 Subject: [PATCH 08/14] fix(memory): mark transport barrier content as feature-bound --- crates/tracedecay-session-memory/src/fact_store/envelope.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/crates/tracedecay-session-memory/src/fact_store/envelope.rs b/crates/tracedecay-session-memory/src/fact_store/envelope.rs index ea7433d855..507178cc03 100644 --- a/crates/tracedecay-session-memory/src/fact_store/envelope.rs +++ b/crates/tracedecay-session-memory/src/fact_store/envelope.rs @@ -259,7 +259,7 @@ impl DatabaseFactStore<'_> { async fn execute_project_memory_write( db: Database, write_control: FactWriteControl, - barrier_content: Option, + _barrier_content: Option, work: impl for<'tx> FnOnce( &'tx Transaction<'_>, ) -> Pin> + Send + 'tx>> @@ -302,7 +302,7 @@ async fn execute_project_memory_write( // expires after the commit point reproducible. #[cfg(feature = "test-transport")] crate::fact_store::commit_barrier::wait_after_durable_fact_commit( - barrier_content.as_deref(), + _barrier_content.as_deref(), ) .await; Ok(value) From bff5690451e01c2f4fea2693b534a98ef1729ffc Mon Sep 17 00:00:00 2001 From: ScriptedAlchemy Date: Thu, 17 Sep 2026 11:55:38 -0700 Subject: [PATCH 09/14] docs(repo): make guidance portable and prune obsolete docs --- .cursor/skills/zack-mode/SKILL.md | 1 - AGENTS.md | 26 +- CLAUDE.md | 4 - CONTRIBUTING.md | 82 ++- NOTES.md | 608 ------------------ docs/CARGO-CONTENTION-POLICY.md | 69 -- docs/DESIGN-DOC.md | 401 ------------ docs/EXTRACTOR-MIGRATION-GUIDE.md | 61 -- docs/MORE-LANGUAGES-SUPPORT.md | 188 ------ docs/PLUGIN-VALIDATION.md | 5 +- docs/PLUGINS-DESIGN.md | 207 ------ docs/QUERY-OPTIMIZATIONS.md | 29 - docs/REBRAND-COMPATIBILITY-POLICY.md | 3 +- docs/TRACEDECAY-ALTERNATIVES.md | 56 -- docs/TRACEDECAY-WHATSNEW.md | 305 --------- docs/plans/tracedecay-v2/00-plan-set-index.md | 5 +- .../39-embedded-grafeo-graph-database.md | 2 +- docs/plans/tracedecay-v2/NEXT.md | 5 +- dogfood-journey.md | 6 +- scripts/agent-worktree.sh | 3 +- 20 files changed, 78 insertions(+), 1988 deletions(-) delete mode 100644 NOTES.md delete mode 100644 docs/CARGO-CONTENTION-POLICY.md delete mode 100644 docs/DESIGN-DOC.md delete mode 100644 docs/EXTRACTOR-MIGRATION-GUIDE.md delete mode 100644 docs/MORE-LANGUAGES-SUPPORT.md delete mode 100644 docs/PLUGINS-DESIGN.md delete mode 100644 docs/QUERY-OPTIMIZATIONS.md delete mode 100644 docs/TRACEDECAY-ALTERNATIVES.md delete mode 100644 docs/TRACEDECAY-WHATSNEW.md diff --git a/.cursor/skills/zack-mode/SKILL.md b/.cursor/skills/zack-mode/SKILL.md index 8bde5becaf..d12229632f 100644 --- a/.cursor/skills/zack-mode/SKILL.md +++ b/.cursor/skills/zack-mode/SKILL.md @@ -59,4 +59,3 @@ matching guide instead of copying its rules here: - Read the `discovering-tracedecay` skill (tracedecay plugin) for TraceDecay operations. - Read the `ripwire-router` skill (Ripwire agent skills) for Ripwire task routing. -- Read the `cargo-hauler` skill (cargo-hauler plugin) for Cargo execution. diff --git a/AGENTS.md b/AGENTS.md index 9d6c86ea83..2ec7fd6918 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -21,12 +21,20 @@ unauthorized external action after completing independent, authorized work. ## Working checkout authority -- Always work in the primary checkout at `/fast/projects/tracedecay`. -- Always work on `codex/tracedecay-total-redesign-plan-reopened`, the head branch - of PR #707. Merge outside work into this branch before continuing. -- Do not create or use linked worktrees or work from another branch. -- Multiple agents may work concurrently in the primary checkout; preserve peer - edits and stage only the paths owned by the current task. +- Use the checkout supplied by the current task. Resolve its root with + `git rev-parse --show-toplevel`; never assume a machine-specific absolute path. +- Inspect `git status --short`, `git branch --show-current`, and + `git worktree list` before changing branches or files. Honor an explicit task + branch; otherwise stay on the current branch. For a PR, resolve its head with + `gh pr view --json headRefName,headRepositoryOwner,headRepository`. +- Do not switch branches, merge other work, or create linked worktrees merely + because an old plan names them. If the requested checkout or branch is + unavailable, report the mismatch instead of substituting an unrelated tree. +- Keep commands and maintained instructions repo-relative. Put machine-local + build caches and host settings in local configuration, not mandatory + repository guidance. +- Multiple agents may work concurrently in a checkout; preserve peer edits and + stage only the paths owned by the current task. ## Layout @@ -57,7 +65,8 @@ unauthorized external action after completing independent, authorized work. ## Build & test -- Edition 2024, resolver 3. +- Edition 2024, resolver 3. Use the toolchain pinned in `rust-toolchain.toml`. + Run `cargo ` normally. - Dashboard: `npm run build` (rsbuild), `npm run typecheck` (`tsc --noEmit`), `npm test` (vitest) from `dashboard/`. - libtest `--exact` requires the full module path and exits 0 when a filter @@ -80,7 +89,8 @@ unauthorized external action after completing independent, authorized work. ## Conventions -- Commits: `(): ` (subject ≤ 72 chars) with one of +- Commits: `(): ` (scope optional; full header ≤ 72 chars) + with one of `build`, `chore`, `ci`, `docs`, `feat`, `fix`, `perf`, `refactor`, `revert`, `style`, `test`. Every non-merge commit message must pass commitlint (`npm run lint:commit`, configured in `commitlint.config.cjs`; the diff --git a/CLAUDE.md b/CLAUDE.md index 99b5abbc08..eb687d5aa6 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -1,7 +1,3 @@ # Claude Notes See [AGENTS.md](AGENTS.md) for repository guidance. - -Run plain `cargo ` through the configured build broker. Do not prefix -with `kache` or set `CARGO_TARGET_DIR`; use the installed cargo-hauler guidance -when submitting or waiting on builds. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 7d8c7cf73a..8343ccf630 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -7,11 +7,17 @@ Thanks for your interest in contributing! This guide covers everything you need ```bash git clone https://github.com/ScriptedAlchemy/tracedecay.git cd tracedecay -cargo build -cargo nextest run --workspace --all-features --no-fail-fast +cargo build -p tracedecay-cli +cargo test-ci ``` -Requires **Rust 1.85+** (edition 2024) and **Node.js 22+ with npm**. +Use the Rust toolchain pinned in `rust-toolchain.toml` (edition 2024) and +**Node.js 22+ with npm**. Install `cargo-nextest` to run the `test-ci` and +`test-all` aliases defined in `.cargo/config.toml`. Commands below run from the +repository root unless noted. + +Use your current checkout; no particular absolute path or historical PR branch +is required. See [AGENTS.md](AGENTS.md) for checkout safety and shared-work rules. The dashboard bundle at `dashboard/app-dist/` is generated output and is git-ignored, so a fresh clone has none. The CLI build script @@ -52,46 +58,52 @@ This tree map is for source orientation only. The precedence and acceptance. ``` -src/ Main tracedecay crate (daemon, MCP tools, sessions, application) -crates/ Workspace members (code-extraction, graph-db, domain, hosts, …) -dashboard/ Embedded React dashboard -plugin/ Host bundles (Claude, Codex, Cursor, Kimi, OpenCode) -tests/ Integration suites -docs/ Design docs and the V2 roadmap +crates/tracedecay/ Composition-root library and its integration suites +crates/tracedecay-cli/ Shipped tracedecay binary and CLI integration suites +crates/ Workspace members (code-extraction, graph-db, domain, hosts, …) +dashboard/ Embedded React dashboard +plugin/ Host bundles (Claude, Codex, Cursor, Kimi, OpenCode) +tests/ Shared fixtures, distribution suites, and cross-crate gates +docs/ Design docs and the V2 roadmap ``` ## Feature Flags -tracedecay supports more than 50 languages. `Cargo.toml` is the source of truth -for the exact feature membership: +tracedecay supports more than 50 languages. The root `Cargo.toml` is a virtual +workspace, not a package. `crates/tracedecay-cli/Cargo.toml` and +`crates/tracedecay/Cargo.toml` expose the language tiers; +`crates/tracedecay-code-extraction/Cargo.toml` owns grammar feature membership: | Feature | Coverage | |---------|----------| | `lite` | Core extractors such as Rust, Go, Java, TypeScript/JS, Python, C/C++, Kotlin, C#, and Swift | | `medium` | `lite` plus Dart, Pascal, PHP, Ruby, Bash, Protobuf, PowerShell, Nix, and VB.NET | -| `full` (default) | `medium` plus all remaining `lang-*` features listed in `Cargo.toml` | +| `full` (default) | `medium` plus the remaining grammars selected by the extractor crate's `full` feature | Build with fewer languages for faster compile times during development: ```bash -cargo build --no-default-features --features lite -cargo nextest run --no-default-features --features lite +cargo build -p tracedecay-cli --no-default-features --features lite +cargo nextest run -p tracedecay-code-extraction --no-default-features --features lite ``` ## Making Changes -1. **Fork and branch** from `master` for stable changes, `beta` for experimental features. -2. **Write tests.** Every extraction change should have a corresponding test in `tests/`. Follow the existing pattern: create a fixture in `tests/fixtures/` and assert on extracted nodes/edges. -3. **Run the full test suite** before submitting: +1. **Use the task's branch**, or branch from `master` for a new contribution. Confirm the target before working on a release-channel branch. +2. **Write tests in the owning crate.** Extraction changes belong with `crates/tracedecay-code-extraction/`; follow nearby inline tests or crate-local integration suites and assert on extracted nodes/edges. +3. **Run focused tests** for the affected behavior, then broaden for unresolved risk. Documentation-only edits do not require application builds. For the hosted acceptance selection: ```bash - cargo nextest run --workspace --all-features --no-fail-fast + cargo test-ci ``` Cargo-launched test processes are isolated from your real `~/.tracedecay` profile: `.cargo/config.toml` pins `TRACEDECAY_DATA_DIR` to `target/test-profile/.tracedecay` (enforced by - `tests/core_cli_suite/test_profile_isolation_test.rs`). Tests that need a private profile + `crates/tracedecay-cli/tests/core_cli_suite/test_profile_isolation_test.rs`). Tests that need a private profile should still override it per-test, e.g. via `common::TraceDecayStorageEnvGuard` or `common::apply_tracedecay_home_env`. + `cargo test-all` additionally enables every optional feature; it is broader + than the hosted selection. Check each suite's `required-features` before + selecting it and confirm that the run executed tests rather than matching zero. 4. **Format your code** with the standard Rust toolchain: ```bash cargo fmt @@ -115,8 +127,8 @@ cargo clippy --workspace --all-targets ``` This check is blocking in CI: the workflow fails if `cargo clippy --workspace ---all-targets` exits non-zero. The crate-level lint policy in `src/lib.rs` -currently denies `clippy::all`, `clippy::unwrap_used`, and +--all-targets` exits non-zero. The composition-root lint policy in +`crates/tracedecay/src/lib.rs` currently denies `clippy::all`, `clippy::unwrap_used`, and `clippy::expect_used`; new violations of those lints must be fixed or justified with the narrowest practical `#[allow(...)]` at the affected item. Do not add a broad allow or weaken the crate policy just to get CI green. @@ -127,8 +139,8 @@ but they do not block CI unless a future policy change promotes a specific lint to `deny`. There is no separate Clippy baseline file today. If a policy change intentionally -promotes additional advisory lints to blocking, update `src/lib.rs`, fix or -narrowly allow the existing violations in the same change, and update this +promotes additional advisory lints to blocking, update the owning crate's lint +policy, fix or narrowly allow the existing violations in the same change, and update this section so the contributor command and blocking/advisory split still match CI. ## Adding a New Language Extractor @@ -136,9 +148,14 @@ section so the contributor command and blocking/advisory split still match CI. 1. Add a tree-sitter grammar dependency (or vendor it under `vendor/`). 2. Create `crates/tracedecay-code-extraction/src/{lang}_extractor.rs` implementing the `LanguageExtractor` trait. 3. Register it in `LanguageRegistry` with a feature flag (e.g., `lang-{name}`) in that crate's `lib.rs` and `Cargo.toml`. -4. Add a test module `crates/tracedecay-code-extraction/tests/{lang}.rs` (inline source or a fixture under `crates/tracedecay-code-extraction/fixtures/`). +4. Add a test module under `crates/tracedecay-code-extraction/tests/main/` and register it in that directory's `main.rs`; follow nearby source and fixture patterns. 5. Update the feature flag tables in that crate's `Cargo.toml` and this document. +When sharing traversal helpers, preserve child ordering, named versus anonymous +node handling, and direct-child versus descendant semantics. Compare against +`crates/tracedecay-code-extraction/src/traversal.rs` and keep language-specific +behavior local; cover the consuming extractor before and after consolidation. + ## Validating Plugins and Skills Changes under `plugin/` or `crates/tracedecay-agent-hosts/` are covered @@ -148,7 +165,7 @@ schema-validation workflow. `plugin/skills/` is the shared source of truth for bundled skills — do not fork host-specific copies. Before submitting, run: ```bash -cargo nextest run -E 'binary(=agent_suite)' +cargo nextest run -p tracedecay --features test-helpers --test agent_suite ``` See [`docs/PLUGIN-VALIDATION.md`](docs/PLUGIN-VALIDATION.md) for the full @@ -160,9 +177,9 @@ ecosystem bundle correctly. `dashboard/src/contracts/generated.ts`, `dashboard/src/contracts/index.ts`, and `dashboard/codegen/schemas/dashboard-contracts.schema.json` are generated, not hand-written. The Rust `schemars` output is authoritative: the codegen CLI -shells out to `cargo test --test dashboard_contract_schema_export -- --ignored -writes_dashboard_contract_schema`, regenerates all three files, and compares -them byte-for-byte with what is committed. +exports the schema through the `tracedecay-dashboard-api` library's ignored +`contract_schema::tests::writes_dashboard_contract_schema` test, regenerates all +three files, and compares them byte-for-byte with what is committed. After changing any Rust type that crosses the dashboard API boundary: @@ -184,14 +201,14 @@ contract change instead. ## Running Specific Tests ```bash -# All extractor tests for a specific language (a module of the `main` test binary) +# All extractor tests for a specific language cargo nextest run -p tracedecay-code-extraction --test main -E 'test(/^rust::/)' # A single test by name -cargo nextest run test_find_stale_files +cargo nextest run -p tracedecay-code-extraction --test main test_rust_file_node_is_root # Only sync-related tests -cargo nextest run sync +cargo nextest run -p tracedecay --features test-helpers sync ``` ## Commit Messages @@ -225,7 +242,8 @@ behavior. ## Pull Requests - Target `master` for bug fixes and stable features. -- Target `beta` for experimental or breaking changes. +- Confirm the target branch with the maintainer for release-channel work; do not + infer it from an archived plan or PR number. - Keep PRs focused — one logical change per PR. - Include test coverage for new behavior. - Do not hand-edit `CHANGELOG.md`; release automation generates it from diff --git a/NOTES.md b/NOTES.md deleted file mode 100644 index f7db595f87..0000000000 --- a/NOTES.md +++ /dev/null @@ -1,608 +0,0 @@ -# Lane notes — fable/test-prune - -## Lane status 2026-09-12 - -Branch `fable/test-prune` (40 prune/style commits on top of the -`fable/test-slop` floor 13452d3ea; `origin/fable/test-slop` is that floor -itself, with no prune commits, and no prune commit had landed on the -integration branch) was merged forward onto -`origin/codex/tracedecay-total-redesign-plan-reopened` in three merge -commits: 03427d2b48 (tip 84f29ec656, 117 conflicted files / 157 hunks), -6fd74ec325 (tip 23dde7695d, 11 files / 14 hunks) and e8c8fa36b2 (tip -0ebeedf7dc after #1241, 3 files / 3 hunks). Pushed as PR #1242. - -Rule applied to every conflict, per test item: a lane deletion stays only -when the item is identical between the lane floor (95893a92a) and the tip, -or the tip's only change was mechanical (rename, extra argument, import -path, constant/prose update, rustfmt). Tests the tip changed to cover new -behaviour keep the tip's version; tests the tip added stay untouched. - -### Kept the tip's version (lane deletion dropped) - -| crate / area | test | why | -| --- | --- | --- | -| tracedecay-code-index | `languages.rs::descriptor_lookups_are_canonical_and_deterministic` | tip pins the extractor revision | -| tracedecay-mcp | `application_output/markdown.rs::canonical_markdown_golden_formats_only_the_supplied_view`, `application_output/view.rs::partial_evidence_extracts_the_typed_coverage_fields` | tip covers the new `Block` payload rendering | -| tracedecay-private-fs | `sharing_violation_is_not_lock_contention` | new `windows()` assertion | -| tracedecay-query | `lexical/tests.rs::candidate_sources_admit_rarest_first_within_the_document_budget` | tip adds the typed `CandidateSourcesPruned` partial outcome | -| tracedecay-runtime-core | `branch_meta.rs::add_and_remove_branch` | new `is_query_eligible` assertions | -| tracedecay-domain | `research/id.rs::integrity_digest_types_accept_supported_algorithms` | tip adds `hex_suffix` assertions | -| tracedecay (mcp_suite) | `memory_facts_test::fact_store_reason_requires_an_entity_selection` | tip adds duplicate-entity denial | -| tracedecay-agent-hosts | `kimi.rs::rendered_plugin_uses_kimi_supported_mcp_command` | tip-new test inside a module the lane had emptied | -| scripts | `test-check-pr-dogfood-output.py::test_strict_accepts_graph_ready_bounded_prefix_with_more_symbols`, `test-linux-test-partitions.py::test_complete_disjoint_partition_passes` | tip refactored / extended them for config summaries and macOS groups | - -### Pruning re-applied at a moved location - -- `daemon/broker_stream_transport.rs` → `daemon/broker_stream_transport_tests.rs`: - `full_close_wait_ignores_request_half_close` (identical; duplicate of the - rmcp receive test). -- `doctor/registry_drift.rs` → `tracedecay-maintenance/src/retention/diagnostics.rs`: - `orphan_finding` + three disposition→kind mapping tests (identical). -- `src/mcp/tools/plugin_conformance_tests.rs` → `tests/product_surface_suite/plugin_conformance.rs`: - tip only renamed it; stays deleted. - -### Pruning dropped because the tip deleted the target itself - -`crates/tracedecay-cli/tests/core_cli_suite/host_cli_fixture.rs` (tip -commit 16e2305a6) and 77 individual items the tip had already removed -(`config/tests.rs`, `store_maintenance/mod.rs`, `handlers/info/status.rs`, -`lifecycle/registry.rs`, … blocks). - -### Measured lib test counts (`cargo test --workspace --lib`, tip 8df152a7b vs merged) - -Crates with no change (capture, host-admission, privacy, sdk, -semantic-contracts, tool-catalog) omitted. - -| crate | tip | merged | pruned | -| --- | ---: | ---: | ---: | -| tracedecay | 1082 | 994 | 88 | -| tracedecay-agent-hosts | 649 | 490 | 159 | -| tracedecay-api | 63 | 51 | 12 | -| tracedecay-application | 649 | 596 | 53 | -| tracedecay-automation | 64 | 42 | 22 | -| tracedecay-automation-runtime | 507 | 442 | 65 | -| tracedecay-code-extraction | 58 | 52 | 6 | -| tracedecay-code-index | 260 | 176 | 84 | -| tracedecay-code-index-retention | 93 | 92 | 1 | -| tracedecay-code-index-runtime | 494 | 468 | 26 | -| tracedecay-configuration | 70 | 63 | 7 | -| tracedecay-contracts | 442 | 379 | 63 | -| tracedecay-daemon-control | 118 | 86 | 32 | -| tracedecay-daemon-identity | 25 | 24 | 1 | -| tracedecay-daemon-protocol | 69 | 61 | 8 | -| tracedecay-daemon-service | 280 | 260 | 20 | -| tracedecay-dashboard-api | 236 | 169 | 67 | -| tracedecay-domain | 238 | 215 | 23 | -| tracedecay-framing | 9 | 6 | 3 | -| tracedecay-global-db | 395 | 365 | 30 | -| tracedecay-graph-db | 187 | 160 | 27 | -| tracedecay-graph-query | 59 | 45 | 14 | -| tracedecay-hooks | 83 | 82 | 1 | -| tracedecay-host-integration | 8 | 7 | 1 | -| tracedecay-lcm | 180 | 158 | 22 | -| tracedecay-lsp | 217 | 183 | 34 | -| tracedecay-maintenance | 161 | 130 | 31 | -| tracedecay-mcp | 372 | 290 | 82 | -| tracedecay-policy | 16 | 14 | 2 | -| tracedecay-private-fs | 20 | 18 | 2 | -| tracedecay-query | 341 | 310 | 31 | -| tracedecay-runtime-core | 480 | 413 | 67 | -| tracedecay-rusqlite-runtime | 374 | 353 | 21 | -| tracedecay-search-eval | 44 | 43 | 1 | -| tracedecay-semantic | 260 | 224 | 36 | -| tracedecay-session-memory | 361 | 296 | 65 | -| tracedecay-session-runtime | 115 | 107 | 8 | -| tracedecay-session-temporal-store | 150 | 137 | 13 | -| tracedecay-sessions | 701 | 563 | 138 | -| tracedecay-source-edit | 93 | 85 | 8 | -| tracedecay-store | 103 | 83 | 20 | -| tracedecay-store-runtime | 149 | 139 | 10 | -| tracedecay-temporal-query | 224 | 177 | 47 | -| **total** | **10744** | **9293** | **1451** | - -Other suites: dashboard vitest 1774 → 1418 (157 files green, -`tsc --noEmit` clean), `sdks/typescript` vitest 36 → 29 green, -`scripts/test-*.py` 117 → 106 green. Integration test binaries -(`crates/*/tests`) were compiled by `cargo check --workspace --all-targets` -but not run or counted here. - -### Verification tickets (hauler) - -- cc-16731 / cc-16946 — `cargo check --workspace --all-targets`: 0 errors, 0 warnings. -- cc-16734 (tip) / cc-16735 (merged) — `cargo test --workspace --lib --no-fail-fast`. - Both ran under load 67–103 on 96 cores and both were red; every - merged-only failure re-ran green in cc-16824, cc-16825, cc-16826, - cc-16827, cc-16828. Failures shared with the tip run - (`code_index_scheduler::*` deadline tests, `extract::tests::canonical_rows_digest_matches_pinned_identity`, - `partitioned_codec::tests::streamed_segment_projection_refuses_an_incomplete_descriptor_set`, - search-eval `report_tests::*`, `accepted_profile_authority::tests::portable_report_requires_the_current_workload_digest`, - `lsp_runtime::advisory_source_tests::incomplete_publication_remains_readable_without_consuming_completed_dedupe`, - `orphan_stores::tests::unregistered_store_sweep_reconciles_interrupted_quarantine`) - are tip-red, not introduced here. -- cc-16947 — `cargo test --lib` agent-hosts / query / semantic / domain after the second merge: green. -- cc-16952 — `cargo test -p tracedecay --lib daemon::core_doctor daemon::maintenance`: 40 passed. - -### Third merge (e8c8fa36b2, tip 0ebeedf7dc) - -Three conflicted files. `orphan_stores/tests.rs`: the lane deletion of -`sweep_unregistered_stores_collects_an_exactly_empty_old_directory` stays -(identical on tip); the tip-added -`unregistered_store_with_a_vanished_manifest_root_is_collected_at_once` -is kept. `host_bundle_acceptance.rs`: use-group only, the lane's trimmed -`agents` import group stays because the two tests it served are unchanged -on tip and remain deleted. `NOTES.md` (both added): this section stays on -top, the #1103 text-build lane notes from tip follow. - -- cc-17119 — `cargo check --workspace --all-targets`: 0 errors, 0 warnings. -- cc-17125 — `product_surface_suite host_bundle_acceptance`: 17 passed. -- cc-17124 — `cargo test -p tracedecay-maintenance --lib`: 128 passed, - 3 failed (`portable_inventory_*` timeouts under load 30–50; two of the - three were already red in the unmodified tip run cc-16734). cc-17153 - re-ran the 6 `portable_inventory` tests single-threaded: 6 passed. - -### Undone - -- Integration suites (`--tests`) were type-checked only; their pruned - counts are not measured. - -# #1103 text-build lane — Fable lane notes - -Worktree: `/fast/tmp/td-text-build-1103` (branch `fable/text-build-1103`). -Target: PR #707 branch `codex/tracedecay-total-redesign-plan-reopened`, merged at `8df152a7b` (measurements) and -again at `9e07045ba0` (clean merge; gate/park/freshness tests, the query freeze test, and clippy re-verified there). -Env: `TRACEDECAY_SKIP_DASHBOARD_BUILD=1 TRACEDECAY_DASHBOARD_BUNDLE_SHA256=806f5649351425b7f80dbb3e088d9032d747dc524922ddc3b823c4ba821f8a1f`. -Not pushed; no PR. - -## Lane status 2026-09-12 - -Journey: `daemon_suite indexing_lifecycle_test::mounted_incremental_lifecycle_preserves_only_complete_compatible_generations` -(`cargo test -p tracedecay --features test-transport --test daemon_suite -- --exact --test-threads=1`), -45 s restart bound untouched. Machine load 40–70 on 96 cores throughout; every number below is under that load -and the before/after pairs are the only fair comparison. - -**Before (merged tip `8df152a7b` + `8cb846989`, hauler cc-16667):** red. At the bound: text 417/772 files, -`phase=bulk_commit`, 26.9 files/s, `last_commit_latency` 0.92 s/64-page batch, `code_graph_serving=pending`. - -**After (`b564e6e3e3`, cc-16936):** still red, but the owner moved. At the bound: text **772/772**, 196,622 chunks, -`phase=index_build` (finalization), `code_graph_serving=ready`. The graph is no longer on the critical path; the -text build (bulk 29 s + finalization ≈ 8 s + verification ≈ 2 s under this load) after a 13–19 s restart reconcile -is what remains. - -Decomposition (Hotpath daemon `target/hotpath/debug/tracedecay` with `test-transport,hotpath`, cold index of the -same 772-file tree = the post-restart phases; report json in `/tmp/td1103/run-hp{2,3}/hotpath.json`): - -| phase | before (hp2, load ~70) | after (hp3, load ~50) | -| --- | --- | --- | -| `code_index.reconcile.pass` (extract + assemble 5.9 s serial + publish) | 17.5 s | 20.6 s | -| text build: `query.artifact.batch.scheduler_wake` ×24 | 64.5 s | 34.6 s | -| of which `batch.sqlite` / `batch.commit` | 27.7 s / 15.3 s | 15.4 s / 4.5 s | -| finalization `advance_wake` (index build; `term_postings_by_term` 3.6–5.6 s) | 18.4 s | 8.1 s | -| `code_graph.activation.total` (`graph_db.sealed_store.build` 17 s, verify 7 s, catalog 3 s) | 40.1 s, **after** text | 29.9 s, **overlapped** | -| wall to `current/complete/fresh` | text ready 85 s → graph ready 123 s | 83 s, graph ready before text | - -Artifacts for the fixture (9.1 MB source, 98,304 one-line fns, 196,620 lexical documents — the `"\n"` window -chunks are already gone): sealed segments 328 MB (772 × 313 KB + 87 MB evidence), lexical artifact **394 MB** -(`term_postings` 91 + `_by_term` 80, `rows` 48, `ngram_postings` 47, dictionary 19, `rows_by_chunk` 17, vocabulary 13+13 -MB via `dbstat`), grafeo 262 MB, interactive catalog read-bundle 117 MB; profile total 1.1 GB. The 2.4 GB of the -issue title does not reproduce; the volume is a data-model question across three stores, not lane-sized. - -Fix in this lane (`b564e6e3e3`): the publication worker drove the whole text projection inline and only then gated -graph prepare/activation on a *ready* text owner, serialising two builds that both consume only the sealed -generation. The projection now runs on its own task, the gate prepares once the owner has *reopened*, and the seat -joins the projection (an unfinished or latched-failed owner still seats nothing; activation is idempotent). Pinned by -`convergence_park_tests::graph_activation_starts_while_the_published_text_owner_is_parked` (times out on the old -ordering — verified by flipping the gate back). `tracedecay-code-index-runtime --lib`: 492 passed, 1 failed — -`verified_empty_source_remains_observable_while_scheduler_is_busy` fails identically on the unmodified tip -(`Elapsed` waiting for `reconciled_without_generation`), not this lane's. Clippy `-D warnings` clean. - -WIP disposition: the four dirty edits were a duplicate of `eda2011eb` (running field-length totals, named -`field_stats_staging` on tip), the parallel-decode test adjustment `683d87d5d` already carries, a tightened -`disk_artifact_defers_statistics_and_serving_indexes_until_freeze` (kept, `8cb846989`, staging-table name fixed), and a -println measurement probe example (checkpointed in `420d7ca9d`, removed in `f2c8d6e597`; `code_lexical_catchup` is -the maintained bench). - -### What remains for #1103 item 4 -1. Text build rate is the wall: 25 batches at 0.6–0.9 s of `batch.sqlite` each (single-row `INSERT` CPU on - `term_postings`/`ngram_postings`, `journal_mode=DELETE synchronous=NORMAL` commit 0.18–0.6 s per batch under load), - plus ~10 s of per-wake sealed-source decode (`restore.file_admit` 6.5 s, `segment_decode` 5.7 s aggregate) that - is not pipelined with the SQLite append, plus 8 s finalization. Under an idle machine the earlier lane measured - 27 s; the restart bound leaves ~25 s for it after a 16–19 s reconcile. -2. Restart reconcile re-seals the recovered batch from scratch: `build.assemble` 5.9 s serial and - `sealed_encode.evidence` 4.4 s serial are the two remaining serial phases in `build.and_publish`. -3. The interactive-catalog read bundle (117 MB) and `graph_db.sealed_store.build` (17 s) are off the critical path now - but still define the graph-ready time when the text build gets faster. -4. Item 5 (hooked saves reconcile only at the 30 s window, `hook_hint_count: 0`) is untouched by this lane. -5. CI: the validated `root-daemon-suite` partition from the issue thread should land with the item-4 fix. - -## Lane status 2026-09-12 — fable/wave2-root (root crate cleanup, ponytail wave 2) - -Worktree `/fast/tmp/td-wave2-root-fable`, base `5292a39d2c` (tip after #1245), target dir -`/fast/tmp/td-target-wave2-root`. Five commits, one per audit item; each compiles under -`cargo check -p tracedecay --lib --tests --features test-helpers` and -`cargo check --workspace --all-targets` (cc-18039) at the branch head. - -| item | outcome | -| --- | --- | -| 1 alias re-exports | `tracedecay::query` had zero consumers; `tracedecay::code_index` had four (workflow handler digest, CLI blocking-thread sizing, one product-surface test, and the scheduler flight test the root compiled via `#[path]`). All retargeted to the sibling crate; CLI gains the edge with `default-features = false`. Both aliases deleted. | -| 2 dispatch table | Forwarding arms for graph (18), info (10), analysis (18), git (8), health (7) moved beside their handlers as one `dispatch_tool` per family in `tracedecay-mcp`; the root lends its admission funnel as `VerifiedGraphOpen`. Root `dispatch_groups.rs` 1478→1104 lines. `LegacyToolCompatibilityOwner` kept (see below); its dead `OWNER` label deleted. | -| 3 in-src tests | 73 `#[cfg(test)]` roots probed by compiling them as an integration target: 66 use private items or sit under private modules, 6 import private `super::` items, 1 (`remote_protocol_tests`, 289 lines) is clean but is the fixture for two blocked siblings. **0 lines movable as pure moves.** Two files the root compiled out of `tracedecay-code-index-runtime/src` via `#[path]` were relocated: the scheduler journeys (2,110 lines) into `tests/daemon_suite/code_index_ignored_dependencies_test`, the census journey (144) into `src/daemon/`. `production_harness` gate moved to the `mod` declaration, deleting 36 per-item repeats and narrowing 3 to `cfg(unix)`. | -| 4 hook runtime | **False claim.** Root `hook_runtime/` (4.9k, 1.4k tests) is the `tracedecay_hook_runtime` action handler composing `tracedecay-hooks` (ledger, config snapshot, envelope types), host-admission, sessions, and `TraceDecay`; `tracedecay-mcp::hook_runtime` is 62 lines of error mapping the root already calls. Zero shared function names; ripwire clone scan across the three trees found no production clone (one 0.82 near-miss: `hook_v2_family_label` vs `HookEventKind::as_key`, different enums). Nothing folded. | -| 5 serve stubs, module name | `ensure_initialized*` had no caller beyond three tombstone assertions: stubs, in-src test, and the two integration assertions deleted; `serve` is now crate-private around the URI decoder. `src/tracedecay.rs` → `src/project.rs`; 236 `crate::tracedecay::` and 71 `tracedecay::tracedecay::` paths retargeted (root, suites, benches, CLI). | - -### Tip-side reds observed (not this lane's) - -- Clippy `-D warnings`: four `clippy::large_futures` errors in `daemon/hook_v2_replay_consumer.rs` - (147, 197, 311) and `project_open_owners/advisory_runtime.rs:1520`; identical bytes in the tip's - CI Clippy job (run 34696973432). Verified this lane with `-A clippy::large_futures`. -- Root lib: `mcp::tools::handlers::search_graph_independence_tests::{tracedecay_search_preserves_lexical_results_when_graph_admission_is_missing, tracedecay_search_refuses_foreign_generation_graph_evidence_without_erasing_results}` - (`node_id` not null) — FAILED in the tip's `Test Linux root-lib` job. -- `code_index_ignored_dependencies_test::flight_tests::aborted_flight_owner_wakes_follower_and_allows_a_fresh_owner` - is a pre-existing intra-test race (fails alone 1/6, module 3/4 in the tip-equivalent in-lib binary of - `/fast/projects/tracedecay` at 5be9a952e7; FAILED in the tip's root-lib CI). Mechanism: after - `owner.abort()` + `hold.release()` the orphaned blocking build still publishes, so the fresh owner's - `expected_generation` is stale (`IgnoredDependency(StaleGeneration)`). Moving the file did not change it. -- `daemon::production_harness::lcm_preserved_profile_journey_test::preserved_profile_lcm_discovery_converges_without_blocking_retrieval` - went `outcome=stale` once under load 43 alongside 25 harness journeys; passes alone 2/2 (46 s each). - -### `LegacyToolCompatibilityOwner` — why it stays - -`admits(name)` is advertised-name membership (`get_tool_definitions()`), consulted after the typed -daemon-surface groups return and before group dispatch. It is not redundant with the binding table: -`MCP_TOOL_BINDING_SPECS` is a static list, so a bound-but-unadvertised name (host-gated -`tracedecay_ast_grep_search`/`_rewrite` when ast-grep is absent) is rejected as unknown only by this -gate. Tools it guards that the root still serves (P0-1 leftovers): `tracedecay_retrieve`, -`tracedecay_remote_status`, `tracedecay_status`, `tracedecay_active_project`, -`tracedecay_project_{list,search,context}`, `tracedecay_admin_sync`, `tracedecay_runtime`, admin -(`hook_runtime`, `admin_cli`, `admin_project`), edit (10 `tracedecay_*_replace/insert/move/rename/rollback/reconcile`), -memory (`automation_run_*`, `analytics`, `skill_*`, `hermes_skill_bridge`), session-workflow -(`diagnose`, `run_affected_tests`, `dashboard`), and every retained-application operation. - -## Lane status 2026-09-12 — fable/wave2-root-collapse (P0-1/P0-2 execution) - -Worktree `/fast/tmp/td-wave2-collapse-fable`, base `14de498296` (tip after #1252), target dir -`/fast/tmp/td-target-wave2-collapse`. One PR per slice; the maintainer merges between slices. - -### Slice 1 — `src/mcp/` → `tracedecay-mcp` (three commits) - -Root `src/mcp/` 42,321 → 30,725 lines (−11,596); `tracedecay-mcp` 36,751 → 48,170. Test attributes -conserved: root 951 → 862, mcp 232 → 320 (89 moved, 1 duplicate test deleted). - -1. **Pure moves** (`git mv`, `pub(crate)`→`pub` on the items the root reads, explicit exports): - -| root module | lines | new home | -| --- | ---: | --- | -| `tools/binding.rs` + `binding/{work,workflow}.rs` | 1,629 | `tracedecay_mcp::tools::binding` | -| `tools/dispatch.rs`, `tools/catalog_discovery.rs` | 928 | `tracedecay_mcp::tools::{dispatch, catalog_discovery}` | -| `tools/handlers/{session_authorities,dashboard_lcm,dashboard_delivery,dashboard_git_correlation}.rs` | 1,787 | `tracedecay_mcp::handlers::*` | -| `tool_analytics.rs`, `scope.rs` | 1,006 | `tracedecay_mcp::{tool_analytics, scope}` | -| `server/{session_refresh,project_host_admission_replay}.rs` | 705 | `tracedecay_mcp::server::*` | -| `dispatch_groups.rs` ceiling block (`TOOL_DISPATCH_CEILING`, `tool_dispatch_{ceiling,budget,deadline_error}`) | 92 | `tracedecay_mcp::tools::dispatch_ceiling` (item cut, needed by the binding table) | -| `project_route.rs::{ProjectRouteFailure, ProjectRouteFailureKind}` | 72 | `tracedecay_mcp::project_route` (item cut, needed by `scope.rs`; the `McpServer`-holding cache stays) | -| `tool_call_support.rs::INTERNAL_DAEMON_TOOL_NAMES` | 7 | beside the binding table | - -2. **Duplicates deleted** (bodies diffed identical before deletion): `binding.rs::multi_root_operation_for_tool` - (ripwire type-2 clone of `handlers::multi_root::operation_for_tool`, 45 tokens — the binding table now - reads the handler-owned lookup like it does for work/workflow), and nine result-shaping helpers plus - `CONTEXT_MEMORY_ANALYTICS_KEY` and one unit test in root `handlers/support.rs` that re-implemented - `tracedecay_mcp::handlers::support` byte-for-byte. Root `support.rs` 243 → 77 lines (selector validation only). - -3. **Handler families whose only root dependency was `TraceDecay`** (`edit`, `workflow` + `workflow/`, - `admin_cli`, `automation_runs`, `skills`; 5,156 lines, 35 tests) → `tracedecay_mcp::handlers::*`. The - root dispatch arms (still bound to `ToolCallRegistryOptions`) call them by path. `json_result` joined - the shared support helpers. Five helpers now borrow an argument they never consumed - (`needless_pass_by_value`; the root allows it, `tracedecay-mcp` does not). - -Edges added to `tracedecay-mcp` (each checked with `cargo tree -p -e normal | rg tracedecay-mcp` -→ empty before adding): agent-hosts, automation, automation-runtime, daemon-service, host-admission, -lcm, maintenance, project, session-runtime; `futures-util`, `hex`, `sha2`, `url`. Dev: dashboard-api -`test-transport`, graph-db, project `test-helpers`. Root re-exports of moved items: none — every root, -CLI, bench, and suite caller retargets to `tracedecay_mcp::…`. - -**Reclassified from the plan's (b) rows — not movable yet:** `server/routing.rs` (holds -`Arc`, `RetainedProjectServerResolver`, and the `WorkspaceProjectRoute` cache), -`server/status_resource.rs` (`impl McpServer`), `hook_runtime/envelope.rs` (calls -`tracedecay_agent_hosts::hooks::protected_native_session_id`; agent-hosts depends on hooks, so a -hooks-crate home would be a cycle — it moves with `hook_runtime/` once `context_scout_lifecycle` -is in daemon-service). `admin_project.rs` waits on the root `bench` runner (used by the CLI too). - -**`LegacyToolCompatibilityOwner` still guards root-only arms:** `tracedecay_retrieve` -(`tool_call_support.rs`, project-route resolver → `McpServer`), `tracedecay_status`/`remote_status`/ -`active_project`/`project_*`/`admin_sync`/`runtime` (`info/`, `CodeIndexReconcileSink` and the other -`crate::mcp::server` port aliases), `tracedecay_hook_runtime` (`crate::daemon::context_scout_lifecycle` -→ slice 2), `tracedecay_admin_project` (`bench`), `tracedecay_analytics` -(`crate::daemon::retained_owner::open_project_retained_memory_target` → slice 2/3), -`tracedecay_dashboard` (`crate::daemon::dashboard_automation`, root `dashboard`/`hooks`, server ports), -and the retained-application family (`retained_catalog.rs`, `ToolCallRegistryOptions`). The owner -itself goes with `handlers/mod.rs` in slice 3. - -### Slice 2 — `src/daemon/` → `tracedecay-daemon-service` (one commit, stacked on slice 1) - -Root `src/daemon/` + `daemon.rs` 98,867 → 92,352 lines (−6,515); `tracedecay-daemon-service` 54,177 → 60,737. -Test attributes conserved: root 862 → 800, daemon-service 262 → 324 (62 moved). - -| root module | lines | new home | -| --- | ---: | --- | -| `shutdown_coordination.rs` (owners, receipts, `ShutdownStatus`) | 588 | `tracedecay_daemon_service::shutdown::owners` (renamed: the crate already has `shutdown_coordination` for `ShutdownCoordinatorV1`) | -| `shutdown_orchestration.rs`, `shutdown_watchdog.rs` | 1,700 | `shutdown::{orchestration, watchdog}` | -| `core_lifecycle.rs` (`DaemonLifecycle`, `DaemonActivity`, drain deadlines) | 399 | `shutdown::lifecycle`; its `McpConnectionLifecyclePort` impl moved beside the port in `tracedecay_mcp::lifecycle` (local trait, foreign type — daemon-service sits below mcp) | -| `context_scout_lifecycle.rs` + `tests.rs` | 1,227 | `tracedecay_daemon_service::context_scout_lifecycle` | -| `doctor_kernel.rs` + `tests.rs` | 1,340 | `tracedecay_daemon_service::doctor_kernel` | -| `core_logging.rs` | 511 | `tracedecay_daemon_service::logging` (CLI retargets `install_stderr_tracing`, `StderrTracingDefault`, `unavailable_error`) | -| `adoption_observation.rs`, `automation_observation.rs` | 345 | same names | - -Edges added to `tracedecay-daemon-service` (each `cargo tree -p -e normal | rg daemon-service` → empty -first): project, automation-runtime, code-index-retention, maintenance, session-runtime, `tracing-subscriber`; -dev: project `test-helpers`. The two `cfg(test)` shutdown join helpers are regated `test-helpers` for the -root's unit tests. `daemon.rs` keeps `pub(crate)` re-exports only for the names its `use super::*` -modules (engine, bootstrap, connection serving, projectless, …) still read; the CLI has no root path left. -The stderr-receipt test now derives its libtest name from `module_path!()` (the hard-coded root path -made the child run vacuous and the assertion falsified it). ripwire `--clones` over daemon-service after the -move: no type-1/2 group touches a moved module (only ≤0.90 type-3 label-map near-misses). - -**The daemon wall (precise).** Every other (b) row of the plan names one of three root authorities: -`DaemonInvocationState` (`invocation_state.rs`, 1,446 lines) holds `StoreAdministration`; -`StoreAdministration` (`branch_admin.rs`, 2,559) holds `Arc`, `SharedHookProjectRouteCache` -and `tracedecay_mcp::{JsonRpcRequest, JsonRpcResponse, McpTransport, ErrorCode}`; `DaemonEngine` -(`engine.rs`) holds both. Because slice 1 made `tracedecay-mcp` depend on `tracedecay-daemon-service`, -nothing that names a `tracedecay-mcp` type can ever live in daemon-service. So `invocation_executor.rs` -(801), `invocation_dispatch.rs` (931), `remote_deletion.rs` (341), `lsp_sessions.rs` (208), -`github_credential_lifecycle.rs` (237), `project_delivery_mount.rs` (46), `http_application_router.rs` -(125), `bootstrap_route.rs` (175), `database_owner_registry.rs` (370, `McpServer`) wait on -`McpServer`/`StoreAdministration`; `wire_io.rs` (448), `core_client.rs` (642), `core_hooks.rs` (137) -write/read MCP JSON-RPC frames and are the *client* seam with `core_handshake.rs` (49) — they stay in the -root (or a client crate), never daemon-service. `core_doctor_schema.rs` is a `#[path]` child of -`core_doctor.rs` (`McpServer`-bound). The plan's "(c) rows as `tracedecay-daemon-service::composition`" -is therefore not reachable: the daemon composition that builds `McpServer` is the root's job (slice 3). - -### Slice 3 — composition (one commit, stacked on slice 2) - -Moves the modules slices 1–2 unblocked, then reclassifies every remaining root module. Root `src/` -144,714 → 116,396 lines over the three slices (−28,318); `src/mcp/` 42,321 → 24,373 (−42%, 14,457 of -what remains is production code); `src/daemon/` + `daemon.rs` 98,867 → 88,735 (−10%). -`tracedecay-mcp` 36,751 → 54,380; `tracedecay-daemon-service` 54,177 → 64,757. Test attributes -conserved in this slice: root 800 → 739, mcp 320 → 360, daemon-service 324 → 345. - -| root module | lines | new home | -| --- | ---: | --- | -| `mcp/tools/handlers/hook_runtime/` (tree) | 4,875 | `tracedecay_mcp::handlers::hook_runtime` (context-scout lifecycle is in daemon-service since slice 2) | -| `mcp/tools/handlers/{analytics,admin_project}.rs`, root `bench.rs` | 1,653 | `tracedecay_mcp::{handlers::analytics, handlers::admin_project, bench}` | -| `daemon/retained_owner.rs` + tests, `dashboard_automation/retained_curator.rs`, `daemon/automation_effect/` (+ journal tests) | 3,637 | `tracedecay_daemon_service::{retained_owner, automation_effect}`; `automation_run_observer` and `scheduler_automation_request_id` (pure identity minting) follow as item cuts | -| `tracedecay_mcp::server::session_refresh` (placed by slice 1) | 393 | `tracedecay_daemon_service::session_refresh` — no MCP coupling, and `StoreAdministration`/`retained_owner` hold it | -| `test_support::host_admission::mcp_session_authorities` | 6 | `tracedecay_mcp::handlers::mcp_session_authorities` behind the new `tracedecay-mcp/test-helpers` feature | - -**Reclassification of what stays (116,396 lines) and why.** Markers: a file is pinned by the first -authority it names. - -| reason | production lines | test lines | files | -| --- | ---: | ---: | --- | -| names `McpServer` (the root server: `mcp/server.rs` + `server/{requests,connection,construction,ledger,rmcp,lifecycle,routing,hook_dispatch,status_resource,project_open_access}`, `mcp/project_route.rs`; and the daemon modules that hold `Arc`: `branch_admin.rs` + `project_retirement.rs`, `project_composition.rs`, `connection_serving.rs`, `engine.rs`, `core_doctor.rs`, `project_open_owners/**`, `project_open_orchestration.rs`, `project_server_lifecycle.rs`, `project_routing.rs`, `database_owner_registry.rs`, `graph_resolution.rs`, `wire_io.rs`, `production_harness.rs`) | 28,728 | 10,819 | 47 | -| names `StoreAdministration` (`scheduler.rs`, `maintenance.rs`, `pr_autotrack.rs` + `runtime.rs`, `branch_admin/remote_deletion_lifecycle*`, `branch_admin/session_runtime_shutdown.rs`, `projectless.rs`, `core_admission.rs`, `project_open_handshake.rs`, `branch_add.rs`, `project_composition/runtime.rs`, `bootstrap_route.rs`, `store_maintenance/`) | 9,894 | 2,619 | 18 | -| names `DaemonInvocationState` (`invocation_state.rs` + `project_invocation.rs`, `bootstrap.rs`, `invocation_dispatch.rs`, `invocation_executor.rs`, `branch_admin/remote_recovery_lifecycle.rs`, `project_composition/code_index_activation.rs`, `daemon.rs`, `remote_deletion.rs`, `github_credential_lifecycle.rs`, `lsp_sessions.rs`, `http_application_router.rs`, `project_delivery_mount.rs`) | 7,236 | 7,080 | 18 | -| names `DaemonEngine` (`scheduler/{combined_effect,effect_admission,host_receipt_review,automation_observation}.rs`, `engine/shutdown.rs`) | 3,047 | 1,323 | 7 | -| MCP JSON-RPC frames (`tracedecay_mcp::{JsonRpcRequest, JsonRpcResponse, McpTransport, BrokerStreamTransport}`): `project_open_admission.rs`, `core_proxy.rs`, `core_client.rs`, `core_hooks.rs`, `tool_call_support.rs` | 3,360 | 5,757 | 10 | -| `ToolCallRegistryOptions` (root dispatch: `handlers/mod.rs`, `dispatch_groups.rs` + `health_dispatch.rs`, `retained_catalog.rs`, `dispatch_controls.rs`, `tools/mod.rs`) and the root server port aliases (`dashboard.rs` handler, `application_surface.rs`, `hook_writes.rs`, `info/`, `support.rs`, `dashboard_automation.rs`, `advisory_runtime/model.rs`) | 4,820 | 3,714 | 24 | -| composition root proper (`lib.rs`, `runtime_ports.rs`, `serve.rs`, `hooks.rs`, `dashboard.rs` + `dashboard/`, `doctor.rs` + `doctor/`, `core_handshake.rs`, `http_application.rs`, `hook_v2_replay_consumer.rs`, `test_support/`, harness/test-only files) | 5,988 | 22,011 | 63 | - -Three authorities pin everything: `McpServer` (root `mcp/server.rs`), `StoreAdministration` -(`branch_admin.rs`, which holds `Arc`, the hook route cache, and MCP frames) and -`DaemonInvocationState` (which holds `StoreAdministration`). They form one strongly connected cluster -with `DaemonEngine`, `project_composition`, and `connection_serving`: the daemon *is* the composition -that builds and serves `McpServer`. With `tracedecay-mcp → tracedecay-daemon-service` fixed, none of it -can enter daemon-service, and moving `McpServer` alone into `tracedecay-mcp` drags the cluster's -daemon half (`branch_admin`, `core_admission`, `dashboard_automation`, the root `dashboard`/`hooks` -composition) along — a ~40k-line move that would make `tracedecay-mcp` the daemon. The honest end -state is therefore not the plan's step 5 ("delete `src/mcp/` and `src/daemon/`") but: the root -`tracedecay` crate **is** the daemon composition crate (engine, connection serving, project -composition, store administration, the root MCP server, the client seam) plus the product -features it composes (`doctor`, `dashboard`, `runtime_ports`, `test_support`). If a smaller -"composition wiring only" root is still wanted, the remaining lever is a `tracedecay-daemon` crate -above mcp and daemon-service that receives the cluster wholesale — a rename of the same lines, not a -reduction — which is a maintainer decision, not a slice. - -Still-movable leftovers not taken here (small, each would need its own edge or a test-fixture -relocation): `serve.rs` (59, `tracedecay-lsp` percent decoding; only `routing.rs` reads it), -`mcp/tools/handlers/info/mod.rs` (43, `TraceDecay` only but `info/status.rs` needs the server ports), -`core_doctor_schema.rs` (22, `#[path]` child of `core_doctor.rs`). - -### Tip-side / environment reds observed (not this lane's) - -- `hooks_lsp_suite::hooks_test::test_codex_{workspace_status_distinguishes_generic_and_project_like_dirs, - user_prompt_submit_records_workspace_status_and_missing_session_hint}` fail on this machine because a - stray `/tmp/package.json` makes every `tempdir()` "project-like"; both pass with - `TMPDIR=/fast/tmp/td-wave2-clean-tmp` (cc-19166). -- Clippy with `--features tracedecay/test-transport` (not CI's shape) hits `clippy::too_many_lines` - on `McpServer::new_with_registered_test_context` (104/100 lines, untouched here); CI's - `cargo clippy --workspace --all-targets -- -D warnings` shape is green. -- `tracedecay-mcp::workflow::test_runner::tests::cargo_runner_*` ×3 fail when the nested cargo they - spawn contends for the target-dir lock with other test binaries (#1251 saw the same); green alone. -- Root lib `daemon::tests::ownership::unborn_git_project_open_retains_lsp_and_starts_hook_replay` fails - alone on any tree since #1252 (`initialize_test_project` opens through the production path, which now - refuses without registered runtime ports); in a full `--lib` run it passes only when another test - registered the process-global ports first. Untouched here (`git diff 14de498296 -- daemon/tests.rs - daemon/tests/ownership.rs tracedecay-project/src` is empty). - -## P0-1/P0-2 collapse plan - -### Dependency direction (cargo tree, normal edges) - -`tracedecay-mcp` ← `tracedecay-agent-hosts` ← `tracedecay-daemon-service` ← `tracedecay` ← `tracedecay-cli`. -`tracedecay-mcp` depends on 25 crates and on none of agent-hosts, daemon-service, daemon-control, -daemon-identity, host-admission, lsp, automation-runtime. The only agent-hosts → mcp edge is -`ports/mcp_tools.rs` (`get_tool_definitions`, `format_capable_tool_names`). - -Two facts block every "move the rest into tracedecay-mcp": - -1. **`TraceDecay` lives in the root** (`src/project/`, 2.8k lines) and is read at 161 sites in - `src/mcp/tools`, 52 in `src/mcp/server`, and throughout `src/daemon`. It depends on agent-hosts - (context-scout owner lookup), configuration, store-runtime, application, graph-query, and the root's - `config`, `runtime_ports`, `project_store_runtime`, `test_support`. -2. **Cycle** if `tracedecay-mcp` took daemon-service or agent-hosts: - `tracedecay-mcp → tracedecay-daemon-service → tracedecay-agent-hosts → tracedecay-mcp`. - -Break it first: replace the two `tracedecay_mcp::` calls in `tracedecay-agent-hosts/src/ports/mcp_tools.rs` -with a tool-name list passed in by the composition root (or sourced from `tracedecay-tool-catalog`). That -removes agent-hosts → mcp and lets `tracedecay-mcp` depend on agent-hosts and host-admission. Then give -`TraceDecay` a home below both consumers: new crate `tracedecay-project` = root `project/` + `config.rs` -+ `project_store_runtime.rs` + `runtime_ports.rs` (test fixtures behind `test-helpers`), depending on -agent-hosts for the scout owner. After that every (c) row below is movable. - -### `tracedecay-project` — done (fable/wave2-project-crate) - -`crates/tracedecay-project` (10,443 lines, all `git mv` from the root; workspace-versioned because it -owns `version::PACKAGE_VERSION`): - -| module | lines | note | -| --- | ---: | --- | -| `project.rs` + `project/**` | 2,865 | `pub(crate)` → `pub` for the items the root reads (`configuration_runtime`, `profile_database`, `project_memory_*`, `context_scout_*`, `init/open*_with_registered_configuration`, `resolve_*_configuration_layout*`, …); `ContextScoutOwnerLookupV1` public | -| `config.rs` + `config/tests.rs` | 2,023 | `PinnedUserDataDir`, `lock_user_data_dir_test_env`, `ensure_runtime_configuration_for_registered_database` regated `cfg(any(test, feature = "test-helpers"))` (root tests reach them through the feature, not `cfg(test)`) | -| `runtime_ports.rs` | 339 | see the seam below | -| `project_store_runtime.rs` | 35 | `join_standalone_session_registry` now *requires* registered ports (typed `Config` refusal) instead of registering them — the project crate cannot build the daemon client | -| `product_runtime.rs` + `version.rs` | 511 | moved because `HostAdmissionTestRuntimeV1::open` registers the fixture product runtime; the root re-exports both at their old paths | -| `test_support/host_admission.rs` + 7 submodules | 4,609 | the registered test runtime; root `test_support::host_admission` re-exports it explicitly (no glob) and keeps the MCP composition | - -Root re-exports: `pub use tracedecay_project::{config, product_runtime, project, version}` plus the item -re-exports `lib.rs` already had, so `tracedecay::project::TraceDecay`, `tracedecay::config::…`, -`tracedecay::hook_runtime()`, `tracedecay::register_runtime_ports()` are unchanged for the CLI, suites and -benches. Root test inventory is byte-identical (983 = 946 root + 37 project; one test renamed). - -**The one production seam** (`runtime_ports`): `HookRuntimeV1` needs the daemon client (`daemon_tool`, -`event_notifier` → `crate::daemon::{handshake_for_current_client, call_default_tool, tool_json_payload, -notify_hook_event}`, i.e. `core_client.rs`/`core_hooks.rs`/`core_handshake.rs` → product runtime and the -wire types), and project open publishes hook bindings through that handle. The project crate therefore -owns `register_runtime_ports(DaemonClientPortsV1)` (sibling ports + one set-once slot for the client), -a fallible `hook_runtime()` (typed refusal when no composition root registered) used only by -`init/open*_with_registered_configuration`, and `hook_runtime_with(client)` for explicit handles. The root -keeps `runtime_ports.rs` (127 lines): the two daemon-client adapters, the no-arg `register_runtime_ports()` -wrapper, the infallible `hook_runtime()` the CLI passes to hook entry points, `session_review_port()`, and -`compose_application_catalog_snapshot()`. The `test-helpers` fixture registers a fixture client (typed -unavailable tool, no-delivery notifier) exactly as it registers the fixture product runtime; the only -in-crate reader of the slot is `publish_hook_bindings`, which consults `scope_resolver` alone. - -**Test dispatch is now explicit for root unit tests.** `TraceDecay::init/open/open_read_only/open_branch -(_with_options)` still route to the registered test runtime under the project crate's -`cfg(any(test, feature = "test-transport"))`, which the root's `test-transport` forwards. Root `cfg(test)` -no longer reaches that gate, and enabling the project crate's `test-transport` from the root's -dev-dependency would flip every `test-helpers`-only suite (`runtime_acceptance_suite`'s "direct -production init", `daemon_runtime_acceptance`) onto the fixture path silently. So the fixture path is also -a named API — `TraceDecay::{init,open,open_read_only,open_branch}_with_options_for_test` under -`test-helpers` — and the 25 root lib-test call sites use it. `test-helpers`-only suites take the production -standalone path, which now needs the suite binary to register the composition root's ports: -`tests/common::register_process_runtime_ports()` runs from `IsolatedEnv::build` (the same choke point as -the product-runtime registration) and from the three fixtures that bypass it. - -**Residual root-only edges** (all composition, none inside the moved code): -- root `runtime_ports.rs::{daemon_tool_json, notify_hook_event}` → `crate::daemon` client (4 fns). -- root `test_support::host_admission::{mcp_session_authorities, call_mcp_tool_for_test, - mcp_server_context_for_test}` → `crate::mcp::tools::{SessionAuthorities, ToolCallRegistryOptions, - handle_tool_call_with_registry_options}`, `crate::mcp::server::McpServerConstructionContext`, - `tracedecay_daemon_service::DaemonProjectRegistryReadService` (11 call sites retargeted to these free - functions). -- With #1251 (agent-hosts cycle break, `tracedecay-mcp-catalog`) merged, `cargo tree -p tracedecay-project - -e normal -i tracedecay-mcp` and `-i tracedecay-daemon-service` both resolve to nothing: the crate reaches - neither consumer. Its only catalog edge is `agent-hosts → tracedecay-mcp-catalog`. - -**Now movable** (`TraceDecay` is below both consumers and the agent-hosts cycle is broken): every (c) row in `src/mcp/` that was blocked only by `TraceDecay` — -`dispatch_groups.rs` (27 `TraceDecay` reads), `handlers/edit.rs` (17), `handlers/hook_runtime/` (38), -`handlers/mod.rs` (7), `construction.rs` (8), `dashboard.rs` (4) — plus the `src/daemon/` (c) rows that -named only `TraceDecay` or root `config`: `scheduler/` (44 reads), `project_open_owners/`, -`project_open_admission/`, `branch_admin/` (12, after its 11 `crate::mcp` uses move), `invocation_state.rs` -and `doctor_kernel/` (root `config` ×2 each, now `tracedecay_project::config`). The two rows that build the -root `McpServer` (`project_composition/`, `connection_serving.rs`) stay until P0-1. Next PR: the (b) rows -of `src/mcp/` (`tools/binding.rs`, `dashboard_lcm.rs`, `catalog_discovery.rs`, `server/routing.rs`, -`tool_analytics.rs`, `scope.rs`, `session_refresh.rs`, …, ≈6.5k lines) into `tracedecay-mcp`, one commit per -target crate, with `TraceDecay` imported from `tracedecay_project`. - -### Root `src/mcp/` — 42,256 lines, 12,390 in test files - -(a) duplicate of an extracted owner → delete: none remain (this lane removed `effective_path`, four -`unknown_tool_error` copies, and the forwarding tables). Checked by name overlap and ripwire `--clones`. - -(b) movable as-is (target crate): - -| module | lines | target | note | -| --- | ---: | --- | --- | -| `tools/binding.rs` + `binding/` | 1,630 | tracedecay-mcp | catalog: contracts + tool-catalog; one `resolve_catalog_tool_binding` call to relocate | -| `tools/handlers/dashboard_lcm.rs` | 1,067 | tracedecay-mcp | needs `tracedecay-lcm`, `tracedecay-session-runtime` edges | -| `tools/catalog_discovery.rs` | 673 | tracedecay-mcp | two daemon-service calls to lift | -| `server/routing.rs` + `serve.rs` | 612+59 | tracedecay-mcp | root-URI decoding + scope routing | -| `tools/handlers/dashboard_delivery.rs` | 564 | tracedecay-daemon-service | already daemon-service shaped | -| `tool_analytics.rs` | 563 | tracedecay-mcp | one agent-hosts type to check | -| `scope.rs` | 443 | tracedecay-mcp | memory/storage scope selection | -| `server/session_refresh.rs` | 393 | tracedecay-mcp | | -| `server/project_host_admission_replay.rs` | 312 | tracedecay-daemon-service | host-admission + sessions | -| `tools/dispatch.rs` | 255 | tracedecay-daemon-service | daemon-protocol + daemon-service | -| `tools/handlers/hook_runtime/envelope.rs` | 222 | tracedecay-hooks | identity minting policy; swap `config_error` for a hooks error | -| `tools/handlers/session_authorities.rs` | 99 | tracedecay-mcp | | -| `tools/handlers/dashboard_git_correlation.rs`, `server/status_resource.rs` | 99 | tracedecay-mcp | | - -(c) composition-root wiring, stays until `TraceDecay` moves (then → tracedecay-mcp, or a new -`tracedecay-mcp-daemon` if daemon-service must stay below mcp): `server.rs` (1,546), -`server/{requests,connection,construction,ledger,rmcp,lifecycle,hook_dispatch,hook_writes}.rs` (6,569), -`tools/handlers/mod.rs` (`ToolCallRegistryOptions`, 50 daemon-owned authorities), `dispatch_groups.rs` -(admission funnel, `McpToolContext` binding, retained protocol), `handlers/{edit,workflow,admin_cli, -admin_project,analytics,skills,automation_runs,dashboard,application_surface,retained_catalog, -tool_call_support,dispatch_controls}.rs`, `handlers/hook_runtime/` minus `envelope.rs`, -`handlers/info/` (admin sync), `project_route.rs` (975). - -(d) blocked by dependency direction: every (c) module that names `TraceDecay` or `tracedecay_daemon_service` -(`dispatch_groups.rs` 27/3, `dashboard.rs` 4/11, `mod.rs` 7/4, `construction.rs` 8/2, `edit.rs` 17, -`hook_runtime/` 38/5) — the cycle above. - -### Root `src/daemon/` — 97,528 lines, 46,370 in test files (51,158 production) - -(a) duplicate → delete: none at module level. Name overlap with daemon-service is zero for all 23 -production modules checked; ripwire `--clones` over both trees (481 groups) finds 27 cross-tree type-3 -near-misses, all ≤160-token observability/receipt helpers (`observe_remote_deletion_receipt` ~ -`record_query_admission_refusal`, `admission_state` ~ `workflow_topology_problem`) — a shared receipt -helper in daemon-service, not deletions. - -(b) movable as-is → tracedecay-daemon-service (no `TraceDecay`, no `crate::mcp`, no root `config`): -`shutdown_orchestration.rs` (1,362), `shutdown_coordination.rs` (588), `shutdown_watchdog.rs` (338), -`invocation_executor.rs` (800; daemon-protocol), `invocation_dispatch.rs` (931), `context_scout_lifecycle/` -(566 + tests), `remote_deletion.rs` (341), `database_owner_registry.rs` (370), `lsp_sessions.rs` (208), -`github_credential_lifecycle.rs` (237), `bootstrap_route.rs` (175), `automation_observation.rs` (66), -`project_delivery_mount.rs` (46), `core_doctor_schema.rs` (22), `http_application_router.rs` (125) — -≈6.2k lines. `core_lifecycle.rs` (399) and `wire_io.rs` (448) go to daemon-protocol/daemon-control once -their two `crate::mcp` uses are typed. - -(c) composition wiring (stays until `TraceDecay` moves): `branch_admin/` (5,571; TraceDecay 12, mcp 11), -`scheduler/` (5,016; TraceDecay 44), `project_open_owners/` (4,441), `project_composition/` (3,458; -`crate::mcp` 64 — it constructs the MCP server), `connection_serving.rs` (2,211), `maintenance.rs` -(2,040), `engine.rs` (1,480), `pr_autotrack/` (1,379), `project_open_admission/` (1,294), -`core_{doctor,proxy,admission,client,logging,hooks,handshake}.rs`, `bootstrap.rs`, `dashboard_automation/`, -`http_application.rs`, `retained_owner/`, `projectless.rs`, `project_open_{orchestration,handshake}.rs`, -`project_server_lifecycle.rs`, `project_routing.rs`, `branch_add.rs`, `hook_v2_replay_consumer.rs`, -`graph_resolution.rs`, `automation_effect/`, `adoption_observation.rs`, `store_maintenance/`, -`invocation_state.rs` (1,662; root `config` ×2), `doctor_kernel/` (1,341; root `config` ×2). - -(d) blocked by dependency direction: `project_composition/` and `connection_serving.rs` build the root -`McpServer`, so they can only move after P0-1; everything naming `TraceDecay` waits on `tracedecay-project`; -`invocation_state.rs`, `doctor_kernel/`, `core_logging.rs`, `bootstrap.rs` wait on root `config.rs` -(786 lines: `PinnedUserDataDir`, `user_data_dir`, `DaemonRuntimeConfiguration`) moving with it. - -### Order - -1. ~~`tracedecay-agent-hosts/src/ports/mcp_tools.rs`: drop the two `tracedecay_mcp::` calls~~ — done in #1251 - (`tracedecay-mcp-catalog`). -2. ~~New `tracedecay-project`~~ — done, see above (also took `product_runtime.rs`, `version.rs`, and the - host-admission test runtime). -3. ~~`src/mcp` (b) rows~~ and the `TraceDecay`-only handler families — done in slice 1 of - fable/wave2-root-collapse (see above); the remaining (c) rows wait on `McpServer` / - `ToolCallRegistryOptions` and move in slice 3. -4. ~~`src/daemon` (b) rows into daemon-service~~ — the `TraceDecay`/`config`-only rows landed in slice 2; - the rest names `McpServer`/`StoreAdministration`/`tracedecay-mcp` frames and cannot enter - daemon-service (see the daemon wall above), so the daemon composition stays in the root. -5. ~~Delete `src/mcp/` and `src/daemon/`~~ — not reachable as written: the daemon composition that builds - and serves `McpServer` cannot live below `tracedecay-mcp` (see slice 3's reclassification). The root - is the daemon composition crate; a `tracedecay-daemon` crate above mcp and daemon-service would only - rename it. - -### Test placement facts (item 3 probe) - -`crates/tracedecay/tests/zz_relocation_probe.rs` (temporary, deleted) mounted each of the 73 `#[cfg(test)]` -module roots under a `pub use tracedecay::::*` shim and compiled with `test-helpers,test-transport`: -507 errors — private modules (`daemon::{automation_effect,context_scout_lifecycle,core_doctor,doctor_kernel, -invocation_executor,production_harness,project_composition,…}` are `pub(crate)`/private), private fields -(`ToolCallRegistryOptions`, server internals), private fns, and `cfg(test)`-only fixtures -(`TraceDecay::init_test_fixture_with_registered_runtime`). A test that reaches those must stay in-src; the -sanctioned fixture surface is `test_support` behind `test-helpers`. diff --git a/docs/CARGO-CONTENTION-POLICY.md b/docs/CARGO-CONTENTION-POLICY.md deleted file mode 100644 index b929c0b93a..0000000000 --- a/docs/CARGO-CONTENTION-POLICY.md +++ /dev/null @@ -1,69 +0,0 @@ -# Cargo invocation policy - -Run plain `cargo ` — cargo-conductor brokers every invocation -(PATH shim). Expect `[cargo-conductor] ticket cc-N` lines on stderr. - -Do **not** prefix with `kache` (bypasses the broker), wrap in `flock`, or set -`CARGO_TARGET_DIR` / isolate builds. The broker serializes per target dir, -dedupes identical runs, and batches compatible checks. Kache caching still -applies automatically via the workspace `rustc-wrapper`. Prefer scoped -commands (`cargo check -p --lib`) — they coalesce and release early. - -```sh -cargo check -p tracedecay-store --lib -cargo test -p tracedecay-store -cargo clippy -p tracedecay-store --all-targets -``` - -Never kill cargo processes. Use `conductor status` (not ps/pgrep) for queue -visibility. If a backgrounded ticket's result will not retrieve, rerun the -command (dedup makes it cheap) — known issue cargo-conductor#16. -`daemon unreachable; running cargo directly` is fail-open: proceed and mention -it. `kache monitor` is a cache dashboard, not a cargo front-end. - -cargo-conductor and kache are machine-local practice. They are not product, -CI, or release architecture, and they are not a revival of the rejected -`cargo-slot` shim. Stock `cargo` remains the portable command for a fresh -checkout, CI, and published contributor instructions. - -## Contended checkouts - -The broker owns serialization. Do not invent a per-lane or `/tmp/...` -`CARGO_TARGET_DIR` merely to avoid contention, and do not redirect -`TRACEDECAY_DATA_DIR` for that reason. Those redirects fragment incremental -artifacts and can bypass the repository's test-profile isolation. The shared -compile-cache key is profile × features × `RUSTFLAGS` × source, not the -worktree path. - -TraceDecay diagnostic commands manage their own private target directories. -Do not reuse or delete those directories while a diagnostic command is active. -Do not reclaim or wipe the machine kache store. - -## Repository rules - -- Do not commit an absolute `[build].target-dir` or any host-specific build - path. -- Keep `.cargo/config.toml` portable. Its checked-in `target-dir = "target"` - is relative to each checkout. Do not edit a machine-local - `rustc-wrapper = "kache"` — that is the compile-cache layer, not a cargo - prefix. -- Do not add a cargo-slot, lock-stealing shim, or any wrapper that changes - Cargo semantics, feature resolution, or `RUSTFLAGS`. cargo-conductor execs - stock Cargo; prefixing `kache cargo` bypasses the broker. -- Novel feature permutations recompile the workspace spine. Stick to the - standard lanes in `AGENTS.md`. -- CI may select a runner-local target directory or cache through its own - environment; that configuration must not leak into published packages or - require cargo-conductor or `kache` for a contributor. - -## Verification - -Before submitting a build-configuration change: - -```sh -cargo check --workspace --all-targets -cargo test --workspace -``` - -Confirm that a fresh shell with a standard Rust toolchain can still run -ordinary `cargo` commands without machine-local aliases, wrappers, or paths. diff --git a/docs/DESIGN-DOC.md b/docs/DESIGN-DOC.md deleted file mode 100644 index 31e05edf42..0000000000 --- a/docs/DESIGN-DOC.md +++ /dev/null @@ -1,401 +0,0 @@ -# TraceDecay Design Document - -This document is a historical snapshot of the pre-crate-split design, not current -layout or ownership authority (`AGENTS.md` and the V2 plan set are). -TraceDecay is a code intelligence tool that builds semantic knowledge graphs from source code. -It parses source files with tree-sitter, extracts symbols and relationships into -`tracedecay-graph-db` (Grafeo), and exposes the graph through a CLI, MCP, LSP, and an -embedded dashboard. Relational SQLite remains for session, memory, and registry -state — not the code graph. -The core insight is that AI coding agents waste tokens reading raw files when a pre-built -graph can answer most questions instantly. - -## Architecture Overview - -The system is structured as a pipeline: source files flow through extraction, resolution, -and storage, then get queried via the CLI or MCP server. - -```mermaid -graph LR - subgraph Indexing - A[Source Files] --> B[Tree-sitter Parsing] - B --> C[Language Extractors] - C --> D[Nodes + Edges + Unresolved Refs] - D --> E[Reference Resolver] - E --> F[Graph store] - end - - subgraph Querying - F --> G[Graph Traversal] - F --> H[Graph Queries] - F --> I[Context Builder] - G --> J[MCP Server] - H --> J - I --> J - G --> K[CLI] - H --> K - I --> K - end - - J --> L[Claude / Gemini / Codex / OpenCode] - K --> M[Terminal] -``` - -The binary (`src/main.rs`) serves as both the CLI frontend and MCP server entry point. -The library (`src/lib.rs`) exposes all internals so the CLI and server share the same code -paths without duplication. - -## Module Map - -This map is a historical snapshot of the pre-crate-split tree. Current layout -is in `AGENTS.md` and `CONTRIBUTING.md` (`src/` daemon/MCP, `crates/` members, -`dashboard/`, `plugin/`). - -``` -src/ - main.rs CLI entry point, subcommand dispatch - lib.rs Crate root, module declarations, lint config - tracedecay.rs TraceDecay facade -- the main public API - branch.rs Git branch resolution (current branch, default detection, merge-base) - branch_meta.rs Branch metadata persistence (branch-meta.json) - config.rs Per-project config (exclude patterns, limits) - errors.rs Error types (thiserror) - sync.rs Content hashing, stale/new/removed file detection - user_config.rs User-level config (~/.tracedecay/config.toml) - cloud.rs Cloudflare Worker counter, GitHub release checks - global_db.rs Cross-project token tracking (~/.tracedecay/global.db) - - extraction/ Tree-sitter based extractors (one per language) - mod.rs Extractor registry, feature-gated language modules - complexity.rs Cyclomatic complexity counting (language-configurable) - rust_extractor.rs ... through qbasic_extractor.rs - - tree_sitter/ Vendored tree-sitter grammars - cobol.rs COBOL (no working crate on crates.io) - protobuf.rs Protobuf (version conflict with tree-sitter 0.26) - - db/ SQLite persistence - connection.rs Database struct, WAL setup, checkpointing - migrations.rs Sequential schema migrations via PRAGMA user_version - queries.rs All SQL queries as async methods on Database - - resolution/ Cross-file symbol resolution - resolver.rs Matches unresolved references to known nodes - - graph/ Higher-level graph algorithms - traversal.rs BFS/DFS, callers/callees, impact radius, path finding - queries.rs Dead code detection, circular deps, file dependencies - - context/ AI-ready context assembly - builder.rs Builds TaskContext from a natural language query - formatter.rs Markdown and JSON output formatters - - vectors/ Embedding storage and brute-force similarity search - search.rs Store/query/delete vectors, cosine similarity - - mcp/ Model Context Protocol server - server.rs McpServer: stdio JSON-RPC loop, lifecycle - tools.rs 37 tool definitions and dispatch - transport.rs JSON-RPC request/response/error types - - agents/ Agent integration (install/uninstall/doctor) - mod.rs Agent trait, registry, shared helpers, git hooks - claude.rs Claude Code: MCP in ~/.claude.json, hooks, permissions - codex.rs Codex CLI: MCP in ~/.codex/config.toml, AGENTS.md - opencode.rs OpenCode: MCP in ~/.config/opencode/opencode.json, ~/.config/opencode/AGENTS.md - gemini.rs Gemini CLI: MCP in ~/.gemini/settings.json, GEMINI.md -``` - -## Core Data Model - -The graph has three primary entities stored in `tracedecay-graph-db`. - -**Nodes** represent code symbols. Each node has: - -- A deterministic ID (content-addressed hash of file path + name + kind) -- A `NodeKind` enum with 50+ variants spanning all supported languages - (Function, Method, Struct, Class, Enum, Trait, Interface, Field, etc.) -- Source location (file, start line, end line) -- Metadata: signature, visibility, docstring, body hash, line count -- Complexity metrics: branches, loops, returns, max nesting, unsafe blocks - -**Edges** represent relationships between nodes: - -| EdgeKind | Meaning | -|--------------|------------------------------------| -| Contains | Parent contains child (file->fn) | -| Calls | Function/method calls another | -| Uses | Symbol references another symbol | -| Implements | Type implements trait/interface | -| TypeOf | Field/param has a type reference | -| Returns | Function returns a type | -| Inherits | Class extends another | -| Overrides | Method overrides a parent method | -| Imports | File imports from another | -| AnnotatedBy | Symbol annotated by annotation | - -**Files** track indexing state per source file: - -- Path, content hash (SHA-256), language, size, last-indexed timestamp -- Content hashing enables incremental sync: only re-extract changed files - -## Indexing Pipeline - -### 1. File Discovery - -`TraceDecay::index_all` walks the project tree, filters by extension (language support) -and config exclude globs. If `git_ignore` is enabled, it additionally filters through -`.gitignore` rules via the `ignore` crate. - -### 2. Extraction - -Each source file is dispatched to a language-specific extractor based on file extension. -Extractors use tree-sitter to parse the file into a concrete syntax tree, then walk it -to produce an `ExtractionResult` containing: - -- **Nodes**: every symbol found in the file -- **Edges**: intra-file relationships (contains, calls, uses, implements, etc.) -- **Unresolved references**: call sites and type references that name symbols - potentially defined in other files - -The extractor architecture is stateless: `XxxExtractor::extract_source(path, source)` -takes a file path and source string, returns nodes and edges. An internal `ExtractionState` -accumulates results during the tree walk, tracks scope nesting for containment edges, -and collects unresolved references. - -Complexity metrics are computed during extraction using a language-configurable walker -(`ComplexityConfig`) that counts branches, loops, nesting depth, unsafe blocks, and -unchecked calls in each function body. - -### 3. Reference Resolution - -After all files are extracted, the `ReferenceResolver` runs a second pass. It loads all -nodes into memory, builds a name-to-node index, and attempts to match each unresolved -reference to a known node. Matched references become typed edges (Calls, Uses, TypeOf, -etc.) that create cross-file connections in the graph. - -### 4. Storage - -The code graph persists through `tracedecay-graph-db` (Grafeo). Relational -SQLite (`tracedecay-rusqlite-runtime`) holds session, memory, and registry -state, not graph nodes. - -### 5. Incremental Sync - -`TraceDecay::sync` compares the current file system state against the stored file records -using SHA-256 content hashes. It identifies three sets: - -- **New files**: on disk but not in DB, need full extraction -- **Modified files**: hash mismatch, re-extract and replace -- **Removed files**: in DB but not on disk, delete nodes and edges - -Only changed files are re-extracted. After re-extraction, reference resolution runs -again on the full graph to pick up any new cross-file edges. - -## Database Layer - -The code graph is Grafeo. Relational catalogs use SQLite through the bundled -`tracedecay-rusqlite-runtime`. Sequential migrations still track relational -schema via `PRAGMA user_version`. - -Key schema features: - -- **FTS5** full-text search index on node names for fuzzy symbol search -- **Covering indexes** on edges `(source_id, kind)` and `(target_id, kind)` for - fast traversal in both directions -- **Content-addressed node IDs** enable deduplication and stable references -- **WAL mode** with `NORMAL` synchronous for concurrent read/write safety - -Domain-specific modules under `src/db/` implement data access through the -engine executor traits. Complex analytical queries (god classes, inheritance -depth, coupling) use CTEs and window functions to avoid pulling large datasets -into Rust. - -## Graph Algorithms - -`graph/traversal.rs` provides BFS and DFS traversal with configurable edge kinds, -max depth, and direction. Built on top of it: - -- **Callers/callees**: follow Calls edges upstream/downstream -- **Impact radius**: BFS from a node following all edge kinds to find the blast radius -- **Call graph**: bidirectional expansion from a function -- **Type hierarchy**: follows Implements and Inherits edges -- **Path finding**: BFS shortest path between two nodes - -`graph/queries.rs` provides higher-level analyses: - -- **Dead code detection**: nodes with zero incoming Calls/Uses edges (excluding files) -- **Circular dependencies**: Tarjan's SCC algorithm on file-level import edges -- **File dependencies/dependents**: which files a file depends on or is depended upon by - -## Context Builder - -The context builder (`context/builder.rs`) is the key integration point for AI agents. -Given a natural language task description, it: - -1. Extracts symbol names from the query using heuristics (camelCase splitting, etc.) -2. Searches for each extracted symbol via FTS5 full-text search -3. Searches for each agent-provided keyword (the `extra_keywords` field) -4. Expands relevant nodes by following edges to include callers, callees, and types -5. Reads source code snippets for the top-ranked nodes -6. Assembles a `TaskContext` with ranked symbols, their code, and relationship summaries - -The output can be formatted as markdown (for humans / LLM prompts) or JSON (for -programmatic consumption). - -### Semantic Search: Keywords vs Embeddings - -The primary search mechanism is FTS5 with BM25 scoring, which matches against -node names, qualified names, signatures, and docstrings. This works well when -query terms appear literally in the code, but fails when concepts don't match -symbol names (e.g. "authentication" won't find `login()` unless a docstring -mentions it). - -Rather than embedding models, tracedecay uses **agent-driven keyword expansion**. -The `tracedecay_context` MCP tool accepts a `keywords` array where the calling -agent provides synonyms: - -```json -{ - "task": "how does authentication work", - "keywords": ["login", "session", "credential", "token", "jwt"] -} -``` - -Each keyword runs as an independent FTS5 query, and results are merged with the -main query's results (deduplicated by node ID). - -**Why keywords instead of embeddings:** - -| | Agent keywords | Local embeddings | -|---|---|---| -| Indexing cost | Zero | ~30s per 1,000 nodes (ONNX inference) | -| Model dependency | None | ~50MB model download | -| Query latency | ~1ms per keyword (FTS5 index hit) | ~200ms (brute-force cosine) | -| Binary size impact | None | +15-20 MB (ONNX runtime) | -| Conceptual match quality | Depends on agent's domain knowledge | Better for truly alien naming | -| Works without an LLM | No (needs an agent to provide keywords) | Yes (standalone) | - -The trade-off: if the codebase uses naming conventions the agent can't predict -(e.g. `guardianGateway` for authentication), keywords miss while embeddings -would catch it via distributional semantics. In practice, the agent is an LLM -that understands programming conventions well enough to supply good synonyms -for the vast majority of cases. - -## MCP Server - -The MCP server (`mcp/server.rs`) runs over stdio using JSON-RPC 2.0. It implements -the Model Context Protocol lifecycle: - -1. **initialize**: returns server capabilities and tool list -2. **tools/list**: returns the 36 available tools with JSON Schema input definitions -3. **tools/call**: dispatches to `handle_tool_call` which routes by tool name - -The server is stateless between calls (each call queries the database independently). -It tracks basic statistics (call counts, tokens saved per tool) for the `tracedecay_status` -tool. - -### Tool Categories - -The MCP tools fall into several categories: - -| Category | Tools | -|-----------------|----------------------------------------------------------| -| Search | search, context, node, files, diff_context | -| Navigation | callers, callees, impact, affected | -| Analysis | complexity, dead_code, god_class, circular, coupling | -| Metrics | rank, hotspots, largest, distribution, inheritance_depth | -| Quality | doc_coverage, unused_imports, recursion | -| Refactoring | rename_preview, similar, module_api | -| Git/CI | changelog, commit_context, pr_context | -| Testing & types | test_map, type_hierarchy | -| Porting | port_status, port_order | -| Branching | branch_search, branch_diff | -| Status | status | - -Each tool is defined in `mcp/tools.rs` with a JSON Schema for its parameters. -`handle_tool_call` deserializes the arguments, calls the appropriate `TraceDecay` -method, and formats the result. - -## Agent Integration - -The `agents/` module implements the `Agent` trait, providing `install`, `uninstall`, -and `healthcheck` operations for each supported coding agent. The registry in `mod.rs` -maps string IDs to agent implementations. - -Each agent's install routine: - -1. Registers the MCP server in the agent's config file -2. Sets up tool permissions / auto-approval where supported -3. Installs a PreToolUse hook (Claude Code only) to block redundant file reads -4. Appends prompt rules to the agent's instructions file - -```mermaid -graph TB - subgraph "tracedecay install --agent X" - I[Install] --> MCP[Register MCP Server] - I --> PERM[Set Tool Permissions] - I --> HOOK[Install Hook] - I --> PROMPT[Append Prompt Rules] - end - - subgraph "Agent Config Locations" - MCP --> C1["Claude: ~/.claude.json"] - MCP --> C2["Codex: ~/.codex/config.toml"] - MCP --> C3["OpenCode: ~/.config/opencode/opencode.json"] - MCP --> C4["Gemini: ~/.gemini/settings.json"] - - PROMPT --> P1["Claude: ~/.claude/CLAUDE.md"] - PROMPT --> P2["Codex: ~/.codex/AGENTS.md"] - PROMPT --> P3["OpenCode: ~/.config/opencode/AGENTS.md"] - PROMPT --> P4["Gemini: ~/.gemini/GEMINI.md"] - end -``` - -The `doctor` command runs healthchecks across all (or a specific) agent, -verifying that the MCP server is registered, permissions are correct, hooks -are installed, and prompt rules are present. - -## Language Support Tiers - -Languages are organized into feature-gated tiers to control binary size and -compile time: - -| Tier | Languages | Feature Flag | -|--------|-------------------------------------------------------------------|--------------| -| Lite | Rust, Go, Java, Scala, TypeScript/JS, Python, C, C++, Kotlin, C#, Swift | always on | -| Medium | Dart, Pascal, PHP, Ruby, Bash, Protobuf, PowerShell, Nix, VB.NET | `medium` | -| Full | Lua, Zig, Obj-C, Perl, Batch, Fortran, COBOL, MSBasic2, GWBasic, QBasic | `full` (default) | - -Two grammars (COBOL and Protobuf) are vendored as C source compiled via `build.rs` -because their crate counterparts are either missing or depend on incompatible -tree-sitter versions. The FFI shims live in `src/tree_sitter/`. - -## Token Tracking - -TraceDecay tracks how many tokens it saves compared to raw file reads. Each MCP tool -call estimates the tokens that would have been consumed by reading the relevant files, -subtracts the size of the tool's response, and accumulates the difference in the -per-project database. - -A global database at `~/.tracedecay/global.db` aggregates totals across all projects -(an existing legacy `~/.tracedecay/` directory is still honored as a fallback). -An opt-in worldwide counter (Cloudflare Worker) lets users contribute their totals -anonymously. Upload is best-effort with 2-second timeouts and never blocks the CLI. - -## Concurrency Model - -The system uses `tokio` for async I/O but most work is CPU-bound (tree-sitter parsing) -or SQLite-bound. Key concurrency points: - -- The MCP server processes one JSON-RPC request at a time (single stdio stream) -- The git post-commit hook runs `tracedecay sync` in the background (`&`). `sync` requires an existing database -- it will not create one. This prevents the hook from silently bootstrapping indexes in repos that were never initialized with `tracedecay init`. -- SQLite WAL mode + busy timeout handles concurrent access gracefully -- Version checks and counter uploads run on background threads during `sync` - -## Build and Distribution - -- **GitHub Releases**: checksummed prebuilt archives and `install.sh` - -The release workflow (`release.yml`) builds reproducible binary archives for -four targets and publishes them only as GitHub Release assets. diff --git a/docs/EXTRACTOR-MIGRATION-GUIDE.md b/docs/EXTRACTOR-MIGRATION-GUIDE.md deleted file mode 100644 index 117f3106c9..0000000000 --- a/docs/EXTRACTOR-MIGRATION-GUIDE.md +++ /dev/null @@ -1,61 +0,0 @@ -# Extractor Traversal Helper Migration Guide - -This guide captures the C/C++ pilot migration strategy for consolidating duplicated tree-sitter traversal helpers under `crates/tracedecay-code-extraction/src/traversal.rs`. - -## Final utility strategy - -Keep the shared traversal module intentionally small and language-agnostic. The current shared helpers are: - -- `find_direct_child_by_kind(node, kind)`: exact `Node::kind()` match over direct children only, preserving source-order traversal and including both named and anonymous children. -- `has_direct_child_kind(node, kind)`: boolean wrapper around `find_direct_child_by_kind`. -- `find_descendant_by_kind(node, kind)`: exact `Node::kind()` match with pre-order depth-first traversal over all children. - -The pilot migrated only the C and C++ extractors to these helpers. That scope is deliberate: both extractors had duplicate local helpers with the same semantics, and the focused C/C++ tests exercise the direct-child and descendant paths through function-pointer typedef extraction. - -## Safe consolidation patterns - -Consolidate a local helper into `extraction::traversal` only when all of these are true: - -1. The helper compares `node.kind()` to a caller-provided string exactly. -2. Traversal visits both named and anonymous children, not just named children. -3. Direct-child searches inspect only immediate children in source order. -4. Descendant searches are pre-order depth-first and return the first matching node. -5. The helper is stateless: it does not read extractor state, source bytes, language config, field names, or parent context. -6. Existing tests cover at least one representative call path in the migrating extractor. - -The C/C++ pilot is the template for this shape: import helpers from `crate::traversal`, replace `Self::find_*_by_kind(...)` calls with the shared functions, remove the now-unused local helper definitions, and keep extractor behavior unchanged. - -## Patterns that should stay local - -Do not consolidate helpers that encode language-specific or extractor-specific behavior, including helpers that: - -- Filter to named children only or intentionally skip anonymous nodes. -- Match tree-sitter field names, grammar aliases, supertypes, or language-specific node families. -- Collect multiple matches rather than returning the first match. -- Depend on source text, byte ranges, comments, docstrings, visibility state, or the extractor's node stack. -- Walk parents or siblings, or otherwise use traversal order different from the direct-child / pre-order descendant helpers. -- Special-case grammar quirks or disambiguate names differently per language. -- Are performance-sensitive enough to need caller-controlled cursors, caching, or a fused extraction pass. - -Keeping these helpers local is preferable to hiding behavior differences behind a shared utility with a misleadingly generic name. - -## Migration checklist - -For each future extractor migration: - -1. Compare the local helper body against `crates/tracedecay-code-extraction/src/traversal.rs`, not just the helper name. -2. Add or identify tests that cover each shared helper path used by the extractor. At minimum, cover direct-child lookup and nested descendant lookup when both are used. -3. Run the focused extractor test module (its `crates/tracedecay-code-extraction/tests/main/.rs` module of the `main` test binary) before and after migration. If behavior changes, either revert the migration or document and test the intended behavior change. -4. Run the shared traversal unit tests and `cargo check --lib` with the same feature profile used by the extractor tests. -5. Leave language-specific traversal helpers local until their semantics are proven identical. - -Validation commands used for the C/C++ pilot: - -```sh -cargo nextest run -p tracedecay-code-extraction --lib --no-default-features extraction::traversal::tests -cargo nextest run -p tracedecay-code-extraction --no-default-features --test main -E 'test(/^c::/)' -cargo nextest run -p tracedecay-code-extraction --no-default-features --test main -E 'test(/^cpp::/)' -cargo check -p tracedecay-code-extraction --lib --no-default-features -``` - -The pilot validation passed with 3/3 traversal unit tests, 25/25 C extractor tests, 30/30 C++ extractor tests, and a clean library check. diff --git a/docs/MORE-LANGUAGES-SUPPORT.md b/docs/MORE-LANGUAGES-SUPPORT.md deleted file mode 100644 index 4df9c1bf15..0000000000 --- a/docs/MORE-LANGUAGES-SUPPORT.md +++ /dev/null @@ -1,188 +0,0 @@ -# Future Language Support - -## Currently Supported (51 languages) - -Counted from the `*_extractor.rs` files in -`crates/tracedecay-code-extraction/src/` and their gating in that crate's -`Cargo.toml`/`src/lib.rs` (`lite`/`medium`/`full` feature sets). - -| Tier | Languages | -|------|-----------| -| **Lite** (always compiled) | Rust, Go, Java, Scala, TypeScript/JavaScript/TSX/JSX, Python, C, C++, Kotlin, C#, Swift, Astro, Svelte, Markdown | -| **Medium** (feature flags) | Dart, Pascal, PHP, Ruby, Bash, Protobuf, PowerShell, Nix, VB.NET | -| **Full** (feature flags) | Lua, Zig, Objective-C, Perl, Batch, Fortran, COBOL, MSBASIC2, GW-BASIC, QBasic, QuickBASIC, Dockerfile, GLSL, WGSL, HLSL, Metal, R, SQL, Julia, Haskell, OCaml, Clojure, Erlang, Elixir, F#, Quint, TOML, Lean | - -## How to add a language - -Each language needs 4 things: - -| # | What | Where | Pattern to follow | -|---|------|-------|-------------------| -| 1 | Tree-sitter grammar | `tracedecay-large-treesitters` crate on crates.io | Add dep + register in `all_languages()` | -| 2 | Extractor | `crates/tracedecay-code-extraction/src/{lang}_extractor.rs` | Implement `LanguageExtractor` trait | -| 3 | Wiring | that crate's `Cargo.toml` + `src/lib.rs` | Feature flag, `mod` decl, `LanguageRegistry` push | -| 4 | Tests | `crates/tracedecay-code-extraction/tests/{lang}.rs` | Sample source + extraction assertions | - -### The `LanguageExtractor` trait - -```rust -pub trait LanguageExtractor: Send + Sync { - fn extensions(&self) -> &[&str]; // e.g. &["svelte"] - fn language_name(&self) -> &str; // e.g. "Svelte" - fn extract(&self, file_path: &str, source: &str) -> ExtractionResult; -} -``` - -### Grammar sourcing - -- **Crate on crates.io:** Add as a dependency to `tracedecay-large-treesitters` and register in `all_languages()`. This is the standard path. -- **Vendor from C source:** If no Rust crate exists, compile the grammar's C source via `build.rs` (same pattern as `protobuf` and `cobol` in the bundled crate). -- **No grammar at all:** Either write a regex-based extractor (skip tree-sitter) or wait for a community grammar. - ---- - -## Proposed languages by tier - -Languages are tiered by a combination of: popularity (TIOBE, Stack Overflow, GitHub usage), relevance to tracedecay's target users (professional developers using AI coding tools), and implementation complexity. - -### High Priority — Web Frameworks - -These produce code graphs that are structurally rich and heavily used in -AI-assisted development. They also tend to generate high tool-call counts -in exploration agents because of their component/template structure. - -| Language | Extensions | Grammar crate | Complexity | Notes | -|----------|-----------|---------------|------------|-------| -| ~~**Svelte**~~ | | | | **Implemented** — see Lite tier above (`svelte_extractor.rs`, always compiled). | -| **Vue** | `.vue` | `tree-sitter-vue3` (0.0.4) | Medium-high | Same embedded-language challenge as Svelte/Astro: `