diff --git a/crates/tracedecay-global-db/src/project_registry/tests.rs b/crates/tracedecay-global-db/src/project_registry/tests.rs index 6c1426684a..b40bb4c825 100644 --- a/crates/tracedecay-global-db/src/project_registry/tests.rs +++ b/crates/tracedecay-global-db/src/project_registry/tests.rs @@ -3,9 +3,9 @@ use std::cell::Cell; use std::collections::HashSet; -#[cfg(unix)] +#[cfg(target_os = "linux")] use std::ffi::OsString; -#[cfg(unix)] +#[cfg(target_os = "linux")] use std::os::unix::ffi::OsStringExt as _; use std::path::{Path, PathBuf}; @@ -971,7 +971,8 @@ async fn anchors_do_not_keep_a_missing_project_alive_but_its_store_does() { ); } -#[cfg(unix)] +// APFS rejects the non-UTF-8 directory name before registry admission. +#[cfg(target_os = "linux")] #[asupersync::test] async fn native_path_aliases_protect_roots_from_lossy_registry_reaping() { let harness = RegisteredGlobalDbHarness::open("project-registry-native-reap").await; diff --git a/crates/tracedecay-global-db/src/registered_lcm.rs b/crates/tracedecay-global-db/src/registered_lcm.rs index fb8891ac4d..aab79b15f0 100644 --- a/crates/tracedecay-global-db/src/registered_lcm.rs +++ b/crates/tracedecay-global-db/src/registered_lcm.rs @@ -455,7 +455,7 @@ impl RegisteredGlobalDb { tracing::trace_span!("sessions.lcm.compress"), ); let bounded = compressed.await?; - if bounded.response.status != "needs_summary" + if bounded.response.outcome()? == tracedecay_lcm::LcmCompressionOutcome::Complete && let Some(candidate) = convergence_candidate { let state = if bounded.has_more { diff --git a/crates/tracedecay-global-db/src/schema_contract/invariants.rs b/crates/tracedecay-global-db/src/schema_contract/invariants.rs index 8cc22406d5..fcbc2b4be0 100644 --- a/crates/tracedecay-global-db/src/schema_contract/invariants.rs +++ b/crates/tracedecay-global-db/src/schema_contract/invariants.rs @@ -44,7 +44,7 @@ use rows::{ validate_source_cursor_authority_rows, }; use triggers::{FOREIGN_KEY_AUDIT_QUERY, replace_trigger, trigger_contracts_intact}; -pub(super) use triggers::{INVARIANTS, Trigger}; +pub(super) use triggers::{INVARIANTS, first_invalid_trigger}; const OPERATION: &str = "ensure global database authority invariants"; const INCOMPLETE_EXHAUSTIVE_PASS: i64 = -1; const FOREIGN_KEY_AUDIT_PROGRESS: &str = "authority-invariants"; diff --git a/crates/tracedecay-global-db/src/schema_contract/invariants/triggers.rs b/crates/tracedecay-global-db/src/schema_contract/invariants/triggers.rs index 6f1daf4011..f81dff2357 100644 --- a/crates/tracedecay-global-db/src/schema_contract/invariants/triggers.rs +++ b/crates/tracedecay-global-db/src/schema_contract/invariants/triggers.rs @@ -1,7 +1,7 @@ use std::collections::HashMap; use tracedecay_domain::errors::{ProjectOpenFailureKind, TraceDecayError}; -use tracedecay_runtime_core::db::engine::{Executor, QueryExecutor}; +use tracedecay_runtime_core::db::engine::{Executor, QueryExecutor, Result}; use super::rows::audit_read_error; @@ -1649,6 +1649,15 @@ pub(super) async fn replace_trigger( pub(super) async fn trigger_contracts_intact( conn: &impl QueryExecutor, ) -> tracedecay_domain::errors::Result { + first_invalid_trigger(conn) + .await + .map(|invalid| invalid.is_none()) + .map_err(audit_read_error) +} + +pub(in crate::schema_contract) async fn first_invalid_trigger( + conn: &impl QueryExecutor, +) -> Result> { // SQLite trigger names are unique under ASCII case folding, which is what // `COLLATE NOCASE` and `to_ascii_lowercase` both apply. let mut rows = conn @@ -1656,20 +1665,19 @@ pub(super) async fn trigger_contracts_intact( "SELECT name, tbl_name, sql FROM sqlite_master WHERE type = 'trigger'", (), ) - .await - .map_err(audit_read_error)?; + .await?; let mut actual = HashMap::new(); - while let Some(row) = rows.next().await.map_err(audit_read_error)? { - let name = row.get::(0).map_err(audit_read_error)?; - let table = row.get::(1).map_err(audit_read_error)?; - let sql = row.get::(2).map_err(audit_read_error)?; + while let Some(row) = rows.next().await? { + let name = row.get::(0)?; + let table = row.get::(1)?; + let sql = row.get::(2)?; actual.insert(name.to_ascii_lowercase(), (table, sql)); } Ok(INVARIANTS .iter() .flat_map(|invariant| invariant.triggers) - .all(|trigger| { - actual + .find(|trigger| { + !actual .get(&trigger.name.to_ascii_lowercase()) .is_some_and(|(table, sql)| { table.eq_ignore_ascii_case(trigger.table) diff --git a/crates/tracedecay-global-db/src/schema_contract/validation.rs b/crates/tracedecay-global-db/src/schema_contract/validation.rs index badd87e934..fb3f0b593e 100644 --- a/crates/tracedecay-global-db/src/schema_contract/validation.rs +++ b/crates/tracedecay-global-db/src/schema_contract/validation.rs @@ -3,7 +3,7 @@ use std::future::Future; use std::pin::Pin; use std::sync::LazyLock; -use tracedecay_runtime_core::db::engine::{QueryExecutor, params}; +use tracedecay_runtime_core::db::engine::QueryExecutor; use tracedecay_runtime_core::db::{DatabaseEngineReadConnection, DatabaseEngineReadSnapshot}; use tracedecay_runtime_core::ports::registered_schema::RegisteredSchemaInstallationV1; use tracedecay_rusqlite_runtime::repository::{ @@ -18,7 +18,7 @@ use super::definitions::{ use super::pragma::{ ActualColumn, ActualForeignKey, ActualIndex, ActualTableMetadata, read_table_metadata, }; -use super::{normalize_trigger_sql, starts_with_ignore_ascii_case}; +use super::starts_with_ignore_ascii_case; const OPERATION: &str = "validate global database authority schema"; @@ -291,45 +291,20 @@ fn validate_indexes_for_table( Ok(()) } -async fn validate_trigger( - conn: &impl QueryExecutor, - trigger: &super::invariants::Trigger, -) -> tracedecay_domain::errors::Result<()> { - let mut rows = conn - .query( - "SELECT tbl_name, sql FROM sqlite_master - WHERE type = 'trigger' AND name = ?1 COLLATE NOCASE", - params![trigger.name], - ) +async fn validate_triggers(conn: &impl QueryExecutor) -> tracedecay_domain::errors::Result<()> { + let invalid = super::invariants::first_invalid_trigger(conn) .await .map_err(|error| global_db_operation_error(OPERATION, error))?; - let actual = match rows - .next() - .await - .map_err(|error| global_db_operation_error(OPERATION, error))? - { - Some(row) => Some(( - row.get::(0) - .map_err(|error| global_db_operation_error(OPERATION, error))?, - row.get::(1) - .map_err(|error| global_db_operation_error(OPERATION, error))?, - )), - None => None, - }; - if actual.as_ref().is_some_and(|(table, sql)| { - table.eq_ignore_ascii_case(trigger.table) - && normalize_trigger_sql(sql) == normalize_trigger_sql(trigger.create_sql) - }) { - Ok(()) - } else { - Err(global_db_operation_message( + if let Some(trigger) = invalid { + return Err(global_db_operation_message( OPERATION, format!( "required trigger '{}' on table '{}' is missing or incompatible", trigger.name, trigger.table ), - )) + )); } + Ok(()) } async fn validate_observation_autoincrement( @@ -559,7 +534,7 @@ pub async fn validate_remote_deletion_schema_contract( /// schema modules; this validator intentionally neither claims nor validates those domains. /// /// Each concrete connection implements this instead of one generic async function: nesting -/// `validate_tables_and_indexes` under the trigger walk overflows the next solver's `Send` +/// table and trigger validation overflows the next solver's `Send` /// proof (rust-lang/rust#159228). The returned future only awaits already-erased children. pub(crate) trait AuthoritySchemaContract: Sync { fn validate_authority_schema<'a>( @@ -579,18 +554,10 @@ macro_rules! impl_authority_schema_contract { Box> + Send + '_>, > = Box::pin(validate_tables_and_indexes(self, TABLES)); tables.await?; - for invariant in super::invariants::INVARIANTS { - for trigger in invariant.triggers { - let checked: Pin< - Box< - dyn Future> - + Send - + '_, - >, - > = Box::pin(validate_trigger(self, trigger)); - checked.await?; - } - } + let triggers: Pin< + Box> + Send + '_>, + > = Box::pin(validate_triggers(self)); + triggers.await?; let autoincrement: Pin< Box> + Send + '_>, > = Box::pin(validate_observation_autoincrement(self)); @@ -615,9 +582,10 @@ pub(crate) fn validate_authority_schema_contract<'a>( mod tests { use std::sync::atomic::{AtomicUsize, Ordering}; - use super::{normalize_default, validate_tables_and_indexes}; + use super::{OPERATION, normalize_default, validate_tables_and_indexes, validate_triggers}; use crate::schema_contract::definitions::TABLES; use crate::tests::harness::open_registered_test_database_fixture; + use tracedecay_domain::errors::TraceDecayError; use tracedecay_runtime_core::db::{ TestDatabaseRuntimeScope, engine::{IntoParams, QueryExecutor, Rows}, @@ -686,4 +654,68 @@ mod tests { "column, foreign-key, and complete index metadata each need one query" ); } + + #[asupersync::test] + async fn trigger_contracts_use_one_query_and_preserve_first_failure() { + let directory = tempfile::tempdir().unwrap(); + let (database, _owner) = open_registered_test_database_fixture( + &directory.path().join("global.db"), + TestDatabaseRuntimeScope::Profile, + ) + .await + .unwrap(); + let connection = database.read_connection(); + let counted = CountingQueryExecutor::new(&connection); + + validate_triggers(&counted).await.unwrap(); + assert_eq!( + counted.queries(), + 1, + "all trigger definitions share one read" + ); + + let writer = database.writer_connection().unwrap(); + writer + .execute_batch( + "DROP TRIGGER observations_immutable_delete; + DROP TRIGGER observations_immutable_update; + CREATE TRIGGER observations_immutable_update + BEFORE UPDATE ON observations BEGIN SELECT 1; END;", + ) + .await + .unwrap(); + let error = validate_triggers(&counted).await.unwrap_err(); + assert!( + matches!( + &error, + TraceDecayError::Database { operation, message } + if operation == OPERATION + && message == "required trigger 'observations_immutable_update' on table 'observations' is missing or incompatible" + ), + "the first invalid contract must win over inventory order: {error}" + ); + assert_eq!(counted.queries(), 2); + + writer + .execute_batch( + "DROP TRIGGER observations_immutable_update; + CREATE TRIGGER OBSERVATIONS_IMMUTABLE_UPDATE + BEFORE UPDATE ON observations BEGIN + SELECT RAISE(ABORT, 'observations are immutable'); + END;", + ) + .await + .unwrap(); + let error = validate_triggers(&counted).await.unwrap_err(); + assert!( + matches!( + &error, + TraceDecayError::Database { operation, message } + if operation == OPERATION + && message == "required trigger 'observations_immutable_delete' on table 'observations' is missing or incompatible" + ), + "case-folded names must match before the next missing contract is reported: {error}" + ); + assert_eq!(counted.queries(), 3); + } } diff --git a/crates/tracedecay-global-db/src/session_project_rebind_tests.rs b/crates/tracedecay-global-db/src/session_project_rebind_tests.rs index b87f65b268..ddfb44fe0a 100644 --- a/crates/tracedecay-global-db/src/session_project_rebind_tests.rs +++ b/crates/tracedecay-global-db/src/session_project_rebind_tests.rs @@ -349,17 +349,7 @@ fn lcm_request(provider: &str, session_id: &str) -> LcmCompressionRequest { stateless_session_patterns: Vec::new(), ignore_message_patterns: Vec::new(), expected_current_frontier_store_id: None, - threshold_tokens: None, - max_assembly_tokens: None, - leaf_chunk_tokens: None, - max_source_messages: None, - summary_fan_in: None, - incremental_max_depth: None, - fresh_tail_count: None, - dynamic_leaf_chunk_enabled: None, - dynamic_leaf_chunk_max: None, - context_length: None, - reserve_tokens_floor: None, + overrides: Default::default(), summarizer: LcmSummarizerMode::Noop, } } diff --git a/crates/tracedecay-lcm/src/compression.rs b/crates/tracedecay-lcm/src/compression.rs index 49bc92b8dc..1c3bc17e28 100644 --- a/crates/tracedecay-lcm/src/compression.rs +++ b/crates/tracedecay-lcm/src/compression.rs @@ -10,14 +10,16 @@ use tracedecay_runtime_core::db::engine::{Executor, QueryExecutor, Value as SqlV use tracedecay_store::{SessionMessageRecord, session_project_fields}; use super::compression_decision::{ - self, AssemblyCapInput, CompressionPlanInput, CondensationCandidateDecision, - CondensationDecision, CondensationDecisionInput, OverflowRecoveryCapInput, - PreflightDecisionInput, + self, CompressionPlanInput, CondensationCandidateDecision, CondensationDecision, + CondensationDecisionInput, OverflowRecoveryCapInput, PreflightDecisionInput, }; use super::compression_policy::{is_policy_anchor_role, source_token_count}; use super::extraction; use super::summarizer::CompressionSummarizerAdapter; -use super::types::{LcmExtractionResult, LcmRelationProjectionStatus, LcmSummarySourceRange}; +use super::types::{ + CRITICAL_PRESSURE_CATCH_UP_RETRY, LcmExtractionResult, LcmRelationProjectionStatus, + LcmSummarySourceRange, NEEDS_SUMMARY_STATUS, +}; use super::{ LCM_DEFAULT_FRESH_TAIL_COUNT, LcmCompressionRequest, LcmCompressionResponse, LcmError, LcmLifecycleState, LcmLifecycleUpdate, LcmMaintenanceDebt, LcmPreflightRequest, @@ -288,12 +290,8 @@ pub async fn preflight( request: LcmPreflightRequest, ) -> Result { let mut request = request; - request.max_assembly_tokens = - compression_decision::effective_assembly_token_cap(AssemblyCapInput { - max_assembly_tokens: request.max_assembly_tokens, - context_length: request.context_length, - reserve_tokens_floor: request.reserve_tokens_floor, - }); + request.overrides.max_assembly_tokens = + compression_decision::effective_assembly_token_cap(&request.overrides); if let Some(reason) = filtered_session_reason( &request.session_id, &request.ignore_session_patterns, @@ -336,19 +334,20 @@ pub async fn preflight( let window = compression_window( &raw_messages, existing_frontier.current_frontier_store_id, - request.fresh_tail_count, + request.overrides.fresh_tail_count, request.current_tokens, - request.threshold_tokens, + request.overrides.threshold_tokens, ); let decision = compression_decision::preflight_decision(PreflightDecisionInput { - request: &request, + current_tokens: request.current_tokens, + overrides: &request.overrides, frontier: &existing_frontier, backlog: &window.backlog, }); Ok(LcmPreflightResponse { status: "ok".to_string(), - should_compress: decision.should_compress, - reason: decision.reason.to_string(), + should_compress: decision.should_compress(), + reason: decision.reason().to_string(), replay_messages: canonical_replay_messages(&raw_messages), }) } @@ -474,11 +473,7 @@ pub async fn begin_compression_pass( payload_rollback: &mut payload::PayloadFileRollback, retained_scan: Option, ) -> Result { - let assembly_token_cap = compression_decision::effective_assembly_token_cap(AssemblyCapInput { - max_assembly_tokens: request.max_assembly_tokens, - context_length: request.context_length, - reserve_tokens_floor: request.reserve_tokens_floor, - }); + let assembly_token_cap = compression_decision::effective_assembly_token_cap(&request.overrides); if let Some(reason) = filtered_session_reason( &request.session_id, &request.ignore_session_patterns, @@ -631,13 +626,13 @@ async fn prepare_compression_context( let window = compression_window( &raw_messages, existing_frontier.current_frontier_store_id, - request.fresh_tail_count, + request.overrides.fresh_tail_count, request.current_tokens, - request.threshold_tokens, + request.overrides.threshold_tokens, ); let plan = compression_decision::compression_plan(CompressionPlanInput { - request, - assembly_token_cap, + current_tokens: request.current_tokens, + overrides: &request.overrides, backlog: &window.backlog, }); let overflow_assembly_cap = @@ -859,7 +854,7 @@ fn auxiliary_summary_response( replay_without_summary(&context.window.pinned_anchors, &context.window.fresh_tail); Some(compression_response( - "needs_summary", + NEEDS_SUMMARY_STATUS, "hermes_auxiliary_not_available", Vec::new(), replay_messages, @@ -950,7 +945,7 @@ async fn persist_and_replay_backlog_compression( let retry_status = context .plan .forced_overflow_recovery - .then_some("critical_pressure_catch_up"); + .then_some(CRITICAL_PRESSURE_CATCH_UP_RETRY); Ok(compression_response_with_attempt_state( CompressionResponseParts { @@ -991,15 +986,13 @@ async fn persist_compression_transaction_writes<'a>( while !remaining_backlog.is_empty() && created_summaries.len() < pass_limit { let leaf_chunk_tokens = compression_decision::effective_leaf_chunk_tokens( - write.request.leaf_chunk_tokens, - write.request.dynamic_leaf_chunk_enabled, - write.request.dynamic_leaf_chunk_max, + &write.request.overrides, source_token_count(remaining_backlog), ); let selected_len = compression_decision::progress_leaf_chunk_len( remaining_backlog, leaf_chunk_tokens, - write.request.max_source_messages, + write.request.overrides.max_source_messages, ); let (selected_backlog, deferred_backlog) = remaining_backlog.split_at(selected_len); @@ -1798,8 +1791,7 @@ async fn condense_summary_nodes_if_ready( let CondensationDecision::QueryCandidates(policy) = compression_decision::condensation_policy_decision(CondensationDecisionInput { has_backlog: !window.backlog.is_empty(), - summary_fan_in: request.summary_fan_in, - incremental_max_depth: request.incremental_max_depth, + overrides: &request.overrides, summarizer, }) else { diff --git a/crates/tracedecay-lcm/src/compression_decision.rs b/crates/tracedecay-lcm/src/compression_decision.rs index 34b2ddc148..64fcbd07ea 100644 --- a/crates/tracedecay-lcm/src/compression_decision.rs +++ b/crates/tracedecay-lcm/src/compression_decision.rs @@ -1,40 +1,60 @@ pub(super) use super::compression_policy::threshold_pressure; -pub use super::compression_policy::{ - AssemblyCapInput, OverflowRecoveryCapInput, bounded_leaf_chunk_len, - effective_assembly_token_cap, effective_leaf_chunk_tokens, has_eligible_backlog, - overflow_recovery_assembly_cap, progress_leaf_chunk_len, -}; pub use super::compression_policy::{ CondensationCandidateDecision, CondensationDecision, CondensationPolicy, CondensationSkipReason, condensation_candidate_decision, incremental_max_depth_limit, }; +pub use super::compression_policy::{ + OverflowRecoveryCapInput, bounded_leaf_chunk_len, effective_assembly_token_cap, + effective_leaf_chunk_tokens, has_eligible_backlog, overflow_recovery_assembly_cap, + progress_leaf_chunk_len, +}; use super::compression_policy::{forced_overflow_pressure, source_token_count}; use super::summarizer::CompressionSummarizerAdapter; use super::{ - LCM_COMPRESSION_BOUNDARY_COOLDOWN_SECONDS, LCM_DEFAULT_SUMMARY_FAN_IN, LcmCompressionRequest, - LcmLifecycleState, LcmMaintenanceDebt, LcmPreflightRequest, LcmRawMessage, - LcmSessionBoundaryRequest, + LCM_COMPRESSION_BOUNDARY_COOLDOWN_SECONDS, LCM_DEFAULT_SUMMARY_FAN_IN, LcmCompressionOverrides, + LcmLifecycleState, LcmMaintenanceDebt, LcmRawMessage, LcmSessionBoundaryRequest, }; #[derive(Debug, Clone, Copy)] pub struct PreflightDecisionInput<'a> { - pub request: &'a LcmPreflightRequest, + pub current_tokens: Option, + pub overrides: &'a LcmCompressionOverrides, pub frontier: &'a LcmLifecycleState, pub backlog: &'a [LcmRawMessage], } #[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub struct PreflightDecision { - pub should_compress: bool, - pub reason: &'static str, +pub enum PreflightDecision { + ForcedOverflowPressure, + MaintenanceDebtReady, + ThresholdBacklogReady, + ThresholdNoEligibleBacklog, + NoCompressionNeeded, +} + +impl PreflightDecision { + pub fn should_compress(self) -> bool { + matches!( + self, + Self::ForcedOverflowPressure | Self::MaintenanceDebtReady | Self::ThresholdBacklogReady + ) + } + + pub fn reason(self) -> &'static str { + match self { + Self::ForcedOverflowPressure => "forced_overflow_pressure", + Self::MaintenanceDebtReady => "maintenance_debt_ready", + Self::ThresholdBacklogReady => "threshold_backlog_ready", + Self::ThresholdNoEligibleBacklog => "threshold_no_eligible_backlog", + Self::NoCompressionNeeded => "no_compression_needed", + } + } } #[derive(Debug, Clone, Copy)] pub struct CompressionPlanInput<'a> { - pub request: &'a LcmCompressionRequest, - /// The effective assembly cap from [`effective_assembly_token_cap`]; the - /// raw `request.max_assembly_tokens` is never read for pressure decisions. - pub assembly_token_cap: Option, + pub current_tokens: Option, + pub overrides: &'a LcmCompressionOverrides, pub backlog: &'a [LcmRawMessage], } @@ -60,8 +80,7 @@ pub enum BoundaryTransitionDecision { #[derive(Debug, Clone, Copy)] pub struct CondensationDecisionInput<'a> { pub has_backlog: bool, - pub summary_fan_in: Option, - pub incremental_max_depth: Option, + pub overrides: &'a LcmCompressionOverrides, pub summarizer: &'a CompressionSummarizerAdapter, } @@ -104,75 +123,49 @@ pub fn condensation_policy_decision(input: CondensationDecisionInput<'_>) -> Con } CondensationDecision::QueryCandidates(CondensationPolicy { fan_in: input + .overrides .summary_fan_in .filter(|fan_in| *fan_in > 1) .unwrap_or(LCM_DEFAULT_SUMMARY_FAN_IN), - incremental_max_depth: incremental_max_depth_limit(input.incremental_max_depth), + incremental_max_depth: incremental_max_depth_limit(input.overrides.incremental_max_depth), }) } pub fn preflight_decision(input: PreflightDecisionInput<'_>) -> PreflightDecision { - if forced_overflow_pressure( - input.request.current_tokens, - input.request.max_assembly_tokens, - ) { - return PreflightDecision { - should_compress: true, - reason: "forced_overflow_pressure", - }; + if forced_overflow_pressure(input.current_tokens, input.overrides.max_assembly_tokens) { + return PreflightDecision::ForcedOverflowPressure; } if frontier_has_maintenance_debt(input.frontier) { - return PreflightDecision { - should_compress: true, - reason: "maintenance_debt_ready", - }; + return PreflightDecision::MaintenanceDebtReady; } - if threshold_pressure(input.request.current_tokens, input.request.threshold_tokens) { + if threshold_pressure(input.current_tokens, input.overrides.threshold_tokens) { if input.backlog.is_empty() { - return PreflightDecision { - should_compress: false, - reason: "threshold_no_eligible_backlog", - }; + return PreflightDecision::ThresholdNoEligibleBacklog; } - let leaf_chunk_tokens = effective_leaf_chunk_tokens( - input.request.leaf_chunk_tokens, - input.request.dynamic_leaf_chunk_enabled, - input.request.dynamic_leaf_chunk_max, - source_token_count(input.backlog), - ); + let leaf_chunk_tokens = + effective_leaf_chunk_tokens(input.overrides, source_token_count(input.backlog)); if has_eligible_backlog(input.backlog, leaf_chunk_tokens) { - return PreflightDecision { - should_compress: true, - reason: "threshold_backlog_ready", - }; + return PreflightDecision::ThresholdBacklogReady; } - return PreflightDecision { - should_compress: false, - reason: "threshold_no_eligible_backlog", - }; + return PreflightDecision::ThresholdNoEligibleBacklog; } - PreflightDecision { - should_compress: false, - reason: "no_compression_needed", - } + PreflightDecision::NoCompressionNeeded } pub fn compression_plan(input: CompressionPlanInput<'_>) -> CompressionPlan { - let forced_overflow_recovery = - forced_overflow_pressure(input.request.current_tokens, input.assembly_token_cap); - let leaf_chunk_tokens = effective_leaf_chunk_tokens( - input.request.leaf_chunk_tokens, - input.request.dynamic_leaf_chunk_enabled, - input.request.dynamic_leaf_chunk_max, - source_token_count(input.backlog), + let forced_overflow_recovery = forced_overflow_pressure( + input.current_tokens, + effective_assembly_token_cap(input.overrides), ); + let leaf_chunk_tokens = + effective_leaf_chunk_tokens(input.overrides, source_token_count(input.backlog)); let selected_len = progress_leaf_chunk_len( input.backlog, leaf_chunk_tokens, - input.request.max_source_messages, + input.overrides.max_source_messages, ); CompressionPlan { selected_len, @@ -191,6 +184,7 @@ pub fn frontier_has_maintenance_debt(frontier: &LcmLifecycleState) -> bool { #[cfg(test)] mod tests { use super::super::LcmSummarizerMode; + use super::super::types::LcmPreflightRequest; use super::*; fn boundary_request() -> LcmSessionBoundaryRequest { @@ -273,12 +267,17 @@ mod tests { let summarizer = CompressionSummarizerAdapter::from_mode(LcmSummarizerMode::Provided { summary_text: "summary".to_string(), route: None, + extraction_result: None, }); + let overrides = LcmCompressionOverrides { + summary_fan_in: Some(6), + incremental_max_depth: Some(-1), + ..Default::default() + }; assert_eq!( condensation_policy_decision(CondensationDecisionInput { has_backlog: false, - summary_fan_in: Some(6), - incremental_max_depth: Some(-1), + overrides: &overrides, summarizer: &summarizer, }), CondensationDecision::QueryCandidates(CondensationPolicy { @@ -290,14 +289,16 @@ mod tests { #[test] fn condensation_policy_skips_backlog_and_auxiliary_modes() { - let persisted = CompressionSummarizerAdapter::from_mode(LcmSummarizerMode::Fake { + let persisted = CompressionSummarizerAdapter::from_mode(LcmSummarizerMode::Provided { summary_text: "summary".to_string(), + route: None, + extraction_result: None, }); + let overrides = LcmCompressionOverrides::default(); assert_eq!( condensation_policy_decision(CondensationDecisionInput { has_backlog: true, - summary_fan_in: None, - incremental_max_depth: None, + overrides: &overrides, summarizer: &persisted, }), CondensationDecision::Skip(CondensationSkipReason::BacklogPresent) @@ -307,8 +308,7 @@ mod tests { assert_eq!( condensation_policy_decision(CondensationDecisionInput { has_backlog: false, - summary_fan_in: None, - incremental_max_depth: None, + overrides: &overrides, summarizer: &auxiliary, }), CondensationDecision::Skip(CondensationSkipReason::AuxiliarySummarizer) @@ -322,17 +322,10 @@ mod tests { session_id: "session-1".to_string(), messages: Vec::new(), current_tokens: Some(12), - threshold_tokens: None, - max_assembly_tokens: Some(8), - leaf_chunk_tokens: None, - max_source_messages: None, - summary_fan_in: None, - incremental_max_depth: None, - fresh_tail_count: None, - dynamic_leaf_chunk_enabled: None, - dynamic_leaf_chunk_max: None, - context_length: None, - reserve_tokens_floor: None, + overrides: crate::LcmCompressionOverrides { + max_assembly_tokens: Some(8), + ..Default::default() + }, ignore_session_patterns: Vec::new(), stateless_session_patterns: Vec::new(), }; @@ -364,12 +357,13 @@ mod tests { }]; let decision = preflight_decision(PreflightDecisionInput { - request: &request, + current_tokens: request.current_tokens, + overrides: &request.overrides, frontier: &frontier, backlog: &backlog, }); - assert!(decision.should_compress); - assert_eq!(decision.reason, "forced_overflow_pressure"); + assert!(decision.should_compress()); + assert_eq!(decision.reason(), "forced_overflow_pressure"); } } diff --git a/crates/tracedecay-lcm/src/compression_policy.rs b/crates/tracedecay-lcm/src/compression_policy.rs index 96b0d061a9..de320bb9d4 100644 --- a/crates/tracedecay-lcm/src/compression_policy.rs +++ b/crates/tracedecay-lcm/src/compression_policy.rs @@ -2,6 +2,7 @@ use serde_json::Value; use super::contracts::LcmRawMessage; use super::replay_transactions; +use super::types::LcmCompressionOverrides; pub const DEFAULT_INCREMENTAL_MAX_DEPTH: i64 = 1; @@ -22,13 +23,6 @@ pub(crate) fn policy_anchor_role_sql_in_list() -> String { .join(", ") } -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub struct AssemblyCapInput { - pub max_assembly_tokens: Option, - pub context_length: Option, - pub reserve_tokens_floor: Option, -} - #[derive(Debug, Clone, Copy)] pub struct OverflowRecoveryCapInput<'a> { pub current_tokens: Option, @@ -79,11 +73,11 @@ pub fn incremental_max_depth_limit(configured: Option) -> i64 { } } -pub fn effective_assembly_token_cap(input: AssemblyCapInput) -> Option { - let explicit_cap = input.max_assembly_tokens.filter(|cap| *cap > 0); +pub fn effective_assembly_token_cap(overrides: &LcmCompressionOverrides) -> Option { + let explicit_cap = overrides.max_assembly_tokens.filter(|cap| *cap > 0); let reserve_cap = match ( - input.context_length.filter(|length| *length > 0), - input.reserve_tokens_floor.filter(|floor| *floor > 0), + overrides.context_length.filter(|length| *length > 0), + overrides.reserve_tokens_floor.filter(|floor| *floor > 0), ) { (Some(context_length), Some(reserve_tokens_floor)) => { Some(context_length - reserve_tokens_floor).filter(|cap| *cap > 0) @@ -127,16 +121,14 @@ pub fn has_eligible_backlog(backlog: &[LcmRawMessage], leaf_chunk_tokens: Option } pub fn effective_leaf_chunk_tokens( - leaf_chunk_tokens: Option, - dynamic_leaf_chunk_enabled: Option, - dynamic_leaf_chunk_max: Option, + overrides: &LcmCompressionOverrides, raw_tokens_outside_tail: i64, ) -> Option { - if !dynamic_leaf_chunk_enabled.unwrap_or(false) { - return leaf_chunk_tokens; + if !overrides.dynamic_leaf_chunk_enabled.unwrap_or(false) { + return overrides.leaf_chunk_tokens; } - let base = leaf_chunk_tokens.unwrap_or(1).max(1); - let ceiling = dynamic_leaf_chunk_max.unwrap_or(base).max(base); + let base = overrides.leaf_chunk_tokens.unwrap_or(1).max(1); + let ceiling = overrides.dynamic_leaf_chunk_max.unwrap_or(base).max(base); let mut working = base; while working < ceiling && raw_tokens_outside_tail > working.saturating_mul(2) { working = ceiling.min(working.saturating_mul(2)); diff --git a/crates/tracedecay-lcm/src/extraction.rs b/crates/tracedecay-lcm/src/extraction.rs index 6dadf89d8a..9da372e986 100644 --- a/crates/tracedecay-lcm/src/extraction.rs +++ b/crates/tracedecay-lcm/src/extraction.rs @@ -16,17 +16,6 @@ If there is nothing worth extracting, respond with exactly: NOTHING_TO_EXTRACT CONTENT: {text}"; -// Route-envelope contract with Hermes plugin: -// - `route`: caller-provided summarizer route string -// - `pre_compaction_extraction`: optional extraction result object -#[derive(Debug, serde::Deserialize)] -struct LcmProvidedSummaryRouteEnvelope { - #[serde(default, skip_serializing_if = "Option::is_none")] - route: Option, - #[serde(default, skip_serializing_if = "Option::is_none")] - pre_compaction_extraction: Option, -} - pub fn build_extraction_request( session_id: &str, source_range: &LcmSummarySourceRange, @@ -44,28 +33,6 @@ pub fn build_extraction_request( }) } -pub fn split_summary_route(route: Option<&str>) -> (Option, Option) { - let route = route.and_then(non_empty).map(str::to_string); - let Some(route) = route else { - return (None, None); - }; - if !(route.starts_with('{') && route.ends_with('}')) { - return (Some(route), None); - } - let parsed = serde_json::from_str::(&route); - if let Ok(envelope) = parsed { - return ( - envelope - .route - .as_deref() - .and_then(non_empty) - .map(str::to_string), - envelope.pre_compaction_extraction, - ); - } - (Some(route), None) -} - pub fn summary_metadata_extraction( extraction_result: Option<&LcmExtractionResult>, condensation: bool, @@ -97,7 +64,7 @@ fn serialize_messages(messages: &[LcmSummarySourceMessage]) -> String { .join("\n\n") } -fn non_empty(value: &str) -> Option<&str> { +pub(crate) fn non_empty(value: &str) -> Option<&str> { let trimmed = value.trim(); if trimmed.is_empty() { None diff --git a/crates/tracedecay-lcm/src/hermes.rs b/crates/tracedecay-lcm/src/hermes.rs index 852a2c9567..9f31721d06 100644 --- a/crates/tracedecay-lcm/src/hermes.rs +++ b/crates/tracedecay-lcm/src/hermes.rs @@ -2,6 +2,8 @@ use serde_json::Value; +use crate::types::{LcmCompressionOverrides, LcmExtractionResult}; + #[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)] pub struct LcmCompressionRequest { pub provider: String, @@ -17,28 +19,8 @@ pub struct LcmCompressionRequest { pub ignore_message_patterns: Vec, #[serde(default, skip_serializing_if = "Option::is_none")] pub expected_current_frontier_store_id: Option, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub threshold_tokens: Option, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub max_assembly_tokens: Option, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub leaf_chunk_tokens: Option, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub max_source_messages: Option, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub summary_fan_in: Option, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub incremental_max_depth: Option, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub fresh_tail_count: Option, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub dynamic_leaf_chunk_enabled: Option, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub dynamic_leaf_chunk_max: Option, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub context_length: Option, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub reserve_tokens_floor: Option, + #[serde(flatten)] + pub overrides: LcmCompressionOverrides, pub summarizer: LcmSummarizerMode, } @@ -46,12 +28,20 @@ pub struct LcmCompressionRequest { #[serde(tag = "mode", rename_all = "snake_case")] pub enum LcmSummarizerMode { Noop, - Fake { - summary_text: String, - }, Provided { summary_text: String, route: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + extraction_result: Option, }, HermesAuxiliary, } + +impl LcmSummarizerMode { + pub fn has_nonempty_provided_summary(&self) -> bool { + matches!( + self, + Self::Provided { summary_text, .. } if !summary_text.trim().is_empty() + ) + } +} diff --git a/crates/tracedecay-lcm/src/lib.rs b/crates/tracedecay-lcm/src/lib.rs index 78bf1242d2..3d4bc888c5 100644 --- a/crates/tracedecay-lcm/src/lib.rs +++ b/crates/tracedecay-lcm/src/lib.rs @@ -55,14 +55,15 @@ pub use raw::derived_text_for_index; pub use schema::LCM_SCHEMA_VERSION; pub use types::{ DERIVED_TRUNCATION_MARKER, LCM_COMPRESSION_BOUNDARY_COOLDOWN_SECONDS, - LCM_DEFAULT_FRESH_TAIL_COUNT, LCM_DEFAULT_SUMMARY_FAN_IN, LcmCompressionResponse, - LcmConfigStatus, LcmContentRange, LcmContentSlice, LcmDagDepthStatus, LcmDagStatus, - LcmDescribeExternalPayload, LcmDescribeRequest, LcmDescribeResponse, LcmDescribeSourceOverview, - LcmDescribeSummaryNode, LcmDescribeTarget, LcmError, LcmExpandQueryBudget, - LcmExpandQueryContextBlock, LcmExpandQueryMatch, LcmExpandQueryPagination, - LcmExpandQueryResponse, LcmExpandQuerySynthesisPrompt, LcmExpandRequest, LcmExpandResponse, - LcmExpandSourcePagination, LcmExpandTarget, LcmExpandedSummarySource, LcmGcConfig, LcmGrepHit, - LcmGrepSort, LcmLifecycleState, LcmLifecycleUpdate, LcmLoadSessionMessage, LcmLoadSessionPage, + LCM_DEFAULT_FRESH_TAIL_COUNT, LCM_DEFAULT_SUMMARY_FAN_IN, LcmCompressionOutcome, + LcmCompressionOverrides, LcmCompressionResponse, LcmConfigStatus, LcmContentRange, + LcmContentSlice, LcmDagDepthStatus, LcmDagStatus, LcmDescribeExternalPayload, + LcmDescribeRequest, LcmDescribeResponse, LcmDescribeSourceOverview, LcmDescribeSummaryNode, + LcmDescribeTarget, LcmError, LcmExpandQueryBudget, LcmExpandQueryContextBlock, + LcmExpandQueryMatch, LcmExpandQueryPagination, LcmExpandQueryResponse, + LcmExpandQuerySynthesisPrompt, LcmExpandRequest, LcmExpandResponse, LcmExpandSourcePagination, + LcmExpandTarget, LcmExpandedSummarySource, LcmGcConfig, LcmGrepHit, LcmGrepSort, + LcmLifecycleState, LcmLifecycleUpdate, LcmLoadSessionMessage, LcmLoadSessionPage, LcmLoadSessionRequest, LcmMaintenanceDebt, LcmPayloadExpansion, LcmPayloadGcStatus, LcmPayloadRef, LcmPreflightRequest, LcmPreflightResponse, LcmRawMessage, LcmRawMessageMetadata, LcmRawMessageOverview, LcmRecentSession, LcmRelationProjectionStatus, LcmReplayMessage, diff --git a/crates/tracedecay-lcm/src/summarizer.rs b/crates/tracedecay-lcm/src/summarizer.rs index 64c18a54a7..077a3b0bcf 100644 --- a/crates/tracedecay-lcm/src/summarizer.rs +++ b/crates/tracedecay-lcm/src/summarizer.rs @@ -21,27 +21,24 @@ pub enum CompressionSummarizerAdapter { impl CompressionSummarizerAdapter { pub fn from_mode(mode: LcmSummarizerMode) -> Self { + let has_nonempty_provided_summary = mode.has_nonempty_provided_summary(); match mode { LcmSummarizerMode::Noop => Self::Noop, LcmSummarizerMode::HermesAuxiliary => Self::HermesAuxiliary, - LcmSummarizerMode::Fake { summary_text } => { - Self::Persisted(PersistedSummaryInvocation { - summary_text, - route: None, - extraction_result: None, - }) - } LcmSummarizerMode::Provided { summary_text, route, + extraction_result, } => { - if summary_text.trim().is_empty() { + if !has_nonempty_provided_summary { return Self::HermesAuxiliary; } - let (route, extraction_result) = extraction::split_summary_route(route.as_deref()); Self::Persisted(PersistedSummaryInvocation { summary_text, - route, + route: route + .as_deref() + .and_then(extraction::non_empty) + .map(str::to_string), extraction_result, }) } @@ -150,21 +147,23 @@ mod tests { } #[test] - fn provided_mode_selects_persisted_summary_and_splits_route_envelope() { - let adapter = CompressionSummarizerAdapter::from_mode(LcmSummarizerMode::Provided { + fn provided_mode_preserves_typed_route_and_extraction() { + let mode = LcmSummarizerMode::Provided { summary_text: "provided summary".into(), - route: Some( - json!({ - "route": "backup", - "pre_compaction_extraction": { - "status": "ok", - "items": ["Decision: keep nightly backups"], - "model": "openai/gpt-5.4-mini" - } - }) - .to_string(), - ), - }); + route: Some(" backup {🧭} ".into()), + extraction_result: Some(LcmExtractionResult { + status: "ok".into(), + items: vec!["Decision: keep nightly backups".into()], + text: None, + model: Some("openai/gpt-5.4-mini".into()), + output_path: None, + error: None, + }), + }; + let encoded = serde_json::to_value(&mode).expect("provided mode should serialize"); + let mode: LcmSummarizerMode = + serde_json::from_value(encoded).expect("provided mode should deserialize"); + let adapter = CompressionSummarizerAdapter::from_mode(mode); assert_eq!( adapter @@ -172,7 +171,7 @@ mod tests { .expect("provided mode should persist a summary"), &PersistedSummaryInvocation { summary_text: "provided summary".into(), - route: Some("backup".into()), + route: Some("backup {🧭}".into()), extraction_result: Some(LcmExtractionResult { status: "ok".into(), items: vec!["Decision: keep nightly backups".into()], @@ -190,6 +189,7 @@ mod tests { let adapter = CompressionSummarizerAdapter::from_mode(LcmSummarizerMode::Provided { summary_text: " \n ".into(), route: Some("obsolete_empty_route".into()), + extraction_result: None, }); assert!(adapter.persisted_summary_invocation().is_none()); @@ -207,6 +207,50 @@ mod tests { assert_eq!(request.source_range.to_store_id, 11); } + #[test] + fn provided_routes_are_trimmed_opaque_text() { + let persisted_route = |route: Option<&str>| { + CompressionSummarizerAdapter::from_mode(LcmSummarizerMode::Provided { + summary_text: "summary".into(), + route: route.map(str::to_string), + extraction_result: None, + }) + .persisted_summary_invocation() + .expect("nonempty provided summary should persist") + .route + .clone() + }; + + assert_eq!(persisted_route(None), None); + assert_eq!(persisted_route(Some(" \t ")), None); + assert_eq!( + persisted_route(Some(" route {with braces} and 雪 ")), + Some("route {with braces} and 雪".to_string()) + ); + let json_text = r#"{"route":"backup","pre_compaction_extraction":{"status":"ok"}}"#; + assert_eq!( + persisted_route(Some(json_text)), + Some(json_text.to_string()) + ); + } + + #[test] + fn malformed_extraction_input_is_rejected_at_deserialization() { + let malformed = json!({ + "mode": "provided", + "summary_text": "summary", + "route": "{\"route\":\"not parsed\"}", + "extraction_result": {"status": "ok", "items": "not a list"} + }); + assert!(serde_json::from_value::(malformed).is_err()); + } + + #[test] + fn retired_fake_mode_is_rejected_at_deserialization() { + let fake = json!({"mode": "fake", "summary_text": "canned summary"}); + assert!(serde_json::from_value::(fake).is_err()); + } + #[test] fn hermes_auxiliary_mode_builds_summary_request_from_backlog_inputs() { let adapter = CompressionSummarizerAdapter::from_mode(LcmSummarizerMode::HermesAuxiliary); diff --git a/crates/tracedecay-lcm/src/types.rs b/crates/tracedecay-lcm/src/types.rs index 92651880dc..1c5a8cbbb7 100644 --- a/crates/tracedecay-lcm/src/types.rs +++ b/crates/tracedecay-lcm/src/types.rs @@ -585,13 +585,8 @@ pub enum LcmMaintenanceDebt { }, } -#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)] -pub struct LcmPreflightRequest { - pub provider: String, - pub session_id: String, - #[serde(default, skip_serializing_if = "Vec::is_empty")] - pub messages: Vec, - pub current_tokens: Option, +#[derive(Debug, Clone, Default, PartialEq, Eq, serde::Serialize, serde::Deserialize)] +pub struct LcmCompressionOverrides { #[serde(default, skip_serializing_if = "Option::is_none")] pub threshold_tokens: Option, #[serde(default, skip_serializing_if = "Option::is_none")] @@ -614,6 +609,17 @@ pub struct LcmPreflightRequest { pub context_length: Option, #[serde(default, skip_serializing_if = "Option::is_none")] pub reserve_tokens_floor: Option, +} + +#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)] +pub struct LcmPreflightRequest { + pub provider: String, + pub session_id: String, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub messages: Vec, + pub current_tokens: Option, + #[serde(flatten)] + pub overrides: LcmCompressionOverrides, #[serde(default, skip_serializing_if = "Vec::is_empty")] pub ignore_session_patterns: Vec, #[serde(default, skip_serializing_if = "Vec::is_empty")] @@ -737,11 +743,111 @@ pub struct LcmCompressionResponse { pub summary_request: Option, } +pub(crate) const NEEDS_SUMMARY_STATUS: &str = "needs_summary"; +const NEEDS_AUTHORITATIVE_SUMMARY_RETRY: &str = "needs_authoritative_summary"; +pub(crate) const CRITICAL_PRESSURE_CATCH_UP_RETRY: &str = "critical_pressure_catch_up"; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum LcmCompressionOutcome<'a> { + Complete, + NeedsSummary(&'a LcmSummaryRequest), + Deferred { reason: &'a str }, +} + +impl LcmCompressionResponse { + pub fn outcome(&self) -> Result, LcmError> { + match ( + self.status.as_str(), + self.summary_request.as_ref(), + self.retry_status.as_deref(), + ) { + (NEEDS_SUMMARY_STATUS, Some(summary_request), None) => { + Ok(LcmCompressionOutcome::NeedsSummary(summary_request)) + } + (NEEDS_SUMMARY_STATUS, Some(_), Some(NEEDS_AUTHORITATIVE_SUMMARY_RETRY)) => { + Ok(LcmCompressionOutcome::Deferred { + reason: &self.reason, + }) + } + ("ok" | "best_effort", None, None | Some(CRITICAL_PRESSURE_CATCH_UP_RETRY)) => { + Ok(LcmCompressionOutcome::Complete) + } + (_, summary_request, retry_status) => Err(LcmError::Db(format!( + "invalid LCM compression outcome: status {:?}, summary request present {}, retry status {:?}", + self.status, + summary_request.is_some(), + retry_status + ))), + } + } + + pub fn defer_for_authoritative_summary(mut self, reason: &str) -> Result { + if !matches!(self.outcome()?, LcmCompressionOutcome::NeedsSummary(_)) { + return Err(LcmError::Db( + "only a pending summary response can be deferred".to_string(), + )); + } + self.reason = reason.to_string(); + self.retry_status = Some(NEEDS_AUTHORITATIVE_SUMMARY_RETRY.to_string()); + Ok(self) + } +} + #[cfg(test)] #[allow(clippy::expect_used)] mod tests { use super::*; + fn summary_request() -> LcmSummaryRequest { + LcmSummaryRequest { + provider: "cursor".to_string(), + session_id: "session-1".to_string(), + focus_topic: None, + prompt: "summarize".to_string(), + source_range: LcmSummarySourceRange { + from_store_id: 1, + to_store_id: 2, + }, + source_messages: vec![LcmSummarySourceMessage { + store_id: 1, + role: "user".to_string(), + content: "source".to_string(), + }], + extraction_request: None, + } + } + + fn compression_response( + status: &str, + reason: &str, + summary_request: Option, + retry_status: Option<&str>, + ) -> LcmCompressionResponse { + LcmCompressionResponse { + status: status.to_string(), + reason: reason.to_string(), + summary_nodes_created: 0, + summary_nodes: Vec::new(), + replay_messages: Vec::new(), + replay_token_estimate: 0, + replay_over_budget: false, + compression_attempts: 0, + context_recovery_hint: None, + retry_status: retry_status.map(str::to_string), + relation_projection_status: LcmRelationProjectionStatus::NotApplicable, + frontier: LcmLifecycleState { + provider: "cursor".to_string(), + conversation_id: "session-1".to_string(), + current_session_id: "session-1".to_string(), + current_frontier_store_id: None, + last_finalized_session_id: None, + last_finalized_frontier_store_id: None, + maintenance_debt: Vec::new(), + }, + summary_request, + } + } + #[test] fn gc_config_clamps_grace_floor_from_serde() { let config: LcmGcConfig = @@ -761,4 +867,106 @@ mod tests { assert!(config.reap_missing_enabled); assert_eq!(config.reap_missing_after, 0); } + + #[test] + fn pending_summary_outcome_borrows_its_required_request() { + let request = summary_request(); + let response = + compression_response("needs_summary", "pending", Some(request.clone()), None); + + assert_eq!( + response.outcome().expect("pending response is valid"), + LcmCompressionOutcome::NeedsSummary(&request) + ); + } + + #[test] + fn completed_outcomes_include_overflow_catch_up_retries() { + for (status, reason) in [ + ("ok", "forced_overflow_recovery"), + ("best_effort", "forced_overflow_recovery_replay_over_budget"), + ] { + let mut response = + compression_response(status, reason, None, Some("critical_pressure_catch_up")); + response.replay_over_budget = status == "best_effort"; + response.compression_attempts = 2; + response.context_recovery_hint = Some("preserve the freshest tail".to_string()); + let original = serde_json::to_value(&response).expect("response should serialize"); + + assert_eq!( + response.outcome().expect("completed response is valid"), + LcmCompressionOutcome::Complete + ); + assert_eq!( + serde_json::to_value(&response).expect("response should serialize"), + original + ); + } + } + + #[test] + fn inconsistent_outcomes_are_contract_failures() { + let inconsistent = [ + compression_response("needs_summary", "pending", None, None), + compression_response( + "needs_summary", + "pending", + Some(summary_request()), + Some("critical_pressure_catch_up"), + ), + compression_response("ok", "done", Some(summary_request()), None), + compression_response("ok", "done", None, Some("needs_authoritative_summary")), + compression_response("future_outcome", "done", None, None), + compression_response( + "unexpected", + "done", + None, + Some("critical_pressure_catch_up"), + ), + ]; + for response in inconsistent { + assert!( + matches!(response.outcome(), Err(LcmError::Db(_))), + "{} / {:?} must not read as a valid outcome", + response.status, + response.retry_status + ); + } + } + + #[test] + fn deferral_keeps_the_request_and_round_trips_through_the_wire() { + let request = summary_request(); + let response = + compression_response("needs_summary", "pending", Some(request.clone()), None) + .defer_for_authoritative_summary("summarizer_configuration_unavailable") + .expect("pending response can be deferred"); + + let wire = serde_json::to_value(&response).expect("response should serialize"); + assert_eq!(wire["status"], "needs_summary"); + assert_eq!(wire["reason"], "summarizer_configuration_unavailable"); + assert_eq!(wire["retry_status"], "needs_authoritative_summary"); + assert_eq!( + wire["summary_request"], + serde_json::to_value(&request).expect("request should serialize") + ); + let decoded: LcmCompressionResponse = + serde_json::from_value(wire).expect("response should deserialize"); + assert_eq!( + decoded.outcome().expect("deferred response is valid"), + LcmCompressionOutcome::Deferred { + reason: "summarizer_configuration_unavailable" + } + ); + } + + #[test] + fn completed_response_cannot_be_rewritten_as_deferred() { + let response = compression_response("ok", "done", None, None); + assert!( + response + .defer_for_authoritative_summary("summarizer_configuration_unavailable") + .is_err() + ); + } } diff --git a/crates/tracedecay-lcm/tests/lcm_suite/compression_policy.rs b/crates/tracedecay-lcm/tests/lcm_suite/compression_policy.rs index 3791cb98b6..fec5a0a6ab 100644 --- a/crates/tracedecay-lcm/tests/lcm_suite/compression_policy.rs +++ b/crates/tracedecay-lcm/tests/lcm_suite/compression_policy.rs @@ -1,8 +1,8 @@ use serde_json::{Value, json}; +use tracedecay_lcm::LcmCompressionOverrides; use tracedecay_lcm::compression_policy::{ - AssemblyCapInput, OverflowRecoveryCapInput, bounded_leaf_chunk_len, - effective_assembly_token_cap, effective_leaf_chunk_tokens, overflow_recovery_assembly_cap, - progress_leaf_chunk_len, + OverflowRecoveryCapInput, bounded_leaf_chunk_len, effective_assembly_token_cap, + effective_leaf_chunk_tokens, overflow_recovery_assembly_cap, progress_leaf_chunk_len, }; use tracedecay_lcm::contracts::{LcmRawMessage, LcmStorageKind}; @@ -56,18 +56,19 @@ fn complete_tool_transaction() -> Vec { #[test] fn assembly_cap_uses_the_smallest_positive_authority() { assert_eq!( - effective_assembly_token_cap(AssemblyCapInput { + effective_assembly_token_cap(&LcmCompressionOverrides { max_assembly_tokens: Some(200), context_length: Some(80), reserve_tokens_floor: Some(30), + ..Default::default() }), Some(50) ); assert_eq!( - effective_assembly_token_cap(AssemblyCapInput { - max_assembly_tokens: None, + effective_assembly_token_cap(&LcmCompressionOverrides { context_length: Some(30), reserve_tokens_floor: Some(30), + ..Default::default() }), None ); @@ -96,11 +97,27 @@ fn overflow_recovery_cap_reserves_non_message_overhead() { #[test] fn dynamic_leaf_size_grows_by_powers_of_two_within_the_ceiling() { assert_eq!( - effective_leaf_chunk_tokens(Some(4), Some(true), Some(16), 20), + effective_leaf_chunk_tokens( + &LcmCompressionOverrides { + leaf_chunk_tokens: Some(4), + dynamic_leaf_chunk_enabled: Some(true), + dynamic_leaf_chunk_max: Some(16), + ..Default::default() + }, + 20, + ), Some(16) ); assert_eq!( - effective_leaf_chunk_tokens(Some(4), Some(false), Some(16), 20), + effective_leaf_chunk_tokens( + &LcmCompressionOverrides { + leaf_chunk_tokens: Some(4), + dynamic_leaf_chunk_enabled: Some(false), + dynamic_leaf_chunk_max: Some(16), + ..Default::default() + }, + 20, + ), Some(4) ); } diff --git a/crates/tracedecay-lcm/tests/lcm_suite/main.rs b/crates/tracedecay-lcm/tests/lcm_suite/main.rs index 15491cc494..5a2f79ec6f 100644 --- a/crates/tracedecay-lcm/tests/lcm_suite/main.rs +++ b/crates/tracedecay-lcm/tests/lcm_suite/main.rs @@ -8,3 +8,4 @@ mod compression_policy; mod lcm_contracts; mod lcm_security; +mod request_wire; diff --git a/crates/tracedecay-lcm/tests/lcm_suite/request_wire.rs b/crates/tracedecay-lcm/tests/lcm_suite/request_wire.rs new file mode 100644 index 0000000000..31406c69b4 --- /dev/null +++ b/crates/tracedecay-lcm/tests/lcm_suite/request_wire.rs @@ -0,0 +1,78 @@ +use tracedecay_lcm::{LcmCompressionRequest, LcmPreflightRequest}; + +const FULL_COMPRESSION_REQUEST: &str = r#"{"provider":"cursor","session_id":"session-1","messages":[{"content":"hi","role":"user"}],"current_tokens":120,"focus_topic":"billing","ignore_session_patterns":["ignored-*"],"stateless_session_patterns":["stateless-*"],"ignore_message_patterns":["^noise"],"expected_current_frontier_store_id":7,"threshold_tokens":1,"max_assembly_tokens":2,"leaf_chunk_tokens":3,"max_source_messages":4,"summary_fan_in":5,"incremental_max_depth":-1,"fresh_tail_count":0,"dynamic_leaf_chunk_enabled":false,"dynamic_leaf_chunk_max":8,"context_length":9,"reserve_tokens_floor":10,"summarizer":{"mode":"provided","summary_text":"summary","route":"backup"}}"#; +const MINIMAL_COMPRESSION_REQUEST: &str = r#"{"provider":"cursor","session_id":"session-1","messages":[],"current_tokens":null,"focus_topic":null,"summarizer":{"mode":"noop"}}"#; +const FULL_PREFLIGHT_REQUEST: &str = r#"{"provider":"cursor","session_id":"session-1","messages":[{"content":"hi","role":"user"}],"current_tokens":120,"threshold_tokens":1,"max_assembly_tokens":2,"leaf_chunk_tokens":3,"max_source_messages":4,"summary_fan_in":5,"incremental_max_depth":-1,"fresh_tail_count":0,"dynamic_leaf_chunk_enabled":true,"dynamic_leaf_chunk_max":8,"context_length":9,"reserve_tokens_floor":10,"ignore_session_patterns":["ignored-*"],"stateless_session_patterns":["stateless-*"]}"#; +const MINIMAL_PREFLIGHT_REQUEST: &str = + r#"{"provider":"cursor","session_id":"session-1","current_tokens":null}"#; + +fn reencoded_compression(json: &str) -> String { + let request: LcmCompressionRequest = + serde_json::from_str(json).expect("compression request should deserialize"); + serde_json::to_string(&request).expect("compression request should serialize") +} + +fn reencoded_preflight(json: &str) -> String { + let request: LcmPreflightRequest = + serde_json::from_str(json).expect("preflight request should deserialize"); + serde_json::to_string(&request).expect("preflight request should serialize") +} + +#[test] +fn compression_request_wire_bytes_round_trip_flat() { + assert_eq!( + reencoded_compression(FULL_COMPRESSION_REQUEST), + FULL_COMPRESSION_REQUEST + ); + assert_eq!( + reencoded_compression(MINIMAL_COMPRESSION_REQUEST), + MINIMAL_COMPRESSION_REQUEST + ); +} + +#[test] +fn preflight_request_wire_bytes_round_trip_flat() { + assert_eq!( + reencoded_preflight(FULL_PREFLIGHT_REQUEST), + FULL_PREFLIGHT_REQUEST + ); + assert_eq!( + reencoded_preflight(MINIMAL_PREFLIGHT_REQUEST), + MINIMAL_PREFLIGHT_REQUEST + ); +} + +#[test] +fn explicit_null_and_unknown_overrides_read_as_absent() { + let with_nulls = r#"{"provider":"cursor","session_id":"session-1","messages":[],"current_tokens":null,"focus_topic":null,"threshold_tokens":null,"max_assembly_tokens":null,"leaf_chunk_tokens":null,"max_source_messages":null,"summary_fan_in":null,"incremental_max_depth":null,"fresh_tail_count":null,"dynamic_leaf_chunk_enabled":null,"dynamic_leaf_chunk_max":null,"context_length":null,"reserve_tokens_floor":null,"future_override":99,"summarizer":{"mode":"noop"}}"#; + assert_eq!( + reencoded_compression(with_nulls), + MINIMAL_COMPRESSION_REQUEST + ); + + let preflight_with_nulls = r#"{"provider":"cursor","session_id":"session-1","current_tokens":null,"threshold_tokens":null,"dynamic_leaf_chunk_enabled":null,"future_override":99}"#; + assert_eq!( + reencoded_preflight(preflight_with_nulls), + MINIMAL_PREFLIGHT_REQUEST + ); +} + +#[test] +fn duplicate_or_mistyped_overrides_are_rejected() { + for rejected in [ + r#"{"provider":"cursor","session_id":"session-1","current_tokens":null,"threshold_tokens":1,"threshold_tokens":2}"#, + r#"{"provider":"cursor","session_id":"session-1","current_tokens":null,"max_source_messages":-1}"#, + r#"{"provider":"cursor","session_id":"session-1","current_tokens":null,"dynamic_leaf_chunk_enabled":"yes"}"#, + ] { + assert!( + serde_json::from_str::(rejected).is_err(), + "{rejected} should be rejected" + ); + } + assert!( + serde_json::from_str::( + r#"{"provider":"cursor","session_id":"session-1","messages":[],"current_tokens":null,"focus_topic":null,"fresh_tail_count":1,"fresh_tail_count":2,"summarizer":{"mode":"noop"}}"# + ) + .is_err() + ); +} diff --git a/crates/tracedecay-lsp/src/analyzer/broker/semantic_authority.rs b/crates/tracedecay-lsp/src/analyzer/broker/semantic_authority.rs index b41ef01459..cdfedd5780 100644 --- a/crates/tracedecay-lsp/src/analyzer/broker/semantic_authority.rs +++ b/crates/tracedecay-lsp/src/analyzer/broker/semantic_authority.rs @@ -503,7 +503,7 @@ impl LspSemanticRequestAuthority for StdioLspSemanticAuthority { root_uri: root.uri().to_owned(), request_id: request_id.clone(), }; - let cancellation = self.inner.operations().get(&key).cloned(); + let cancellation = self.inner.operations().remove(&key); cancellation.is_some_and(|cancellation| { cancellation.cancel(); true @@ -907,37 +907,36 @@ mod tests { } #[test] - fn stale_registration_cannot_remove_its_replacement() { - let authority = unspawnable_authority(); - let key = SemanticOperationKey { - root_uri: "file:///project".to_owned(), - request_id: LspRequestId::Number(6), - }; - let original = Arc::new(CancellationToken::new()); - authority - .inner - .operations() - .insert(key.clone(), Arc::clone(&original)); - let guard = SemanticOperationRegistration { - inner: Arc::clone(&authority.inner), - key: key.clone(), - cancellation: original, - }; - let replacement = Arc::new(CancellationToken::new()); - authority - .inner - .operations() - .insert(key.clone(), Arc::clone(&replacement)); - - drop(guard); + fn cancelled_semantic_request_id_is_reusable_before_its_future_settles() { + native_runtime::block_on(async { + let authority = unspawnable_authority(); + let root = AdmittedRoot::new("file:///project"); + let request_id = LspRequestId::Number(6); + let held_client = authority.inner.shared.client().await.unwrap(); + let mut original = + authority.start(root.clone(), request_id.clone(), document_symbol_request()); + assert!(futures_util::poll!(original.as_mut()).is_pending()); + assert!(authority.cancel_request(&root, &request_id)); - let operations = authority.inner.operations(); - assert!( - operations - .get(&key) - .is_some_and(|current| Arc::ptr_eq(current, &replacement)), - "the stale guard leaves the current registration intact" - ); + let mut replacement = + authority.start(root.clone(), request_id.clone(), document_symbol_request()); + assert!( + futures_util::poll!(replacement.as_mut()).is_pending(), + "the cancelled request ID admits replacement work before the old future settles" + ); + drop(original); + assert!(futures_util::poll!(replacement.as_mut()).is_pending()); + assert!(authority.cancel_request(&root, &request_id)); + assert!(!authority.cancel_request(&root, &request_id)); + assert!(matches!( + replacement.await, + LspSemanticOperationOutcome::Partial { coverage, .. } + if coverage == "semantic-cancelled" + )); + assert!(authority.inner.operations().is_empty()); + drop(held_client); + }) + .expect("native runtime"); } /// Another request registering or retiring its operation holds the map diff --git a/crates/tracedecay-lsp/src/analyzer/client.rs b/crates/tracedecay-lsp/src/analyzer/client.rs index 59a872ae7f..168a8819fb 100644 --- a/crates/tracedecay-lsp/src/analyzer/client.rs +++ b/crates/tracedecay-lsp/src/analyzer/client.rs @@ -1241,7 +1241,10 @@ impl StdioLspClient { return Err(LspSemanticRequestError::TimedOut); } SemanticMessagePoll::Closed => { - return Err(self.closed_semantic_error().await); + return Err(tokio::select! { + () = cancellation.cancelled() => LspSemanticRequestError::Cancelled, + error = self.closed_semantic_error(deadline) => error, + }); } }; match message { @@ -1394,7 +1397,9 @@ impl StdioLspClient { match read_refresh_message_until(&self.process, deadline, timeouts).await? { RefreshMessagePoll::Message(message) => message, RefreshMessagePoll::Quiet => break, - RefreshMessagePoll::Closed => return Err(self.closed_refresh_error().await), + RefreshMessagePoll::Closed => { + return Err(self.closed_refresh_error(deadline).await); + } }; let params = match message { JsonRpcMessage::Request { id, method, params } => { @@ -1504,8 +1509,8 @@ impl StdioLspClient { Ok(result) } - async fn closed_refresh_error(&self) -> LspRefreshError { - match self.process.try_wait().await { + async fn closed_refresh_error(&self, deadline: Instant) -> LspRefreshError { + match self.process.exit_status_until(deadline).await { Ok(Some(status)) => LspRefreshError::ProcessExited { message: format!( "LSP server '{}' exited while collecting diagnostics ({status})", @@ -1527,8 +1532,8 @@ impl StdioLspClient { } } - async fn closed_semantic_error(&self) -> LspSemanticRequestError { - match self.process.try_wait().await { + async fn closed_semantic_error(&self, deadline: Instant) -> LspSemanticRequestError { + match self.process.exit_status_until(deadline).await { Ok(Some(status)) => LspSemanticRequestError::ProcessExited { status: status.to_string(), }, diff --git a/crates/tracedecay-lsp/src/analyzer/process.rs b/crates/tracedecay-lsp/src/analyzer/process.rs index 7c49b6d0c3..482ea0bb0d 100644 --- a/crates/tracedecay-lsp/src/analyzer/process.rs +++ b/crates/tracedecay-lsp/src/analyzer/process.rs @@ -13,7 +13,7 @@ use std::process::{Child, ChildStderr, ChildStdin, ChildStdout, Command, ExitSta use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::{Arc, Mutex, mpsc}; use std::task::Poll; -use std::time::Instant; +use std::time::{Duration, Instant}; use tokio_util::bytes::BytesMut; use tokio_util::codec::Decoder; @@ -170,7 +170,7 @@ impl AnalyzerProcess { } } - /// The child's exit status if it has exited, without reaping it. + /// The child's exit status, reaped and cached by the supervisor if it exited. pub(super) async fn try_wait(&self) -> io::Result> { let lifeline = self.lifeline.as_ref().ok_or_else(supervisor_gone)?; let (reply, status) = async_channel::bounded(1); @@ -178,6 +178,28 @@ impl AnalyzerProcess { status.recv().await.map_err(|_| supervisor_gone())? } + /// EOF can precede the OS publishing the child's exit status. + pub(super) async fn exit_status_until( + &self, + deadline: Instant, + ) -> io::Result> { + loop { + match until(deadline, self.try_wait()).await? { + Some(Ok(Some(status))) => return Ok(Some(status)), + Some(Err(error)) => return Err(error), + None => return Ok(None), + Some(Ok(None)) => {} + } + if Instant::now() >= deadline { + return Ok(None); + } + // std::process has no portable async exit notification. + native_runtime::sleep_until(deadline.min(Instant::now() + Duration::from_millis(10))) + .await + .map_err(io::Error::other)?; + } + } + /// Kills the child, then waits until it is reaped and stderr has closed. pub(super) async fn terminate(&mut self) -> io::Result { self.begin_terminate(); @@ -348,7 +370,6 @@ async fn until(deadline: Instant, work: F) -> io::Result&-; read line; exit 17"]); + native_runtime::block_on(async { + let mut process = AnalyzerProcess::spawn(command, "delayed-exit") + .await + .unwrap(); + let deadline = Instant::now() + Duration::from_secs(5); + assert_eq!( + process.read_frame_until(deadline).await.unwrap(), + FramePoll::Closed + ); + assert!(process.try_wait().await.unwrap().is_none()); + { + let mut waiting = pin!(process.exit_status_until(deadline)); + assert!( + until(Instant::now() + Duration::from_millis(20), waiting.as_mut()) + .await + .unwrap() + .is_none() + ); + process.write_frame(b"{}", deadline).await.unwrap(); + assert_eq!(waiting.await.unwrap().unwrap().code(), Some(17)); + } + assert_eq!(process.terminate().await.unwrap().code(), Some(17)); + }) + .unwrap(); + } + + #[cfg(unix)] + #[test] + fn expired_exit_observation_leaves_a_live_child_terminable() { + let mut command = Command::new("/bin/sh"); + command.args(["-c", "exec 1>&-; read line"]); + native_runtime::block_on(async { + let mut process = AnalyzerProcess::spawn(command, "live-eof").await.unwrap(); + assert_eq!( + process + .read_frame_until(Instant::now() + Duration::from_secs(5)) + .await + .unwrap(), + FramePoll::Closed + ); + assert!( + process + .exit_status_until(Instant::now() + Duration::from_millis(20)) + .await + .unwrap() + .is_none() + ); + assert!(process.try_wait().await.unwrap().is_none()); + let terminated = until(Instant::now() + Duration::from_secs(2), process.terminate()) + .await + .unwrap() + .expect("status observation must not block termination") + .unwrap(); + assert!(!terminated.success()); + }) + .unwrap(); + } + #[cfg(unix)] #[test] fn reap_collects_a_child_that_already_exited() { diff --git a/crates/tracedecay-lsp/tests/lsp_suite/analyzer_runtime/broker_refresh.rs b/crates/tracedecay-lsp/tests/lsp_suite/analyzer_runtime/broker_refresh.rs index 66a0e29774..f96336f169 100644 --- a/crates/tracedecay-lsp/tests/lsp_suite/analyzer_runtime/broker_refresh.rs +++ b/crates/tracedecay-lsp/tests/lsp_suite/analyzer_runtime/broker_refresh.rs @@ -1645,6 +1645,15 @@ while True: #[tokio::test] async fn semantic_eof_is_retired_as_transport_failure_not_timeout() { + assert_semantic_eof_outcome(false).await; +} + +#[tokio::test] +async fn semantic_cancellation_after_eof_releases_the_registered_operation() { + assert_semantic_eof_outcome(true).await; +} + +async fn assert_semantic_eof_outcome(cancel_after_eof: bool) { let temp = tempfile::tempdir().unwrap(); let document_path = temp.path().join(FAKE_PATH); std::fs::create_dir_all(document_path.parent().unwrap()).unwrap(); @@ -1655,6 +1664,7 @@ async fn semantic_eof_is_retired_as_transport_failure_not_timeout() { r#" import json import os +from pathlib import Path import sys import time @@ -1684,6 +1694,7 @@ while True: send({"jsonrpc": "2.0", "id": message["id"], "result": {"capabilities": {}}}) elif message.get("method") == "textDocument/documentSymbol": os.close(sys.stdout.fileno()) + Path(__file__).with_suffix(".closed").write_text("closed") time.sleep(60) "#, ) @@ -1706,16 +1717,45 @@ while True: .unwrap() .expect("fake analyzer is executable"); - let outcome = authority - .start( - AdmittedRoot::new(root_uri.clone()), - LspRequestId::Number(3), - tracedecay_lsp::lsp_semantic_request(&SemanticRequest::DocumentSymbols { - document_uri: format!("{root_uri}{FAKE_PATH}"), - }) - .unwrap(), - ) - .await; + let operation = authority.start( + AdmittedRoot::new(root_uri.clone()), + LspRequestId::Number(3), + tracedecay_lsp::lsp_semantic_request(&SemanticRequest::DocumentSymbols { + document_uri: format!("{root_uri}{FAKE_PATH}"), + }) + .unwrap(), + ); + let outcome = if cancel_after_eof { + let operation = tokio::spawn(operation); + tokio::time::timeout(FAKE_LSP_RECOVERY_TIMEOUT, async { + while !script_path.with_extension("closed").exists() { + tokio::task::yield_now().await; + } + assert!(authority.cancel_request( + &AdmittedRoot::new(root_uri.clone()), + &LspRequestId::Number(3), + )); + operation.await.unwrap() + }) + .await + .expect("cancellation after EOF must not wait for the response deadline") + } else { + operation.await + }; + if cancel_after_eof { + assert_eq!( + outcome, + tracedecay_lsp::LspSemanticOperationOutcome::Partial { + value: serde_json::Value::Null, + coverage: "analyzer-cancelled".to_owned(), + detail: Some( + tracedecay_lsp::LspSemanticOperationOutcome::ANALYZER_CANCELLED_DETAIL, + ), + }, + ); + assert!(!authority.cancel_request(&AdmittedRoot::new(root_uri), &LspRequestId::Number(3),)); + return; + } assert!(matches!( outcome, @@ -1739,7 +1779,9 @@ async fn semantic_process_exit_is_classified_before_the_response_deadline() { &script_path, r#" import json +import os import sys +import time def read_message(): headers = {} @@ -1766,6 +1808,8 @@ while True: if message.get("method") == "initialize": send({"jsonrpc": "2.0", "id": message["id"], "result": {"capabilities": {}}}) elif message.get("method") == "textDocument/documentSymbol": + os.close(sys.stdout.fileno()) + time.sleep(0.1) sys.exit(17) "#, ) diff --git a/crates/tracedecay-mcp/src/handlers/hook_runtime/ingest.rs b/crates/tracedecay-mcp/src/handlers/hook_runtime/ingest.rs index 2e223d2b08..2a8d9bd1dc 100644 --- a/crates/tracedecay-mcp/src/handlers/hook_runtime/ingest.rs +++ b/crates/tracedecay-mcp/src/handlers/hook_runtime/ingest.rs @@ -521,17 +521,12 @@ fn pressure_only_command( current_tokens, ignore_session_patterns: Vec::new(), stateless_session_patterns: Vec::new(), - threshold_tokens: None, - max_assembly_tokens: None, - leaf_chunk_tokens: None, - max_source_messages, - summary_fan_in: None, - incremental_max_depth: None, - fresh_tail_count, - dynamic_leaf_chunk_enabled: None, - dynamic_leaf_chunk_max: None, - context_length, - reserve_tokens_floor: None, + overrides: tracedecay_lcm::LcmCompressionOverrides { + max_source_messages, + fresh_tail_count, + context_length, + ..Default::default() + }, }, evidence: LcmCompressionEvidence::PressureOnly { protocol }, })) diff --git a/crates/tracedecay-session-memory/src/session/lcm/authority.rs b/crates/tracedecay-session-memory/src/session/lcm/authority.rs index 0c3f62eee6..c30f82a9d8 100644 --- a/crates/tracedecay-session-memory/src/session/lcm/authority.rs +++ b/crates/tracedecay-session-memory/src/session/lcm/authority.rs @@ -187,7 +187,6 @@ pub enum LcmAuthorityPayload { #[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] #[serde(rename_all = "snake_case")] pub enum LcmAuthorityUnavailableReason { - StoreAuthorityUnavailable, HostProtocolUnavailable, } diff --git a/crates/tracedecay-session-runtime/src/lcm_authority.rs b/crates/tracedecay-session-runtime/src/lcm_authority.rs index 48b2c37cdd..9e388e4453 100644 --- a/crates/tracedecay-session-runtime/src/lcm_authority.rs +++ b/crates/tracedecay-session-runtime/src/lcm_authority.rs @@ -5,7 +5,6 @@ use std::future::Future; use std::pin::Pin; -use std::sync::Arc; use tracedecay_contracts::{ CancellationSignal, CancellationStage, OperationTermination, RequestAdmission, RequestContext, @@ -38,61 +37,6 @@ pub use mount::{ }; use receipt::{terminal, terminal_failure, terminal_interruption, unavailable}; -type StoreFuture<'a, T> = Pin> + Send + 'a>>; - -trait LcmDaemonStore: Send + Sync { - fn compact(&self, request: LcmCompressionRequest) -> StoreFuture<'_, LcmCompressionResponse>; - fn status(&self, query: LcmStatusQuery) -> StoreFuture<'_, LcmStatus>; - fn doctor(&self, query: LcmDoctorQuery) -> StoreFuture<'_, serde_json::Value>; -} - -struct RegisteredLcmDaemonStore { - database: RegisteredGlobalDbLeaseV1, -} - -impl RegisteredLcmDaemonStore { - fn new(database: RegisteredGlobalDbLeaseV1) -> Self { - Self { database } - } -} - -impl LcmDaemonStore for RegisteredLcmDaemonStore { - fn compact(&self, request: LcmCompressionRequest) -> StoreFuture<'_, LcmCompressionResponse> { - let database = self.database.clone(); - Box::pin(async move { - super::lcm_effects::DaemonLcmEffectService::new(database, None, None) - .compress(request) - .await - }) - } - - fn status(&self, query: LcmStatusQuery) -> StoreFuture<'_, LcmStatus> { - let database = self.database.clone(); - Box::pin(async move { - let gc_config = LcmGcConfig::default(); - let status: Pin> + Send>> = - Box::pin(database.lcm_status_with_options( - &query.provider, - query.session_id.as_deref(), - query.deep, - &gc_config, - )); - status.await - }) - } - - fn doctor(&self, _query: LcmDoctorQuery) -> StoreFuture<'_, serde_json::Value> { - Box::pin(async move { - serde_json::to_value( - SessionTemporalAccess::new(&*self.database) - .session_temporal_doctor_health() - .await, - ) - .map_err(|error| LcmError::Db(error.to_string())) - }) - } -} - /// Compression driven purely by host pressure evidence. /// /// The daemon compresses already-ingested canonical content: host-supplied @@ -110,17 +54,7 @@ fn pressure_compression_request(preflight: LcmPreflightRequest) -> LcmCompressio stateless_session_patterns: preflight.stateless_session_patterns, ignore_message_patterns: Vec::new(), expected_current_frontier_store_id: None, - threshold_tokens: preflight.threshold_tokens, - max_assembly_tokens: preflight.max_assembly_tokens, - leaf_chunk_tokens: preflight.leaf_chunk_tokens, - max_source_messages: preflight.max_source_messages, - summary_fan_in: preflight.summary_fan_in, - incremental_max_depth: preflight.incremental_max_depth, - fresh_tail_count: preflight.fresh_tail_count, - dynamic_leaf_chunk_enabled: preflight.dynamic_leaf_chunk_enabled, - dynamic_leaf_chunk_max: preflight.dynamic_leaf_chunk_max, - context_length: preflight.context_length, - reserve_tokens_floor: preflight.reserve_tokens_floor, + overrides: preflight.overrides, summarizer: LcmSummarizerMode::HermesAuxiliary, } } @@ -131,24 +65,39 @@ fn pressure_compression_request(preflight: LcmPreflightRequest) -> LcmCompressio /// restart binds a newly fenced handle to the same durable shard and reads the /// committed LCM frontier/status from storage. pub(crate) struct DaemonLcmAuthority { - store: Option>, + database: RegisteredGlobalDbLeaseV1, } impl DaemonLcmAuthority { pub(crate) fn registered(database: RegisteredGlobalDbLeaseV1) -> Self { - Self { - store: Some(Arc::new(RegisteredLcmDaemonStore::new(database))), - } + Self { database } } - #[cfg(test)] - fn unavailable() -> Self { - Self { store: None } + async fn compact( + &self, + request: LcmCompressionRequest, + ) -> Result { + super::lcm_effects::DaemonLcmEffectService::new(self.database.clone(), None, None) + .compress(request) + .await } - #[cfg(test)] - fn with_store(store: Arc) -> Self { - Self { store: Some(store) } + async fn status(&self, query: &LcmStatusQuery) -> Result { + self.database + .lcm_status_with_options( + &query.provider, + query.session_id.as_deref(), + query.deep, + &LcmGcConfig::default(), + ) + .await + } + + async fn doctor(&self) -> Result { + let health = SessionTemporalAccess::new(&*self.database) + .session_temporal_doctor_health() + .await; + serde_json::to_value(health).map_err(|error| LcmError::Db(error.to_string())) } #[tracing::instrument(name = "daemon.lcm.execute", level = "trace", skip_all)] @@ -401,16 +350,8 @@ impl DaemonLcmAuthority { started_at: UtcMicros, query: LcmStatusQuery, ) -> LcmAuthorityResponse { - let Some(store) = self.store.as_ref() else { - return unavailable( - context, - LcmAuthorityOperation::Status, - started_at, - LcmAuthorityUnavailableReason::StoreAuthorityUnavailable, - ); - }; tokio::select! { - result = store.status(query) => status_read_response(context, started_at, result), + result = self.status(&query) => status_read_response(context, started_at, result), () = cancellation.cancelled() => terminal_interruption( context, LcmAuthorityOperation::Status, @@ -427,18 +368,10 @@ impl DaemonLcmAuthority { context: &RequestContext, cancellation: &CancellationSignal, started_at: UtcMicros, - query: LcmDoctorQuery, + _query: LcmDoctorQuery, ) -> LcmAuthorityResponse { - let Some(store) = self.store.as_ref() else { - return unavailable( - context, - LcmAuthorityOperation::Doctor, - started_at, - LcmAuthorityUnavailableReason::StoreAuthorityUnavailable, - ); - }; tokio::select! { - result = store.doctor(query) => doctor_read_response(context, started_at, result), + result = self.doctor() => doctor_read_response(context, started_at, result), () = cancellation.cancelled() => terminal_interruption( context, LcmAuthorityOperation::Doctor, @@ -458,14 +391,6 @@ impl DaemonLcmAuthority { started_at: UtcMicros, command: LcmCompactionCommand, ) -> LcmAuthorityResponse { - let Some(store) = self.store.as_ref() else { - return unavailable( - context, - LcmAuthorityOperation::Compact, - started_at, - LcmAuthorityUnavailableReason::StoreAuthorityUnavailable, - ); - }; if command.evidence.protocol().provider() != command.preflight.provider { return unavailable( context, @@ -479,7 +404,7 @@ impl DaemonLcmAuthority { let result = run_application_request_interruptible( context, cancellation, - store.compact(request), + self.compact(request), || {}, ) .await; @@ -549,18 +474,10 @@ impl DaemonLcmAuthority { started_at: UtcMicros, query: LcmStatusQuery, ) -> LcmAuthorityResponse { - let Some(store) = self.store.as_ref() else { - return unavailable( - context, - LcmAuthorityOperation::Status, - started_at, - LcmAuthorityUnavailableReason::StoreAuthorityUnavailable, - ); - }; let result = run_application_request_interruptible( context, cancellation, - store.status(query), + self.status(&query), || {}, ) .await; @@ -583,23 +500,11 @@ impl DaemonLcmAuthority { context: &RequestContext, cancellation: &CancellationToken, started_at: UtcMicros, - query: LcmDoctorQuery, + _query: LcmDoctorQuery, ) -> LcmAuthorityResponse { - let Some(store) = self.store.as_ref() else { - return unavailable( - context, - LcmAuthorityOperation::Doctor, - started_at, - LcmAuthorityUnavailableReason::StoreAuthorityUnavailable, - ); - }; - let result = run_application_request_interruptible( - context, - cancellation, - store.doctor(query), - || {}, - ) - .await; + let result = + run_application_request_interruptible(context, cancellation, self.doctor(), || {}) + .await; match result { Ok(result) => doctor_read_response(context, started_at, result), Err(interruption) => terminal_interruption( diff --git a/crates/tracedecay-session-runtime/src/lcm_authority/tests.rs b/crates/tracedecay-session-runtime/src/lcm_authority/tests.rs index c94be43625..8db842455b 100644 --- a/crates/tracedecay-session-runtime/src/lcm_authority/tests.rs +++ b/crates/tracedecay-session-runtime/src/lcm_authority/tests.rs @@ -1,5 +1,4 @@ use std::collections::BTreeSet; -use std::sync::Mutex; use tracedecay_contracts::{ CancellationContext, CapabilityGrantId, CapabilityGrantSnapshot, Deadline, DisclosureClass, @@ -21,77 +20,6 @@ use tracedecay_tool_catalog::{CapabilityId, UseCaseId}; use super::*; -#[derive(Default)] -struct FakeStore { - calls: Mutex>, - compact_requests: Mutex>, - status: Mutex>, -} - -impl FakeStore { - fn calls(&self) -> Vec { - self.calls.lock().map_or_default(|calls| calls.clone()) - } - - fn compact_requests(&self) -> Vec { - self.compact_requests - .lock() - .map_or_default(|requests| requests.clone()) - } -} - -impl LcmDaemonStore for FakeStore { - fn compact(&self, request: LcmCompressionRequest) -> StoreFuture<'_, LcmCompressionResponse> { - if let Ok(mut calls) = self.calls.lock() { - calls.push(LcmAuthorityOperation::Compact); - } - if let Ok(mut requests) = self.compact_requests.lock() { - requests.push(request.clone()); - } - Box::pin(async move { - Ok(LcmCompressionResponse { - status: "skipped".to_owned(), - reason: "below_threshold".to_owned(), - summary_nodes_created: 0, - summary_nodes: Vec::new(), - replay_messages: Vec::new(), - replay_token_estimate: 0, - replay_over_budget: false, - compression_attempts: 0, - context_recovery_hint: None, - retry_status: None, - relation_projection_status: - tracedecay_lcm::LcmRelationProjectionStatus::NotApplicable, - frontier: tracedecay_lcm::LcmLifecycleState { - provider: request.provider, - conversation_id: request.session_id.clone(), - current_session_id: request.session_id, - current_frontier_store_id: None, - last_finalized_session_id: None, - last_finalized_frontier_store_id: None, - maintenance_debt: Vec::new(), - }, - summary_request: None, - }) - }) - } - - fn status(&self, _query: LcmStatusQuery) -> StoreFuture<'_, LcmStatus> { - if let Ok(mut calls) = self.calls.lock() { - calls.push(LcmAuthorityOperation::Status); - } - let status = self.status.lock().ok().and_then(|status| status.clone()); - Box::pin(async move { status.ok_or_else(|| LcmError::Db("unavailable".to_owned())) }) - } - - fn doctor(&self, _query: LcmDoctorQuery) -> StoreFuture<'_, serde_json::Value> { - if let Ok(mut calls) = self.calls.lock() { - calls.push(LcmAuthorityOperation::Doctor); - } - Box::pin(async { Ok(serde_json::json!({"status": "healthy", "findings": []})) }) - } -} - async fn table_row_count(database: &RegisteredGlobalDb, table: &'static str) -> i64 { let snapshot = database.read_snapshot().await.unwrap(); let mut rows = snapshot @@ -101,6 +29,21 @@ async fn table_row_count(database: &RegisteredGlobalDb, table: &'static str) -> rows.next().await.unwrap().unwrap().get(0).unwrap() } +async fn registered_authority() -> ( + tempfile::TempDir, + tracedecay_global_db::tests::harness::RegisteredGlobalDbTestRuntime, + DaemonLcmAuthority, +) { + let directory = tempfile::tempdir().unwrap(); + let runtime = tracedecay_global_db::tests::harness::RegisteredGlobalDbTestRuntime::profile( + directory.path(), + ) + .await + .unwrap(); + let authority = DaemonLcmAuthority::registered(runtime.profile_database_arc()); + (directory, runtime, authority) +} + fn request_context( operation: LcmAuthorityOperation, allowed: bool, @@ -209,17 +152,10 @@ fn preflight(provider: &str) -> LcmPreflightRequest { session_id: "session.lcm-test".to_owned(), messages: Vec::new(), current_tokens: Some(1), - threshold_tokens: Some(100), - max_assembly_tokens: None, - leaf_chunk_tokens: None, - max_source_messages: None, - summary_fan_in: None, - incremental_max_depth: None, - fresh_tail_count: None, - dynamic_leaf_chunk_enabled: None, - dynamic_leaf_chunk_max: None, - context_length: None, - reserve_tokens_floor: None, + overrides: tracedecay_lcm::LcmCompressionOverrides { + threshold_tokens: Some(100), + ..Default::default() + }, ignore_session_patterns: Vec::new(), stateless_session_patterns: Vec::new(), } @@ -241,9 +177,8 @@ fn status_request(provider: &str, session_id: Option<&str>) -> LcmAuthorityReque } #[tokio::test] -async fn denied_command_never_reaches_daemon_store() { - let store = Arc::new(FakeStore::default()); - let authority = DaemonLcmAuthority::with_store(store.clone()); +async fn denied_command_is_rejected_before_registered_store_access() { + let (_directory, _runtime, authority) = registered_authority().await; let (context, binding, cancellation) = request_context(LcmAuthorityOperation::Status, false); let response = authority @@ -257,13 +192,11 @@ async fn denied_command_never_reaches_daemon_store() { .await; assert_eq!(response.outcome, LcmAuthorityOutcome::Denied); - assert!(store.calls().is_empty()); } #[tokio::test] async fn mismatched_live_cancellation_identity_is_denied_before_store_effect() { - let store = Arc::new(FakeStore::default()); - let authority = DaemonLcmAuthority::with_store(store.clone()); + let (_directory, _runtime, authority) = registered_authority().await; let (context, binding, _) = request_context(LcmAuthorityOperation::Status, true); let cancellation = CancellationToken::for_application_request("request.other"); @@ -278,13 +211,11 @@ async fn mismatched_live_cancellation_identity_is_denied_before_store_effect() { .await; assert_eq!(response.outcome, LcmAuthorityOutcome::Denied); - assert!(store.calls().is_empty()); } #[tokio::test] async fn request_target_not_bound_into_grant_is_denied_before_store_effect() { - let store = Arc::new(FakeStore::default()); - let authority = DaemonLcmAuthority::with_store(store.clone()); + let (_directory, _runtime, authority) = registered_authority().await; let (context, binding, cancellation) = request_context(LcmAuthorityOperation::Status, true); let response = authority @@ -298,41 +229,11 @@ async fn request_target_not_bound_into_grant_is_denied_before_store_effect() { .await; assert_eq!(response.outcome, LcmAuthorityOutcome::Denied); - assert!(store.calls().is_empty()); -} - -#[tokio::test] -async fn unavailable_store_is_typed_and_never_fabricates_empty_success() { - let authority = DaemonLcmAuthority::unavailable(); - let (context, binding, cancellation) = request_context(LcmAuthorityOperation::Status, true); - - let response = authority - .execute(LcmAuthorityInvocation { - context, - binding, - target: target("claude", None), - cancellation, - request: LcmAuthorityRequest::Status(LcmStatusQuery { - provider: "claude".to_owned(), - session_id: None, - deep: false, - }), - }) - .await; - - assert_eq!( - response.outcome, - LcmAuthorityOutcome::Unavailable { - reason: LcmAuthorityUnavailableReason::StoreAuthorityUnavailable - } - ); - assert!(response.payload.is_none()); } #[tokio::test] async fn live_cancellation_stops_before_store_effect() { - let store = Arc::new(FakeStore::default()); - let authority = DaemonLcmAuthority::with_store(store.clone()); + let (_directory, _runtime, authority) = registered_authority().await; let (context, binding, cancellation) = request_context(LcmAuthorityOperation::Status, true); cancellation.cancel(); @@ -347,7 +248,6 @@ async fn live_cancellation_stops_before_store_effect() { .await; assert_eq!(response.outcome, LcmAuthorityOutcome::Cancelled); - assert!(store.calls().is_empty()); assert_eq!( response.receipt.execution.termination, OperationTermination::Cancelled @@ -356,8 +256,7 @@ async fn live_cancellation_stops_before_store_effect() { #[tokio::test] async fn expired_deadline_stops_before_store_effect() { - let store = Arc::new(FakeStore::default()); - let authority = DaemonLcmAuthority::with_store(store.clone()); + let (_directory, _runtime, authority) = registered_authority().await; let (context, binding, cancellation) = request_context_until(LcmAuthorityOperation::Status, true, UtcMicros(2)); @@ -372,7 +271,6 @@ async fn expired_deadline_stops_before_store_effect() { .await; assert_eq!(response.outcome, LcmAuthorityOutcome::TimedOut); - assert!(store.calls().is_empty()); assert_eq!( response.receipt.execution.termination, OperationTermination::TimedOut @@ -381,8 +279,8 @@ async fn expired_deadline_stops_before_store_effect() { #[tokio::test] async fn pressure_only_event_compacts_daemon_owned_without_caller_payload() { - let store = Arc::new(FakeStore::default()); - let authority = DaemonLcmAuthority::with_store(store.clone()); + let (_directory, runtime, authority) = registered_authority().await; + let database = runtime.profile_database_arc(); let (context, binding, cancellation) = request_context(LcmAuthorityOperation::Compact, true); let mut pressure_preflight = preflight("cursor"); pressure_preflight.messages = vec![serde_json::json!({ @@ -414,28 +312,17 @@ async fn pressure_only_event_compacts_daemon_owned_without_caller_payload() { // summarization source is the daemon's canonical store, never // caller-authored preflight text. assert_eq!(response.outcome, LcmAuthorityOutcome::Ready); - assert_eq!(store.calls(), vec![LcmAuthorityOperation::Compact]); - let requests = store.compact_requests(); - assert_eq!(requests.len(), 1); - assert!( - requests[0].messages.is_empty(), - "pressure evidence must not carry caller-authored messages into the store" - ); - assert!(matches!( - requests[0].summarizer, - LcmSummarizerMode::HermesAuxiliary - )); assert!(matches!( response.payload, Some(LcmAuthorityPayload::Compaction(_)) )); assert!(response.receipt.committed_state.is_some()); + assert_eq!(table_row_count(&database, "lcm_raw_messages").await, 0); } #[tokio::test] async fn pressure_protocol_provider_mismatch_is_typed_before_ingest() { - let store = Arc::new(FakeStore::default()); - let authority = DaemonLcmAuthority::with_store(store.clone()); + let (_directory, _runtime, authority) = registered_authority().await; let (context, binding, cancellation) = request_context(LcmAuthorityOperation::Compact, true); let command = LcmCompactionCommand { preflight: preflight("cursor"), @@ -463,7 +350,6 @@ async fn pressure_protocol_provider_mismatch_is_typed_before_ingest() { reason: LcmAuthorityUnavailableReason::HostProtocolUnavailable } ); - assert!(store.calls().is_empty()); assert!(response.receipt.committed_state.is_none()); } @@ -636,3 +522,49 @@ async fn mounted_authority_rejects_identity_that_does_not_own_registered_shard() ); assert!(mount_registered_lcm_authority(database, project_identity, &shard).is_none()); } + +#[test] +fn pressure_conversion_keeps_every_override_and_drops_host_content() { + let overrides = tracedecay_lcm::LcmCompressionOverrides { + threshold_tokens: Some(11), + max_assembly_tokens: Some(12), + leaf_chunk_tokens: Some(13), + max_source_messages: Some(14), + summary_fan_in: Some(15), + incremental_max_depth: Some(-1), + fresh_tail_count: Some(0), + dynamic_leaf_chunk_enabled: Some(true), + dynamic_leaf_chunk_max: Some(18), + context_length: Some(19), + reserve_tokens_floor: Some(20), + }; + let request = pressure_compression_request(LcmPreflightRequest { + provider: "cursor".to_owned(), + session_id: "session.lcm-test".to_owned(), + messages: vec![serde_json::json!({ + "role": "user", + "content": "host-authored text that must not be compacted", + })], + current_tokens: Some(42), + overrides: overrides.clone(), + ignore_session_patterns: vec!["ignored-*".to_owned()], + stateless_session_patterns: vec!["stateless-*".to_owned()], + }); + + assert_eq!( + request, + LcmCompressionRequest { + provider: "cursor".to_owned(), + session_id: "session.lcm-test".to_owned(), + messages: Vec::new(), + current_tokens: Some(42), + focus_topic: None, + ignore_session_patterns: vec!["ignored-*".to_owned()], + stateless_session_patterns: vec!["stateless-*".to_owned()], + ignore_message_patterns: Vec::new(), + expected_current_frontier_store_id: None, + overrides, + summarizer: LcmSummarizerMode::HermesAuxiliary, + } + ); +} diff --git a/crates/tracedecay-session-runtime/src/lcm_effects.rs b/crates/tracedecay-session-runtime/src/lcm_effects.rs index 26f0afe5af..26938666f4 100644 --- a/crates/tracedecay-session-runtime/src/lcm_effects.rs +++ b/crates/tracedecay-session-runtime/src/lcm_effects.rs @@ -6,7 +6,10 @@ use tracedecay_contracts::{CancellationSignal, Deadline}; use tracedecay_temporal_query::execution::ExecutionControl; use tracedecay_global_db::RegisteredGlobalDbLeaseV1; -use tracedecay_lcm::{LcmCompressionRequest, LcmCompressionResponse, LcmError, LcmSummarizerMode}; +use tracedecay_lcm::{ + LcmCompressionOutcome, LcmCompressionRequest, LcmCompressionResponse, LcmError, + LcmSummarizerMode, +}; #[cfg(any(test, feature = "test-helpers"))] use tracedecay_lcm::{LcmSessionBoundaryRequest, LcmSessionBoundaryResponse}; use tracedecay_session_temporal_store::SessionTemporalAccess; @@ -120,11 +123,7 @@ impl DaemonLcmEffectService { )), ) .await?; - if matches!( - &request.summarizer, - LcmSummarizerMode::Provided { summary_text, .. } if !summary_text.trim().is_empty() - ) || matches!(&request.summarizer, LcmSummarizerMode::Fake { .. }) - { + if request.summarizer.has_nonempty_provided_summary() { return self.commit_compression(&request).await; } @@ -132,11 +131,11 @@ impl DaemonLcmEffectService { // changes between the planning pass and the final commit. request.summarizer = LcmSummarizerMode::HermesAuxiliary; let pending = self.commit_compression(&request).await?; - if pending.status != "needs_summary" { - return Ok(pending); - } - let Some(summary_request) = pending.summary_request.as_ref() else { - return Ok(pending); + let summary_request = match pending.outcome()? { + LcmCompressionOutcome::NeedsSummary(summary_request) => summary_request, + LcmCompressionOutcome::Complete | LcmCompressionOutcome::Deferred { .. } => { + return Ok(pending); + } }; let summary = match super::lcm_summarization::resolve_authoritative_summary( &self.db, @@ -154,13 +153,14 @@ impl DaemonLcmEffectService { } Err(super::lcm_summarization::SummaryResolutionError::Unavailable(reason)) => { self.control.checkpoint()?; - return Ok(summary_unavailable(pending, reason)); + return pending.defer_for_authoritative_summary(reason); } }; self.control.checkpoint()?; request.summarizer = LcmSummarizerMode::Provided { summary_text: summary.text, route: Some(summary.route), + extraction_result: None, }; self.commit_compression(&request).await } @@ -170,11 +170,7 @@ impl DaemonLcmEffectService { mut request: LcmCompressionRequest, convergence_candidate: &tracedecay_lcm::summary_convergence::LcmSummaryConvergenceCandidate, ) -> Result { - if matches!( - &request.summarizer, - LcmSummarizerMode::Provided { summary_text, .. } if !summary_text.trim().is_empty() - ) || matches!(&request.summarizer, LcmSummarizerMode::Fake { .. }) - { + if request.summarizer.has_nonempty_provided_summary() { return self .commit_retained_compression(&request, Some(convergence_candidate), None) .await; @@ -184,11 +180,11 @@ impl DaemonLcmEffectService { let pending = self .commit_retained_compression(&request, Some(convergence_candidate), None) .await?; - if pending.response.status != "needs_summary" { - return Ok(pending); - } - let Some(summary_request) = pending.response.summary_request.as_ref() else { - return Ok(pending); + let summary_request = match pending.response.outcome()? { + LcmCompressionOutcome::NeedsSummary(summary_request) => summary_request, + LcmCompressionOutcome::Complete | LcmCompressionOutcome::Deferred { .. } => { + return Ok(pending); + } }; // Host-native compaction text is usable for retained convergence only // when its evidence binds the exact raw source range selected by this @@ -214,13 +210,12 @@ impl DaemonLcmEffectService { let mut pending = pending; // An absent interval is its own typed state; only a present // interval that binds a different range is a mismatch. - pending.response = summary_unavailable( - pending.response, + pending.response = pending.response.defer_for_authoritative_summary( summary .source_range .absent_reason() .unwrap_or("authoritative_summary_source_mismatch"), - ); + )?; return Ok(pending); } Err(super::lcm_summarization::SummaryResolutionError::Storage(error)) => { @@ -229,7 +224,7 @@ impl DaemonLcmEffectService { Err(super::lcm_summarization::SummaryResolutionError::Unavailable(reason)) => { self.control.checkpoint()?; let mut pending = pending; - pending.response = summary_unavailable(pending.response, reason); + pending.response = pending.response.defer_for_authoritative_summary(reason)?; return Ok(pending); } }; @@ -237,6 +232,7 @@ impl DaemonLcmEffectService { request.summarizer = LcmSummarizerMode::Provided { summary_text: summary.text, route: Some(summary.route), + extraction_result: None, }; let mut committed = self .commit_retained_compression( @@ -391,14 +387,5 @@ pub async fn lcm_session_boundary_for_test( .await } -fn summary_unavailable( - mut response: LcmCompressionResponse, - reason: &'static str, -) -> LcmCompressionResponse { - response.reason = reason.to_string(); - response.retry_status = Some("needs_authoritative_summary".to_string()); - response -} - #[cfg(test)] mod tests; diff --git a/crates/tracedecay-session-runtime/src/lcm_effects/tests.rs b/crates/tracedecay-session-runtime/src/lcm_effects/tests.rs index 2dba676951..3b8549b87b 100644 --- a/crates/tracedecay-session-runtime/src/lcm_effects/tests.rs +++ b/crates/tracedecay-session-runtime/src/lcm_effects/tests.rs @@ -74,20 +74,21 @@ fn compression_request(session_id: &str) -> LcmCompressionRequest { stateless_session_patterns: Vec::new(), ignore_message_patterns: Vec::new(), expected_current_frontier_store_id: None, - threshold_tokens: None, - max_assembly_tokens: None, - leaf_chunk_tokens: Some(1), - max_source_messages: Some(8), - summary_fan_in: None, - incremental_max_depth: None, - fresh_tail_count: Some(1), - dynamic_leaf_chunk_enabled: None, - dynamic_leaf_chunk_max: None, - context_length: None, - reserve_tokens_floor: None, - summarizer: LcmSummarizerMode::Fake { - summary_text: "fixture summary preserving canonical historical context".to_string(), + overrides: tracedecay_lcm::LcmCompressionOverrides { + leaf_chunk_tokens: Some(1), + max_source_messages: Some(8), + fresh_tail_count: Some(1), + ..Default::default() }, + summarizer: provided_summary("fixture summary preserving canonical historical context"), + } +} + +fn provided_summary(summary_text: impl Into) -> LcmSummarizerMode { + LcmSummarizerMode::Provided { + summary_text: summary_text.into(), + route: None, + extraction_result: None, } } @@ -408,17 +409,7 @@ async fn preflight_reads_canonical_state_without_creating_or_ingesting_a_session session_id: "missing-session".to_string(), messages: Vec::new(), current_tokens: Some(1_000), - threshold_tokens: None, - max_assembly_tokens: None, - leaf_chunk_tokens: None, - max_source_messages: None, - summary_fan_in: None, - incremental_max_depth: None, - fresh_tail_count: None, - dynamic_leaf_chunk_enabled: None, - dynamic_leaf_chunk_max: None, - context_length: None, - reserve_tokens_floor: None, + overrides: Default::default(), ignore_session_patterns: Vec::new(), stateless_session_patterns: Vec::new(), }) @@ -1284,6 +1275,7 @@ async fn interrupted_atomic_summary_settlement_resumes_without_duplicate() { interrupted_request.summarizer = LcmSummarizerMode::Provided { summary_text: summary_text.to_string(), route: Some("codex_native_compaction".to_string()), + extraction_result: None, }; let interrupted = db .lcm_compress_retained_page_guarded( @@ -1319,6 +1311,57 @@ async fn interrupted_atomic_summary_settlement_resumes_without_duplicate() { assert_eq!(status.summary_convergence.current_session_count, 1); } +#[tokio::test] +async fn retained_pressure_recovery_commits_ok_and_best_effort_summaries() { + for (cap, expected_status) in [(1, "best_effort"), (10_000, "ok")] { + let harness = RegisteredGlobalDbHarness::open("lcm-retained-pressure-recovery").await; + let db = harness.registered.clone(); + let storage_root = db.db_path().parent().unwrap(); + let session_id = "retained-pressure-recovery-session"; + assert!(db.upsert_session(&session("cursor", session_id)).await); + for ordinal in 1..=4 { + db.lcm_ingest_raw_message(storage_root, &message(session_id, ordinal)) + .await + .unwrap(); + } + let mut request = compression_request(session_id); + request.current_tokens = Some(20_000); + request.overrides.max_assembly_tokens = Some(cap); + let bounded = db + .lcm_compress_retained_page_guarded( + &request, + &execution_control(), + || Ok(()), + retained_guard(None), + None, + ) + .await + .unwrap(); + assert_eq!(bounded.response.status, expected_status); + assert_eq!( + bounded.response.retry_status.as_deref(), + Some("critical_pressure_catch_up") + ); + assert!(bounded.response.summary_nodes_created > 0); + assert_eq!( + bounded.response.replay_over_budget, + expected_status == "best_effort" + ); + + drop(db); + let restarted = harness.restart().await; + let status = restarted + .registered + .lcm_status("cursor", Some(session_id)) + .await + .unwrap(); + assert_eq!( + status.summary_node_count, + bounded.response.summary_nodes_created as i64 + ); + } +} + #[tokio::test] async fn retained_commit_defers_whole_session_replay_and_relation_application() { let harness = RegisteredGlobalDbHarness::open("lcm-retained-deferred-maintenance").await; @@ -1334,9 +1377,7 @@ async fn retained_commit_defers_whole_session_replay_and_relation_application() .unwrap(); } let mut initial = compression_request(session_id); - initial.summarizer = LcmSummarizerMode::Fake { - summary_text: "old whole-session replay sentinel".to_string(), - }; + initial.summarizer = provided_summary("old whole-session replay sentinel"); DaemonLcmEffectService::new(db.clone(), None, None) .compress(initial) .await @@ -1355,9 +1396,7 @@ async fn retained_commit_defers_whole_session_replay_and_relation_application() .unwrap(); drop(snapshot); let mut retained = compression_request(session_id); - retained.summarizer = LcmSummarizerMode::Fake { - summary_text: "new bounded-page summary".to_string(), - }; + retained.summarizer = provided_summary("new bounded-page summary"); let bounded = db .lcm_compress_retained_page_guarded( &retained, @@ -1959,6 +1998,7 @@ async fn partial_revision_invalidation_hides_replay_and_yields_to_a_due_peer() { request.summarizer = LcmSummarizerMode::Provided { summary_text: "summary before the raw revision".to_string(), route: Some("test_partial_invalidation_fairness".to_string()), + extraction_result: None, }; db.lcm_compress_retained_page_guarded( &request, @@ -2089,6 +2129,7 @@ async fn malformed_relation_receipt_is_permanent_without_starving_summary_work() poison_request.summarizer = LcmSummarizerMode::Provided { summary_text: "summary with a poisoned relation receipt".to_string(), route: Some("test_relation_poison".to_string()), + extraction_result: None, }; let poison = db .lcm_compress_retained_page_guarded( @@ -2440,6 +2481,7 @@ fn protected_in_place_revision_stales_old_summary_before_reconvergence() { initial_request.summarizer = LcmSummarizerMode::Provided { summary_text: old_text.to_string(), route: Some("test_exact_source_summary".to_string()), + extraction_result: None, }; let initial = db .lcm_compress_retained_page_guarded( @@ -2592,11 +2634,12 @@ fn disjoint_published_summary_revisions_both_reconverge_across_restart() { .unwrap(); drop(snapshot); let mut request = daemon_summary_request("cursor", session_id); - request.max_source_messages = Some(4); - request.fresh_tail_count = Some(0); + request.overrides.max_source_messages = Some(4); + request.overrides.fresh_tail_count = Some(0); request.summarizer = LcmSummarizerMode::Provided { summary_text: summary_text.to_string(), route: Some("test_disjoint_revision".to_string()), + extraction_result: None, }; let compressed = db .lcm_compress_retained_page_guarded( @@ -2871,6 +2914,7 @@ async fn raw_message_identity_cannot_move_between_sessions_after_summary_publica request.summarizer = LcmSummarizerMode::Provided { summary_text: "summary owned by the original session".to_string(), route: Some("test_exact_source_summary".to_string()), + extraction_result: None, }; let initial = db .lcm_compress_retained_page_guarded( @@ -2955,6 +2999,7 @@ async fn retained_summary_publication_requires_the_exact_planned_source_range() request.summarizer = LcmSummarizerMode::Provided { summary_text: "summary generated for a different page".to_string(), route: Some("test_exact_source_summary".to_string()), + extraction_result: None, }; let error = db .lcm_compress_retained_page_guarded( diff --git a/crates/tracedecay-session-runtime/src/lcm_effects/tests/compression_ownership.rs b/crates/tracedecay-session-runtime/src/lcm_effects/tests/compression_ownership.rs index 7331aaf6b6..1f948d6fbb 100644 --- a/crates/tracedecay-session-runtime/src/lcm_effects/tests/compression_ownership.rs +++ b/crates/tracedecay-session-runtime/src/lcm_effects/tests/compression_ownership.rs @@ -102,8 +102,8 @@ async fn authoritative_summary_journey_owns_the_message_corpus_once() { let mut request = daemon_summary_request("claude", session_id); request.messages = (0..CORPUS_MESSAGES).map(corpus_message).collect(); - request.leaf_chunk_tokens = Some(1); - request.fresh_tail_count = Some(1); + request.overrides.leaf_chunk_tokens = Some(1); + request.overrides.fresh_tail_count = Some(1); for message in &request.messages { assert_eq!( message["content"].as_str().map(str::len), diff --git a/crates/tracedecay-session-runtime/src/lcm_summary_convergence.rs b/crates/tracedecay-session-runtime/src/lcm_summary_convergence.rs index 05257306b3..9a890fcba3 100644 --- a/crates/tracedecay-session-runtime/src/lcm_summary_convergence.rs +++ b/crates/tracedecay-session-runtime/src/lcm_summary_convergence.rs @@ -8,7 +8,7 @@ use tracedecay_global_db::RegisteredGlobalDbLeaseV1; use tracedecay_lcm::summary_convergence::{ LcmSummaryConvergenceCandidate, LcmSummaryConvergenceQueueState, }; -use tracedecay_lcm::{LcmCompressionRequest, LcmError, LcmSummarizerMode}; +use tracedecay_lcm::{LcmCompressionOutcome, LcmCompressionRequest, LcmError, LcmSummarizerMode}; use crate::session_temporal_refresh_scheduler::registry::session_refresh_retry_delay; use crate::session_temporal_refresh_scheduler::wake::SessionTemporalRefreshRetryClass; @@ -439,34 +439,45 @@ async fn process_candidate( } }; let response = &bounded.response; - let (state, disposition, failure_code) = - if response.retry_status.as_deref() == Some("needs_authoritative_summary") { - ( - LcmSummaryConvergenceQueueState::Unavailable, - LcmSummaryConvergenceDisposition::Pending { - reason: response.reason.clone(), - }, - Some(response.reason.as_str()), - ) - } else if bounded.has_more { - ( - LcmSummaryConvergenceQueueState::Pending, - LcmSummaryConvergenceDisposition::Summarized, - None, - ) - } else if response.summary_nodes_created > 0 { - ( - LcmSummaryConvergenceQueueState::Current, - LcmSummaryConvergenceDisposition::Summarized, - None, - ) - } else { - ( - LcmSummaryConvergenceQueueState::Current, - LcmSummaryConvergenceDisposition::Current, - None, - ) - }; + let deferred_reason = match response.outcome() { + Ok(LcmCompressionOutcome::Complete) => None, + Ok(LcmCompressionOutcome::Deferred { reason }) => Some(reason), + Ok(LcmCompressionOutcome::NeedsSummary(_)) => { + let error = + LcmError::Db("retained compression left an unresolved summary request".to_string()); + return record_candidate_error(database, candidate, error, now_unix_ms).await; + } + Err(error) => { + return record_candidate_error(database, candidate, error, now_unix_ms).await; + } + }; + let (state, disposition, failure_code) = if let Some(reason) = deferred_reason { + ( + LcmSummaryConvergenceQueueState::Unavailable, + LcmSummaryConvergenceDisposition::Pending { + reason: reason.to_string(), + }, + Some(reason), + ) + } else if bounded.has_more { + ( + LcmSummaryConvergenceQueueState::Pending, + LcmSummaryConvergenceDisposition::Summarized, + None, + ) + } else if response.summary_nodes_created > 0 { + ( + LcmSummaryConvergenceQueueState::Current, + LcmSummaryConvergenceDisposition::Summarized, + None, + ) + } else { + ( + LcmSummaryConvergenceQueueState::Current, + LcmSummaryConvergenceDisposition::Current, + None, + ) + }; if state == LcmSummaryConvergenceQueueState::Unavailable && !persist_outcome(database, &candidate, state, failure_code, 0, 0).await? { @@ -639,17 +650,7 @@ fn compression_request(candidate: &LcmSummaryConvergenceCandidate) -> LcmCompres stateless_session_patterns: Vec::new(), ignore_message_patterns: Vec::new(), expected_current_frontier_store_id: None, - threshold_tokens: None, - max_assembly_tokens: None, - leaf_chunk_tokens: None, - max_source_messages: None, - summary_fan_in: None, - incremental_max_depth: None, - fresh_tail_count: None, - dynamic_leaf_chunk_enabled: None, - dynamic_leaf_chunk_max: None, - context_length: None, - reserve_tokens_floor: None, + overrides: Default::default(), summarizer: LcmSummarizerMode::HermesAuxiliary, } } diff --git a/crates/tracedecay-session-runtime/src/retained/lcm.rs b/crates/tracedecay-session-runtime/src/retained/lcm.rs index cf446f19a0..0adcd90435 100644 --- a/crates/tracedecay-session-runtime/src/retained/lcm.rs +++ b/crates/tracedecay-session-runtime/src/retained/lcm.rs @@ -732,7 +732,6 @@ fn execution_error(outcome: LcmAuthorityOutcome) -> RetainedSurfaceExecutionErro const fn lcm_unavailable_reason(reason: LcmAuthorityUnavailableReason) -> &'static str { match reason { - LcmAuthorityUnavailableReason::StoreAuthorityUnavailable => "store_authority_unavailable", LcmAuthorityUnavailableReason::HostProtocolUnavailable => "host_protocol_unavailable", } } diff --git a/crates/tracedecay-session-temporal-store/src/retrieval/tests/relation_graph_tests.rs b/crates/tracedecay-session-temporal-store/src/retrieval/tests/relation_graph_tests.rs index 8a6110500f..d7e45865c2 100644 --- a/crates/tracedecay-session-temporal-store/src/retrieval/tests/relation_graph_tests.rs +++ b/crates/tracedecay-session-temporal-store/src/retrieval/tests/relation_graph_tests.rs @@ -260,19 +260,16 @@ async fn seed_production_summary( stateless_session_patterns: Vec::new(), ignore_message_patterns: Vec::new(), expected_current_frontier_store_id: None, - threshold_tokens: None, - max_assembly_tokens: None, - leaf_chunk_tokens: Some(1), - max_source_messages: Some(8), - summary_fan_in: None, - incremental_max_depth: None, - fresh_tail_count: Some(1), - dynamic_leaf_chunk_enabled: None, - dynamic_leaf_chunk_max: None, - context_length: None, - reserve_tokens_floor: None, - summarizer: tracedecay_lcm::LcmSummarizerMode::Fake { + overrides: tracedecay_lcm::LcmCompressionOverrides { + leaf_chunk_tokens: Some(1), + max_source_messages: Some(8), + fresh_tail_count: Some(1), + ..Default::default() + }, + summarizer: tracedecay_lcm::LcmSummarizerMode::Provided { summary_text: "production dirty summary sentinel".to_string(), + route: None, + extraction_result: None, }, }, &ExecutionControl::default(), diff --git a/crates/tracedecay/src/daemon/tests/rmcp_route/host_index_handoff.rs b/crates/tracedecay/src/daemon/tests/rmcp_route/host_index_handoff.rs index 06cc07fb88..648ade299d 100644 --- a/crates/tracedecay/src/daemon/tests/rmcp_route/host_index_handoff.rs +++ b/crates/tracedecay/src/daemon/tests/rmcp_route/host_index_handoff.rs @@ -30,7 +30,7 @@ async fn committed_save_remains_noncurrent_until_its_broker_handoff() { let project = canonical_existing_identity(fixture.handshake.project_path.as_deref().unwrap()).unwrap(); wait_for_host_ingest_publication(&fixture).await; - let initial = ready_reading(&fixture, &project, "initial").await; + let initial_text = ready_reading(&fixture, &project, "initial text").await; let graph = fixture.server.cg_snapshot().await; let project_id = graph.store_layout().identity.project_id.as_deref().unwrap(); let scope = tracedecay_code_index_runtime::resolved_scope_for_project( @@ -66,11 +66,14 @@ async fn committed_save_remains_noncurrent_until_its_broker_handoff() { std::time::Instant::now() + PHASE_TIMEOUT, ) .await; - assert!(owner.is_some(), "initial ready reading: {initial:?}"); + assert!(owner.is_some(), "initial ready reading: {initial_text:?}"); owner } else { None }; + // Complete-owner demand can queue a freshness pass after text readiness. + // Settle that setup work before pinning the generation the save must retain. + let initial = ready_reading(&fixture, &project, "after initial graph admission").await; let replay = broker.begin_replay().await.unwrap(); assert_eq!(broker.pending_replay_count().await.unwrap(), 0); std::fs::write( diff --git a/crates/tracedecay/tests/hooks_lsp_suite/hint_settlement_test.rs b/crates/tracedecay/tests/hooks_lsp_suite/hint_settlement_test.rs index 2457ab0e88..a4960f4e90 100644 --- a/crates/tracedecay/tests/hooks_lsp_suite/hint_settlement_test.rs +++ b/crates/tracedecay/tests/hooks_lsp_suite/hint_settlement_test.rs @@ -46,12 +46,12 @@ const HINT_TS: i64 = 1_000_000; const AFTER_HORIZON: i64 = HINT_TS + 2_000; struct ProjectSettlementFixture { - _dir: TempDir, runtime: HostAdmissionTestRuntimeV1, project_root: std::path::PathBuf, /// Scope the observability envelopes are attributed to: the registered /// project session binding, not the analytics project key. binding_project_id: ProjectId, + _dir: TempDir, } impl ProjectSettlementFixture { @@ -68,13 +68,24 @@ impl ProjectSettlementFixture { .await .expect("open registered project runtime"); Self { - _dir: dir, runtime, project_root, binding_project_id, + _dir: dir, } } + async fn shutdown(self) { + let Self { + runtime, _dir: dir, .. + } = self; + runtime + .shutdown() + .await + .expect("close registered project runtime"); + drop(dir); + } + fn analytics(&self) -> &RegisteredGlobalDb { self.runtime.profile_database_for_test() } @@ -261,6 +272,7 @@ async fn settlement_records_settled_hints_as_a_linked_adoption_outcome() { } ); assert_eq!(fixture.adoption_outcome_events().await.len(), 1); + fixture.shutdown().await; } #[tokio::test] @@ -283,6 +295,7 @@ async fn settlement_with_only_open_hints_emits_no_adoption_outcome() { fixture.adoption_outcome_events().await.is_empty(), "an all-open pass settled nothing and must not fabricate funnel mass" ); + fixture.shutdown().await; } #[tokio::test] @@ -321,6 +334,7 @@ async fn fully_settled_pass_records_a_known_coverage_census() { CoverageStateV1::Known, "a pass that settled every scanned hint is a complete census" ); + fixture.shutdown().await; } /// The daemon's delivery row for one intent-pass suggestion, built by the @@ -520,4 +534,5 @@ async fn daemon_intent_delivery_rows_settle_against_their_intent_key() { ), ] ); + fixture.shutdown().await; } diff --git a/crates/tracedecay/tests/hooks_lsp_suite/hook_branch_routing_test.rs b/crates/tracedecay/tests/hooks_lsp_suite/hook_branch_routing_test.rs index abb7e7fc7c..411da49cc1 100644 --- a/crates/tracedecay/tests/hooks_lsp_suite/hook_branch_routing_test.rs +++ b/crates/tracedecay/tests/hooks_lsp_suite/hook_branch_routing_test.rs @@ -300,6 +300,7 @@ async fn hook_branch_write_lands_in_a_sealed_single_store_generation() { !shard_root.join("branches").exists(), "no write may create a per-branch database" ); + harness.shutdown().await; } #[cfg(not(unix))] diff --git a/crates/tracedecay/tests/session_suite/lcm_compression/boundary.rs b/crates/tracedecay/tests/session_suite/lcm_compression/boundary.rs index 17feed0e84..12b9d6aeb8 100644 --- a/crates/tracedecay/tests/session_suite/lcm_compression/boundary.rs +++ b/crates/tracedecay/tests/session_suite/lcm_compression/boundary.rs @@ -33,7 +33,7 @@ async fn boundary_skip_starts_preflight_compression_cooldown() { vec![json!({"id": "fresh-user", "role": "user", "content": "fresh preflight payload"})], Some(120), ); - request.threshold_tokens = Some(100); + request.overrides.threshold_tokens = Some(100); let response = db.lcm_preflight(request).await.unwrap(); @@ -103,7 +103,7 @@ async fn boundary_cooldown_expires_after_sixty_seconds() { assert!(recorded.recorded); let mut request = preflight_request("cursor", "session-b", Vec::new(), Some(120)); - request.threshold_tokens = Some(100); + request.overrides.threshold_tokens = Some(100); let response = db.lcm_preflight(request).await.unwrap(); @@ -139,7 +139,7 @@ async fn boundary_continuation_with_matching_bound_session_records_no_cooldown() assert_eq!(boundary.reason, "compression_boundary_carried_over"); let mut request = preflight_request("cursor", "session-b", Vec::new(), Some(120)); - request.threshold_tokens = Some(100); + request.overrides.threshold_tokens = Some(100); let response = db.lcm_preflight(request).await.unwrap(); @@ -184,9 +184,7 @@ async fn compression_boundary_links_without_reassigning_lcm_data() { .lcm_compress(compress_request( "cursor", "session-old", - LcmSummarizerMode::Fake { - summary_text: "old summary".into(), - }, + provided_summary("old summary"), )) .await .unwrap(); @@ -271,9 +269,7 @@ async fn compression_boundary_links_without_reassigning_lcm_data() { .lcm_compress(compress_request( "cursor", "session-new", - LcmSummarizerMode::Fake { - summary_text: "unused".into(), - }, + provided_summary("unused"), )) .await .unwrap(); @@ -369,9 +365,7 @@ async fn compression_boundary_link_preserves_payload_owner_and_projects_debt() { .lcm_compress(limited_compress_request( "cursor", "session-old", - LcmSummarizerMode::Fake { - summary_text: "first chunk summary".into(), - }, + provided_summary("first chunk summary"), Some(2 * budget_tokens("old-1 token")), Some(2), None, @@ -473,9 +467,7 @@ async fn boundary_link_to_existing_target_leaves_source_session_state_intact() { .lcm_compress(limited_compress_request( "cursor", "session-old", - LcmSummarizerMode::Fake { - summary_text: "first chunk summary".into(), - }, + provided_summary("first chunk summary"), Some(2 * budget_tokens("old-1 token")), Some(2), None, diff --git a/crates/tracedecay/tests/session_suite/lcm_compression/compaction.rs b/crates/tracedecay/tests/session_suite/lcm_compression/compaction.rs index 327355858e..eb9ca7d05e 100644 --- a/crates/tracedecay/tests/session_suite/lcm_compression/compaction.rs +++ b/crates/tracedecay/tests/session_suite/lcm_compression/compaction.rs @@ -16,9 +16,7 @@ async fn compress_noops_for_sub_threshold_backlog_in_threshold_mode() { .lcm_compress(limited_compress_request( "cursor", "session-1", - LcmSummarizerMode::Fake { - summary_text: "should not be written".into(), - }, + provided_summary("should not be written"), Some(10), None, None, @@ -89,9 +87,7 @@ async fn compress_proceeds_at_exact_leaf_chunk_threshold() { .lcm_compress(limited_compress_request( "cursor", "session-1", - LcmSummarizerMode::Fake { - summary_text: "boundary summary".into(), - }, + provided_summary("boundary summary"), Some(4), None, None, @@ -125,9 +121,7 @@ async fn maintenance_debt_bypasses_sub_threshold_noop_guard() { .lcm_compress(limited_compress_request( "cursor", "session-1", - LcmSummarizerMode::Fake { - summary_text: "first chunk summary".into(), - }, + provided_summary("first chunk summary"), Some(2 * budget_tokens("old-1 token")), Some(2), None, @@ -148,9 +142,7 @@ async fn maintenance_debt_bypasses_sub_threshold_noop_guard() { .lcm_compress(limited_compress_request( "cursor", "session-1", - LcmSummarizerMode::Fake { - summary_text: "debt catch-up summary".into(), - }, + provided_summary("debt catch-up summary"), Some(50), None, None, @@ -186,20 +178,12 @@ async fn zero_leaf_chunk_tokens_disables_threshold_guard() { stateless_session_patterns: Vec::new(), ignore_message_patterns: Vec::new(), expected_current_frontier_store_id: None, - threshold_tokens: None, - max_assembly_tokens: None, - leaf_chunk_tokens: Some(100_000), - max_source_messages: None, - summary_fan_in: None, - incremental_max_depth: None, - fresh_tail_count: Some(1), - dynamic_leaf_chunk_enabled: None, - dynamic_leaf_chunk_max: None, - context_length: None, - reserve_tokens_floor: None, - summarizer: LcmSummarizerMode::Fake { - summary_text: "should not be used".into(), + overrides: tracedecay_lcm::LcmCompressionOverrides { + leaf_chunk_tokens: Some(100_000), + fresh_tail_count: Some(1), + ..Default::default() }, + summarizer: provided_summary("should not be used"), }) .await .unwrap(); @@ -217,20 +201,12 @@ async fn zero_leaf_chunk_tokens_disables_threshold_guard() { stateless_session_patterns: Vec::new(), ignore_message_patterns: Vec::new(), expected_current_frontier_store_id: None, - threshold_tokens: None, - max_assembly_tokens: None, - leaf_chunk_tokens: Some(0), - max_source_messages: None, - summary_fan_in: None, - incremental_max_depth: None, - fresh_tail_count: Some(1), - dynamic_leaf_chunk_enabled: None, - dynamic_leaf_chunk_max: None, - context_length: None, - reserve_tokens_floor: None, - summarizer: LcmSummarizerMode::Fake { - summary_text: "zero leaf summary".into(), + overrides: tracedecay_lcm::LcmCompressionOverrides { + leaf_chunk_tokens: Some(0), + fresh_tail_count: Some(1), + ..Default::default() }, + summarizer: provided_summary("zero leaf summary"), }) .await .unwrap(); @@ -261,20 +237,11 @@ async fn zero_fresh_tail_count_keeps_no_raw_tail() { stateless_session_patterns: Vec::new(), ignore_message_patterns: Vec::new(), expected_current_frontier_store_id: None, - threshold_tokens: None, - max_assembly_tokens: None, - leaf_chunk_tokens: None, - max_source_messages: None, - summary_fan_in: None, - incremental_max_depth: None, - fresh_tail_count: Some(0), - dynamic_leaf_chunk_enabled: None, - dynamic_leaf_chunk_max: None, - context_length: None, - reserve_tokens_floor: None, - summarizer: LcmSummarizerMode::Fake { - summary_text: "zero tail summary".into(), + overrides: tracedecay_lcm::LcmCompressionOverrides { + fresh_tail_count: Some(0), + ..Default::default() }, + summarizer: provided_summary("zero tail summary"), }) .await .unwrap(); @@ -325,9 +292,7 @@ async fn dynamic_chunking_compacts_bounded_oldest_leaf_chunk_and_records_backlog .lcm_compress(limited_compress_request( "cursor", "session-1", - LcmSummarizerMode::Fake { - summary_text: "first chunk summary".into(), - }, + provided_summary("first chunk summary"), Some(2 * budget_tokens("old-1 token")), Some(2), None, @@ -395,9 +360,7 @@ async fn fake_summarizer_compacts_backlog_and_preserves_fresh_tail() { .lcm_compress(compress_request( "cursor", "session-1", - LcmSummarizerMode::Fake { - summary_text: "old summary".into(), - }, + provided_summary("old summary"), )) .await .unwrap(); @@ -447,9 +410,7 @@ async fn compression_preserves_leading_system_developer_tool_anchor_outside_summ .lcm_compress(compress_request( "cursor", "session-1", - LcmSummarizerMode::Fake { - summary_text: "old exchange summary".into(), - }, + provided_summary("old exchange summary"), )) .await .unwrap(); @@ -524,9 +485,7 @@ async fn compression_summarizes_historical_tool_messages_instead_of_pinning_all( .lcm_compress(compress_request( "cursor", "session-1", - LcmSummarizerMode::Fake { - summary_text: "tool result summary".into(), - }, + provided_summary("tool result summary"), )) .await .unwrap(); @@ -583,9 +542,7 @@ async fn compression_preserves_interleaved_policy_anchor_outside_summary() { .lcm_compress(compress_request( "cursor", "session-1", - LcmSummarizerMode::Fake { - summary_text: "old exchange summary".into(), - }, + provided_summary("old exchange summary"), )) .await .unwrap(); diff --git a/crates/tracedecay/tests/session_suite/lcm_compression/condensation.rs b/crates/tracedecay/tests/session_suite/lcm_compression/condensation.rs index b5bc88fa5d..05f15b919c 100644 --- a/crates/tracedecay/tests/session_suite/lcm_compression/condensation.rs +++ b/crates/tracedecay/tests/session_suite/lcm_compression/condensation.rs @@ -51,20 +51,11 @@ async fn condensation_creates_higher_depth_summary_from_existing_leaf_nodes() { stateless_session_patterns: Vec::new(), ignore_message_patterns: Vec::new(), expected_current_frontier_store_id: None, - threshold_tokens: None, - max_assembly_tokens: None, - leaf_chunk_tokens: None, - max_source_messages: None, - summary_fan_in: Some(3), - incremental_max_depth: None, - fresh_tail_count: None, - dynamic_leaf_chunk_enabled: None, - dynamic_leaf_chunk_max: None, - context_length: None, - reserve_tokens_floor: None, - summarizer: LcmSummarizerMode::Fake { - summary_text: "depth one condensed".into(), + overrides: tracedecay_lcm::LcmCompressionOverrides { + summary_fan_in: Some(3), + ..Default::default() }, + summarizer: provided_summary("depth one condensed"), }) .await .unwrap(); @@ -169,20 +160,11 @@ async fn condensation_waits_for_one_depth_with_enough_unparented_nodes() { stateless_session_patterns: Vec::new(), ignore_message_patterns: Vec::new(), expected_current_frontier_store_id: None, - threshold_tokens: None, - max_assembly_tokens: None, - leaf_chunk_tokens: None, - max_source_messages: None, - summary_fan_in: Some(3), - incremental_max_depth: None, - fresh_tail_count: None, - dynamic_leaf_chunk_enabled: None, - dynamic_leaf_chunk_max: None, - context_length: None, - reserve_tokens_floor: None, - summarizer: LcmSummarizerMode::Fake { - summary_text: "should not mix depths".into(), + overrides: tracedecay_lcm::LcmCompressionOverrides { + summary_fan_in: Some(3), + ..Default::default() }, + summarizer: provided_summary("should not mix depths"), }) .await .unwrap(); @@ -257,20 +239,11 @@ async fn condensation_orders_same_depth_candidates_by_source_time() { stateless_session_patterns: Vec::new(), ignore_message_patterns: Vec::new(), expected_current_frontier_store_id: None, - threshold_tokens: None, - max_assembly_tokens: None, - leaf_chunk_tokens: None, - max_source_messages: None, - summary_fan_in: Some(3), - incremental_max_depth: None, - fresh_tail_count: None, - dynamic_leaf_chunk_enabled: None, - dynamic_leaf_chunk_max: None, - context_length: None, - reserve_tokens_floor: None, - summarizer: LcmSummarizerMode::Fake { - summary_text: "depth one condensed".into(), + overrides: tracedecay_lcm::LcmCompressionOverrides { + summary_fan_in: Some(3), + ..Default::default() }, + summarizer: provided_summary("depth one condensed"), }) .await .unwrap(); @@ -336,11 +309,9 @@ async fn condensation_respects_default_incremental_max_depth() { let mut request = compress_request( "cursor", "session-1", - LcmSummarizerMode::Fake { - summary_text: "should not condense above max depth".into(), - }, + provided_summary("should not condense above max depth"), ); - request.summary_fan_in = Some(3); + request.overrides.summary_fan_in = Some(3); let response = db.lcm_compress(request).await.unwrap(); assert_eq!(response.status, "ok"); @@ -402,12 +373,10 @@ async fn condensation_honors_non_default_incremental_max_depth() { let mut request = compress_request( "cursor", "session-1", - LcmSummarizerMode::Fake { - summary_text: "condensed depth one summaries".into(), - }, + provided_summary("condensed depth one summaries"), ); - request.summary_fan_in = Some(3); - request.incremental_max_depth = Some(2); + request.overrides.summary_fan_in = Some(3); + request.overrides.incremental_max_depth = Some(2); let response = db.lcm_compress(request).await.unwrap(); assert_eq!(response.status, "ok"); diff --git a/crates/tracedecay/tests/session_suite/lcm_compression/frontier.rs b/crates/tracedecay/tests/session_suite/lcm_compression/frontier.rs index 690aa6eafe..a10ef44286 100644 --- a/crates/tracedecay/tests/session_suite/lcm_compression/frontier.rs +++ b/crates/tracedecay/tests/session_suite/lcm_compression/frontier.rs @@ -30,9 +30,7 @@ async fn compress_frontier_changed_preserves_existing_transaction_state() { let mut request = compress_request( "cursor", "session-frontier-noop", - LcmSummarizerMode::Fake { - summary_text: "should not be written".into(), - }, + provided_summary("should not be written"), ); request.expected_current_frontier_store_id = Some(0); @@ -71,9 +69,7 @@ async fn compress_persists_summary_frontier_and_remaining_backlog_debt() { .lcm_compress(limited_compress_request( "cursor", "session-write-success", - LcmSummarizerMode::Fake { - summary_text: "writer summary".into(), - }, + provided_summary("writer summary"), None, Some(1), None, @@ -130,9 +126,7 @@ async fn compress_rolls_back_summary_and_lifecycle_when_debt_write_fails() { .lcm_compress_for_test(limited_compress_request( "cursor", "session-write-rollback", - LcmSummarizerMode::Fake { - summary_text: "writer summary".into(), - }, + provided_summary("writer summary"), None, Some(1), None, @@ -170,15 +164,13 @@ async fn late_summary_projection_failure_rolls_back_payload_files_and_canonical_ let mut request = limited_compress_request( "cursor", "session-late-payload-rollback", - LcmSummarizerMode::Fake { - summary_text: "must roll back".into(), - }, + provided_summary("must roll back"), Some(1), Some(1), None, ); - request.threshold_tokens = Some(1); - request.fresh_tail_count = Some(2); + request.overrides.threshold_tokens = Some(1); + request.overrides.fresh_tail_count = Some(2); request.messages = with_authoritative_timestamps(vec![ json!({ "id": "large-tool-result", @@ -298,20 +290,8 @@ async fn compression_noops_when_expected_frontier_is_stale() { stateless_session_patterns: Vec::new(), ignore_message_patterns: Vec::new(), expected_current_frontier_store_id: Some(0), - threshold_tokens: None, - max_assembly_tokens: None, - leaf_chunk_tokens: None, - max_source_messages: None, - summary_fan_in: None, - incremental_max_depth: None, - fresh_tail_count: None, - dynamic_leaf_chunk_enabled: None, - dynamic_leaf_chunk_max: None, - context_length: None, - reserve_tokens_floor: None, - summarizer: LcmSummarizerMode::Fake { - summary_text: "stale summary".into(), - }, + overrides: Default::default(), + summarizer: provided_summary("stale summary"), }) .await .unwrap(); @@ -359,17 +339,7 @@ async fn repeated_active_ingest_preserves_existing_message_ordinals() { stateless_session_patterns: Vec::new(), ignore_message_patterns: Vec::new(), expected_current_frontier_store_id: None, - threshold_tokens: None, - max_assembly_tokens: None, - leaf_chunk_tokens: None, - max_source_messages: None, - summary_fan_in: None, - incremental_max_depth: None, - fresh_tail_count: None, - dynamic_leaf_chunk_enabled: None, - dynamic_leaf_chunk_max: None, - context_length: None, - reserve_tokens_floor: None, + overrides: Default::default(), summarizer: LcmSummarizerMode::Noop, }) .await diff --git a/crates/tracedecay/tests/session_suite/lcm_compression/mod.rs b/crates/tracedecay/tests/session_suite/lcm_compression/mod.rs index a766605f20..9814bbe339 100644 --- a/crates/tracedecay/tests/session_suite/lcm_compression/mod.rs +++ b/crates/tracedecay/tests/session_suite/lcm_compression/mod.rs @@ -160,17 +160,7 @@ fn preflight_request( session_id: session_id.to_string(), messages, current_tokens, - threshold_tokens: None, - max_assembly_tokens: None, - leaf_chunk_tokens: None, - max_source_messages: None, - summary_fan_in: None, - incremental_max_depth: None, - fresh_tail_count: None, - dynamic_leaf_chunk_enabled: None, - dynamic_leaf_chunk_max: None, - context_length: None, - reserve_tokens_floor: None, + overrides: Default::default(), ignore_session_patterns: Vec::new(), stateless_session_patterns: Vec::new(), } @@ -198,7 +188,7 @@ async fn ingest_active_messages( let message_count = messages.len(); let mut request = compress_request(provider, session_id, LcmSummarizerMode::Noop); request.messages = messages; - request.fresh_tail_count = Some(message_count.max(1)); + request.overrides.fresh_tail_count = Some(message_count.max(1)); let response = db .lcm_compress(request) .await @@ -223,21 +213,19 @@ fn compress_request( stateless_session_patterns: Vec::new(), ignore_message_patterns: Vec::new(), expected_current_frontier_store_id: None, - threshold_tokens: None, - max_assembly_tokens: None, - leaf_chunk_tokens: None, - max_source_messages: None, - summary_fan_in: None, - incremental_max_depth: None, - fresh_tail_count: None, - dynamic_leaf_chunk_enabled: None, - dynamic_leaf_chunk_max: None, - context_length: None, - reserve_tokens_floor: None, + overrides: Default::default(), summarizer, } } +fn provided_summary(summary_text: impl Into) -> LcmSummarizerMode { + LcmSummarizerMode::Provided { + summary_text: summary_text.into(), + route: None, + extraction_result: None, + } +} + fn limited_compress_request( provider: &str, session_id: &str, @@ -256,17 +244,12 @@ fn limited_compress_request( stateless_session_patterns: Vec::new(), ignore_message_patterns: Vec::new(), expected_current_frontier_store_id: None, - threshold_tokens: None, - max_assembly_tokens, - leaf_chunk_tokens, - max_source_messages, - summary_fan_in: None, - incremental_max_depth: None, - fresh_tail_count: None, - dynamic_leaf_chunk_enabled: None, - dynamic_leaf_chunk_max: None, - context_length: None, - reserve_tokens_floor: None, + overrides: tracedecay_lcm::LcmCompressionOverrides { + max_assembly_tokens, + leaf_chunk_tokens, + max_source_messages, + ..Default::default() + }, summarizer, } } diff --git a/crates/tracedecay/tests/session_suite/lcm_compression/overflow.rs b/crates/tracedecay/tests/session_suite/lcm_compression/overflow.rs index 168a1b0a7b..116d762ce8 100644 --- a/crates/tracedecay/tests/session_suite/lcm_compression/overflow.rs +++ b/crates/tracedecay/tests/session_suite/lcm_compression/overflow.rs @@ -15,15 +15,13 @@ async fn compress_forces_overflow_recovery_with_reserve_derived_cap() { let mut request = limited_compress_request( "cursor", "session-1", - LcmSummarizerMode::Fake { - summary_text: "forced summary".into(), - }, + provided_summary("forced summary"), Some(10), None, None, ); - request.context_length = Some(150); - request.reserve_tokens_floor = Some(50); + request.overrides.context_length = Some(150); + request.overrides.reserve_tokens_floor = Some(50); let response = db.lcm_compress(request).await.unwrap(); @@ -52,15 +50,13 @@ async fn forced_overflow_summary_replay_keeps_latest_message_when_nothing_fits() let mut request = limited_compress_request( "cursor", "session-1", - LcmSummarizerMode::Fake { - summary_text: "summary too large for cap".into(), - }, + provided_summary("summary too large for cap"), Some(1), None, Some(1), ); request.current_tokens = Some(20_000); - request.fresh_tail_count = Some(2); + request.overrides.fresh_tail_count = Some(2); let response = db.lcm_compress(request).await.unwrap(); @@ -91,9 +87,7 @@ async fn forced_overflow_compresses_sub_threshold_backlog() { .lcm_compress(limited_compress_request( "cursor", "session-1", - LcmSummarizerMode::Fake { - summary_text: "forced summary".into(), - }, + provided_summary("forced summary"), Some(10), None, Some(100), @@ -128,9 +122,7 @@ async fn forced_overflow_recovery_compacts_additional_backlog_and_reports_reason let mut request = limited_compress_request( "cursor", "session-1", - LcmSummarizerMode::Fake { - summary_text: "forced overflow summary".into(), - }, + provided_summary("forced overflow summary"), Some(2), Some(1), Some(20), @@ -196,9 +188,7 @@ async fn forced_overflow_triggers_at_configured_cap_and_catches_up_in_passes() { let mut request = limited_compress_request( "cursor", "session-1", - LcmSummarizerMode::Fake { - summary_text: "catchup summary".into(), - }, + provided_summary("catchup summary"), Some(2 * budget_tokens("old-1 token")), Some(2), Some(40), @@ -234,9 +224,7 @@ async fn forced_overflow_without_backlog_reports_irreducible_best_effort() { let mut request = limited_compress_request( "cursor", "session-1", - LcmSummarizerMode::Fake { - summary_text: "unused summary".into(), - }, + provided_summary("unused summary"), None, None, Some(3), @@ -287,9 +275,7 @@ async fn forced_overflow_trims_replay_to_assembly_cap() { let mut request = limited_compress_request( "cursor", "session-1", - LcmSummarizerMode::Fake { - summary_text: "small summary".into(), - }, + provided_summary("small summary"), None, None, Some(6), @@ -334,9 +320,7 @@ async fn forced_overflow_replay_budget_accounts_for_prompt_overhead_delta() { let mut request = limited_compress_request( "cursor", "session-1", - LcmSummarizerMode::Fake { - summary_text: "unused summary".into(), - }, + provided_summary("unused summary"), None, None, Some(12), @@ -381,9 +365,7 @@ async fn oversized_authoritative_summary_is_preserved_exactly() { .lcm_compress(compress_request( "cursor", "session-1", - LcmSummarizerMode::Fake { - summary_text: exact_summary.clone(), - }, + provided_summary(exact_summary.clone()), )) .await .unwrap(); @@ -415,9 +397,7 @@ async fn authoritative_summary_reports_attempt_state() { .lcm_compress(compress_request( "cursor", "session-1", - LcmSummarizerMode::Fake { - summary_text: "oversized ".repeat(100), - }, + provided_summary("oversized ".repeat(100)), )) .await .unwrap(); @@ -455,9 +435,7 @@ async fn critical_pressure_catch_up_reports_attempts_debt_and_budget_state() { let mut request = limited_compress_request( "cursor", "session-1", - LcmSummarizerMode::Fake { - summary_text: "catchup summary".into(), - }, + provided_summary("catchup summary"), Some(budget_tokens("old-1 token")), Some(1), Some(budget_tokens("fresh-1") + budget_tokens("fresh-2") + 1), @@ -522,9 +500,7 @@ async fn maintenance_debt_clears_when_retry_compacts_remaining_backlog() { .lcm_compress(limited_compress_request( "cursor", "session-1", - LcmSummarizerMode::Fake { - summary_text: "first retry summary".into(), - }, + provided_summary("first retry summary"), Some(2 * budget_tokens("old-1 token")), Some(2), None, @@ -543,9 +519,7 @@ async fn maintenance_debt_clears_when_retry_compacts_remaining_backlog() { .lcm_compress(limited_compress_request( "cursor", "session-1", - LcmSummarizerMode::Fake { - summary_text: "second retry summary".into(), - }, + provided_summary("second retry summary"), Some(2 * budget_tokens("old-1 token")), Some(2), None, @@ -585,9 +559,7 @@ async fn compression_reinjects_latest_user_objective_when_tail_is_tool_heavy() { .lcm_compress(compress_request( "cursor", "session-1", - LcmSummarizerMode::Fake { - summary_text: "historical summary".into(), - }, + provided_summary("historical summary"), )) .await .unwrap(); @@ -633,9 +605,7 @@ async fn overflow_recovery_keeps_preserved_objective_scaffold_when_evicting_tail let mut request = limited_compress_request( "cursor", "session-1", - LcmSummarizerMode::Fake { - summary_text: "unused summary".into(), - }, + provided_summary("unused summary"), None, None, Some( @@ -688,9 +658,7 @@ async fn overflow_recovery_without_backlog_evicts_droppable_tail() { let mut request = limited_compress_request( "cursor", "session-1", - LcmSummarizerMode::Fake { - summary_text: "unused summary".into(), - }, + provided_summary("unused summary"), None, None, Some(5), diff --git a/crates/tracedecay/tests/session_suite/lcm_compression/patterns.rs b/crates/tracedecay/tests/session_suite/lcm_compression/patterns.rs index f2468bdeaf..3719bca273 100644 --- a/crates/tracedecay/tests/session_suite/lcm_compression/patterns.rs +++ b/crates/tracedecay/tests/session_suite/lcm_compression/patterns.rs @@ -21,20 +21,8 @@ async fn ignored_session_pattern_skips_active_ingest_and_compression() { stateless_session_patterns: Vec::new(), ignore_message_patterns: Vec::new(), expected_current_frontier_store_id: None, - threshold_tokens: None, - max_assembly_tokens: None, - leaf_chunk_tokens: None, - max_source_messages: None, - summary_fan_in: None, - incremental_max_depth: None, - fresh_tail_count: None, - dynamic_leaf_chunk_enabled: None, - dynamic_leaf_chunk_max: None, - context_length: None, - reserve_tokens_floor: None, - summarizer: LcmSummarizerMode::Fake { - summary_text: "should not be used".into(), - }, + overrides: Default::default(), + summarizer: provided_summary("should not be used"), }) .await .unwrap(); @@ -71,17 +59,7 @@ async fn stateless_session_pattern_keeps_replay_but_does_not_persist_lcm_rows() "content": "throwaway one-shot prompt" })], current_tokens: Some(100), - threshold_tokens: None, - max_assembly_tokens: None, - leaf_chunk_tokens: None, - max_source_messages: None, - summary_fan_in: None, - incremental_max_depth: None, - fresh_tail_count: None, - dynamic_leaf_chunk_enabled: None, - dynamic_leaf_chunk_max: None, - context_length: None, - reserve_tokens_floor: None, + overrides: Default::default(), ignore_session_patterns: Vec::new(), stateless_session_patterns: vec!["scratch-shell-*".into()], }) @@ -112,20 +90,8 @@ async fn stateless_session_pattern_keeps_replay_but_does_not_persist_lcm_rows() stateless_session_patterns: vec!["scratch-shell-*".into()], ignore_message_patterns: Vec::new(), expected_current_frontier_store_id: None, - threshold_tokens: None, - max_assembly_tokens: None, - leaf_chunk_tokens: None, - max_source_messages: None, - summary_fan_in: None, - incremental_max_depth: None, - fresh_tail_count: None, - dynamic_leaf_chunk_enabled: None, - dynamic_leaf_chunk_max: None, - context_length: None, - reserve_tokens_floor: None, - summarizer: LcmSummarizerMode::Fake { - summary_text: "should not be used".into(), - }, + overrides: Default::default(), + summarizer: provided_summary("should not be used"), }) .await .unwrap(); @@ -169,17 +135,7 @@ async fn ignore_message_patterns_skip_storage_but_heartbeat_noise_is_stored() { stateless_session_patterns: Vec::new(), ignore_message_patterns: vec!["Cronjob Response:*".into()], expected_current_frontier_store_id: None, - threshold_tokens: None, - max_assembly_tokens: None, - leaf_chunk_tokens: None, - max_source_messages: None, - summary_fan_in: None, - incremental_max_depth: None, - fresh_tail_count: None, - dynamic_leaf_chunk_enabled: None, - dynamic_leaf_chunk_max: None, - context_length: None, - reserve_tokens_floor: None, + overrides: Default::default(), summarizer: LcmSummarizerMode::Noop, }) .await diff --git a/crates/tracedecay/tests/session_suite/lcm_compression/preflight.rs b/crates/tracedecay/tests/session_suite/lcm_compression/preflight.rs index 96bedce8d5..07af73b315 100644 --- a/crates/tracedecay/tests/session_suite/lcm_compression/preflight.rs +++ b/crates/tracedecay/tests/session_suite/lcm_compression/preflight.rs @@ -21,17 +21,7 @@ async fn preflight_is_read_only_and_never_ingests_active_messages() { "content": format!("data:image/png;base64,{}", "A".repeat(100_000)) })], current_tokens: Some(100), - threshold_tokens: None, - max_assembly_tokens: None, - leaf_chunk_tokens: None, - max_source_messages: None, - summary_fan_in: None, - incremental_max_depth: None, - fresh_tail_count: None, - dynamic_leaf_chunk_enabled: None, - dynamic_leaf_chunk_max: None, - context_length: None, - reserve_tokens_floor: None, + overrides: Default::default(), ignore_session_patterns: Vec::new(), stateless_session_patterns: Vec::new(), }) @@ -221,12 +211,12 @@ async fn preflight_decides_compression_from_threshold_and_overflow_pressure() { let mut request = preflight_request("cursor", &session_id, Vec::new(), Some(case.current_tokens)); - request.threshold_tokens = case.threshold_tokens; - request.leaf_chunk_tokens = case.leaf_chunk_tokens; - request.max_source_messages = case.max_source_messages; - request.max_assembly_tokens = case.max_assembly_tokens; - request.context_length = case.context_length; - request.reserve_tokens_floor = case.reserve_tokens_floor; + request.overrides.threshold_tokens = case.threshold_tokens; + request.overrides.leaf_chunk_tokens = case.leaf_chunk_tokens; + request.overrides.max_source_messages = case.max_source_messages; + request.overrides.max_assembly_tokens = case.max_assembly_tokens; + request.overrides.context_length = case.context_length; + request.overrides.reserve_tokens_floor = case.reserve_tokens_floor; let response = db.lcm_preflight(request).await.unwrap(); @@ -262,9 +252,7 @@ async fn preflight_requests_compression_for_maintenance_debt() { .lcm_compress(limited_compress_request( "cursor", "session-1", - LcmSummarizerMode::Fake { - summary_text: "first chunk summary".into(), - }, + provided_summary("first chunk summary"), Some(2 * budget_tokens("old-1 token")), Some(2), None, diff --git a/crates/tracedecay/tests/session_suite/lcm_compression/replay.rs b/crates/tracedecay/tests/session_suite/lcm_compression/replay.rs index 7fde938e6e..72d17d0eff 100644 --- a/crates/tracedecay/tests/session_suite/lcm_compression/replay.rs +++ b/crates/tracedecay/tests/session_suite/lcm_compression/replay.rs @@ -24,17 +24,7 @@ async fn ingest_only_compress_stores_messages_without_summary_nodes() { stateless_session_patterns: Vec::new(), ignore_message_patterns: Vec::new(), expected_current_frontier_store_id: None, - threshold_tokens: None, - max_assembly_tokens: None, - leaf_chunk_tokens: None, - max_source_messages: None, - summary_fan_in: None, - incremental_max_depth: None, - fresh_tail_count: None, - dynamic_leaf_chunk_enabled: None, - dynamic_leaf_chunk_max: None, - context_length: None, - reserve_tokens_floor: None, + overrides: Default::default(), summarizer: LcmSummarizerMode::Noop, }) .await @@ -144,17 +134,7 @@ async fn replay_assembly_terminates_when_existing_summary_sources_contain_cycle( stateless_session_patterns: Vec::new(), ignore_message_patterns: Vec::new(), expected_current_frontier_store_id: None, - threshold_tokens: None, - max_assembly_tokens: None, - leaf_chunk_tokens: None, - max_source_messages: None, - summary_fan_in: None, - incremental_max_depth: None, - fresh_tail_count: None, - dynamic_leaf_chunk_enabled: None, - dynamic_leaf_chunk_max: None, - context_length: None, - reserve_tokens_floor: None, + overrides: Default::default(), summarizer: LcmSummarizerMode::Noop, }), ) @@ -204,17 +184,12 @@ async fn threshold_pressure_summarizes_short_huge_active_context() { stateless_session_patterns: Vec::new(), ignore_message_patterns: Vec::new(), expected_current_frontier_store_id: None, - threshold_tokens: Some(1_000), - max_assembly_tokens: None, - leaf_chunk_tokens: Some(100), - max_source_messages: None, - summary_fan_in: None, - incremental_max_depth: None, - fresh_tail_count: Some(64), - dynamic_leaf_chunk_enabled: None, - dynamic_leaf_chunk_max: None, - context_length: None, - reserve_tokens_floor: None, + overrides: tracedecay_lcm::LcmCompressionOverrides { + threshold_tokens: Some(1_000), + leaf_chunk_tokens: Some(100), + fresh_tail_count: Some(64), + ..Default::default() + }, summarizer: LcmSummarizerMode::HermesAuxiliary, }) .await @@ -331,9 +306,7 @@ async fn idless_compression_replay_does_not_reingest_existing_raw_messages() { let mut request = compress_request( "cursor", "session-idless-replay", - LcmSummarizerMode::Fake { - summary_text: "condensed old context".into(), - }, + provided_summary("condensed old context"), ); request.messages = initial.replay_messages; let compressed = db.lcm_compress(request).await.unwrap(); @@ -420,20 +393,8 @@ async fn active_replay_preserves_top_level_fields_that_collide_with_storage_meta stateless_session_patterns: Vec::new(), ignore_message_patterns: Vec::new(), expected_current_frontier_store_id: None, - threshold_tokens: None, - max_assembly_tokens: None, - leaf_chunk_tokens: None, - max_source_messages: None, - summary_fan_in: None, - incremental_max_depth: None, - fresh_tail_count: None, - dynamic_leaf_chunk_enabled: None, - dynamic_leaf_chunk_max: None, - context_length: None, - reserve_tokens_floor: None, - summarizer: LcmSummarizerMode::Fake { - summary_text: "unused".into(), - }, + overrides: Default::default(), + summarizer: provided_summary("unused"), }) .await .unwrap(); @@ -495,9 +456,7 @@ async fn raw_replay_strips_disposable_provider_reasoning_sidecars() { .lcm_compress(compress_request( "cursor", "session-sidecars", - LcmSummarizerMode::Fake { - summary_text: "unused".into(), - }, + provided_summary("unused"), )) .await .unwrap(); @@ -559,20 +518,8 @@ async fn raw_replay_preserves_assistant_tool_calls_and_tool_result_linking() { stateless_session_patterns: Vec::new(), ignore_message_patterns: Vec::new(), expected_current_frontier_store_id: None, - threshold_tokens: None, - max_assembly_tokens: None, - leaf_chunk_tokens: None, - max_source_messages: None, - summary_fan_in: None, - incremental_max_depth: None, - fresh_tail_count: None, - dynamic_leaf_chunk_enabled: None, - dynamic_leaf_chunk_max: None, - context_length: None, - reserve_tokens_floor: None, - summarizer: LcmSummarizerMode::Fake { - summary_text: "unused".into(), - }, + overrides: Default::default(), + summarizer: provided_summary("unused"), }) .await .unwrap(); @@ -659,9 +606,7 @@ async fn active_replay_tool_calls_apply_ingest_protection_and_externalize_media_ .lcm_compress(compress_request( "cursor", "session-tool-calls-protection", - LcmSummarizerMode::Fake { - summary_text: "unused".into(), - }, + provided_summary("unused"), )) .await .unwrap(); @@ -786,9 +731,7 @@ async fn no_backlog_compress_replays_persisted_uncondensed_summaries() { .lcm_compress(compress_request( "cursor", "session-1", - LcmSummarizerMode::Fake { - summary_text: "old summary".into(), - }, + provided_summary("old summary"), )) .await .unwrap(); @@ -798,9 +741,7 @@ async fn no_backlog_compress_replays_persisted_uncondensed_summaries() { .lcm_compress(compress_request( "cursor", "session-1", - LcmSummarizerMode::Fake { - summary_text: "unused".into(), - }, + provided_summary("unused"), )) .await .unwrap(); diff --git a/crates/tracedecay/tests/session_suite/lcm_compression/summarizer_modes.rs b/crates/tracedecay/tests/session_suite/lcm_compression/summarizer_modes.rs index 5a6e187c45..a110fa384f 100644 --- a/crates/tracedecay/tests/session_suite/lcm_compression/summarizer_modes.rs +++ b/crates/tracedecay/tests/session_suite/lcm_compression/summarizer_modes.rs @@ -1,3 +1,5 @@ +use tracedecay_lcm::dag::sanitize_summary_draft; + use super::*; #[tokio::test] @@ -26,17 +28,7 @@ async fn hermes_auxiliary_request_mode_returns_summary_contract() { stateless_session_patterns: Vec::new(), ignore_message_patterns: Vec::new(), expected_current_frontier_store_id: None, - threshold_tokens: None, - max_assembly_tokens: None, - leaf_chunk_tokens: None, - max_source_messages: None, - summary_fan_in: None, - incremental_max_depth: None, - fresh_tail_count: None, - dynamic_leaf_chunk_enabled: None, - dynamic_leaf_chunk_max: None, - context_length: None, - reserve_tokens_floor: None, + overrides: Default::default(), summarizer: LcmSummarizerMode::HermesAuxiliary, }) .await @@ -95,6 +87,7 @@ async fn provided_summarizer_advances_frontier_consistently() { LcmSummarizerMode::Provided { summary_text: "one two three".into(), route: Some("test-route".into()), + extraction_result: None, }, )) .await @@ -112,6 +105,7 @@ async fn provided_summarizer_advances_frontier_consistently() { LcmSummarizerMode::Provided { summary_text: "four five".into(), route: Some("test-route".into()), + extraction_result: None, }, )) .await @@ -129,64 +123,89 @@ async fn provided_summarizer_advances_frontier_consistently() { ); } -#[tokio::test] -async fn provided_route_envelope_persists_extraction_metadata() { +const WITH_EXTRACTION_METADATA: &str = r#"{"pre_compaction_extraction":{"items":["Decision: keep nightly backups","Commitment: rotate keys weekly"],"model":"openai/gpt-5.4-mini","output_path":"/tmp/extractions","status":"ok"},"summary_route":"backup"}"#; +const ROUTE_ONLY_METADATA: &str = r#"{"pre_compaction_extraction":{"status":"not_requested"},"summary_route":"plain {route} 雪"}"#; +const NO_ROUTE_METADATA: &str = r#"{"pre_compaction_extraction":{"status":"not_requested"}}"#; + +async fn persisted_summary_metadata(summarizer: LcmSummarizerMode) -> String { let tmp = TempDir::new().unwrap(); let db = open_lcm_db(&tmp).await; insert_raw_messages(&db, "cursor", "session-1", &["old-1", "old-2", "fresh-1"]).await; let response = db - .lcm_compress(compress_request( - "cursor", - "session-1", - LcmSummarizerMode::Provided { - summary_text: "summary with extraction".into(), - route: Some( - json!({ - "route": "backup", - "pre_compaction_extraction": { - "status": "ok", - "items": [ - "Decision: keep nightly backups", - "Commitment: rotate keys weekly" - ], - "model": "openai/gpt-5.4-mini", - "output_path": "/tmp/extractions" - } - }) - .to_string(), - ), - }, - )) + .lcm_compress(compress_request("cursor", "session-1", summarizer)) .await .unwrap(); - assert_eq!(response.status, "ok"); assert_eq!(response.summary_nodes_created, 1); - let metadata: Value = serde_json::from_str( - response.summary_nodes[0] + let node = &response.summary_nodes[0]; + assert_eq!(node.source_refs.len(), 1); + assert_eq!(node.expand_hint.as_deref(), Some("1 raw messages")); + let expanded = db + .lcm_expand_summary_node("cursor", "session-1", &node.node_id) + .await + .unwrap(); + let persisted = expanded + .summary + .metadata_json + .expect("persisted summary metadata"); + assert_eq!(node.metadata_json.as_deref(), Some(persisted.as_str())); + persisted +} + +fn provided( + route: Option<&str>, + extraction: Option, +) -> LcmSummarizerMode { + LcmSummarizerMode::Provided { + summary_text: "summary with extraction".into(), + route: route.map(str::to_string), + extraction_result: extraction, + } +} + +#[tokio::test] +async fn provided_summary_metadata_rows_are_byte_stable() { + let extraction = tracedecay_lcm::types::LcmExtractionResult { + status: "ok".into(), + items: vec![ + "Decision: keep nightly backups".into(), + "Commitment: rotate keys weekly".into(), + ], + text: None, + model: Some("openai/gpt-5.4-mini".into()), + output_path: Some("/tmp/extractions".into()), + error: None, + }; + let cases = [ + ( + provided(Some(" backup "), Some(extraction)), + WITH_EXTRACTION_METADATA, + ), + ( + provided(Some("plain {route} 雪"), None), + ROUTE_ONLY_METADATA, + ), + (provided(Some(" "), None), NO_ROUTE_METADATA), + (provided(None, None), NO_ROUTE_METADATA), + ]; + for (summarizer, expected) in cases { + // Pin compression's metadata bytes independently, then pass them + // through the same privacy authority every persisted summary requires. + // Compare the entire protected value, including all receipt bindings. + let mut draft = summary_draft( + "cursor", + "session-1", + 0, + "summary with extraction", + Vec::new(), + ); + draft.expand_hint = Some("1 raw messages".to_owned()); + draft.metadata_json = Some(expected.to_owned()); + let expected = sanitize_summary_draft(draft) + .expect("canonical summary protection") .metadata_json - .as_deref() - .expect("summary metadata"), - ) - .unwrap(); - assert_eq!( - metadata["summary_route"], - Value::String("backup".to_string()) - ); - assert_eq!( - metadata["pre_compaction_extraction"]["status"], - Value::String("ok".to_string()) - ); - assert_eq!( - metadata["pre_compaction_extraction"]["items"], - json!([ - "Decision: keep nightly backups", - "Commitment: rotate keys weekly" - ]) - ); - assert_eq!( - metadata["pre_compaction_extraction"]["model"], - Value::String("openai/gpt-5.4-mini".to_string()) - ); + .expect("protected summary metadata"); + assert_eq!(persisted_summary_metadata(summarizer).await, expected); + } } diff --git a/crates/tracedecay/tests/session_suite/lcm_compression/tool_transactions.rs b/crates/tracedecay/tests/session_suite/lcm_compression/tool_transactions.rs index 44a2499847..4d7781399a 100644 --- a/crates/tracedecay/tests/session_suite/lcm_compression/tool_transactions.rs +++ b/crates/tracedecay/tests/session_suite/lcm_compression/tool_transactions.rs @@ -16,17 +16,15 @@ async fn fresh_tail_boundary_keeps_multi_tool_transaction_atomic_and_shrinking() let mut request = limited_compress_request( "cursor", "session-atomic-tail", - LcmSummarizerMode::Fake { - summary_text: "compact summary".into(), - }, + provided_summary("compact summary"), None, None, Some(400), ); request.messages = with_authoritative_timestamps(messages); request.current_tokens = Some(107); - request.threshold_tokens = Some(80); - request.fresh_tail_count = Some(3); + request.overrides.threshold_tokens = Some(80); + request.overrides.fresh_tail_count = Some(3); let response = db.lcm_compress(request).await.unwrap(); let assistant = response @@ -68,15 +66,13 @@ async fn bounded_leaf_chunk_backs_off_before_multi_tool_transaction() { let mut request = limited_compress_request( "cursor", "session-atomic-leaf", - LcmSummarizerMode::Fake { - summary_text: "transaction summary".into(), - }, + provided_summary("transaction summary"), None, Some(2), None, ); request.messages = with_authoritative_timestamps(messages); - request.fresh_tail_count = Some(1); + request.overrides.fresh_tail_count = Some(1); let response = db.lcm_compress(request).await.unwrap(); assert_eq!(response.summary_nodes_created, 1); @@ -106,16 +102,14 @@ async fn budget_and_overflow_replay_never_split_tool_transaction() { let mut request = limited_compress_request( "cursor", "session-atomic-budget", - LcmSummarizerMode::Fake { - summary_text: "summary".into(), - }, + provided_summary("summary"), None, None, Some(8), ); request.messages = with_authoritative_timestamps(messages); request.current_tokens = Some(107); - request.fresh_tail_count = Some(3); + request.overrides.fresh_tail_count = Some(3); let response = db.lcm_compress(request).await.unwrap(); let assistant_count = response @@ -169,15 +163,10 @@ async fn legacy_partial_tool_transaction_drops_invalid_group_and_repairs_orphans "role": "user", "content": "the unmatched call is now a closed legacy group", })); - let mut request = compress_request( - "cursor", - "session-legacy-tools", - LcmSummarizerMode::Fake { - summary_text: "unused".into(), - }, - ); + let mut request = + compress_request("cursor", "session-legacy-tools", provided_summary("unused")); request.messages = transaction; - request.fresh_tail_count = Some(10); + request.overrides.fresh_tail_count = Some(10); let response = db.lcm_compress(request).await.unwrap(); assert!( diff --git a/crates/tracedecay/tests/session_suite/lcm_raw.rs b/crates/tracedecay/tests/session_suite/lcm_raw.rs index dc5b762e9b..8cf2e6a1ff 100644 --- a/crates/tracedecay/tests/session_suite/lcm_raw.rs +++ b/crates/tracedecay/tests/session_suite/lcm_raw.rs @@ -60,17 +60,7 @@ async fn active_replay_metadata_namespaces_original_fields_from_storage_metadata stateless_session_patterns: Vec::new(), ignore_message_patterns: Vec::new(), expected_current_frontier_store_id: None, - threshold_tokens: None, - max_assembly_tokens: None, - leaf_chunk_tokens: None, - max_source_messages: None, - summary_fan_in: None, - incremental_max_depth: None, - fresh_tail_count: None, - dynamic_leaf_chunk_enabled: None, - dynamic_leaf_chunk_max: None, - context_length: None, - reserve_tokens_floor: None, + overrides: Default::default(), summarizer: LcmSummarizerMode::Noop, }) .await diff --git a/crates/tracedecay/tests/transcript_ingest_suite/hermes.rs b/crates/tracedecay/tests/transcript_ingest_suite/hermes.rs index b202a5cf4b..b5488ed647 100644 --- a/crates/tracedecay/tests/transcript_ingest_suite/hermes.rs +++ b/crates/tracedecay/tests/transcript_ingest_suite/hermes.rs @@ -571,17 +571,7 @@ async fn hermes_projection_sweep_does_not_mutate_runtime_owned_raw_messages() { stateless_session_patterns: Vec::new(), ignore_message_patterns: Vec::new(), expected_current_frontier_store_id: None, - threshold_tokens: None, - max_assembly_tokens: None, - leaf_chunk_tokens: None, - max_source_messages: None, - summary_fan_in: None, - incremental_max_depth: None, - fresh_tail_count: None, - dynamic_leaf_chunk_enabled: None, - dynamic_leaf_chunk_max: None, - context_length: None, - reserve_tokens_floor: None, + overrides: Default::default(), summarizer: LcmSummarizerMode::Noop, }) .await