From 2b00553fcfba4a5438586765cf049a79c0b5cb34 Mon Sep 17 00:00:00 2001 From: ScriptedAlchemy Date: Tue, 20 Oct 2026 16:54:30 -0700 Subject: [PATCH 01/15] fix(store): converge reopened writers on persisted authority --- .../tests/core_cli_suite/main.rs | 2 + .../session_store_reopen_test.rs | 170 ++++++++++++++++ .../src/observation_adapter.rs | 25 +++ .../src/observation_projection/rebuild.rs | 4 +- crates/tracedecay-host-admission/src/lib.rs | 43 ++++ .../src/shard_runtime/registry.rs | 8 + .../src/shard_runtime/registry/open.rs | 27 ++- .../src/shard_runtime/registry/ports.rs | 43 ++++ .../registry/tests/production_routes.rs | 190 ++++++++++++++++++ .../tracedecay-rusqlite-runtime/src/ledger.rs | 1 + .../src/ledger/checkpoint.rs | 100 +++++++++ crates/tracedecay-rusqlite-runtime/src/lib.rs | 5 +- .../src/persistence.rs | 102 ++++++++-- .../src/writer/transaction.rs | 24 ++- .../rusqlite_suite/storage_contention.rs | 106 ++++++++++ .../worker.rs | 28 ++- .../src/runtime/ingest/failure.rs | 54 +++++ crates/tracedecay-store/src/lib.rs | 7 +- crates/tracedecay-store/src/projection.rs | 12 ++ crates/tracedecay-store/src/runtime/error.rs | 27 ++- crates/tracedecay-store/src/runtime/mod.rs | 3 +- 21 files changed, 955 insertions(+), 26 deletions(-) create mode 100644 crates/tracedecay-cli/tests/core_cli_suite/session_store_reopen_test.rs diff --git a/crates/tracedecay-cli/tests/core_cli_suite/main.rs b/crates/tracedecay-cli/tests/core_cli_suite/main.rs index 4ac196eee9..f319aa39e6 100644 --- a/crates/tracedecay-cli/tests/core_cli_suite/main.rs +++ b/crates/tracedecay-cli/tests/core_cli_suite/main.rs @@ -26,6 +26,8 @@ mod monitor_test; mod provision_host_cli_fixture; #[cfg(unix)] mod remote_cli_test; +#[cfg(unix)] +mod session_store_reopen_test; mod source_provenance_test; mod sync_test; mod test_profile_isolation_test; diff --git a/crates/tracedecay-cli/tests/core_cli_suite/session_store_reopen_test.rs b/crates/tracedecay-cli/tests/core_cli_suite/session_store_reopen_test.rs new file mode 100644 index 0000000000..86a600a8eb --- /dev/null +++ b/crates/tracedecay-cli/tests/core_cli_suite/session_store_reopen_test.rs @@ -0,0 +1,170 @@ +//! A daemon that restarts without its authority record starts at authority +//! epoch 1 again and so reuses the previous generation's store incarnation. +//! The project sessions store then already holds a writer authority above the +//! epoch the new daemon allocates for it. The restarted daemon must converge +//! on that persisted authority and keep projecting; it must not retry a write +//! the store fences on every pass. + +use std::fs; +use std::path::{Path, PathBuf}; +use std::process::Command; +use std::time::{Duration, Instant}; + +use tempfile::TempDir; + +use crate::common::{self, canonical_existing_path, tracedecay_command_with_home}; + +const THREAD_ID: &str = "codex-reopen-thread"; +const FIRST_MARKER: &str = "ledgerquartz"; +const RESTART_MARKER: &str = "restartquartz"; +/// A projection pass runs in the background; this is how long one import may +/// take to become searchable on a loaded runner. +const PROJECTION_BUDGET: Duration = Duration::from_secs(120); + +fn git(project: &Path, args: &[&str]) { + let output = Command::new(common::git_program()) + .args(args) + .current_dir(project) + .output() + .expect("git should run"); + assert!( + output.status.success(), + "git {args:?} failed: {}", + String::from_utf8_lossy(&output.stderr) + ); +} + +fn rollout_line(timestamp: &str, kind: &str, payload: serde_json::Value) -> String { + serde_json::json!({ "timestamp": timestamp, "type": kind, "payload": payload }).to_string() +} + +fn write_rollout(home: &Path, project: &Path, messages: &[&str]) -> PathBuf { + let dir = home.join(".codex/sessions/2026/01/01"); + fs::create_dir_all(&dir).unwrap(); + let path = dir.join(format!("rollout-2026-01-01T00-00-00-{THREAD_ID}.jsonl")); + let mut lines = vec![rollout_line( + "2026-01-01T00:00:00.000Z", + "session_meta", + serde_json::json!({ + "id": THREAD_ID, + "cwd": project.to_string_lossy(), + "model": "gpt-5.5" + }), + )]; + for (index, message) in messages.iter().enumerate() { + lines.push(rollout_line( + &format!("2026-01-01T00:00:{:02}.000Z", index + 1), + "event_msg", + serde_json::json!({ "type": "agent_message", "message": message }), + )); + } + fs::write(&path, format!("{}\n", lines.join("\n"))).unwrap(); + path +} + +fn import_sessions(home: &Path, project: &Path) { + let output = tracedecay_command_with_home(home) + .current_dir(project) + .args(["sessions", "import", "--project-path"]) + .arg(project) + .output() + .expect("tracedecay sessions import should run"); + assert!( + output.status.success(), + "sessions import failed\nstdout:\n{}\nstderr:\n{}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ); +} + +fn wait_until_searchable(home: &Path, project: &Path, marker: &str, context: &str) { + common::poll_until( + Instant::now() + PROJECTION_BUDGET, + Duration::from_millis(500), + || { + let output = tracedecay_command_with_home(home) + .current_dir(project) + .args([ + "sessions", + "search", + marker, + "--provider", + "codex", + "--project-path", + ]) + .arg(project) + .args(["--limit", "20", "--json"]) + .output() + .expect("tracedecay sessions search should run"); + (output.status.success() && String::from_utf8_lossy(&output.stdout).contains(marker)) + .then_some(()) + }, + || format!("{context}: {marker} never became searchable"), + ); +} + +#[test] +fn a_daemon_restarted_on_a_reused_incarnation_converges_on_the_persisted_writer_authority() { + let home = TempDir::new().unwrap(); + let project = TempDir::new().unwrap(); + let home_path = canonical_existing_path(home.path()); + let project_path = canonical_existing_path(project.path()); + fs::write(project_path.join("main.rs"), "fn main() {}\n").unwrap(); + git(&project_path, &["init", "--quiet", "--initial-branch=main"]); + git(&project_path, &["add", "."]); + git( + &project_path, + &[ + "-c", + "user.email=reopen@tracedecay.invalid", + "-c", + "user.name=Reopen", + "commit", + "--quiet", + "-m", + "fixture", + ], + ); + + let first_log = home_path.join("daemon-first.log"); + let mut first = common::spawn_tracedecay_daemon_logged(&home_path, &first_log, |_| {}); + let init = tracedecay_command_with_home(&home_path) + .current_dir(&project_path) + .arg("init") + .output() + .expect("tracedecay init should run"); + assert!( + init.status.success(), + "tracedecay init failed: {}", + String::from_utf8_lossy(&init.stderr) + ); + write_rollout(&home_path, &project_path, &[FIRST_MARKER]); + import_sessions(&home_path, &project_path); + wait_until_searchable(&home_path, &project_path, FIRST_MARKER, "first daemon"); + first.kill_and_wait().expect("stop the first daemon"); + + // The managed-daemon fixture and the work-loop journey both drop this + // record between restarts, as does an operator clearing stale daemon + // state. The next daemon then takes authority epoch 1 again and the + // same store incarnation as the generation that just wrote the store. + fs::remove_file(common::daemon_authority_path( + &home_path.join(".tracedecay"), + )) + .expect("remove the first daemon's authority record"); + let second_log = home_path.join("daemon-second.log"); + let _second = common::spawn_tracedecay_daemon_logged(&home_path, &second_log, |_| {}); + write_rollout(&home_path, &project_path, &[FIRST_MARKER, RESTART_MARKER]); + import_sessions(&home_path, &project_path); + wait_until_searchable( + &home_path, + &project_path, + RESTART_MARKER, + "restarted daemon", + ); + + let second_stderr = fs::read_to_string(&second_log).expect("read the second daemon's log"); + assert!( + !second_stderr.contains("stale writer authority epoch"), + "the restarted daemon retried writes its store had fenced:\n{second_stderr}" + ); +} diff --git a/crates/tracedecay-global-db/src/observation_adapter.rs b/crates/tracedecay-global-db/src/observation_adapter.rs index 5bc6062563..6f2cf6ca39 100644 --- a/crates/tracedecay-global-db/src/observation_adapter.rs +++ b/crates/tracedecay-global-db/src/observation_adapter.rs @@ -2505,6 +2505,13 @@ fn rejected_submit_error( RuntimeSubmitOutcomeV1::Saturated { .. } => { ObservationStoreError::WriterSaturated { operation } } + RuntimeSubmitOutcomeV1::Fenced { expected, actual } => ObservationStoreError::Storage { + operation, + source: Box::new(tracedecay_store::StoreWriterFencedV1 { + requested: *expected, + current: *actual, + }), + }, outcome => runtime_storage_error(operation, format!("{rejection}: {outcome:?}")), } } @@ -2597,6 +2604,24 @@ mod tests { use super::*; + #[test] + fn writer_fence_survives_observation_and_projection_error_wrapping() { + let requested = tracedecay_store::StoreAuthorityEpochV1::new(2).unwrap(); + let current = tracedecay_store::StoreAuthorityEpochV1::new(3).unwrap(); + let error = rejected_submit_error( + "submit anchored observation", + "runtime rejected observation write", + &RuntimeSubmitOutcomeV1::Fenced { + expected: requested, + actual: current, + }, + ); + assert_eq!( + projection_runtime_error(error).writer_fence(), + Some(tracedecay_store::StoreWriterFencedV1 { requested, current }), + ); + } + #[test] fn writer_saturation_is_backpressure_not_a_storage_failure() { for scope in [ diff --git a/crates/tracedecay-global-db/src/observation_projection/rebuild.rs b/crates/tracedecay-global-db/src/observation_projection/rebuild.rs index 773df53880..e0685201e6 100644 --- a/crates/tracedecay-global-db/src/observation_projection/rebuild.rs +++ b/crates/tracedecay-global-db/src/observation_projection/rebuild.rs @@ -101,7 +101,9 @@ fn json_extract_neq_predicates(left_alias: &str, json_column: &str, fields: &[&s } fn records_transient_projection_retry(error: &ProjectionStoreError) -> bool { - matches!(error, ProjectionStoreError::Storage { .. }) && !error.is_sqlite_disk_image_corrupt() + matches!(error, ProjectionStoreError::Storage { .. }) + && !error.is_sqlite_disk_image_corrupt() + && error.writer_fence().is_none() } /// Projects one queued observation through the guarded registered database diff --git a/crates/tracedecay-host-admission/src/lib.rs b/crates/tracedecay-host-admission/src/lib.rs index d17fcd9ccd..b4cb09dc0a 100644 --- a/crates/tracedecay-host-admission/src/lib.rs +++ b/crates/tracedecay-host-admission/src/lib.rs @@ -1140,6 +1140,9 @@ fn projection_error_outcome(error: &ProjectionStoreError) -> HostAdmissionOutcom ProjectionStoreError::Storage { .. } if error.is_sqlite_disk_image_corrupt() => { HostAdmissionOutcome::terminal_unavailable("projection_store_corrupt") } + ProjectionStoreError::Storage { .. } if error.writer_fence().is_some() => { + HostAdmissionOutcome::terminal_unavailable("projection_store_writer_fenced") + } ProjectionStoreError::Storage { .. } => { HostAdmissionOutcome::retained_unavailable("projection_storage_retry_scheduled") } @@ -1244,6 +1247,29 @@ mod projection_error_outcome_tests { ); assert!(outcome.retryable); } + + /// A lease whose authority epoch is behind the store's can never commit + /// again; the drain must stop instead of rescheduling the same write. + #[test] + fn a_fenced_writer_classifies_as_blocked_not_retryable() { + let fence = tracedecay_store::StoreWriterFencedV1 { + requested: tracedecay_store::StoreAuthorityEpochV1::new(2).unwrap(), + current: tracedecay_store::StoreAuthorityEpochV1::new(3).unwrap(), + }; + let error = ProjectionStoreError::Storage { + operation: "dispatch observation projection runtime operation", + source: Box::new(ObservationStoreError::Storage { + operation: "submit anchored observation", + source: Box::new(fence), + }), + }; + + let outcome = projection_error_outcome(&error); + + assert_eq!(outcome.reason_code, Some("projection_store_writer_fenced")); + assert!(!outcome.retryable, "a fenced writer must not be retried"); + assert_eq!(outcome.status, HostAdmissionStatus::Unavailable); + } } fn host_scope(scope: &ObservationScopeV1) -> HostAdmissionScope { @@ -1607,6 +1633,23 @@ fn classify_error(error: &ObservationApplicationError) -> HostAdmissionOutcome { ObservationApplicationError::Store(ObservationStoreError::WriterSaturated { .. }) => { HostAdmissionOutcome::retained_backpressured("writer_lane_saturated") } + // The store fenced this lease: its authority epoch is behind the one + // the store holds, so re-driving the same batch through it can never + // commit. The owner must reopen the store, not retry. + ObservationApplicationError::Store(ObservationStoreError::Storage { + operation, + source, + }) if tracedecay_store::error_chain_writer_fence(source.as_ref()).is_some() => { + tracing::warn!( + operation, + error = %source, + reason_code = "authority_writer_fenced", + "observation store fenced this writer; its lease must be reopened" + ); + let mut outcome = HostAdmissionOutcome::terminal_unavailable("authority_writer_fenced"); + outcome.cause = Some(format!("{operation}: {source}")); + outcome + } ObservationApplicationError::Store(ObservationStoreError::Storage { operation, source, diff --git a/crates/tracedecay-runtime-core/src/shard_runtime/registry.rs b/crates/tracedecay-runtime-core/src/shard_runtime/registry.rs index 495177b8c3..27a87972ac 100644 --- a/crates/tracedecay-runtime-core/src/shard_runtime/registry.rs +++ b/crates/tracedecay-runtime-core/src/shard_runtime/registry.rs @@ -1189,6 +1189,14 @@ pub enum StoreRuntimeRegistryFailure { operation_leases: u32, }, AuthorityEpochExhausted, + /// The store's ledger already holds a newer writer authority for this + /// binding's shard and incarnation, so a writer published at the binding + /// could never commit. `open` raises the process epoch floor past + /// `persisted` and opens once more before it reports this. + WriterAuthorityBehindPersisted { + binding: Box, + persisted: StoreAuthorityEpochV1, + }, OpenAttemptExhausted, EvictionAttemptExhausted, EvictionReservationLost { diff --git a/crates/tracedecay-runtime-core/src/shard_runtime/registry/open.rs b/crates/tracedecay-runtime-core/src/shard_runtime/registry/open.rs index a725ed23f4..8cb6ca3375 100644 --- a/crates/tracedecay-runtime-core/src/shard_runtime/registry/open.rs +++ b/crates/tracedecay-runtime-core/src/shard_runtime/registry/open.rs @@ -415,6 +415,7 @@ impl StoreRuntimeRegistry { }, ); }; + let mut converged_on_persisted_authority = false; loop { if let Some(path) = request .database_authority @@ -443,7 +444,31 @@ impl StoreRuntimeRegistry { StoreRuntimeOpenBegin::Rejected( StoreRuntimeRegistryFailure::DestructiveMaintenanceInProgress { .. }, ) => {} - begin => return begin.wait().await, + begin => match begin.wait().await { + // A store the same incarnation already wrote under a newer + // authority fences every writer below it. Allocating above + // the persisted epoch converges this process on the store's + // authority; a second fence means the store moved again + // while this open ran, which the caller must see. + StoreRuntimeOpenResult::Failed( + StoreRuntimeRegistryFailure::WriterAuthorityBehindPersisted { + binding, + persisted, + }, + ) if !converged_on_persisted_authority => { + converged_on_persisted_authority = true; + retain_authority_epoch_floor(persisted); + tracing::warn!( + event = "store_writer_authority_converged", + shard = ?binding.shard_id, + incarnation = binding.incarnation.get(), + requested_epoch = binding.authority_epoch.get(), + persisted_epoch = persisted.get(), + "store ledger holds a newer writer authority; reopening above it" + ); + } + result => return result, + }, } } } diff --git a/crates/tracedecay-runtime-core/src/shard_runtime/registry/ports.rs b/crates/tracedecay-runtime-core/src/shard_runtime/registry/ports.rs index 9253d0c629..7687a009a2 100644 --- a/crates/tracedecay-runtime-core/src/shard_runtime/registry/ports.rs +++ b/crates/tracedecay-runtime-core/src/shard_runtime/registry/ports.rs @@ -174,6 +174,14 @@ async fn publish_lifecycle_runtime( attachment.abort(request.locator.is_prospective()); return Err(error); } + if request.mode == StoreRuntimeOpenMode::Existing + && request.access == StoreRuntimeAccessMode::ReadWrite + && let Err(error) = + verify_writer_authority_before_existing_publication(&request, attachment.as_physical()) + { + attachment.abort(request.locator.is_prospective()); + return Err(error); + } if let Err(error) = runtime.transition(RuntimeMaintenanceStateV1::Ready) { attachment.abort(request.locator.is_prospective()); return Err(runtime_lifecycle_failure(error)); @@ -190,6 +198,41 @@ async fn publish_lifecycle_runtime( Ok(PublishedShardRuntime::new(runtime, attachment.into_box())) } +/// A shard whose ledger already holds a newer writer authority for this +/// incarnation must not be published at the stale binding: every submit +/// through it would be fenced. The registry raises its epoch floor past the +/// persisted authority and opens again. +fn verify_writer_authority_before_existing_publication( + request: &ShardRuntimeBuildRequest, + attachment: &dyn PhysicalRuntimeAttachment, +) -> Result<(), StoreRuntimeRegistryFailure> { + const OPERATION: &str = "read persisted writer authority for existing-runtime admission"; + let handle = attachment.exact_sql_handle().map_err(|message| { + StoreRuntimeRegistryFailure::PhysicalRuntimeFailed { + operation: OPERATION, + message, + } + })?; + let persisted = tracedecay_rusqlite_runtime::runtime_ledger::persisted_writer_authority_epoch( + &handle.read_only_clone(), + &request.binding, + Duration::ZERO, + ) + .map_err(|error| StoreRuntimeRegistryFailure::PhysicalRuntimeFailed { + operation: OPERATION, + message: error.to_string(), + })?; + match persisted { + Some(persisted) if persisted > request.binding.authority_epoch => Err( + StoreRuntimeRegistryFailure::WriterAuthorityBehindPersisted { + binding: Box::new(request.binding.clone()), + persisted, + }, + ), + _ => Ok(()), + } +} + fn runtime_core_final_schema_applies(scope: &StoreShardScopeV1) -> bool { matches!( scope, diff --git a/crates/tracedecay-runtime-core/src/shard_runtime/registry/tests/production_routes.rs b/crates/tracedecay-runtime-core/src/shard_runtime/registry/tests/production_routes.rs index eecb8233ef..943b5e0e6a 100644 --- a/crates/tracedecay-runtime-core/src/shard_runtime/registry/tests/production_routes.rs +++ b/crates/tracedecay-runtime-core/src/shard_runtime/registry/tests/production_routes.rs @@ -649,3 +649,193 @@ async fn existing_store_at_an_older_schema_is_refused_even_with_write_authority( "a refused open must leave the stamp alone" ); } + +struct SubmitProbe { + cancellation: RuntimeCancellationIdentityV1, + deadline: RuntimeDeadlineV1, +} + +impl RuntimeRequestProbeV1 for SubmitProbe { + fn cancellation_identity(&self) -> &RuntimeCancellationIdentityV1 { + &self.cancellation + } + + fn deadline_identity(&self) -> &RuntimeDeadlineV1 { + &self.deadline + } + + fn interruption(&self) -> Option { + None + } + + fn try_begin_commit(&self) -> bool { + true + } +} + +fn diagnostics_submit( + binding: &tracedecay_store::StoreRuntimeBindingV1, + name: &str, +) -> (tracedecay_store::RuntimeSubmitRequestV1, SubmitProbe) { + let metadata: tracedecay_store::StoreOperationMetadataV1 = + serde_json::from_value(serde_json::json!({ + "operation_id": format!("operation.{name}"), + "client_id": "client.reopen", + "shard_id": binding.shard_id, + "incarnation": binding.incarnation.get(), + "authority_epoch": binding.authority_epoch.get(), + "idempotency": { + "key": format!("key.{name}"), + "command_digest": format!("sha256:{}", "e".repeat(64)), + }, + "durability": "full", + "priority": "foreground", + "admission_bytes": 128, + "admitted_at": 1 + })) + .unwrap(); + let scope = tracedecay_store::RuntimeTransactionScopeV1 { + transaction_id: tracedecay_store::RuntimeTransactionIdV1::new(format!( + "transaction.{name}" + )) + .unwrap(), + compatibility: tracedecay_store::RuntimeBatchCompatibilityV1::from_operation(&metadata) + .unwrap(), + opened_at: metadata.admitted_at, + }; + let cancellation = RuntimeCancellationIdentityV1 { + cancellation_id: RuntimeCancellationIdV1::new(format!("cancel.{name}")).unwrap(), + generation: 1, + }; + let deadline = RuntimeDeadlineV1 { + deadline_id: RuntimeDeadlineIdV1::new(format!("deadline.{name}")).unwrap(), + }; + let control = RuntimeRequestControlV1 { + requested_at: UtcMicros(1), + deadline: deadline.clone(), + cancellation: cancellation.clone(), + }; + let request = tracedecay_store::RuntimeSubmitRequestV1::new( + tracedecay_store::RepositoryOperationEnvelopeV1 { + metadata, + payload: tracedecay_store::RepositoryWritePayloadV1::Diagnostics(Box::new( + tracedecay_store::SanitizedCleanDiagnosticSnapshotV1::new( + tracedecay_domain::CodeGenerationId::new(format!("generation.{name}")).unwrap(), + Vec::new(), + ) + .unwrap(), + )), + }, + scope, + control, + ) + .unwrap(); + ( + request, + SubmitProbe { + cancellation, + deadline, + }, + ) +} + +/// A store this incarnation already wrote under a newer authority (a previous +/// daemon of the same incarnation, or a sibling runtime on the file) fences +/// every writer below that authority. The reopen converges above it instead of +/// publishing a runtime whose every submit would be fenced. +#[asupersync::test] +async fn reopening_a_store_behind_its_persisted_writer_authority_converges_above_it() { + let root = TempDir::new().unwrap(); + let resolver = Arc::new(FileResolver::default()); + resolver.push(seed_db(&root, "profile.db")); + let sessions_path = seed_db(&root, "sessions.db"); + Connection::open(&sessions_path) + .unwrap() + .execute_batch(tracedecay_store::GENERATION_DIAGNOSTICS_SCHEMA_DDL) + .unwrap(); + resolver.push(sessions_path.clone()); + resolver.push(sessions_path.clone()); + let shard = StoreShardIdV1::project_sessions( + id::("brain.registry"), + id::("profile.registry"), + id::("project.reopen"), + ); + let ahead = tracedecay_store::StoreRuntimeBindingV1::new( + shard.clone(), + incarnation(), + tracedecay_store::StoreAuthorityEpochV1::new(1 << 40).unwrap(), + ); + let runtime = asupersync::runtime::Runtime::current_handle().unwrap(); + let previous = tracedecay_rusqlite_runtime::PersistentWriter::start( + &runtime, + tracedecay_rusqlite_runtime::ExistingWriterLocator::new( + ahead.clone(), + VerifiedStoreLocatorV1::new( + shard.clone(), + incarnation(), + LocatorDigest::new(format!("sha256:{}", "c".repeat(64))).unwrap(), + ), + sessions_path.clone(), + ) + .unwrap(), + tracedecay_store::AdmissionConfigV1::default(), + tracedecay_rusqlite_runtime::repository::ConcreteRepositoryWriteExecutor::default(), + ) + .unwrap(); + let (request, probe) = diagnostics_submit(&ahead, "previous-authority"); + let outcome = previous + .submit( + &crate::native_runtime::cx().unwrap(), + request, + Arc::new(probe), + ) + .await + .unwrap(); + assert!( + matches!( + outcome, + tracedecay_store::RuntimeSubmitOutcomeV1::Committed { .. } + ), + "the previous authority commits: {outcome:?}" + ); + previous.shutdown_and_join().unwrap(); + + let registry = + StoreRuntimeRegistry::new(runtime, resolver, Arc::new(LifecycleShardRuntimePublisher)) + .unwrap(); + let _profile = open_published( + ®istry, + StoreRuntimeOpenRequest::new(profile_shard(), incarnation(), None), + ) + .await; + let pin = match registry.profile_authority_pin(&profile_shard()) { + ProfileAuthorityPinResult::Pinned(pin) => pin, + other => panic!("profile was not pinned: {other:?}"), + }; + let authority = crate::db::DatabaseAuthority::for_runtime(&sessions_path, "reopen sessions") + .expect("sessions database authority"); + let sessions = open_published( + ®istry, + StoreRuntimeOpenRequest::new_authorized(shard, incarnation(), Some(pin), authority.clone()), + ) + .await; + + assert!( + sessions.binding().authority_epoch > ahead.authority_epoch, + "the reopened runtime publishes above the persisted authority: {:?} vs {:?}", + sessions.binding().authority_epoch, + ahead.authority_epoch + ); + let (request, probe) = diagnostics_submit(sessions.binding(), "converged-authority"); + let outcome = sessions + .dispatch_submit_authorized(request, Arc::new(probe), authority) + .await + .expect("converged runtime accepts writes"); + assert!( + matches!( + outcome, + tracedecay_store::RuntimeSubmitOutcomeV1::Committed { .. } + ), + "a write through the converged runtime commits: {outcome:?}" + ); +} diff --git a/crates/tracedecay-rusqlite-runtime/src/ledger.rs b/crates/tracedecay-rusqlite-runtime/src/ledger.rs index bc9e79291e..8b03992db6 100644 --- a/crates/tracedecay-rusqlite-runtime/src/ledger.rs +++ b/crates/tracedecay-rusqlite-runtime/src/ledger.rs @@ -12,6 +12,7 @@ mod schema; mod sqlite; pub(crate) use checkpoint::current_watermark; +pub use checkpoint::{PersistedWriterAuthorityError, persisted_writer_authority_epoch}; #[cfg(test)] pub(crate) use commit::record_commit; pub(crate) use commit::record_runtime_commit; diff --git a/crates/tracedecay-rusqlite-runtime/src/ledger/checkpoint.rs b/crates/tracedecay-rusqlite-runtime/src/ledger/checkpoint.rs index d1539565df..6c5c690fdd 100644 --- a/crates/tracedecay-rusqlite-runtime/src/ledger/checkpoint.rs +++ b/crates/tracedecay-rusqlite-runtime/src/ledger/checkpoint.rs @@ -1,3 +1,6 @@ +use std::fmt; +use std::time::Duration; + use rusqlite::{Row, params}; use tracedecay_store::{ CommitSequenceV1, DurabilityClassV1, RuntimeTransactionScopeV1, ShardWatermarkV1, @@ -8,8 +11,105 @@ use super::{ LedgerError, sqlite::{BindingKey, LedgerTransaction, Submission, decode_json, encode_json, sqlite_u64}, }; +use crate::exact_sql::{ExactSqlError, ExactSqlHandle, ExactSqlStatement, ExactSqlValue}; const CHECKPOINT_TABLE: &str = "td_runtime_writer_checkpoint_v1"; +const SELECT_PERSISTED_AUTHORITY_EPOCH: &str = r#" +SELECT authority_epoch +FROM td_runtime_writer_checkpoint_v1 +WHERE shard_json = ?1 AND incarnation = ?2 +"#; + +/// Why the persisted writer authority of a shard incarnation could not be read. +#[derive(Debug)] +pub enum PersistedWriterAuthorityError { + Sql(ExactSqlError), + Corrupt { field: &'static str }, +} + +impl fmt::Display for PersistedWriterAuthorityError { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::Sql(error) => { + write!(formatter, "persisted writer authority read failed: {error}") + } + Self::Corrupt { field } => write!( + formatter, + "persisted writer authority row is corrupt in {CHECKPOINT_TABLE}.{field}" + ), + } + } +} + +impl std::error::Error for PersistedWriterAuthorityError { + fn source(&self) -> Option<&(dyn std::error::Error + 'static)> { + match self { + Self::Sql(error) => Some(error), + Self::Corrupt { .. } => None, + } + } +} + +impl From for PersistedWriterAuthorityError { + fn from(error: ExactSqlError) -> Self { + Self::Sql(error) + } +} + +/// The authority epoch the ledger last checkpointed for `binding`'s shard and +/// incarnation, or `None` for a store no writer of that incarnation has +/// committed to. Read through the attached runtime's own readers, before the +/// runtime is published, so a reopen can learn whether its allocated epoch +/// is already behind the store. +pub fn persisted_writer_authority_epoch( + handle: &ExactSqlHandle, + binding: &StoreRuntimeBindingV1, + max_wait: Duration, +) -> Result, PersistedWriterAuthorityError> { + if handle.binding() != binding { + return Err(ExactSqlError::AuthorityMismatch.into()); + } + let key = + BindingKey::from_binding(binding).map_err(|_| PersistedWriterAuthorityError::Corrupt { + field: "shard_json", + })?; + let schema = handle.query( + ExactSqlStatement::new( + "SELECT EXISTS(SELECT 1 FROM sqlite_schema WHERE type = 'table' AND name = ?1)" + .to_owned(), + vec![ExactSqlValue::Text(CHECKPOINT_TABLE.to_owned())], + )?, + max_wait, + )?; + match schema.rows.first().and_then(|row| row.values.first()) { + Some(ExactSqlValue::Integer(0)) => return Ok(None), + Some(ExactSqlValue::Integer(1)) => {} + _ => return Err(PersistedWriterAuthorityError::Corrupt { field: "schema" }), + } + let rows = handle.query( + ExactSqlStatement::new( + SELECT_PERSISTED_AUTHORITY_EPOCH.to_owned(), + vec![ + ExactSqlValue::Text(key.shard_json), + ExactSqlValue::Integer(key.incarnation_sql), + ], + )?, + max_wait, + )?; + let Some(row) = rows.rows.first() else { + return Ok(None); + }; + match row.values.first() { + Some(ExactSqlValue::Integer(raw)) => decode_authority_epoch(*raw, "authority_epoch") + .map(Some) + .map_err(|_| PersistedWriterAuthorityError::Corrupt { + field: "authority_epoch", + }), + _ => Err(PersistedWriterAuthorityError::Corrupt { + field: "authority_epoch", + }), + } +} const SELECT_CHECKPOINT: &str = r#" SELECT authority_epoch, commit_sequence, watermark_json, transaction_scope_json, original_receipt_json, operation_id, durability_json, committed_at_micros diff --git a/crates/tracedecay-rusqlite-runtime/src/lib.rs b/crates/tracedecay-rusqlite-runtime/src/lib.rs index b22ccf9ec3..4749a19e4f 100644 --- a/crates/tracedecay-rusqlite-runtime/src/lib.rs +++ b/crates/tracedecay-rusqlite-runtime/src/lib.rs @@ -18,7 +18,10 @@ mod ledger; /// Canonical schema for the runtime writer ledger installed in registered /// SQLite stores. pub mod runtime_ledger { - pub use crate::ledger::{RETIRE_RUNTIME_EFFECTS_V1, RUNTIME_LEDGER_SCHEMA}; + pub use crate::ledger::{ + PersistedWriterAuthorityError, RETIRE_RUNTIME_EFFECTS_V1, RUNTIME_LEDGER_SCHEMA, + persisted_writer_authority_epoch, + }; } pub mod native_runtime; #[cfg(test)] diff --git a/crates/tracedecay-rusqlite-runtime/src/persistence.rs b/crates/tracedecay-rusqlite-runtime/src/persistence.rs index 09761c19cf..b19e7c2fdb 100644 --- a/crates/tracedecay-rusqlite-runtime/src/persistence.rs +++ b/crates/tracedecay-rusqlite-runtime/src/persistence.rs @@ -4,7 +4,7 @@ use rusqlite::{Savepoint, Transaction}; use tracedecay_store::{ CorruptionClassV1, IdempotencyIdentityV1, RuntimeSubmitRequestV1, StorageRuntimeErrorV1, - StoreCommitReceiptV1, StoreRuntimeBindingV1, + StoreAuthorityEpochV1, StoreCommitReceiptV1, StoreRuntimeBindingV1, }; use crate::{ @@ -64,33 +64,65 @@ impl RuntimeWriterPersistence { savepoint: &mut Savepoint<'_>, binding: &StoreRuntimeBindingV1, request: &RuntimeSubmitRequestV1, - ) -> Result { + ) -> Result { if binding != request.binding() { - return Err(infrastructure("apply-and-record binding mismatch")); + return Err(infrastructure("apply-and-record binding mismatch").into()); } operation::execute(savepoint, request, &mut self.executor).map_err(map_operation_error)?; - match ledger::record_runtime_commit( + let disposition = match ledger::record_runtime_commit( savepoint, &request.envelope().metadata, request.transaction_scope(), &request.envelope().payload, - ) - .map_err(map_ledger_error)? - { + ) { + Ok(disposition) => disposition, + Err(LedgerError::StaleAuthority { + persisted, + requested, + }) => { + return Err(ApplyRecordError::Fenced { + requested, + persisted, + }); + } + Err(error) => return Err(map_ledger_error(error).into()), + }; + match disposition { LedgerDisposition::Committed(receipt) => Ok(receipt), LedgerDisposition::Replay(receipt) | LedgerDisposition::Conflict(receipt) => { drop(receipt); - Err(infrastructure( - "idempotency disposition changed after the writer lookup", - )) + Err( + infrastructure("idempotency disposition changed after the writer lookup") + .into(), + ) + } + LedgerDisposition::New => { + Err(infrastructure("runtime ledger did not record the applied operation").into()) } - LedgerDisposition::New => Err(infrastructure( - "runtime ledger did not record the applied operation", - )), } } } +/// Why one request could not be applied and recorded in its savepoint. +/// +/// A fence is a submit decision, not a storage failure: the ledger already +/// holds a newer writer authority for this shard and incarnation, so the same +/// request can never commit through this writer. +#[derive(Debug)] +pub(crate) enum ApplyRecordError { + Fenced { + requested: StoreAuthorityEpochV1, + persisted: StoreAuthorityEpochV1, + }, + Storage(StorageRuntimeErrorV1), +} + +impl From for ApplyRecordError { + fn from(error: StorageRuntimeErrorV1) -> Self { + Self::Storage(error) + } +} + fn map_ledger_error(error: LedgerError) -> StorageRuntimeErrorV1 { match error { LedgerError::Corrupt { .. } => StorageRuntimeErrorV1::Corrupt { @@ -292,4 +324,48 @@ mod tests { ); transaction.commit().unwrap(); } + + #[test] + fn a_writer_behind_the_persisted_authority_is_fenced_not_failed() { + let mut connection = Connection::open_in_memory().unwrap(); + let newer = request(metadata("operation.newer", "key.newer", 'a')); + let mut stale_metadata = metadata("operation.stale", "key.stale", 'b'); + stale_metadata.authority_epoch = StoreAuthorityEpochV1::new(2).unwrap(); + let stale = request(stale_metadata); + let mut persistence = RuntimeWriterPersistence::new(MarkerExecutor); + let mut transaction = connection.transaction().unwrap(); + assert!( + persistence + .lookup_idempotency( + &transaction, + newer.binding(), + &newer.envelope().metadata.idempotency, + ) + .unwrap() + .is_none() + ); + { + let mut savepoint = transaction.savepoint().unwrap(); + persistence + .apply_and_record(&mut savepoint, newer.binding(), &newer) + .unwrap(); + savepoint.commit().unwrap(); + } + + let mut savepoint = transaction.savepoint().unwrap(); + let fenced = persistence + .apply_and_record(&mut savepoint, stale.binding(), &stale) + .unwrap_err(); + + assert!( + matches!( + fenced, + ApplyRecordError::Fenced { + requested, + persisted, + } if requested.get() == 2 && persisted.get() == 7 + ), + "a stale writer is fenced, not failed: {fenced:?}" + ); + } } diff --git a/crates/tracedecay-rusqlite-runtime/src/writer/transaction.rs b/crates/tracedecay-rusqlite-runtime/src/writer/transaction.rs index d3391acf42..3a62990511 100644 --- a/crates/tracedecay-rusqlite-runtime/src/writer/transaction.rs +++ b/crates/tracedecay-rusqlite-runtime/src/writer/transaction.rs @@ -16,7 +16,7 @@ use crate::{ RuntimeWriteAuthorityStage, StorageOperationExecutor, admission::QueueItem, connection, - persistence::RuntimeWriterPersistence, + persistence::{ApplyRecordError, RuntimeWriterPersistence}, telemetry::{ LockWorkScope, WriterBatchMetrics, WriterLockWorkSnapshot, WriterTelemetry, WriterTransactionMetrics, WriterTransactionOutcome, take_observed_vm, @@ -364,7 +364,27 @@ fn apply_new( // One operation-and-ledger boundary: the receipt is the one this savepoint recorded. let receipt = match persistence.apply_and_record(&mut savepoint, binding, &item.request) { Ok(receipt) => receipt, - Err(error) => { + Err(ApplyRecordError::Fenced { + requested, + persisted, + }) => { + return match savepoint.rollback() { + Ok(()) => processed( + item, + Ok(RuntimeSubmitOutcomeV1::Fenced { + expected: requested, + actual: persisted, + }), + false, + ), + Err(error) => { + let result = + driver_failure(error, "rollback fenced request").result(&item.request); + processed(item, result, false) + } + }; + } + Err(ApplyRecordError::Storage(error)) => { if let Some(outcome) = interruption_outcome( &item.request, item.probe.as_ref(), diff --git a/crates/tracedecay-rusqlite-runtime/tests/rusqlite_suite/storage_contention.rs b/crates/tracedecay-rusqlite-runtime/tests/rusqlite_suite/storage_contention.rs index 1567455f40..009918cd98 100644 --- a/crates/tracedecay-rusqlite-runtime/tests/rusqlite_suite/storage_contention.rs +++ b/crates/tracedecay-rusqlite-runtime/tests/rusqlite_suite/storage_contention.rs @@ -26,6 +26,7 @@ use tracedecay_rusqlite_runtime::{ ConcreteRepositoryReadExecutor, ConcreteRepositoryWriteExecutor, RepositoryPhysicalAttachmentFactory, }, + runtime_ledger::{PersistedWriterAuthorityError, persisted_writer_authority_epoch}, }; use tracedecay_store::{ AdmissionConfigV1, CommitSequenceV1, ConsistencyModeV1, DiagnosticReadOperationV1, @@ -1166,3 +1167,108 @@ fn at_least_reads_under_a_superseded_authority_epoch_are_typed_unavailable() { attachment.drain().unwrap(); attachment.close_and_join().unwrap(); } + +/// Two live writers on one file under one incarnation. Once the newer authority +/// commits, a submit from the older writer is the typed fence every caller can +/// classify as terminal, not a storage failure they would retry forever. +#[test] +fn a_writer_behind_the_persisted_authority_epoch_is_fenced_on_submit() { + let database = TestDatabase::new("write-superseded-epoch.db"); + seed(&database); + let superseded = binding(0); + let mut current = superseded.clone(); + current.authority_epoch = + StoreAuthorityEpochV1::new(superseded.authority_epoch.get() + 1).unwrap(); + let stale_writer = start_writer( + &database, + &superseded, + ConcreteRepositoryWriteExecutor::default(), + ); + let current_writer = start_writer( + &database, + ¤t, + ConcreteRepositoryWriteExecutor::default(), + ); + commit( + ¤t_writer, + publication(¤t, "generation.current", 1), + ); + + let request = publication(&superseded, "generation.stale", 1); + let probe = Probe::for_submit(&request); + let outcome = run(stale_writer.submit(&native_runtime::cx().unwrap(), request, probe)).unwrap(); + + assert!( + matches!( + outcome, + RuntimeSubmitOutcomeV1::Fenced { expected, actual } + if expected == superseded.authority_epoch && actual == current.authority_epoch + ), + "stale writer submit is a typed fence: {outcome:?}" + ); + assert_eq!( + persisted_rows(&database.connect(), "generation_diagnostics"), + 1, + "the fenced write rolled back and only the current authority's rows remain" + ); + stale_writer.shutdown_and_join().unwrap(); + current_writer.shutdown_and_join().unwrap(); +} + +#[test] +fn persisted_writer_authority_is_scoped_to_the_shard_and_incarnation() { + let database = TestDatabase::new("persisted-authority.db"); + seed(&database); + let current = binding(0); + let writer = start_writer( + &database, + ¤t, + ConcreteRepositoryWriteExecutor::default(), + ); + commit(&writer, publication(¤t, "generation.persisted", 1)); + writer.shutdown_and_join().unwrap(); + let mut other_incarnation = current.clone(); + other_incarnation.incarnation = + tracedecay_store::StoreIncarnationV1::new(current.incarnation.get() + 1).unwrap(); + + for (binding, expected) in [ + (current.clone(), Some(current.authority_epoch)), + (binding(1), None), + (other_incarnation, None), + ] { + let attachment = RepositoryPhysicalAttachmentFactory + .attach_read_only( + binding.clone(), + verified_locator(&binding), + database.path.canonicalize().unwrap(), + AdmissionConfigV1::default(), + ) + .unwrap(); + let handle = attachment.exact_sql_handle().unwrap(); + assert_eq!( + persisted_writer_authority_epoch(&handle, &binding, Duration::ZERO).unwrap(), + expected + ); + attachment.drain().unwrap(); + attachment.close_and_join().unwrap(); + } + + database.connect().execute_batch("PRAGMA ignore_check_constraints = ON; UPDATE td_runtime_writer_checkpoint_v1 SET authority_epoch = -1;").unwrap(); + let attachment = RepositoryPhysicalAttachmentFactory + .attach_read_only( + current.clone(), + verified_locator(¤t), + database.path.canonicalize().unwrap(), + AdmissionConfigV1::default(), + ) + .unwrap(); + let handle = attachment.exact_sql_handle().unwrap(); + assert!(matches!( + persisted_writer_authority_epoch(&handle, ¤t, Duration::ZERO), + Err(PersistedWriterAuthorityError::Corrupt { + field: "authority_epoch" + }) + )); + attachment.drain().unwrap(); + attachment.close_and_join().unwrap(); +} diff --git a/crates/tracedecay-session-runtime/src/session_temporal_refresh_scheduler/worker.rs b/crates/tracedecay-session-runtime/src/session_temporal_refresh_scheduler/worker.rs index 5427dd0fa8..e6c90edf14 100644 --- a/crates/tracedecay-session-runtime/src/session_temporal_refresh_scheduler/worker.rs +++ b/crates/tracedecay-session-runtime/src/session_temporal_refresh_scheduler/worker.rs @@ -800,9 +800,12 @@ fn is_deterministic_refusal(error: &SessionStoreError) -> bool { SessionStoreError::Cancelled | SessionStoreError::DeadlineExceeded | SessionStoreError::BudgetExceeded { .. } => false, - SessionStoreError::Storage { source, .. } => source - .downcast_ref::() - .is_some_and(EngineError::is_deterministic_refusal), + SessionStoreError::Storage { source, .. } => { + source + .downcast_ref::() + .is_some_and(EngineError::is_deterministic_refusal) + || tracedecay_store::error_chain_writer_fence(source.as_ref()).is_some() + } _ => true, } } @@ -1915,6 +1918,25 @@ mod tests { use tracedecay_runtime_core::db::engine::params; use tracedecay_sessions::runtime::{SessionMessageRecord, SessionRecord}; + /// A lease the store has fenced resubmits the identical request on every + /// pass and can never commit through it, so the pass retires the refresh + /// instead of backing off forever. + #[test] + fn a_fenced_writer_is_a_deterministic_refusal_not_a_retry() { + let fence = tracedecay_store::StoreWriterFencedV1 { + requested: tracedecay_store::StoreAuthorityEpochV1::new(2).unwrap(), + current: tracedecay_store::StoreAuthorityEpochV1::new(3).unwrap(), + }; + let wrapped = tracedecay_store::ObservationStoreError::Storage { + operation: "submit anchored observation", + source: Box::new(fence), + }; + let error = SessionStoreError::storage("persist session history batch", wrapped); + + assert!(is_deterministic_refusal(&error)); + assert!(!is_retryable_storage(&error)); + } + /// Commits `item_count` items. A group of several items never commits, and /// neither does a `slow` item alone: its transaction ends in the /// `DeadlineExceeded` that storage reports when its own deadline passes. diff --git a/crates/tracedecay-sessions/src/runtime/ingest/failure.rs b/crates/tracedecay-sessions/src/runtime/ingest/failure.rs index baf618deb5..0e623efa8f 100644 --- a/crates/tracedecay-sessions/src/runtime/ingest/failure.rs +++ b/crates/tracedecay-sessions/src/runtime/ingest/failure.rs @@ -339,6 +339,11 @@ pub fn classify_transcript_ingest_disposition( source::TranscriptIngestError::Store(TranscriptStoreError::Conflict { .. }) => { ("transcript_cursor_conflict", true, Backpressured) } + source::TranscriptIngestError::Store(TranscriptStoreError::Storage { source, .. }) + if tracedecay_store::error_chain_writer_fence(source.as_ref()).is_some() => + { + ("transcript_writer_fenced", false, Unavailable) + } source::TranscriptIngestError::Store(TranscriptStoreError::Storage { .. }) => { ("transcript_storage_failed", true, Unavailable) } @@ -496,6 +501,17 @@ pub fn classify_claude_observation_failure( Store::CursorCoverageMismatch => permanent("observation_cursor_coverage_gap"), Store::InvalidReplayLimit { .. } => permanent("observation_replay_limit_invalid"), Store::Contract(_) => permanent("observation_contract_invalid"), + // The lease's authority is behind the store's: no replay through this + // lease can commit, so the pass stops instead of spinning on it. + Store::Storage { source, .. } + if tracedecay_store::error_chain_writer_fence(source.as_ref()).is_some() => + { + ClaudeObservationFailureClass { + reason_code: "observation_writer_fenced", + retryable: false, + status: Unavailable, + } + } _ => unavailable("observation_storage_failed"), }; let projection = |error: &Projection| { @@ -506,6 +522,13 @@ pub fn classify_claude_observation_failure( status: Unavailable, }; } + if error.writer_fence().is_some() { + return ClaudeObservationFailureClass { + reason_code: "projection_store_writer_fenced", + retryable: false, + status: Unavailable, + }; + } match error { Projection::Corrupt { .. } => ClaudeObservationFailureClass { reason_code: "projection_store_corrupt", @@ -703,4 +726,35 @@ mod cancellation_tests { ); assert_eq!(disposition.status, HostAdmissionStatus::Degraded); } + + /// A store that fenced the ingest lease rejects every replay through it; + /// the pass must not keep retrying the same write. + #[test] + fn a_fenced_writer_is_not_retryable() { + let fence = tracedecay_store::StoreWriterFencedV1 { + requested: tracedecay_store::StoreAuthorityEpochV1::new(2).unwrap(), + current: tracedecay_store::StoreAuthorityEpochV1::new(3).unwrap(), + }; + let fenced = + source::TranscriptIngestError::Store(tracedecay_store::TranscriptStoreError::Storage { + operation: "persist transcript observations", + source: Box::new(fence), + }); + + let disposition = classify_transcript_ingest_disposition(&fenced); + + assert_eq!(disposition.reason_code, "transcript_writer_fenced"); + assert!(!disposition.retryable, "a fenced lease never commits again"); + assert_eq!(disposition.status, HostAdmissionStatus::Unavailable); + + let transient = + source::TranscriptIngestError::Store(tracedecay_store::TranscriptStoreError::Storage { + operation: "persist transcript observations", + source: Box::new(std::io::Error::other("database is locked")), + }); + assert!( + classify_transcript_ingest_disposition(&transient).retryable, + "ordinary storage failures stay retryable" + ); + } } diff --git a/crates/tracedecay-store/src/lib.rs b/crates/tracedecay-store/src/lib.rs index 1dd1f488b2..1c2cfffeb5 100644 --- a/crates/tracedecay-store/src/lib.rs +++ b/crates/tracedecay-store/src/lib.rs @@ -220,10 +220,11 @@ pub use runtime::{ StorageRuntimeErrorV1, StoreAuthorityEpochV1, StoreClientIdV1, StoreCommitReceiptV1, StoreIdempotencyKeyV1, StoreIncarnationV1, StoreOperationIdV1, StoreOperationMetadataV1, StoreRuntimeBindingV1, StoreRuntimeRegistryPublicationV1, StoreShardIdV1, StoreShardScopeV1, - StoreSnapshotIdV1, StoredObservationRowV1, UnavailableReasonV1, UserProfileId, - VerifiedStoreLocatorV1, WAL_HARD_LIMIT_BYTES, WAL_SOFT_LIMIT_BYTES, + StoreSnapshotIdV1, StoreWriterFencedV1, StoredObservationRowV1, UnavailableReasonV1, + UserProfileId, VerifiedStoreLocatorV1, WAL_HARD_LIMIT_BYTES, WAL_SOFT_LIMIT_BYTES, WORKSTATION_GLOBAL_QUEUE_BYTES, WalBudgetV1, WatermarkCoverageStatusV1, WorktreeId, - canonical_store_locator_digest, graph_store_locator_path, single_shard_required_coverage_v1, + canonical_store_locator_digest, error_chain_writer_fence, graph_store_locator_path, + single_shard_required_coverage_v1, }; pub use schema::{ CLEAR_CURRENT_DIAGNOSTIC_PUBLICATION_SQL, CLEAR_CURRENT_DIAGNOSTICS_SQL, diff --git a/crates/tracedecay-store/src/projection.rs b/crates/tracedecay-store/src/projection.rs index 02db331c26..aacfdfe6cb 100644 --- a/crates/tracedecay-store/src/projection.rs +++ b/crates/tracedecay-store/src/projection.rs @@ -665,6 +665,18 @@ impl ProjectionStoreError { } } + /// The store's writer fence when this failure is one: the lease's + /// authority epoch is behind the authority the store holds, so replaying + /// the request on that lease can never commit. + pub fn writer_fence(&self) -> Option { + match self { + Self::Storage { source, .. } => { + crate::runtime::error_chain_writer_fence(source.as_ref()) + } + _ => None, + } + } + pub fn durable_detail(&self) -> String { durable_error_detail(self) } diff --git a/crates/tracedecay-store/src/runtime/error.rs b/crates/tracedecay-store/src/runtime/error.rs index 83c1c69e7a..84d5830db4 100644 --- a/crates/tracedecay-store/src/runtime/error.rs +++ b/crates/tracedecay-store/src/runtime/error.rs @@ -2,9 +2,34 @@ use serde::{Deserialize, Serialize}; use thiserror::Error; use tracedecay_domain::ObservationSourceCursorV1; -use super::{DurabilityClassV1, StoreIncarnationV1}; +use super::{DurabilityClassV1, StoreAuthorityEpochV1, StoreIncarnationV1}; use crate::observation::CursorAdvanceLedgerDisagreementV1; +/// A writer whose authority epoch is behind the authority its store already +/// holds for the same shard and incarnation. The fence is permanent for that +/// writer: replaying the same request on the same lease can never commit, so +/// every retry loop must treat it as terminal and reopen the store instead. +#[derive(Clone, Copy, Debug, Error, PartialEq, Eq)] +#[error("store writer authority epoch {requested:?} is fenced by authority epoch {current:?}")] +pub struct StoreWriterFencedV1 { + pub requested: StoreAuthorityEpochV1, + pub current: StoreAuthorityEpochV1, +} + +/// The writer fence carried anywhere in `error`'s source chain. +pub fn error_chain_writer_fence( + error: &(dyn std::error::Error + 'static), +) -> Option { + let mut current: Option<&(dyn std::error::Error + 'static)> = Some(error); + while let Some(err) = current { + if let Some(fence) = err.downcast_ref::() { + return Some(*fence); + } + current = err.source(); + } + None +} + /// Validation failures for pure runtime contract DTOs. #[derive(Clone, Debug, Error, PartialEq, Eq)] pub enum StorageRuntimeContractErrorV1 { diff --git a/crates/tracedecay-store/src/runtime/mod.rs b/crates/tracedecay-store/src/runtime/mod.rs index a79fd358be..0ed0560236 100644 --- a/crates/tracedecay-store/src/runtime/mod.rs +++ b/crates/tracedecay-store/src/runtime/mod.rs @@ -26,7 +26,8 @@ pub use consistency::{ }; pub use error::{ CorruptionClassV1, RuntimeCancellationStageV1, SaturationScopeV1, - StorageRuntimeContractErrorV1, StorageRuntimeErrorV1, UnavailableReasonV1, + StorageRuntimeContractErrorV1, StorageRuntimeErrorV1, StoreWriterFencedV1, UnavailableReasonV1, + error_chain_writer_fence, }; pub use graph_publication::{ GraphCanonicalReplaySourceDigestV1, GraphDependencyGenerationClosureDigestV1, From 6b9c4e2d96cea73b6d553b4e1fbd347396805755 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Tue, 20 Oct 2026 18:22:16 -0700 Subject: [PATCH 02/15] fix(store): preserve graph leases during writer epoch recovery --- .../src/shard_runtime/registry.rs | 5 +- .../src/shard_runtime/registry/open.rs | 25 +- .../src/shard_runtime/registry/ports.rs | 30 ++- .../registry/tests/production_routes.rs | 236 +++++++++++++++--- .../src/ledger/checkpoint.rs | 45 ++-- .../rusqlite_suite/storage_contention.rs | 13 +- 6 files changed, 286 insertions(+), 68 deletions(-) diff --git a/crates/tracedecay-runtime-core/src/shard_runtime/registry.rs b/crates/tracedecay-runtime-core/src/shard_runtime/registry.rs index 27a87972ac..da1e161500 100644 --- a/crates/tracedecay-runtime-core/src/shard_runtime/registry.rs +++ b/crates/tracedecay-runtime-core/src/shard_runtime/registry.rs @@ -1191,8 +1191,9 @@ pub enum StoreRuntimeRegistryFailure { AuthorityEpochExhausted, /// The store's ledger already holds a newer writer authority for this /// binding's shard and incarnation, so a writer published at the binding - /// could never commit. `open` raises the process epoch floor past - /// `persisted` and opens once more before it reports this. + /// could never commit. `open` retains `persisted` as the process epoch + /// floor and retries once unless a live graph publication still owns the + /// stale binding; its owners must release that binding before reopening. WriterAuthorityBehindPersisted { binding: Box, persisted: StoreAuthorityEpochV1, diff --git a/crates/tracedecay-runtime-core/src/shard_runtime/registry/open.rs b/crates/tracedecay-runtime-core/src/shard_runtime/registry/open.rs index 8cb6ca3375..28b718ebf2 100644 --- a/crates/tracedecay-runtime-core/src/shard_runtime/registry/open.rs +++ b/crates/tracedecay-runtime-core/src/shard_runtime/registry/open.rs @@ -456,15 +456,34 @@ impl StoreRuntimeRegistry { persisted, }, ) if !converged_on_persisted_authority => { - converged_on_persisted_authority = true; retain_authority_epoch_floor(persisted); + let retained_graph_is_stale = self + .lock_state() + .graph_publications + .get(request.key()) + .is_some_and(|graph| graph.binding.authority_epoch < persisted); + if retained_graph_is_stale { + // Graph owner and operation leases retain this + // exact immutable binding. A retry would reuse the + // same stale epoch, and rebinding here would bypass + // their retirement authority. Fail once; after the + // owners release, the retained floor admits a new + // shared graph/relational binding above the ledger. + return StoreRuntimeOpenResult::Failed( + StoreRuntimeRegistryFailure::WriterAuthorityBehindPersisted { + binding, + persisted, + }, + ); + } + converged_on_persisted_authority = true; tracing::warn!( - event = "store_writer_authority_converged", + event = "store_writer_authority_floor_retained", shard = ?binding.shard_id, incarnation = binding.incarnation.get(), requested_epoch = binding.authority_epoch.get(), persisted_epoch = persisted.get(), - "store ledger holds a newer writer authority; reopening above it" + "store ledger holds a newer writer authority; retrying above its retained epoch floor" ); } result => return result, diff --git a/crates/tracedecay-runtime-core/src/shard_runtime/registry/ports.rs b/crates/tracedecay-runtime-core/src/shard_runtime/registry/ports.rs index 7687a009a2..033c2b8e59 100644 --- a/crates/tracedecay-runtime-core/src/shard_runtime/registry/ports.rs +++ b/crates/tracedecay-runtime-core/src/shard_runtime/registry/ports.rs @@ -178,8 +178,27 @@ async fn publish_lifecycle_runtime( && request.access == StoreRuntimeAccessMode::ReadWrite && let Err(error) = verify_writer_authority_before_existing_publication(&request, attachment.as_physical()) + .await { - attachment.abort(request.locator.is_prospective()); + // This failed attachment must finish closing before the registry can + // retry at a new epoch. Joining its physical workers is blocking work. + let created = request.locator.is_prospective(); + crate::native_runtime::spawn_blocking(move || { + if created { + attachment.0.abort_initialization() + } else { + attachment.0.drain()?; + attachment.0.close_and_join() + } + }) + .await + .and_then(|closed| closed) + .map_err( + |message| StoreRuntimeRegistryFailure::PhysicalRuntimeFailed { + operation: "close runtime after writer authority refusal", + message, + }, + )?; return Err(error); } if let Err(error) = runtime.transition(RuntimeMaintenanceStateV1::Ready) { @@ -200,9 +219,9 @@ async fn publish_lifecycle_runtime( /// A shard whose ledger already holds a newer writer authority for this /// incarnation must not be published at the stale binding: every submit -/// through it would be fenced. The registry raises its epoch floor past the -/// persisted authority and opens again. -fn verify_writer_authority_before_existing_publication( +/// through it would be fenced. The registry retains the persisted epoch floor +/// and retries only if no live graph publication still owns the stale binding. +async fn verify_writer_authority_before_existing_publication( request: &ShardRuntimeBuildRequest, attachment: &dyn PhysicalRuntimeAttachment, ) -> Result<(), StoreRuntimeRegistryFailure> { @@ -216,8 +235,11 @@ fn verify_writer_authority_before_existing_publication( let persisted = tracedecay_rusqlite_runtime::runtime_ledger::persisted_writer_authority_epoch( &handle.read_only_clone(), &request.binding, + // Publication has not exposed this attachment's fresh reader pool to + // any clients, so this admission cannot contend with user snapshots. Duration::ZERO, ) + .await .map_err(|error| StoreRuntimeRegistryFailure::PhysicalRuntimeFailed { operation: OPERATION, message: error.to_string(), diff --git a/crates/tracedecay-runtime-core/src/shard_runtime/registry/tests/production_routes.rs b/crates/tracedecay-runtime-core/src/shard_runtime/registry/tests/production_routes.rs index 943b5e0e6a..22f68b8dbd 100644 --- a/crates/tracedecay-runtime-core/src/shard_runtime/registry/tests/production_routes.rs +++ b/crates/tracedecay-runtime-core/src/shard_runtime/registry/tests/production_routes.rs @@ -5,6 +5,7 @@ use std::path::{Path, PathBuf}; use std::sync::atomic::{AtomicUsize, Ordering}; use std::sync::{Arc, Mutex}; +use std::time::{Duration, Instant}; use rusqlite::Connection; use tempfile::TempDir; @@ -12,11 +13,16 @@ use tracedecay_domain::{ BrainId, BrainNodeId, LocatorDigest, ProjectId, RepositoryId, UserProfileId, UtcMicros, WorktreeId, }; +use tracedecay_graph_db::{ + GraphDbOwnerRegistrationV1, GraphDbRegistration, GraphDbRegistry, GraphDbRegistryConfig, + NeverCancelled, +}; use tracedecay_store::{ - CodeShardScopeV1, ConsistencyModeV1, OperationPriorityV1, RuntimeCancellationIdV1, - RuntimeCancellationIdentityV1, RuntimeDeadlineIdV1, RuntimeDeadlineV1, RuntimeReadOperationV1, - RuntimeReadRequestV1, RuntimeReadResultV1, RuntimeRequestControlV1, RuntimeRequestProbeV1, - StoreShardIdV1, StoreShardScopeV1, VerifiedStoreLocatorV1, + CodeShardScopeV1, ConsistencyModeV1, OperationPriorityV1, RetainedGraphStoreLeaseV1, + RetainedGraphStoreOwnerAttachmentV1, RuntimeCancellationIdV1, RuntimeCancellationIdentityV1, + RuntimeDeadlineIdV1, RuntimeDeadlineV1, RuntimeReadOperationV1, RuntimeReadRequestV1, + RuntimeReadResultV1, RuntimeRequestControlV1, RuntimeRequestProbeV1, StoreShardIdV1, + StoreShardScopeV1, VerifiedStoreLocatorV1, canonical_store_locator_digest, }; use super::super::*; @@ -134,6 +140,26 @@ impl StoreRuntimeResolver for FileResolver { ); Box::pin(async move { Ok(ResolvedStoreLocator::new(locator, path)) }) } + + fn resolve_graph<'a>( + &'a self, + key: &'a StoreRuntimeKey, + ) -> StoreRuntimeRegistryFuture<'a, Result> + { + Box::pin(async move { + let resolved = self + .resolve(key, StoreRuntimeOpenMode::Existing, None) + .await?; + Ok(ResolvedStoreLocator::new( + VerifiedStoreLocatorV1::new( + key.shard_id.clone(), + key.incarnation, + canonical_store_locator_digest(resolved.path()).unwrap(), + ), + resolved.path().to_path_buf(), + )) + }) + } } fn health_request( @@ -739,6 +765,49 @@ fn diagnostics_submit( ) } +async fn commit_diagnostics_with_writer( + path: &Path, + binding: &tracedecay_store::StoreRuntimeBindingV1, + name: &str, +) { + let writer = tracedecay_rusqlite_runtime::PersistentWriter::start( + &crate::native_runtime::handle().unwrap(), + tracedecay_rusqlite_runtime::ExistingWriterLocator::new( + binding.clone(), + VerifiedStoreLocatorV1::new( + binding.shard_id.clone(), + binding.incarnation, + LocatorDigest::new(format!("sha256:{}", "c".repeat(64))).unwrap(), + ), + path.to_path_buf(), + ) + .unwrap(), + tracedecay_store::AdmissionConfigV1::default(), + tracedecay_rusqlite_runtime::repository::ConcreteRepositoryWriteExecutor::default(), + ) + .unwrap(); + let (request, probe) = diagnostics_submit(binding, name); + let outcome = writer + .submit( + &crate::native_runtime::cx().unwrap(), + request, + Arc::new(probe), + ) + .await + .unwrap(); + assert!( + matches!( + outcome, + tracedecay_store::RuntimeSubmitOutcomeV1::Committed { .. } + ), + "the writer must durably establish its authority: {outcome:?}" + ); + crate::native_runtime::spawn_blocking(move || writer.shutdown_and_join()) + .await + .unwrap() + .unwrap(); +} + /// A store this incarnation already wrote under a newer authority (a previous /// daemon of the same incarnation, or a sibling runtime on the file) fences /// every writer below that authority. The reopen converges above it instead of @@ -766,39 +835,7 @@ async fn reopening_a_store_behind_its_persisted_writer_authority_converges_above tracedecay_store::StoreAuthorityEpochV1::new(1 << 40).unwrap(), ); let runtime = asupersync::runtime::Runtime::current_handle().unwrap(); - let previous = tracedecay_rusqlite_runtime::PersistentWriter::start( - &runtime, - tracedecay_rusqlite_runtime::ExistingWriterLocator::new( - ahead.clone(), - VerifiedStoreLocatorV1::new( - shard.clone(), - incarnation(), - LocatorDigest::new(format!("sha256:{}", "c".repeat(64))).unwrap(), - ), - sessions_path.clone(), - ) - .unwrap(), - tracedecay_store::AdmissionConfigV1::default(), - tracedecay_rusqlite_runtime::repository::ConcreteRepositoryWriteExecutor::default(), - ) - .unwrap(); - let (request, probe) = diagnostics_submit(&ahead, "previous-authority"); - let outcome = previous - .submit( - &crate::native_runtime::cx().unwrap(), - request, - Arc::new(probe), - ) - .await - .unwrap(); - assert!( - matches!( - outcome, - tracedecay_store::RuntimeSubmitOutcomeV1::Committed { .. } - ), - "the previous authority commits: {outcome:?}" - ); - previous.shutdown_and_join().unwrap(); + commit_diagnostics_with_writer(&sessions_path, &ahead, "previous-authority").await; let registry = StoreRuntimeRegistry::new(runtime, resolver, Arc::new(LifecycleShardRuntimePublisher)) @@ -838,4 +875,129 @@ async fn reopening_a_store_behind_its_persisted_writer_authority_converges_above ), "a write through the converged runtime commits: {outcome:?}" ); + assert!( + tracedecay_rusqlite_runtime::exact_sql::native_thread_storage_wait_sites() + .iter() + .all(|site| !site.call_site.contains("/ledger/checkpoint.rs:")), + "persisted writer authority reads must not park native runtime workers" + ); +} + +/// A graph-first owner cannot be rebound while any of its graph leases remain +/// live. Reopening refuses once with the exact writer fence, then succeeds +/// above the ledger after the ordinary graph close releases that authority. +#[asupersync::test] +async fn a_stale_graph_writer_authority_refuses_once_then_reopens_after_graph_close() { + let root = TempDir::new().unwrap(); + let resolver = Arc::new(FileResolver::default()); + resolver.push(seed_db(&root, "profile.db")); + let graph_path = root.path().join("project.grafeo"); + resolver.push(graph_path.clone()); + let project_path = seed_final_graph_db(&root, "project.db").await; + Connection::open(&project_path) + .unwrap() + .execute_batch(tracedecay_store::GENERATION_DIAGNOSTICS_SCHEMA_DDL) + .unwrap(); + resolver.push(project_path.clone()); + resolver.push(project_path.clone()); + resolver.push(graph_path); + let registry = StoreRuntimeRegistry::new( + asupersync::runtime::Runtime::current_handle().unwrap(), + resolver.clone(), + Arc::new(LifecycleShardRuntimePublisher), + ) + .unwrap(); + let _profile = open_published( + ®istry, + StoreRuntimeOpenRequest::new(profile_shard(), incarnation(), None), + ) + .await; + let pin = match registry.profile_authority_pin(&profile_shard()) { + ProfileAuthorityPinResult::Pinned(pin) => pin, + other => panic!("profile was not pinned: {other:?}"), + }; + let shard = project_shard("project.graph-first-reopen"); + let key = StoreRuntimeKey::new(shard.clone(), incarnation()); + let (store_attachment, store_target) = registry + .attach_graph_store_owner(key.clone()) + .await + .unwrap(); + let stale = store_attachment.binding().clone(); + let graph_locator = store_attachment.verified_locator().clone(); + let operation = store_attachment.issue_operation_lease().unwrap(); + let graphs = GraphDbRegistry::new(GraphDbRegistryConfig { max_open: 1 }).unwrap(); + let graph_owner = graphs + .resolve_owner_attachment(GraphDbOwnerRegistrationV1 { + operation: GraphDbRegistration { + authority_lease: operation, + cancellation: Arc::new(NeverCancelled), + lifecycle_cancellation: Arc::new(NeverCancelled), + deadline: Instant::now() + Duration::from_secs(30), + }, + authority_attachment: Box::new(store_attachment), + }) + .unwrap(); + let client = graph_owner.issue_lease().unwrap(); + + let mut ahead = stale.clone(); + ahead.authority_epoch = tracedecay_store::StoreAuthorityEpochV1::new( + stale.authority_epoch.get().checked_add(1).unwrap(), + ) + .unwrap(); + commit_diagnostics_with_writer(&project_path, &ahead, "graph-newer-writer").await; + let authority = + crate::db::DatabaseAuthority::for_runtime(&project_path, "reopen project").unwrap(); + let request = + StoreRuntimeOpenRequest::new_authorized(shard, incarnation(), Some(pin), authority.clone()); + let calls = resolver.calls.load(Ordering::SeqCst); + assert!(matches!( + registry.open(request.clone()).await, + StoreRuntimeOpenResult::Failed( + StoreRuntimeRegistryFailure::WriterAuthorityBehindPersisted { binding, persisted } + ) if *binding == stale && persisted == ahead.authority_epoch + )); + assert_eq!( + resolver.calls.load(Ordering::SeqCst), + calls + 1, + "a live stale graph binding cannot converge through another identical open" + ); + assert_eq!(graph_owner.binding(), &stale); + assert!(graph_owner.shares_runtime_with(&client)); + drop( + client + .snapshot() + .expect("the old graph client stays readable"), + ); + assert!( + graphs.close_retained(&stale, &graph_locator).is_err(), + "the refused writer must preserve the graph owner's live close blockers" + ); + drop( + graph_owner + .issue_lease() + .expect("the old exact graph owner stays usable"), + ); + + drop(client); + drop(graph_owner); + assert!(graphs.close_retained(&stale, &graph_locator).unwrap()); + drop(store_target); + assert_eq!(registry.retained_graph_publications_for_test(), 0); + + let project = open_published(®istry, request).await; + assert!(project.binding().authority_epoch > ahead.authority_epoch); + let graph = registry.retain_graph_store(key).await.unwrap(); + assert_eq!(graph.binding(), project.binding()); + let (request, probe) = diagnostics_submit(project.binding(), "graph-reopened-writer"); + let outcome = project + .dispatch_submit_authorized(request, Arc::new(probe), authority) + .await + .unwrap(); + assert!( + matches!( + outcome, + tracedecay_store::RuntimeSubmitOutcomeV1::Committed { .. } + ), + "the reopened shared authority accepts writes: {outcome:?}" + ); } diff --git a/crates/tracedecay-rusqlite-runtime/src/ledger/checkpoint.rs b/crates/tracedecay-rusqlite-runtime/src/ledger/checkpoint.rs index 6c5c690fdd..e4b876d505 100644 --- a/crates/tracedecay-rusqlite-runtime/src/ledger/checkpoint.rs +++ b/crates/tracedecay-rusqlite-runtime/src/ledger/checkpoint.rs @@ -60,8 +60,9 @@ impl From for PersistedWriterAuthorityError { /// incarnation, or `None` for a store no writer of that incarnation has /// committed to. Read through the attached runtime's own readers, before the /// runtime is published, so a reopen can learn whether its allocated epoch -/// is already behind the store. -pub fn persisted_writer_authority_epoch( +/// is already behind the store. Reader admission and execution are awaited +/// off the native runtime worker through the existing exact-SQL async port. +pub async fn persisted_writer_authority_epoch( handle: &ExactSqlHandle, binding: &StoreRuntimeBindingV1, max_wait: Duration, @@ -73,29 +74,33 @@ pub fn persisted_writer_authority_epoch( BindingKey::from_binding(binding).map_err(|_| PersistedWriterAuthorityError::Corrupt { field: "shard_json", })?; - let schema = handle.query( - ExactSqlStatement::new( - "SELECT EXISTS(SELECT 1 FROM sqlite_schema WHERE type = 'table' AND name = ?1)" - .to_owned(), - vec![ExactSqlValue::Text(CHECKPOINT_TABLE.to_owned())], - )?, - max_wait, - )?; + let schema = handle + .query_async( + ExactSqlStatement::new( + "SELECT EXISTS(SELECT 1 FROM sqlite_schema WHERE type = 'table' AND name = ?1)" + .to_owned(), + vec![ExactSqlValue::Text(CHECKPOINT_TABLE.to_owned())], + )?, + max_wait, + ) + .await?; match schema.rows.first().and_then(|row| row.values.first()) { Some(ExactSqlValue::Integer(0)) => return Ok(None), Some(ExactSqlValue::Integer(1)) => {} _ => return Err(PersistedWriterAuthorityError::Corrupt { field: "schema" }), } - let rows = handle.query( - ExactSqlStatement::new( - SELECT_PERSISTED_AUTHORITY_EPOCH.to_owned(), - vec![ - ExactSqlValue::Text(key.shard_json), - ExactSqlValue::Integer(key.incarnation_sql), - ], - )?, - max_wait, - )?; + let rows = handle + .query_async( + ExactSqlStatement::new( + SELECT_PERSISTED_AUTHORITY_EPOCH.to_owned(), + vec![ + ExactSqlValue::Text(key.shard_json), + ExactSqlValue::Integer(key.incarnation_sql), + ], + )?, + max_wait, + ) + .await?; let Some(row) = rows.rows.first() else { return Ok(None); }; diff --git a/crates/tracedecay-rusqlite-runtime/tests/rusqlite_suite/storage_contention.rs b/crates/tracedecay-rusqlite-runtime/tests/rusqlite_suite/storage_contention.rs index 009918cd98..cb2ed1e38b 100644 --- a/crates/tracedecay-rusqlite-runtime/tests/rusqlite_suite/storage_contention.rs +++ b/crates/tracedecay-rusqlite-runtime/tests/rusqlite_suite/storage_contention.rs @@ -1246,7 +1246,12 @@ fn persisted_writer_authority_is_scoped_to_the_shard_and_incarnation() { .unwrap(); let handle = attachment.exact_sql_handle().unwrap(); assert_eq!( - persisted_writer_authority_epoch(&handle, &binding, Duration::ZERO).unwrap(), + run(persisted_writer_authority_epoch( + &handle, + &binding, + Duration::ZERO + )) + .unwrap(), expected ); attachment.drain().unwrap(); @@ -1264,7 +1269,11 @@ fn persisted_writer_authority_is_scoped_to_the_shard_and_incarnation() { .unwrap(); let handle = attachment.exact_sql_handle().unwrap(); assert!(matches!( - persisted_writer_authority_epoch(&handle, ¤t, Duration::ZERO), + run(persisted_writer_authority_epoch( + &handle, + ¤t, + Duration::ZERO + )), Err(PersistedWriterAuthorityError::Corrupt { field: "authority_epoch" }) From b3ce079e4e80e080efe25b430e8474cb342ec64d Mon Sep 17 00:00:00 2001 From: Zack Jackson Date: Tue, 20 Oct 2026 21:51:00 -0700 Subject: [PATCH 03/15] test(store): reopen diagnostics on their project shard --- .../registry/tests/production_routes.rs | 26 +++++++++---------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/crates/tracedecay-runtime-core/src/shard_runtime/registry/tests/production_routes.rs b/crates/tracedecay-runtime-core/src/shard_runtime/registry/tests/production_routes.rs index 22f68b8dbd..93c943b690 100644 --- a/crates/tracedecay-runtime-core/src/shard_runtime/registry/tests/production_routes.rs +++ b/crates/tracedecay-runtime-core/src/shard_runtime/registry/tests/production_routes.rs @@ -817,14 +817,14 @@ async fn reopening_a_store_behind_its_persisted_writer_authority_converges_above let root = TempDir::new().unwrap(); let resolver = Arc::new(FileResolver::default()); resolver.push(seed_db(&root, "profile.db")); - let sessions_path = seed_db(&root, "sessions.db"); - Connection::open(&sessions_path) + let project_path = seed_final_graph_db(&root, "project.db").await; + Connection::open(&project_path) .unwrap() .execute_batch(tracedecay_store::GENERATION_DIAGNOSTICS_SCHEMA_DDL) .unwrap(); - resolver.push(sessions_path.clone()); - resolver.push(sessions_path.clone()); - let shard = StoreShardIdV1::project_sessions( + resolver.push(project_path.clone()); + resolver.push(project_path.clone()); + let shard = StoreShardIdV1::project( id::("brain.registry"), id::("profile.registry"), id::("project.reopen"), @@ -835,7 +835,7 @@ async fn reopening_a_store_behind_its_persisted_writer_authority_converges_above tracedecay_store::StoreAuthorityEpochV1::new(1 << 40).unwrap(), ); let runtime = asupersync::runtime::Runtime::current_handle().unwrap(); - commit_diagnostics_with_writer(&sessions_path, &ahead, "previous-authority").await; + commit_diagnostics_with_writer(&project_path, &ahead, "previous-authority").await; let registry = StoreRuntimeRegistry::new(runtime, resolver, Arc::new(LifecycleShardRuntimePublisher)) @@ -849,22 +849,22 @@ async fn reopening_a_store_behind_its_persisted_writer_authority_converges_above ProfileAuthorityPinResult::Pinned(pin) => pin, other => panic!("profile was not pinned: {other:?}"), }; - let authority = crate::db::DatabaseAuthority::for_runtime(&sessions_path, "reopen sessions") - .expect("sessions database authority"); - let sessions = open_published( + let authority = crate::db::DatabaseAuthority::for_runtime(&project_path, "reopen project") + .expect("project database authority"); + let project_runtime = open_published( ®istry, StoreRuntimeOpenRequest::new_authorized(shard, incarnation(), Some(pin), authority.clone()), ) .await; assert!( - sessions.binding().authority_epoch > ahead.authority_epoch, + project_runtime.binding().authority_epoch > ahead.authority_epoch, "the reopened runtime publishes above the persisted authority: {:?} vs {:?}", - sessions.binding().authority_epoch, + project_runtime.binding().authority_epoch, ahead.authority_epoch ); - let (request, probe) = diagnostics_submit(sessions.binding(), "converged-authority"); - let outcome = sessions + let (request, probe) = diagnostics_submit(project_runtime.binding(), "converged-authority"); + let outcome = project_runtime .dispatch_submit_authorized(request, Arc::new(probe), authority) .await .expect("converged runtime accepts writes"); From c46fac5b097032d23478b49299a25183eb49aa24 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Wed, 21 Oct 2026 10:34:38 -0700 Subject: [PATCH 04/15] test(sessions): synchronize rekey request assertions --- .../registry_tests.rs | 65 ++++++++++++++++--- 1 file changed, 56 insertions(+), 9 deletions(-) diff --git a/crates/tracedecay-session-runtime/src/session_temporal_refresh_scheduler/registry_tests.rs b/crates/tracedecay-session-runtime/src/session_temporal_refresh_scheduler/registry_tests.rs index ad4495ee87..06453b690c 100644 --- a/crates/tracedecay-session-runtime/src/session_temporal_refresh_scheduler/registry_tests.rs +++ b/crates/tracedecay-session-runtime/src/session_temporal_refresh_scheduler/registry_tests.rs @@ -377,10 +377,11 @@ async fn a_retried_rekey_recovers_the_dropped_attempts_route_and_requests() { .project_state(&old_owner) .await .expect("project entry"); - old_state.requeue_request(tracedecay_store::SessionRefreshBeginOrJoinRequestV1::new( + let request = tracedecay_store::SessionRefreshBeginOrJoinRequestV1::new( tracedecay_domain::SessionId::new("session.dropped-rekey").unwrap(), tracedecay_store::SessionRefreshFrontierV1::new(0, 0).unwrap(), - )); + ); + old_state.requeue_request(request.clone()); assert_eq!(old_state.queued_request_count(), 1); assert!( @@ -395,6 +396,10 @@ async fn a_retried_rekey_recovers_the_dropped_attempts_route_and_requests() { "no worker owns the route of a dropped rekey, so a wake must not be accepted" ); + // Keep the destination in history until the transferred request and route + // are inspected; its worker may otherwise consume them before the test. + let destination_history = + mount_held_project(®istry, &harness, &new_owner, HeldHistory::default()).await; let mut rekey = std::pin::pin!(registry.rekey_project( &old_owner, new_owner.clone(), @@ -419,12 +424,23 @@ async fn a_retried_rekey_recovers_the_dropped_attempts_route_and_requests() { 1, "the retry must land the retired owner's queued request on the new owner" ); + let transferred = new_state.take_requests(1); + assert!( + transferred + .iter() + .any(|transferred| transferred.is_equivalent_to(&request)), + "the retry must retain the exact request and frontier" + ); + for request in transferred { + new_state.requeue_request(request); + } assert!( old_wake.wake(), "the old route must reach the new owner once the rekey completes" ); assert!(new_state.take_dirty()); + destination_history.release.notify_one(); registry.shutdown().await; } @@ -540,10 +556,11 @@ async fn a_rekey_after_a_readmission_lands_every_route_and_request() { .project_state(&a_owner) .await .expect("project entry"); - a_state.requeue_request(tracedecay_store::SessionRefreshBeginOrJoinRequestV1::new( + let parked_request = tracedecay_store::SessionRefreshBeginOrJoinRequestV1::new( tracedecay_domain::SessionId::new("session.readmitted-a").unwrap(), tracedecay_store::SessionRefreshFrontierV1::new(0, 0).unwrap(), - )); + ); + a_state.requeue_request(parked_request.clone()); { let mut rekey = std::pin::pin!(registry.rekey_project( @@ -601,18 +618,35 @@ async fn a_rekey_after_a_readmission_lands_every_route_and_request() { !a_wake.wake(), "the parked handoff stays owed to its own destination" ); + // Keep both live workers in history while inspecting transferred work. + // Otherwise either owner can consume its queue before the handoff assertion. + let readmitted_history = + mount_held_project(®istry, &harness, &a_owner, HeldHistory::default()).await; + let destination_history = + mount_held_project(®istry, &harness, &d_owner, HeldHistory::default()).await; let readmitted = registry .project_state(&a_owner) .await .expect("readmitted project entry"); - readmitted.requeue_request(tracedecay_store::SessionRefreshBeginOrJoinRequestV1::new( + let live_request = tracedecay_store::SessionRefreshBeginOrJoinRequestV1::new( tracedecay_domain::SessionId::new("session.readmitted-live").unwrap(), tracedecay_store::SessionRefreshFrontierV1::new(0, 0).unwrap(), - )); + ); + readmitted.requeue_request(live_request.clone()); - registry - .rekey_project(&a_owner, d_owner.clone(), harness.registered.clone()) - .await; + { + let mut rekey = std::pin::pin!(registry.rekey_project( + &a_owner, + d_owner.clone(), + harness.registered.clone(), + )); + assert!( + rekey.as_mut().now_or_never().is_none(), + "the readmitted worker must be cancelled before releasing its history pass" + ); + readmitted_history.release.notify_one(); + rekey.await; + } assert!(registry.project_state(&a_owner).await.is_none()); let landed = registry @@ -624,6 +658,18 @@ async fn a_rekey_after_a_readmission_lands_every_route_and_request() { 2, "the parked request and the readmitted entry's request must both reach the final owner" ); + let transferred = landed.take_requests(2); + for expected in [&parked_request, &live_request] { + assert!( + transferred + .iter() + .any(|request| request.is_equivalent_to(expected)), + "the handoff must retain both exact requests and frontiers" + ); + } + for request in transferred { + landed.requeue_request(request); + } assert!( a_wake.wake(), "the parked route must reach the new owner once the rekey completes" @@ -636,6 +682,7 @@ async fn a_rekey_after_a_readmission_lands_every_route_and_request() { // The destination retains the complete route set it absorbed: a later // D-to-E rekey must still land both handles, or older callers die at // the next move. + destination_history.release.notify_one(); registry .rekey_project(&d_owner, e_owner.clone(), harness.registered.clone()) .await; From db3f7529f963dbdfce839de7c16f007fcb46ebb9 Mon Sep 17 00:00:00 2001 From: Zack Jackson Date: Wed, 21 Oct 2026 14:01:25 -0700 Subject: [PATCH 05/15] test(cli): follow profile authority in host daemon journeys --- .../sweep_outcomes.rs | 29 +++++++++++++++---- 1 file changed, 24 insertions(+), 5 deletions(-) diff --git a/crates/tracedecay-cli/tests/host_journeys_suite/host_lifecycle_cli_acceptance/sweep_outcomes.rs b/crates/tracedecay-cli/tests/host_journeys_suite/host_lifecycle_cli_acceptance/sweep_outcomes.rs index e447a37183..43d0713c74 100644 --- a/crates/tracedecay-cli/tests/host_journeys_suite/host_lifecycle_cli_acceptance/sweep_outcomes.rs +++ b/crates/tracedecay-cli/tests/host_journeys_suite/host_lifecycle_cli_acceptance/sweep_outcomes.rs @@ -7,6 +7,12 @@ use std::process::Output; use tracedecay_agent_hosts::agents::host_bundle::{HostComponentV1, HostKindV1}; #[cfg(unix)] +use tracedecay_daemon_identity::current_daemon_connection; +#[cfg(unix)] +use tracedecay_daemon_protocol::DaemonEndpoint; +#[cfg(target_os = "linux")] +use tracedecay_daemon_protocol::unix_socket_path_within_limit; +#[cfg(unix)] use tracedecay_runtime_core::test_executable::write_executable_script; #[cfg(unix)] @@ -387,13 +393,21 @@ impl ProfileDaemon { .stdout(std::process::Stdio::null()) .stderr(std::process::Stdio::null()); let daemon = Self(command.spawn().unwrap()); - let socket = cli.profile.join("daemon.sock"); let deadline = std::time::Instant::now() + std::time::Duration::from_secs(30); - while std::os::unix::net::UnixStream::connect(&socket).is_err() { + // Deep worker paths can move the socket outside the profile. Follow + // the same profile-owned authority that production clients discover. + while !current_daemon_connection(&cli.profile).is_ok_and(|connection| { + match connection.endpoint() { + DaemonEndpoint::Unix(socket) => { + std::os::unix::net::UnixStream::connect(socket).is_ok() + } + DaemonEndpoint::Loopback(address) => std::net::TcpStream::connect(address).is_ok(), + } + }) { assert!( std::time::Instant::now() < deadline, - "the profile daemon never listened at {}", - socket.display() + "the registered daemon never listened for profile {}", + cli.profile.display() ); std::thread::sleep(std::time::Duration::from_millis(25)); } @@ -864,7 +878,12 @@ fn doctor_observes_copilots_vscode_settings_without_failing_on_them() { #[cfg(target_os = "linux")] #[test] fn doctor_warns_on_detected_hosts_without_a_tracedecay_integration() { - let cli = IsolatedCli::new(); + let mut cli = IsolatedCli::new(); + cli.profile = cli.home.path().join("profile".repeat(20)); + assert!( + !unix_socket_path_within_limit(&cli.profile.join("daemon.sock")), + "this journey must exercise production's short endpoint for a deep profile" + ); install_cline(&cli); fs::create_dir_all(cli.home.path().join(".config/zed")).unwrap(); fs::create_dir_all(cli.home.path().join(".gemini/antigravity")).unwrap(); From 9d22f0243405bd9b51d40a70fdc8bb214b3caed0 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Wed, 21 Oct 2026 16:20:03 -0700 Subject: [PATCH 06/15] fix(sessions): join temporal publication independently of summaries --- .../src/session_temporal_refresh.rs | 12 +- .../src/session_refresh.rs | 5 +- .../src/server/live_transcript_refresh.rs | 4 +- .../wake.rs | 84 ++++-- .../worker.rs | 283 +++++++++++++++-- .../worker_tests.rs | 285 +++++++++++++++++- .../session_refresh_cancel_test.rs | 2 +- .../session_runtime/temporal_refresh.rs | 5 +- 8 files changed, 620 insertions(+), 60 deletions(-) diff --git a/crates/tracedecay-contracts/src/session_temporal_refresh.rs b/crates/tracedecay-contracts/src/session_temporal_refresh.rs index b1cc77c845..e62f8064e8 100644 --- a/crates/tracedecay-contracts/src/session_temporal_refresh.rs +++ b/crates/tracedecay-contracts/src/session_temporal_refresh.rs @@ -8,7 +8,12 @@ pub type SessionTemporalRefreshWakeFuture<'a> = Pin bool; fn is_unavailable(&self) -> bool; - fn wake_and_wait_until_idle(&self, timeout: Duration) -> SessionTemporalRefreshWakeFuture<'_>; + /// Join temporal publication caused by this wake. Background history and + /// summary convergence may continue after the projection has committed. + fn wake_and_wait_for_projection( + &self, + timeout: Duration, + ) -> SessionTemporalRefreshWakeFuture<'_>; } /// Typed missing-worker implementation for unmounted servers and tests. @@ -24,7 +29,10 @@ impl SessionTemporalRefreshWakePort for UnavailableSessionTemporalRefreshWake { true } - fn wake_and_wait_until_idle(&self, _timeout: Duration) -> SessionTemporalRefreshWakeFuture<'_> { + fn wake_and_wait_for_projection( + &self, + _timeout: Duration, + ) -> SessionTemporalRefreshWakeFuture<'_> { Box::pin(async { false }) } } diff --git a/crates/tracedecay-daemon-service/src/session_refresh.rs b/crates/tracedecay-daemon-service/src/session_refresh.rs index b878505890..a49649a40b 100644 --- a/crates/tracedecay-daemon-service/src/session_refresh.rs +++ b/crates/tracedecay-daemon-service/src/session_refresh.rs @@ -295,7 +295,10 @@ impl SessionTemporalRefreshWakePort for FixedSessionTemporalRefreshWake { !self.0 } - fn wake_and_wait_until_idle(&self, _timeout: Duration) -> SessionTemporalRefreshWakeFuture<'_> { + fn wake_and_wait_for_projection( + &self, + _timeout: Duration, + ) -> SessionTemporalRefreshWakeFuture<'_> { let accepted = self.0; Box::pin(async move { accepted }) } diff --git a/crates/tracedecay-mcp/src/server/live_transcript_refresh.rs b/crates/tracedecay-mcp/src/server/live_transcript_refresh.rs index de4897782a..f61c6ff56d 100644 --- a/crates/tracedecay-mcp/src/server/live_transcript_refresh.rs +++ b/crates/tracedecay-mcp/src/server/live_transcript_refresh.rs @@ -97,7 +97,7 @@ async fn join_scope( } .ok_or_else(&unavailable)?; if wake - .wake_and_wait_until_idle(LIVE_TRANSCRIPT_REFRESH_DEADLINE) + .wake_and_wait_for_projection(LIVE_TRANSCRIPT_REFRESH_DEADLINE) .await { Ok(LiveTranscriptRefreshJoin::PublicationJoined) @@ -149,7 +149,7 @@ mod tests { false } - fn wake_and_wait_until_idle( + fn wake_and_wait_for_projection( &self, _timeout: Duration, ) -> SessionTemporalRefreshWakeFuture<'_> { diff --git a/crates/tracedecay-session-runtime/src/session_temporal_refresh_scheduler/wake.rs b/crates/tracedecay-session-runtime/src/session_temporal_refresh_scheduler/wake.rs index 8f37a1a1ed..5cf49c106e 100644 --- a/crates/tracedecay-session-runtime/src/session_temporal_refresh_scheduler/wake.rs +++ b/crates/tracedecay-session-runtime/src/session_temporal_refresh_scheduler/wake.rs @@ -2,7 +2,6 @@ use std::collections::{HashMap, HashSet, VecDeque}; use std::sync::Arc; use std::sync::PoisonError; use std::sync::atomic::{AtomicBool, Ordering}; -use std::time::Instant; use tracedecay_runtime_core::native_runtime; use tracedecay_store::SessionRefreshBeginOrJoinRequestV1; use tracedecay_temporal_query::execution::ExecutionControl; @@ -139,6 +138,7 @@ macro_rules! define_wake_state { /// is still unpublished. The next history window waits. pub(super) history_after_projection: AtomicBool, pub(super) pass_count: std::sync::atomic::AtomicUsize, + projection_joins: std::sync::Mutex>>, pub(super) wake: tokio::sync::Notify, pub(super) idle: tokio::sync::Notify, pub(super) cancelled: AtomicBool, @@ -171,6 +171,7 @@ impl Default for SessionTemporalRefreshWakeState { history_retry_pending: AtomicBool::new(false), history_after_projection: AtomicBool::new(false), pass_count: std::sync::atomic::AtomicUsize::new(0), + projection_joins: std::sync::Mutex::new(Vec::new()), wake: tokio::sync::Notify::new(), idle: tokio::sync::Notify::new(), cancelled: AtomicBool::new(false), @@ -341,6 +342,34 @@ impl SessionTemporalRefreshWakeState { self.wake.notify_one(); } + fn request_projection_join(&self) -> Option> { + let mut joins = self + .projection_joins + .lock() + .unwrap_or_else(PoisonError::into_inner); + if self.cancelled.load(Ordering::Acquire) { + return None; + } + // Expired callers retain no obligation. The worker owns only live + // joins, and each result is delivered once even if a later pass fails. + joins.retain(|join| !join.is_closed()); + let (completion, receiver) = tokio::sync::oneshot::channel(); + joins.push(completion); + self.wake(); + Some(receiver) + } + + /// Take the joins covered by the projection about to start. A join that + /// arrives during that projection remains queued for the next pass. + pub(super) fn take_projection_joins(&self) -> Vec> { + std::mem::take( + &mut *self + .projection_joins + .lock() + .unwrap_or_else(PoisonError::into_inner), + ) + } + pub fn requeue_projection(&self) { self.dirty.swap(true, Ordering::AcqRel); self.mark_converging(); @@ -668,6 +697,9 @@ impl SessionTemporalRefreshWakeState { self.completion_control.cancel(); self.clear_worker_instrumentation(); self.mark_stopped(); + for join in self.take_projection_joins() { + let _ = join.send(false); + } self.cancellation.notify_waiters(); self.wake.notify_waiters(); } @@ -848,6 +880,7 @@ struct SessionTemporalRefreshWakeRoute { target: std::sync::RwLock>, } +#[cfg(any(test, feature = "test-helpers"))] pub(super) fn enabled_idle_notification( state: &SessionTemporalRefreshWakeState, ) -> std::pin::Pin>> { @@ -942,39 +975,32 @@ impl SessionTemporalRefreshWake { true } - /// Wake the refresh worker and wait until it finishes the resulting pass. + /// Wake the refresh worker and wait until the resulting projection drains. /// /// Live LCM transcript projection writes must become temporally searchable /// before the mutating tool returns so Hermes `sync_turn` → `lcm_grep` /// stays available on the same route without inventing stores or weakening - /// empty/unavailable semantics. - pub async fn wake_and_wait_until_idle(&self, timeout: std::time::Duration) -> bool { + /// empty/unavailable semantics. Historical discovery and derived summaries + /// may continue after that publication; their idle/admission state is not + /// evidence that a temporal generation has committed. + pub async fn wake_and_wait_for_projection(&self, timeout: std::time::Duration) -> bool { let Some(state) = self.target() else { return false; }; - let before = state.pass_count.load(Ordering::Acquire); - state.wake(); - let deadline = Instant::now() + timeout; - loop { - let idle = tracing::Instrument::instrument( - enabled_idle_notification(&state), - tracing::trace_span!("daemon.scheduler.session_temporal.idle_wait"), - ); - let pass_count = state.pass_count.load(Ordering::Acquire); - let busy = state.busy.load(Ordering::Acquire); - let dirty = state.dirty.load(Ordering::Acquire); - let pending = state.has_requests(); - if pass_count > before && !busy && !dirty && !pending { - return true; - } - if native_runtime::timeout_at(deadline, idle).await.is_err() { - let pass_count = state.pass_count.load(Ordering::Acquire); - return pass_count > before - && !state.busy.load(Ordering::Acquire) - && !state.dirty.load(Ordering::Acquire) - && !state.has_requests(); - } - } + let Some(completion) = state.request_projection_join() else { + return false; + }; + matches!( + native_runtime::timeout( + timeout, + tracing::Instrument::instrument( + completion, + tracing::trace_span!("daemon.scheduler.session_temporal.publication_wait"), + ), + ) + .await, + Ok(Ok(true)) + ) } /// Marks historical catch-up pending and wakes its worker, unless the @@ -1014,11 +1040,11 @@ impl tracedecay_contracts::SessionTemporalRefreshWakePort for SessionTemporalRef self.status().unavailable_reason.is_some() } - fn wake_and_wait_until_idle( + fn wake_and_wait_for_projection( &self, timeout: std::time::Duration, ) -> tracedecay_contracts::SessionTemporalRefreshWakeFuture<'_> { - Box::pin(SessionTemporalRefreshWake::wake_and_wait_until_idle( + Box::pin(SessionTemporalRefreshWake::wake_and_wait_for_projection( self, timeout, )) } diff --git a/crates/tracedecay-session-runtime/src/session_temporal_refresh_scheduler/worker.rs b/crates/tracedecay-session-runtime/src/session_temporal_refresh_scheduler/worker.rs index e6c90edf14..013fd9143d 100644 --- a/crates/tracedecay-session-runtime/src/session_temporal_refresh_scheduler/worker.rs +++ b/crates/tracedecay-session-runtime/src/session_temporal_refresh_scheduler/worker.rs @@ -235,6 +235,13 @@ pub(super) async fn run_session_temporal_refresh_scheduler( let mut history_priority_passes = 0u32; let mut history_release = None; let mut granted_history_admission = None; + let mut projection_joins: Vec> = Vec::new(); + let live_projection = LiveProjectionDuringSummary { + database: &database, + state: &state, + projector: projector.as_ref(), + policy, + }; let _instrumentation = SessionTemporalRefreshWorkerInstrumentation::new(&state); state.mark_running(); loop { @@ -289,8 +296,7 @@ pub(super) async fn run_session_temporal_refresh_scheduler( None }; // A wake that lands after this pass started is served by its - // projection instead of a pass of its own, so it counts as a - // pass: `wake_and_wait_until_idle` joins the pass after its wake. + // projection instead of a pass of its own. if state.take_dirty() { projection_requested = true; state.pass_count.fetch_add(1, Ordering::AcqRel); @@ -328,7 +334,7 @@ pub(super) async fn run_session_temporal_refresh_scheduler( ) || history_outcome .as_ref() .is_some_and(SessionHistoricalIngestOutcome::made_progress); - let report = if projection_requested + let mut report = if projection_requested || state.has_requests() || history_requires_projection { @@ -339,6 +345,7 @@ pub(super) async fn run_session_temporal_refresh_scheduler( &state, projector.as_ref(), policy, + &mut projection_joins, )); let pass = tracing::Instrument::instrument( projection, @@ -368,24 +375,34 @@ pub(super) async fn run_session_temporal_refresh_scheduler( } else { history_outcome.clone() }; - let convergence_admission = + let mut convergence_admission = lcm_convergence_admission(convergence_outcome, &mut history_priority_passes); - // Derived from the admission so the pass report can never disagree - // with what this pass actually ran. let history_needs_another_pass = !matches!( convergence_admission, LcmConvergenceAdmission::Admitted(LcmConvergencePage::Full) ); + // Finish the temporal generation before spending the next pass on + // derived summaries. In particular, waiting for summary admission + // must not strand a live hook's partially published projection. + // The bounded predecessor rewrite retains its history fairness + // turn; it performs no model call. + if matches!( + convergence_admission, + LcmConvergenceAdmission::Admitted(LcmConvergencePage::Full) + ) && projection_still_unpublished(&report) + && projection_published_work(&report) + { + convergence_admission = LcmConvergenceAdmission::Deferred; + } let ( summary_convergence_made_progress, summary_convergence_has_more, summary_retry_delay, ) = match convergence_admission { LcmConvergenceAdmission::Deferred => { - // Historical continuation owns the next bounded pass. LCM - // summaries are independent derived work and can run after - // the raw frontier is terminal; placing a model call between - // source windows delays both project and profile readiness. + // Historical continuation or temporal publication owns the + // next bounded pass. Summaries can run after those frontiers + // settle without delaying live projection joins. (false, false, None) } LcmConvergenceAdmission::Admitted(convergence_page) => { @@ -410,10 +427,11 @@ pub(super) async fn run_session_temporal_refresh_scheduler( // registered to receive one. state.mark_worker_idle(); state.idle.notify_waiters(); - let permit = tokio::select! { - biased; - () = tracing::Instrument::instrument(state.wait_for_cancellation(), tracing::trace_span!("daemon.scheduler.lcm_summary.admission_cancel")) => return, - permit = &mut admission => permit, + let Some(permit) = live_projection + .wait_for(&mut admission, &mut projection_joins, &mut report, true) + .await + else { + return; }; state.mark_worker_busy(); permit @@ -449,10 +467,11 @@ pub(super) async fn run_session_temporal_refresh_scheduler( ), }; tokio::pin!(page); - let result = tokio::select! { - biased; - () = tracing::Instrument::instrument(state.wait_for_cancellation(), tracing::trace_span!("daemon.scheduler.lcm_summary.cancel")) => return, - result = &mut page => result, + let Some(result) = live_projection + .wait_for(&mut page, &mut projection_joins, &mut report, false) + .await + else { + return; }; drop(permit); result @@ -772,8 +791,153 @@ async fn session_projection_refresh( state: &Arc, projector: &dyn SessionTemporalRefreshProjector, policy: SessionTemporalRefreshPolicy, + joins: &mut Vec>, ) -> SessionTemporalRefreshPassReport { - run_session_temporal_refresh_pass(database, state, projector, policy).await + joins.retain(|join| !join.is_closed()); + joins.extend(state.take_projection_joins()); + let report = run_session_temporal_refresh_pass(database, state, projector, policy).await; + if state.cancelled.load(Ordering::Acquire) { + return report; + } + let result = if report.failed > 0 || report.cancelled > 0 || report.terminal_errors > 0 { + Some(false) + } else if report.backlog == Some(0) + && !report.saturated + && report.retry_class.is_none() + && report.retryable_errors == 0 + && report.deadline_errors == 0 + && report.deferred == 0 + && !state.has_requests() + { + Some(true) + } else { + None + }; + if let Some(published) = result { + for join in joins.drain(..) { + let _ = join.send(published); + } + } + report +} + +/// Keeps one fair summary admission or in-flight summary page alive while the +/// same worker services live temporal projection. Both futures keep polling: +/// suspending a summary while it owns a writer could block the projection. +struct LiveProjectionDuringSummary<'a> { + database: &'a RegisteredGlobalDbLeaseV1, + state: &'a Arc, + projector: &'a dyn SessionTemporalRefreshProjector, + policy: SessionTemporalRefreshPolicy, +} + +impl LiveProjectionDuringSummary<'_> { + async fn wait_for( + &self, + stage: F, + joins: &mut Vec>, + report: &mut SessionTemporalRefreshPassReport, + idle_between_projections: bool, + ) -> Option + where + F: Future + Send, + F::Output: Send, + { + tokio::pin!(stage); + let mut retry_attempt = 0u32; + let mut retry_at = None; + loop { + let projection_wake = async { + let wake = self.state.wake.notified(); + if let Some(deadline) = retry_at { + tokio::select! { + () = wake => true, + fired = native_runtime::sleep_until(deadline) => !stop_without_timer(fired), + } + } else { + if !self.state.dirty.load(Ordering::Acquire) { + wake.await; + } + true + } + }; + let woke = tokio::select! { + biased; + () = self.state.wait_for_cancellation() => return None, + result = &mut stage => return Some(result), + woke = projection_wake => woke, + }; + if !woke { + return None; + } + if !self.state.take_dirty() { + continue; + } + self.state.mark_worker_busy(); + self.state.begin_pass(); + self.state.pass_count.fetch_add(1, Ordering::AcqRel); + let (next_report, completed_stage) = { + let projection: Pin< + Box + Send + '_>, + > = Box::pin(session_projection_refresh( + self.database, + self.state, + self.projector, + self.policy, + joins, + )); + let projection = tracing::Instrument::instrument( + projection, + tracing::trace_span!("daemon.scheduler.session_temporal.projection"), + ); + tokio::pin!(projection); + tokio::select! { + biased; + () = self.state.wait_for_cancellation() => return None, + next = &mut projection => (next, None), + result = &mut stage => { + // Finish the admitted projection rather than cancelling + // it when the summary stage happens to complete first. + let next = tokio::select! { + biased; + () = self.state.wait_for_cancellation() => return None, + next = &mut projection => next, + }; + (next, Some(result)) + } + } + }; + *report = next_report; + if let Some(backlog) = report.backlog { + self.state + .record_pass(backlog, projection_published_work(report)); + } + if let Some(class) = report.retry_class { + retry_attempt = retry_attempt.saturating_add(1); + self.state.mark_recovering(class.into(), class); + self.state.requeue_projection(); + retry_at = Some(Instant::now() + session_refresh_retry_delay(class, retry_attempt)); + } else { + retry_attempt = 0; + retry_at = None; + self.state.mark_running(); + if self.state.has_requests() + || report.begun > 0 + || report.saturated + || report.projected_batches > 0 + { + self.state.requeue_projection(); + } + } + if let Some(result) = completed_stage { + return Some(result); + } + if idle_between_projections { + self.state.mark_worker_idle(); + self.state.idle.notify_waiters(); + } + } + } } /// True when replaying this store failure unchanged could still succeed. @@ -1913,10 +2077,91 @@ pub async fn run_session_temporal_refresh_pass( #[cfg(test)] mod tests { use super::*; + use crate::session_temporal_refresh_scheduler::projector::CanonicalSessionTemporalProjector; use tracedecay_domain::SessionId; use tracedecay_global_db::tests::harness::RegisteredGlobalDbHarness; use tracedecay_runtime_core::db::engine::params; use tracedecay_sessions::runtime::{SessionMessageRecord, SessionRecord}; + use tracedecay_store::{SessionRefreshReceiptRequestV1, SessionRefreshTerminalStateV1}; + + #[tokio::test] + async fn an_in_flight_summary_stage_keeps_polling_while_a_live_wake_projects() { + let harness = RegisteredGlobalDbHarness::open("refresh-during-summary-writer").await; + let store = SessionTemporalStore::new(harness.registered.as_ref()); + let begun = store + .begin_or_join_session_refresh(SessionRefreshBeginOrJoinRequestV1::new( + SessionId::new("refresh-during-summary-writer").expect("session id"), + SessionRefreshFrontierV1::new(0, 0).expect("empty frontier"), + )) + .await + .expect("running refresh"); + let state = Arc::new(SessionTemporalRefreshWakeState::default()); + let entered = Arc::new(tokio::sync::Notify::new()); + let release_writer = Arc::new(tokio::sync::Notify::new()); + let finish = Arc::new(tokio::sync::Notify::new()); + let worker = tokio::spawn({ + let database = harness.registered.clone(); + let state = Arc::clone(&state); + let entered = Arc::clone(&entered); + let release_writer = Arc::clone(&release_writer); + let finish = Arc::clone(&finish); + async move { + let stage = async { + let transaction = database.begin_write_transaction().await.expect("writer"); + entered.notify_one(); + release_writer.notified().await; + transaction.commit().await.expect("release summary writer"); + finish.notified().await; + 17usize + }; + let live_projection = LiveProjectionDuringSummary { + database: &database, + state: &state, + projector: &CanonicalSessionTemporalProjector, + policy: SessionTemporalRefreshPolicy::default(), + }; + let mut joins = Vec::new(); + let mut report = SessionTemporalRefreshPassReport::default(); + live_projection + .wait_for(stage, &mut joins, &mut report, false) + .await + } + }); + entered.notified().await; + + let handle = state.handle(); + let join = handle.wake_and_wait_for_projection(Duration::from_secs(5)); + tokio::pin!(join); + assert!( + std::future::poll_fn(|cx| std::task::Poll::Ready(join.as_mut().poll(cx))) + .await + .is_pending() + ); + tokio::time::timeout(Duration::from_secs(5), async { + while state.pass_count.load(Ordering::Acquire) == 0 { + tokio::task::yield_now().await; + } + }) + .await + .expect("live projection starts while the summary stage owns the writer"); + release_writer.notify_one(); + assert!( + join.await, + "summary must keep polling to release the writer needed by projection" + ); + let receipt = store + .session_refresh_receipt(SessionRefreshReceiptRequestV1::new( + begun.operation_id().clone(), + begun.session_id().clone(), + )) + .await + .expect("read terminal receipt") + .expect("projection publishes before summary finishes"); + assert_eq!(receipt.state(), SessionRefreshTerminalStateV1::Complete); + assert!(!worker.is_finished(), "the summary stage remains in flight"); + finish.notify_one(); + assert_eq!(worker.await.expect("summary stage completes"), Some(17)); + } /// A lease the store has fenced resubmits the identical request on every /// pass and can never commit through it, so the pass retires the refresh diff --git a/crates/tracedecay-session-runtime/src/session_temporal_refresh_scheduler/worker_tests.rs b/crates/tracedecay-session-runtime/src/session_temporal_refresh_scheduler/worker_tests.rs index 27f2c43eaf..14454612e1 100644 --- a/crates/tracedecay-session-runtime/src/session_temporal_refresh_scheduler/worker_tests.rs +++ b/crates/tracedecay-session-runtime/src/session_temporal_refresh_scheduler/worker_tests.rs @@ -1,4 +1,5 @@ use std::sync::Arc; +use std::sync::atomic::{AtomicBool, Ordering}; use std::time::Duration; use tracedecay_domain::{SessionId, UtcMicros}; @@ -7,7 +8,8 @@ use tracedecay_global_db::tests::harness::RegisteredGlobalDbHarness; use tracedecay_session_temporal_store::{SessionRefreshRecoveryV1, SessionTemporalStore}; use tracedecay_store::{ SessionRefreshBeginOrJoinRequestV1, SessionRefreshFrontierV1, SessionRefreshProgressV1, - SessionRefreshStore, SessionTemporalProjectionBatchV1, + SessionRefreshReceiptRequestV1, SessionRefreshStore, SessionRefreshTerminalStateV1, + SessionTemporalProjectionBatchV1, }; use super::history::{ @@ -16,10 +18,13 @@ use super::history::{ }; use super::projector::{ CanonicalSessionTemporalProjector, SessionTemporalRefreshEffect, SessionTemporalRefreshPolicy, - SessionTemporalRefreshProjectionFuture, SessionTemporalRefreshProjector, zero_refresh_coverage, + SessionTemporalRefreshProjectionFuture, SessionTemporalRefreshProjector, + SessionTemporalRefreshProjectorError, zero_refresh_coverage, }; use super::registry::SessionTemporalRefreshPassReport; -use super::wake::{SessionTemporalRefreshRetryClass, SessionTemporalRefreshWakeState}; +use super::wake::{ + SessionTemporalRefreshRetryClass, SessionTemporalRefreshWakeState, enabled_idle_notification, +}; use super::worker::{ apply_refresh_effect, run_session_temporal_refresh_pass, run_session_temporal_refresh_scheduler, }; @@ -216,9 +221,9 @@ async fn a_wake_served_by_the_running_pass_joins_that_pass() { history.entered.notified().await; let handle = state.handle(); - let join = handle.wake_and_wait_until_idle(Duration::from_secs(5)); + let join = handle.wake_and_wait_for_projection(Duration::from_secs(5)); tokio::pin!(join); - // The first poll records the pass count and delivers the wake. + // The first poll registers the join and delivers the wake. assert!( std::future::poll_fn(|cx| std::task::Poll::Ready(join.as_mut().poll(cx))) .await @@ -233,3 +238,273 @@ async fn a_wake_served_by_the_running_pass_joins_that_pass() { state.cancel(); worker.await.expect("refresh worker exits on cancellation"); } + +/// Every history window advances but waits for the test before returning. +/// This leaves the next window running after the preceding projection drains. +struct ContinuingHistory { + entered: tokio::sync::Notify, + release: tokio::sync::Notify, +} + +impl SessionHistoricalIngestor for ContinuingHistory { + fn run_pass(&self) -> SessionHistoricalIngestPass<'_> { + Box::pin(async move { + self.entered.notify_one(); + self.release.notified().await; + SessionHistoricalIngestOutcome::Pending { + made_progress: true, + } + }) + } + + fn capacity_release(&self) -> SessionHistoricalCapacityRelease { + SessionHistoricalCapacityRelease::new(Vec::new()) + } + + fn cancel(&self) { + self.release.notify_one(); + } +} + +#[tokio::test] +async fn a_published_wake_does_not_wait_for_the_next_history_window() { + let harness = RegisteredGlobalDbHarness::open("refresh-join-before-next-history").await; + let state = Arc::new(SessionTemporalRefreshWakeState::default()); + let history = Arc::new(ContinuingHistory { + entered: tokio::sync::Notify::new(), + release: tokio::sync::Notify::new(), + }); + let ingestor: SharedSessionHistoricalIngestor = history.clone(); + state.wake_history(); + let worker = tokio::spawn(run_session_temporal_refresh_scheduler( + harness.registered.clone(), + Arc::clone(&state), + Arc::new(CanonicalSessionTemporalProjector), + Arc::new(std::sync::RwLock::new(Some(ingestor))), + Arc::new(tokio::sync::Semaphore::new(1)), + SessionTemporalRefreshPolicy::default(), + )); + history.entered.notified().await; + + let handle = state.handle(); + let join = handle.wake_and_wait_for_projection(Duration::from_secs(1)); + tokio::pin!(join); + assert!( + std::future::poll_fn(|cx| std::task::Poll::Ready(join.as_mut().poll(cx))) + .await + .is_pending() + ); + history.release.notify_one(); + history.entered.notified().await; + + assert!( + join.await, + "the published wake must join while the next history window is still running" + ); + state.cancel(); + history.release.notify_one(); + worker.await.expect("refresh worker exits on cancellation"); +} + +#[tokio::test] +async fn a_live_wake_publishes_before_waiting_for_summary_admission() { + let harness = RegisteredGlobalDbHarness::open("refresh-join-before-summary-admission").await; + let store = SessionTemporalStore::new(harness.registered.as_ref()); + let begun = store + .begin_or_join_session_refresh(SessionRefreshBeginOrJoinRequestV1::new( + SessionId::new("refresh-before-summary").expect("session id"), + SessionRefreshFrontierV1::new(0, 0).expect("empty frontier"), + )) + .await + .expect("running refresh"); + let state = Arc::new(SessionTemporalRefreshWakeState::default()); + let handle = state.handle(); + let join = handle.wake_and_wait_for_projection(Duration::from_secs(5)); + tokio::pin!(join); + assert!( + std::future::poll_fn(|cx| std::task::Poll::Ready(join.as_mut().poll(cx))) + .await + .is_pending() + ); + let worker = tokio::spawn(run_session_temporal_refresh_scheduler( + harness.registered.clone(), + Arc::clone(&state), + Arc::new(CanonicalSessionTemporalProjector), + Arc::new(std::sync::RwLock::new(None)), + Arc::new(tokio::sync::Semaphore::new(0)), + SessionTemporalRefreshPolicy::default(), + )); + + assert!( + join.await, + "summary admission must not hold a live projection" + ); + let receipt = store + .session_refresh_receipt(SessionRefreshReceiptRequestV1::new( + begun.operation_id().clone(), + begun.session_id().clone(), + )) + .await + .expect("read terminal receipt") + .expect("joining the wake must publish a terminal receipt"); + assert_eq!(receipt.state(), SessionRefreshTerminalStateV1::Complete); + + state.cancel(); + worker.await.expect("refresh worker exits on cancellation"); +} + +#[tokio::test] +async fn a_live_wake_during_summary_admission_preserves_the_queued_turn() { + let harness = RegisteredGlobalDbHarness::open("refresh-join-during-summary-admission").await; + let state = Arc::new(SessionTemporalRefreshWakeState::default()); + let admission = Arc::new(tokio::sync::Semaphore::new(0)); + state.wake(); + let worker = tokio::spawn(run_session_temporal_refresh_scheduler( + harness.registered.clone(), + Arc::clone(&state), + Arc::new(CanonicalSessionTemporalProjector), + Arc::new(std::sync::RwLock::new(None)), + Arc::clone(&admission), + SessionTemporalRefreshPolicy::default(), + )); + tokio::time::timeout(Duration::from_secs(5), async { + loop { + let idle = enabled_idle_notification(&state); + if state.is_idle() { + break; + } + idle.await; + } + }) + .await + .expect("worker queues for summary admission before advertising idle"); + + // Register another profile behind this worker before the live wake. A + // projection must not surrender the existing FIFO turn and queue anew. + let later = admission.acquire(); + tokio::pin!(later); + assert!( + std::future::poll_fn(|cx| std::task::Poll::Ready(later.as_mut().poll(cx))) + .await + .is_pending() + ); + let store = SessionTemporalStore::new(harness.registered.as_ref()); + let begun = store + .begin_or_join_session_refresh(SessionRefreshBeginOrJoinRequestV1::new( + SessionId::new("refresh-during-summary").expect("session id"), + SessionRefreshFrontierV1::new(0, 0).expect("empty frontier"), + )) + .await + .expect("running refresh"); + assert!( + state + .handle() + .wake_and_wait_for_projection(Duration::from_secs(5)) + .await, + "a new wake must publish while summary admission has no permits" + ); + let receipt = store + .session_refresh_receipt(SessionRefreshReceiptRequestV1::new( + begun.operation_id().clone(), + begun.session_id().clone(), + )) + .await + .expect("read terminal receipt") + .expect("joining the wake must publish a terminal receipt"); + assert_eq!(receipt.state(), SessionRefreshTerminalStateV1::Complete); + assert_eq!(admission.available_permits(), 0); + + admission.add_permits(1); + assert!( + std::future::poll_fn(|cx| std::task::Poll::Ready(later.as_mut().poll(cx))) + .await + .is_pending(), + "the worker's original summary admission must retain the first turn" + ); + state.cancel(); + worker.await.expect("refresh worker exits on cancellation"); +} + +struct ControlledProjector { + reject: AtomicBool, +} + +impl SessionTemporalRefreshProjector for ControlledProjector { + fn project<'a>( + &'a self, + database: &'a RegisteredGlobalDbLeaseV1, + recovery: SessionRefreshRecoveryV1, + ) -> SessionTemporalRefreshProjectionFuture<'a> { + Box::pin(async move { + if self.reject.load(Ordering::Acquire) { + Err(SessionTemporalRefreshProjectorError::terminal( + "fixture_projection_refused", + )) + } else { + CanonicalSessionTemporalProjector + .project(database, recovery) + .await + } + }) + } +} + +#[tokio::test] +async fn projection_joins_keep_their_result_when_later_work_settles_differently() { + for reject_first in [false, true] { + let label = format!("refresh-join-result-reject-first-{reject_first}"); + let harness = RegisteredGlobalDbHarness::open(&label).await; + let store = SessionTemporalStore::new(harness.registered.as_ref()); + begin_empty_refreshes(&store, ["first-projection"]).await; + let state = Arc::new(SessionTemporalRefreshWakeState::default()); + let history = Arc::new(ContinuingHistory { + entered: tokio::sync::Notify::new(), + release: tokio::sync::Notify::new(), + }); + let ingestor: SharedSessionHistoricalIngestor = history.clone(); + let projector = Arc::new(ControlledProjector { + reject: AtomicBool::new(reject_first), + }); + state.wake_history(); + let worker = tokio::spawn(run_session_temporal_refresh_scheduler( + harness.registered.clone(), + Arc::clone(&state), + projector.clone(), + Arc::new(std::sync::RwLock::new(Some(ingestor))), + Arc::new(tokio::sync::Semaphore::new(1)), + SessionTemporalRefreshPolicy::default(), + )); + history.entered.notified().await; + + let handle = state.handle(); + let first = handle.wake_and_wait_for_projection(Duration::from_secs(5)); + tokio::pin!(first); + assert!( + std::future::poll_fn(|cx| std::task::Poll::Ready(first.as_mut().poll(cx))) + .await + .is_pending() + ); + history.release.notify_one(); + history.entered.notified().await; + + // Do not poll the first join again until a different projection has + // settled with the opposite result. Neither result may overwrite the + // one already delivered to the earlier request. + projector.reject.store(!reject_first, Ordering::Release); + begin_empty_refreshes(&store, ["second-projection"]).await; + let second = handle.wake_and_wait_for_projection(Duration::from_secs(5)); + tokio::pin!(second); + assert!( + std::future::poll_fn(|cx| std::task::Poll::Ready(second.as_mut().poll(cx))) + .await + .is_pending() + ); + history.release.notify_one(); + assert_eq!(second.await, reject_first); + assert_eq!(first.await, !reject_first); + + state.cancel(); + history.release.notify_one(); + worker.await.expect("refresh worker exits on cancellation"); + } +} diff --git a/crates/tracedecay/tests/mcp_suite/mcp_handler_test/session_refresh_cancel_test.rs b/crates/tracedecay/tests/mcp_suite/mcp_handler_test/session_refresh_cancel_test.rs index 9785b6eb53..b9bd80e133 100644 --- a/crates/tracedecay/tests/mcp_suite/mcp_handler_test/session_refresh_cancel_test.rs +++ b/crates/tracedecay/tests/mcp_suite/mcp_handler_test/session_refresh_cancel_test.rs @@ -46,7 +46,7 @@ impl SessionTemporalRefreshWakePort for AcceptedIdleSessionRefreshWake { false } - fn wake_and_wait_until_idle( + fn wake_and_wait_for_projection( &self, _timeout: Duration, ) -> tracedecay_contracts::SessionTemporalRefreshWakeFuture<'_> { diff --git a/crates/tracedecay/tests/session_suite/session_runtime/temporal_refresh.rs b/crates/tracedecay/tests/session_suite/session_runtime/temporal_refresh.rs index aade4e52f4..0147039cc9 100644 --- a/crates/tracedecay/tests/session_suite/session_runtime/temporal_refresh.rs +++ b/crates/tracedecay/tests/session_suite/session_runtime/temporal_refresh.rs @@ -1490,7 +1490,10 @@ async fn project_rekey_retires_old_owner_before_rebinding_wake() { .rekey_project(®istry, &old_owner, new_owner.clone()) .await; // The replacement worker owns the dirty flag; join its actual pass. - assert!(wake.wake_and_wait_until_idle(Duration::from_secs(1)).await); + assert!( + wake.wake_and_wait_for_projection(Duration::from_secs(1)) + .await + ); assert!(old_state.is_cancelled()); assert!(registry.project_state(&old_owner).await.is_none()); From 5b8e317217aae769becce457952ee34ba1010bbe Mon Sep 17 00:00:00 2001 From: Zack Jackson Date: Wed, 21 Oct 2026 17:38:02 -0700 Subject: [PATCH 07/15] fix(sessions): isolate live transcript publication joins --- .../src/hooks/cursor.rs | 2 +- .../tracedecay-agent-hosts/src/hooks/mod.rs | 9 +- .../src/hooks/pi_tests.rs | 1 + .../src/session_temporal_refresh.rs | 5 + .../src/session_refresh.rs | 1 + .../src/server/live_transcript_refresh.rs | 72 ++++++-- .../registry.rs | 1 + .../wake.rs | 35 +++- .../worker.rs | 167 ++++++++++++++++-- .../worker_tests.rs | 32 +++- .../src/projection.rs | 52 ++++++ crates/tracedecay/src/daemon/profile_owner.rs | 16 +- .../src/mcp/server/graph_tool_owner.rs | 6 + crates/tracedecay/src/mcp/server/requests.rs | 28 +++ .../session_refresh_cancel_test.rs | 1 + .../session_runtime/temporal_refresh.rs | 4 +- 16 files changed, 385 insertions(+), 47 deletions(-) diff --git a/crates/tracedecay-agent-hosts/src/hooks/cursor.rs b/crates/tracedecay-agent-hosts/src/hooks/cursor.rs index 3e6406be8a..ff9268f928 100644 --- a/crates/tracedecay-agent-hosts/src/hooks/cursor.rs +++ b/crates/tracedecay-agent-hosts/src/hooks/cursor.rs @@ -445,7 +445,7 @@ mod tests { "action": "ingest_transcript", "provider": "cursor", "user_scope": true, - "session_id": null, + "session_id": "cursor-budget", "event_json": event, "messages": null, "max_new_bytes": 4_096, diff --git a/crates/tracedecay-agent-hosts/src/hooks/mod.rs b/crates/tracedecay-agent-hosts/src/hooks/mod.rs index 407366d414..4522ad000a 100644 --- a/crates/tracedecay-agent-hosts/src/hooks/mod.rs +++ b/crates/tracedecay-agent-hosts/src/hooks/mod.rs @@ -742,10 +742,17 @@ pub(crate) async fn ingest_transcript_for_event( budget: Duration, telemetry: Option<&analytics::HookTimingSpan>, ) { + let parsed = match serde_json::from_str::(event_json) { + Ok(parsed) => parsed, + Err(error) => { + tracing::warn!(provider, %error, "transcript ingest event is invalid"); + return; + } + }; let args = HookRuntimeSurfaceRequestV1::IngestTranscript(HookIngestTranscriptRequestV1 { provider: provider.to_owned(), user_scope: project_root.is_none(), - session_id: None, + session_id: event_session_id(&parsed), event_json: Some(event_json.to_owned()), messages: None, max_new_bytes, diff --git a/crates/tracedecay-agent-hosts/src/hooks/pi_tests.rs b/crates/tracedecay-agent-hosts/src/hooks/pi_tests.rs index 53892ced4e..c92891c2b0 100644 --- a/crates/tracedecay-agent-hosts/src/hooks/pi_tests.rs +++ b/crates/tracedecay-agent-hosts/src/hooks/pi_tests.rs @@ -63,6 +63,7 @@ async fn pi_session_boundaries_land_their_session() { for (root, args) in &ingests { assert_eq!(root.as_deref(), Some(project_root.as_path())); assert_eq!(args["provider"], "pi"); + assert_eq!(args["session_id"], "pi-session-1"); assert_eq!(args["user_scope"], false); let event: Value = serde_json::from_str(args["event_json"].as_str().unwrap()).unwrap(); assert_eq!(event["session_id"], "pi-session-1"); diff --git a/crates/tracedecay-contracts/src/session_temporal_refresh.rs b/crates/tracedecay-contracts/src/session_temporal_refresh.rs index e62f8064e8..26cb73b7ae 100644 --- a/crates/tracedecay-contracts/src/session_temporal_refresh.rs +++ b/crates/tracedecay-contracts/src/session_temporal_refresh.rs @@ -1,6 +1,7 @@ use std::future::Future; use std::pin::Pin; use std::time::Duration; +use tracedecay_domain::SessionId; pub type SessionTemporalRefreshWakeFuture<'a> = Pin + Send + 'a>>; @@ -10,8 +11,11 @@ pub trait SessionTemporalRefreshWakePort: Send + Sync { fn is_unavailable(&self) -> bool; /// Join temporal publication caused by this wake. Background history and /// summary convergence may continue after the projection has committed. + /// A session target joins only that session; `None` joins a batch source + /// scan across the store. fn wake_and_wait_for_projection( &self, + session_id: Option, timeout: Duration, ) -> SessionTemporalRefreshWakeFuture<'_>; } @@ -31,6 +35,7 @@ impl SessionTemporalRefreshWakePort for UnavailableSessionTemporalRefreshWake { fn wake_and_wait_for_projection( &self, + _session_id: Option, _timeout: Duration, ) -> SessionTemporalRefreshWakeFuture<'_> { Box::pin(async { false }) diff --git a/crates/tracedecay-daemon-service/src/session_refresh.rs b/crates/tracedecay-daemon-service/src/session_refresh.rs index a49649a40b..f8515eeacd 100644 --- a/crates/tracedecay-daemon-service/src/session_refresh.rs +++ b/crates/tracedecay-daemon-service/src/session_refresh.rs @@ -297,6 +297,7 @@ impl SessionTemporalRefreshWakePort for FixedSessionTemporalRefreshWake { fn wake_and_wait_for_projection( &self, + _session_id: Option, _timeout: Duration, ) -> SessionTemporalRefreshWakeFuture<'_> { let accepted = self.0; diff --git a/crates/tracedecay-mcp/src/server/live_transcript_refresh.rs b/crates/tracedecay-mcp/src/server/live_transcript_refresh.rs index f61c6ff56d..33911df0da 100644 --- a/crates/tracedecay-mcp/src/server/live_transcript_refresh.rs +++ b/crates/tracedecay-mcp/src/server/live_transcript_refresh.rs @@ -2,6 +2,7 @@ use std::time::Duration; use serde_json::Value; +use tracedecay_domain::SessionId; use tracedecay_domain::errors::{Result, TraceDecayError}; use tracedecay_sessions::serving::SessionRefreshWorkerPort; @@ -54,8 +55,19 @@ pub async fn join_required_live_transcript_refresh( let Some(scope) = required_refresh_scope(tool_name, arguments) else { return Ok(LiveTranscriptRefreshJoin::NotRequired); }; - join_scope(scope, project_wake, user_wake, || TraceDecayError::Config { - message: REFRESH_UNAVAILABLE.to_owned(), + let session_id = arguments + .get("session_id") + .and_then(Value::as_str) + .ok_or_else(|| TraceDecayError::Config { + message: "transcript projection requires session_id".to_owned(), + })?; + let session_id = SessionId::new(session_id).map_err(|error| TraceDecayError::Config { + message: format!("invalid transcript session identity: {error}"), + })?; + join_scope(scope, Some(session_id), project_wake, user_wake, || { + TraceDecayError::Config { + message: REFRESH_UNAVAILABLE.to_owned(), + } }) .await } @@ -64,6 +76,7 @@ pub async fn join_required_live_transcript_refresh( /// wrote: the user store's for a `user_scope` ingest, the project's /// otherwise. pub async fn join_hook_ingest_refresh( + session_id: Option<&str>, user_scope: bool, project_wake: Option<&dyn SessionRefreshWorkerPort>, user_wake: Option<&dyn SessionRefreshWorkerPort>, @@ -73,7 +86,13 @@ pub async fn join_hook_ingest_refresh( } else { LiveTranscriptRefreshScope::Project }; - join_scope(scope, project_wake, user_wake, || { + let session_id = session_id + .map(SessionId::new) + .transpose() + .map_err(|error| TraceDecayError::Config { + message: format!("invalid transcript session identity: {error}"), + })?; + join_scope(scope, session_id, project_wake, user_wake, || { TraceDecayError::hook_runtime_with_status( "temporal_refresh_unavailable", true, @@ -87,6 +106,7 @@ pub async fn join_hook_ingest_refresh( async fn join_scope( scope: LiveTranscriptRefreshScope, + session_id: Option, project_wake: Option<&dyn SessionRefreshWorkerPort>, user_wake: Option<&dyn SessionRefreshWorkerPort>, unavailable: impl Fn() -> TraceDecayError, @@ -97,7 +117,7 @@ async fn join_scope( } .ok_or_else(&unavailable)?; if wake - .wake_and_wait_for_projection(LIVE_TRANSCRIPT_REFRESH_DEADLINE) + .wake_and_wait_for_projection(session_id, LIVE_TRANSCRIPT_REFRESH_DEADLINE) .await { Ok(LiveTranscriptRefreshJoin::PublicationJoined) @@ -108,11 +128,13 @@ async fn join_scope( #[cfg(test)] mod tests { + use std::sync::Mutex; use std::sync::atomic::{AtomicBool, Ordering}; use std::time::Duration; use serde_json::json; use tracedecay_contracts::{SessionTemporalRefreshWakeFuture, SessionTemporalRefreshWakePort}; + use tracedecay_domain::SessionId; use tracedecay_sessions::serving::{ SessionConvergenceStatus, SessionProjectionServingState, SessionProjectionServingStatus, SessionProjectionServingStatusPort, @@ -125,12 +147,14 @@ mod tests { /// Refresh owner whose publication is the state after `wake_and_wait`. struct PublishingRefresh { published: AtomicBool, + session_id: Mutex>, } impl PublishingRefresh { fn idle() -> Self { Self { published: AtomicBool::new(false), + session_id: Mutex::new(None), } } @@ -151,8 +175,10 @@ mod tests { fn wake_and_wait_for_projection( &self, + session_id: Option, _timeout: Duration, ) -> SessionTemporalRefreshWakeFuture<'_> { + *self.session_id.lock().unwrap() = session_id; let published = self.wake(); Box::pin(async move { published }) } @@ -174,6 +200,7 @@ mod tests { #[tokio::test] async fn completed_hook_ingest_fails_when_its_refresh_owner_is_unavailable() { let error = super::join_hook_ingest_refresh( + Some("test-session"), false, Some(&UnavailableSessionTemporalRefreshWake), None, @@ -197,9 +224,10 @@ mod tests { #[tokio::test] async fn user_scope_never_falls_back_to_the_project_refresh_owner() { let project = PublishingRefresh::idle(); - let error = super::join_hook_ingest_refresh(true, Some(&project), None) - .await - .expect_err("user ingest must require the user refresh owner"); + let error = + super::join_hook_ingest_refresh(Some("test-session"), true, Some(&project), None) + .await + .expect_err("user ingest must require the user refresh owner"); assert_eq!( error.hook_runtime_context().map(|context| context.0), @@ -214,7 +242,7 @@ mod tests { #[tokio::test] async fn project_ingest_does_not_publish_through_the_user_refresh_owner() { let user = PublishingRefresh::idle(); - let error = super::join_hook_ingest_refresh(false, None, Some(&user)) + let error = super::join_hook_ingest_refresh(Some("test-session"), false, None, Some(&user)) .await .expect_err("project ingest must require the project refresh owner"); @@ -232,7 +260,7 @@ mod tests { async fn hook_ingest_joins_the_project_refresh_owner() { let project = PublishingRefresh::idle(); let user = PublishingRefresh::idle(); - super::join_hook_ingest_refresh(false, Some(&project), Some(&user)) + super::join_hook_ingest_refresh(Some("test-session"), false, Some(&project), Some(&user)) .await .expect("project hook ingest must join its refresh owner"); @@ -252,7 +280,7 @@ mod tests { let user = PublishingRefresh::idle(); let joined = super::join_required_live_transcript_refresh( "tracedecay_lcm_preflight", - &json!({"transcript_projection": true, "storage_scope": "user"}), + &json!({"transcript_projection": true, "storage_scope": "user", "session_id": "test-session"}), Some(&project), Some(&user), ) @@ -261,6 +289,30 @@ mod tests { assert_eq!(joined, LiveTranscriptRefreshJoin::PublicationJoined); assert!(user.published()); + assert_eq!( + user.session_id.lock().unwrap().as_ref().unwrap().as_str(), + "test-session" + ); + assert!(!project.published()); + } + #[tokio::test] + async fn invalid_session_identity_never_wakes_a_refresh_owner() { + let project = PublishingRefresh::idle(); + assert!( + super::join_hook_ingest_refresh(Some(""), false, Some(&project), None) + .await + .is_err() + ); + assert!( + super::join_required_live_transcript_refresh( + "tracedecay_lcm_preflight", + &json!({"transcript_projection": true}), + Some(&project), + None, + ) + .await + .is_err() + ); assert!(!project.published()); } } diff --git a/crates/tracedecay-session-runtime/src/session_temporal_refresh_scheduler/registry.rs b/crates/tracedecay-session-runtime/src/session_temporal_refresh_scheduler/registry.rs index 7a58d40ece..579633ae7d 100644 --- a/crates/tracedecay-session-runtime/src/session_temporal_refresh_scheduler/registry.rs +++ b/crates/tracedecay-session-runtime/src/session_temporal_refresh_scheduler/registry.rs @@ -41,6 +41,7 @@ macro_rules! define_pass_report { $visibility terminal_errors: usize, $visibility deadline_errors: usize, $visibility saturated: bool, + $visibility discovery_advanced: bool, $visibility backlog: Option, $visibility retry_class: Option, $visibility last_error: Option, diff --git a/crates/tracedecay-session-runtime/src/session_temporal_refresh_scheduler/wake.rs b/crates/tracedecay-session-runtime/src/session_temporal_refresh_scheduler/wake.rs index 5cf49c106e..a63256c633 100644 --- a/crates/tracedecay-session-runtime/src/session_temporal_refresh_scheduler/wake.rs +++ b/crates/tracedecay-session-runtime/src/session_temporal_refresh_scheduler/wake.rs @@ -2,6 +2,7 @@ use std::collections::{HashMap, HashSet, VecDeque}; use std::sync::Arc; use std::sync::PoisonError; use std::sync::atomic::{AtomicBool, Ordering}; +use tracedecay_domain::SessionId; use tracedecay_runtime_core::native_runtime; use tracedecay_store::SessionRefreshBeginOrJoinRequestV1; use tracedecay_temporal_query::execution::ExecutionControl; @@ -18,6 +19,11 @@ use tracedecay_sessions::serving::{ SessionProjectionWorkerRetryClass, }; +pub(super) struct ProjectionJoin { + pub(super) session_id: Option, + pub(super) completion: tokio::sync::oneshot::Sender, +} + #[derive(Clone, Copy, Debug, Eq, PartialEq)] pub enum SessionTemporalRefreshRetryClass { Storage, @@ -138,7 +144,7 @@ macro_rules! define_wake_state { /// is still unpublished. The next history window waits. pub(super) history_after_projection: AtomicBool, pub(super) pass_count: std::sync::atomic::AtomicUsize, - projection_joins: std::sync::Mutex>>, + projection_joins: std::sync::Mutex>, pub(super) wake: tokio::sync::Notify, pub(super) idle: tokio::sync::Notify, pub(super) cancelled: AtomicBool, @@ -342,7 +348,10 @@ impl SessionTemporalRefreshWakeState { self.wake.notify_one(); } - fn request_projection_join(&self) -> Option> { + fn request_projection_join( + &self, + session_id: Option, + ) -> Option> { let mut joins = self .projection_joins .lock() @@ -352,16 +361,19 @@ impl SessionTemporalRefreshWakeState { } // Expired callers retain no obligation. The worker owns only live // joins, and each result is delivered once even if a later pass fails. - joins.retain(|join| !join.is_closed()); + joins.retain(|join| !join.completion.is_closed()); let (completion, receiver) = tokio::sync::oneshot::channel(); - joins.push(completion); + joins.push(ProjectionJoin { + session_id, + completion, + }); self.wake(); Some(receiver) } /// Take the joins covered by the projection about to start. A join that /// arrives during that projection remains queued for the next pass. - pub(super) fn take_projection_joins(&self) -> Vec> { + pub(super) fn take_projection_joins(&self) -> Vec { std::mem::take( &mut *self .projection_joins @@ -698,7 +710,7 @@ impl SessionTemporalRefreshWakeState { self.clear_worker_instrumentation(); self.mark_stopped(); for join in self.take_projection_joins() { - let _ = join.send(false); + let _ = join.completion.send(false); } self.cancellation.notify_waiters(); self.wake.notify_waiters(); @@ -983,11 +995,15 @@ impl SessionTemporalRefreshWake { /// empty/unavailable semantics. Historical discovery and derived summaries /// may continue after that publication; their idle/admission state is not /// evidence that a temporal generation has committed. - pub async fn wake_and_wait_for_projection(&self, timeout: std::time::Duration) -> bool { + pub async fn wake_and_wait_for_projection( + &self, + session_id: Option, + timeout: std::time::Duration, + ) -> bool { let Some(state) = self.target() else { return false; }; - let Some(completion) = state.request_projection_join() else { + let Some(completion) = state.request_projection_join(session_id) else { return false; }; matches!( @@ -1042,10 +1058,11 @@ impl tracedecay_contracts::SessionTemporalRefreshWakePort for SessionTemporalRef fn wake_and_wait_for_projection( &self, + session_id: Option, timeout: std::time::Duration, ) -> tracedecay_contracts::SessionTemporalRefreshWakeFuture<'_> { Box::pin(SessionTemporalRefreshWake::wake_and_wait_for_projection( - self, timeout, + self, session_id, timeout, )) } } diff --git a/crates/tracedecay-session-runtime/src/session_temporal_refresh_scheduler/worker.rs b/crates/tracedecay-session-runtime/src/session_temporal_refresh_scheduler/worker.rs index 013fd9143d..b13fed3784 100644 --- a/crates/tracedecay-session-runtime/src/session_temporal_refresh_scheduler/worker.rs +++ b/crates/tracedecay-session-runtime/src/session_temporal_refresh_scheduler/worker.rs @@ -25,8 +25,8 @@ use super::projector::{ }; use super::registry::{SessionTemporalRefreshPassReport, session_refresh_retry_delay}; use super::wake::{ - PendingBeginRequestGuard, RecoverySelectionGuard, SessionTemporalRefreshRetryClass, - SessionTemporalRefreshWakeState, TerminalAttemptGuard, + PendingBeginRequestGuard, ProjectionJoin, RecoverySelectionGuard, + SessionTemporalRefreshRetryClass, SessionTemporalRefreshWakeState, TerminalAttemptGuard, }; use tracedecay_global_db::{RegisteredGlobalDb, RegisteredGlobalDbLeaseV1}; use tracedecay_runtime_core::db::engine::Error as EngineError; @@ -235,7 +235,7 @@ pub(super) async fn run_session_temporal_refresh_scheduler( let mut history_priority_passes = 0u32; let mut history_release = None; let mut granted_history_admission = None; - let mut projection_joins: Vec> = Vec::new(); + let mut projection_joins: Vec = Vec::new(); let live_projection = LiveProjectionDuringSummary { database: &database, state: &state, @@ -390,7 +390,8 @@ pub(super) async fn run_session_temporal_refresh_scheduler( convergence_admission, LcmConvergenceAdmission::Admitted(LcmConvergencePage::Full) ) && projection_still_unpublished(&report) - && projection_published_work(&report) + && (projection_published_work(&report) + || (!projection_joins.is_empty() && report.discovery_advanced)) { convergence_admission = LcmConvergenceAdmission::Deferred; } @@ -562,7 +563,8 @@ pub(super) async fn run_session_temporal_refresh_scheduler( match window_follow_up( history_continuation(history_outcome), projection_still_unpublished(&report), - projection_published_work(&report), + projection_published_work(&report) + || (!projection_joins.is_empty() && report.discovery_advanced), holding_publication, ) { // The admitted window is not searchable until this backlog @@ -791,9 +793,9 @@ async fn session_projection_refresh( state: &Arc, projector: &dyn SessionTemporalRefreshProjector, policy: SessionTemporalRefreshPolicy, - joins: &mut Vec>, + joins: &mut Vec, ) -> SessionTemporalRefreshPassReport { - joins.retain(|join| !join.is_closed()); + joins.retain(|join| !join.completion.is_closed()); joins.extend(state.take_projection_joins()); let report = run_session_temporal_refresh_pass(database, state, projector, policy).await; if state.cancelled.load(Ordering::Acquire) { @@ -813,11 +815,35 @@ async fn session_projection_refresh( } else { None }; - if let Some(published) = result { - for join in joins.drain(..) { - let _ = join.send(published); + let access = SessionTemporalAccess::new(database.as_ref()); + let mut pending = Vec::new(); + for join in joins.drain(..) { + let published = match &join.session_id { + Some(session_id) => match access + .session_projection_publication_result(session_id) + .await + { + Ok(published) => published, + Err(error) => { + tracing::warn!(%error, "could not verify session projection publication"); + Some(false) + } + }, + None => result, + }; + let published = if state.cancelled.load(Ordering::Acquire) { + Some(false) + } else { + published + }; + match published { + Some(published) => { + let _ = join.completion.send(published); + } + None => pending.push(join), } } + *joins = pending; report } @@ -835,7 +861,7 @@ impl LiveProjectionDuringSummary<'_> { async fn wait_for( &self, stage: F, - joins: &mut Vec>, + joins: &mut Vec, report: &mut SessionTemporalRefreshPassReport, idle_between_projections: bool, ) -> Option @@ -1117,6 +1143,7 @@ pub async fn begin_admitted_session_refreshes<'a>( } }; let (requests, next_cursor, has_more) = page.into_parts(); + report.discovery_advanced |= next_cursor != cursor; for request in requests.into_iter().rev() { if !state.suppresses_discovered_request(&request) { state.requeue_request(request); @@ -2077,7 +2104,9 @@ pub async fn run_session_temporal_refresh_pass( #[cfg(test)] mod tests { use super::*; - use crate::session_temporal_refresh_scheduler::projector::CanonicalSessionTemporalProjector; + use crate::session_temporal_refresh_scheduler::projector::{ + CanonicalSessionTemporalProjector, SessionTemporalRefreshProjectionFuture, + }; use tracedecay_domain::SessionId; use tracedecay_global_db::tests::harness::RegisteredGlobalDbHarness; use tracedecay_runtime_core::db::engine::params; @@ -2130,7 +2159,7 @@ mod tests { entered.notified().await; let handle = state.handle(); - let join = handle.wake_and_wait_for_projection(Duration::from_secs(5)); + let join = handle.wake_and_wait_for_projection(None, Duration::from_secs(5)); tokio::pin!(join); assert!( std::future::poll_fn(|cx| std::task::Poll::Ready(join.as_mut().poll(cx))) @@ -2701,4 +2730,116 @@ mod tests { row.get::(1).expect("to store id"), )) } + struct SessionSelectiveProjector; + + impl SessionTemporalRefreshProjector for SessionSelectiveProjector { + fn project<'a>( + &'a self, + database: &'a RegisteredGlobalDbLeaseV1, + recovery: SessionRefreshRecoveryV1, + ) -> SessionTemporalRefreshProjectionFuture<'a> { + Box::pin(async move { + if recovery.session_id().as_str() == "rejected-session" { + Err(SessionTemporalRefreshProjectorError::terminal( + "fixture_projection_refused", + )) + } else { + CanonicalSessionTemporalProjector + .project(database, recovery) + .await + } + }) + } + } + + #[tokio::test] + async fn a_published_session_join_survives_an_unrelated_terminal_failure() { + let harness = RegisteredGlobalDbHarness::open("refresh-session-isolated-publication").await; + let store = SessionTemporalStore::new(harness.registered.as_ref()); + for session in ["published-session", "rejected-session"] { + store + .begin_or_join_session_refresh(SessionRefreshBeginOrJoinRequestV1::new( + SessionId::new(session).unwrap(), + SessionRefreshFrontierV1::new(0, 0).unwrap(), + )) + .await + .unwrap(); + } + let state = Arc::new(SessionTemporalRefreshWakeState::default()); + let handle = state.handle(); + let published_id = SessionId::new("published-session").unwrap(); + let rejected_id = SessionId::new("rejected-session").unwrap(); + let published_operation = store + .session_refresh_recovery(&published_id) + .await + .unwrap() + .unwrap(); + let rejected_operation = store + .session_refresh_recovery(&rejected_id) + .await + .unwrap() + .unwrap(); + let published = + handle.wake_and_wait_for_projection(Some(published_id.clone()), Duration::from_secs(5)); + let rejected = + handle.wake_and_wait_for_projection(Some(rejected_id.clone()), Duration::from_secs(5)); + tokio::pin!(published, rejected); + assert!( + std::future::poll_fn(|cx| std::task::Poll::Ready(published.as_mut().poll(cx))) + .await + .is_pending() + ); + assert!( + std::future::poll_fn(|cx| std::task::Poll::Ready(rejected.as_mut().poll(cx))) + .await + .is_pending() + ); + let mut joins = Vec::new(); + let first = session_projection_refresh( + &harness.registered, + &state, + &SessionSelectiveProjector, + SessionTemporalRefreshPolicy::default(), + &mut joins, + ) + .await; + assert_eq!(first.failed, 1); + assert_eq!(first.projected_batches, 1); + assert!( + !rejected.await, + "the caller's own terminal failure must remain unavailable" + ); + assert!( + std::future::poll_fn(|cx| std::task::Poll::Ready(published.as_mut().poll(cx))) + .await + .is_pending() + ); + let second = session_projection_refresh( + &harness.registered, + &state, + &SessionSelectiveProjector, + SessionTemporalRefreshPolicy::default(), + &mut joins, + ) + .await; + assert_eq!(second.completed, 1); + assert!( + published.await, + "an unrelated failure must not reject the committed session" + ); + for (operation, expected) in [ + (published_operation, SessionRefreshTerminalStateV1::Complete), + (rejected_operation, SessionRefreshTerminalStateV1::Failed), + ] { + let receipt = store + .session_refresh_receipt(SessionRefreshReceiptRequestV1::new( + operation.operation_id().clone(), + operation.session_id().clone(), + )) + .await + .unwrap() + .unwrap(); + assert_eq!(receipt.state(), expected); + } + } } diff --git a/crates/tracedecay-session-runtime/src/session_temporal_refresh_scheduler/worker_tests.rs b/crates/tracedecay-session-runtime/src/session_temporal_refresh_scheduler/worker_tests.rs index 14454612e1..22291400cb 100644 --- a/crates/tracedecay-session-runtime/src/session_temporal_refresh_scheduler/worker_tests.rs +++ b/crates/tracedecay-session-runtime/src/session_temporal_refresh_scheduler/worker_tests.rs @@ -221,7 +221,7 @@ async fn a_wake_served_by_the_running_pass_joins_that_pass() { history.entered.notified().await; let handle = state.handle(); - let join = handle.wake_and_wait_for_projection(Duration::from_secs(5)); + let join = handle.wake_and_wait_for_projection(None, Duration::from_secs(5)); tokio::pin!(join); // The first poll registers the join and delivers the wake. assert!( @@ -287,7 +287,7 @@ async fn a_published_wake_does_not_wait_for_the_next_history_window() { history.entered.notified().await; let handle = state.handle(); - let join = handle.wake_and_wait_for_projection(Duration::from_secs(1)); + let join = handle.wake_and_wait_for_projection(None, Duration::from_secs(1)); tokio::pin!(join); assert!( std::future::poll_fn(|cx| std::task::Poll::Ready(join.as_mut().poll(cx))) @@ -319,7 +319,7 @@ async fn a_live_wake_publishes_before_waiting_for_summary_admission() { .expect("running refresh"); let state = Arc::new(SessionTemporalRefreshWakeState::default()); let handle = state.handle(); - let join = handle.wake_and_wait_for_projection(Duration::from_secs(5)); + let join = handle.wake_and_wait_for_projection(None, Duration::from_secs(5)); tokio::pin!(join); assert!( std::future::poll_fn(|cx| std::task::Poll::Ready(join.as_mut().poll(cx))) @@ -337,7 +337,16 @@ async fn a_live_wake_publishes_before_waiting_for_summary_admission() { assert!( join.await, - "summary admission must not hold a live projection" + "summary admission must not hold a live projection: status={:?}, passes={}, receipt={:?}", + state.handle().status(), + state.pass_count.load(Ordering::Acquire), + store + .session_refresh_receipt(SessionRefreshReceiptRequestV1::new( + begun.operation_id().clone(), + begun.session_id().clone(), + )) + .await + .unwrap() ); let receipt = store .session_refresh_receipt(SessionRefreshReceiptRequestV1::new( @@ -399,7 +408,7 @@ async fn a_live_wake_during_summary_admission_preserves_the_queued_turn() { assert!( state .handle() - .wake_and_wait_for_projection(Duration::from_secs(5)) + .wake_and_wait_for_projection(None, Duration::from_secs(5)) .await, "a new wake must publish while summary admission has no permits" ); @@ -477,7 +486,7 @@ async fn projection_joins_keep_their_result_when_later_work_settles_differently( history.entered.notified().await; let handle = state.handle(); - let first = handle.wake_and_wait_for_projection(Duration::from_secs(5)); + let first = handle.wake_and_wait_for_projection(None, Duration::from_secs(5)); tokio::pin!(first); assert!( std::future::poll_fn(|cx| std::task::Poll::Ready(first.as_mut().poll(cx))) @@ -492,7 +501,7 @@ async fn projection_joins_keep_their_result_when_later_work_settles_differently( // one already delivered to the earlier request. projector.reject.store(!reject_first, Ordering::Release); begin_empty_refreshes(&store, ["second-projection"]).await; - let second = handle.wake_and_wait_for_projection(Duration::from_secs(5)); + let second = handle.wake_and_wait_for_projection(None, Duration::from_secs(5)); tokio::pin!(second); assert!( std::future::poll_fn(|cx| std::task::Poll::Ready(second.as_mut().poll(cx))) @@ -500,7 +509,14 @@ async fn projection_joins_keep_their_result_when_later_work_settles_differently( .is_pending() ); history.release.notify_one(); - assert_eq!(second.await, reject_first); + assert_eq!( + second.await, + reject_first, + "status={:?}, passes={}, running={:?}", + state.handle().status(), + state.pass_count.load(Ordering::Acquire), + store.running_session_refreshes().await.unwrap() + ); assert_eq!(first.await, !reject_first); state.cancel(); diff --git a/crates/tracedecay-session-temporal-store/src/projection.rs b/crates/tracedecay-session-temporal-store/src/projection.rs index 53165af2aa..d6f66f544a 100644 --- a/crates/tracedecay-session-temporal-store/src/projection.rs +++ b/crates/tracedecay-session-temporal-store/src/projection.rs @@ -402,6 +402,58 @@ async fn skip_trailing_history_effects( } impl SessionTemporalAccess<'_, D> { + /// Verify one live writer's publication without borrowing other sessions' + /// pass outcomes. Pending source effects or a running refresh remain pending; + /// terminal generation failures remain failures even beside a successful pass. + pub async fn session_projection_publication_result( + &self, + session_id: &SessionId, + ) -> SessionStoreResult> { + let snapshot = self + .read_snapshot() + .await + .map_err(|error| storage(DISCOVER_REFRESH, error))?; + let discovered = discover_session(&snapshot, session_id.as_str()).await?; + if matches!(discovered, DiscoveredSession::Running) { + return Ok(None); + } + let mut rows = snapshot + .query( + "SELECT generation.state, EXISTS ( + SELECT 1 FROM session_relation_receipts AS receipt + WHERE receipt.session_id = generation.session_id + AND receipt.generation = generation.generation + ) + FROM session_temporal_generations AS generation + WHERE generation.session_id = ?1 + ORDER BY generation.generation DESC LIMIT 1", + params![session_id.as_str()], + ) + .await + .map_err(|error| storage(DISCOVER_REFRESH, error))?; + let generation = rows + .next() + .await + .map_err(|error| storage(DISCOVER_REFRESH, error))?; + if let Some(row) = generation { + let state: String = row + .get(0) + .map_err(|error| storage(DISCOVER_REFRESH, error))?; + if matches!(state.as_str(), "failed" | "cancelled") { + return Ok(Some(false)); + } + let relations: i64 = row + .get(1) + .map_err(|error| storage(DISCOVER_REFRESH, error))?; + return Ok((matches!(discovered, DiscoveredSession::Current) + && state == "active" + && relations != 0) + .then_some(true)); + } + // A no-op ingest can have no effect or generation to publish. + Ok(matches!(discovered, DiscoveredSession::Current).then_some(true)) + } + /// Discovers sessions that need temporal projection: sessions whose /// output-producing effects committed past `cursor` left them behind their /// projection frontier, plus active generations that lack an applied diff --git a/crates/tracedecay/src/daemon/profile_owner.rs b/crates/tracedecay/src/daemon/profile_owner.rs index f625653e85..5dba39c4e4 100644 --- a/crates/tracedecay/src/daemon/profile_owner.rs +++ b/crates/tracedecay/src/daemon/profile_owner.rs @@ -23,8 +23,8 @@ use serde_json::{Map, Value}; use tracedecay_contracts::graph_tool::{GraphToolCompletionV1, GraphToolResultV1}; use tracedecay_contracts::retrieval::{ AdminCliSurfaceRequestV1, AdminProjectResultV1, AdminProjectSurfaceRequestV1, - AutomationReconcileScope, HookRuntimeResultV1, ProfileAutomationReconcileReport, - UncachedProjectReconcileOutcome, + AutomationReconcileScope, HookRuntimeResultV1, HookRuntimeSurfaceRequestV1, + ProfileAutomationReconcileReport, UncachedProjectReconcileOutcome, }; use tracedecay_contracts::{ ApplicationProblem, CancellationContext, CancellationStage, CancellationState, Deadline, @@ -197,6 +197,10 @@ async fn profile_hook_runtime( user_session_db.clone(), )) .await; + let transcript_session_id = match &request { + HookRuntimeSurfaceRequestV1::IngestTranscript(ingest) => ingest.session_id.clone(), + _ => None, + }; let result = Box::pin(hook_runtime::compute_projectless_hook_runtime( request, profile_identity.profile_root(), @@ -208,7 +212,13 @@ async fn profile_hook_runtime( )) .await?; if let HookRuntimeResultV1::IngestTranscript(ingest) = &result { - join_hook_ingest_refresh(ingest.user_scope, None, Some(&refresh_wake)).await?; + join_hook_ingest_refresh( + transcript_session_id.as_deref(), + ingest.user_scope, + None, + Some(&refresh_wake), + ) + .await?; } else { refresh_wake.wake(); } diff --git a/crates/tracedecay/src/mcp/server/graph_tool_owner.rs b/crates/tracedecay/src/mcp/server/graph_tool_owner.rs index 912e89af5a..0c4b383571 100644 --- a/crates/tracedecay/src/mcp/server/graph_tool_owner.rs +++ b/crates/tracedecay/src/mcp/server/graph_tool_owner.rs @@ -169,6 +169,11 @@ impl McpServer { application_cancellation: Some(invocation.cancellation), ..self.tool_call_registry_options(session_sync_service.as_deref()) }; + let transcript_session_id = invocation + .arguments + .get("session_id") + .and_then(serde_json::Value::as_str) + .map(str::to_owned); let computed = compute_graph_tool_for_owner( cg.as_ref(), invocation.operation, @@ -190,6 +195,7 @@ impl McpServer { // The ingest wrote through this server's session stores; answer // only once their refresh owner has published it. join_hook_ingest_refresh( + transcript_session_id.as_deref(), ingest.user_scope, self.project_session_refresh_wake.as_deref(), self.user_session_refresh_wake.as_deref(), diff --git a/crates/tracedecay/src/mcp/server/requests.rs b/crates/tracedecay/src/mcp/server/requests.rs index af60e2c63c..04abc7135f 100644 --- a/crates/tracedecay/src/mcp/server/requests.rs +++ b/crates/tracedecay/src/mcp/server/requests.rs @@ -87,6 +87,7 @@ fn analytics_arguments_snapshot(tool_name: &str, arguments: &Value) -> Value { const ANALYTICS_ARGUMENT_KEYS: &[&str] = &[ "action", "transcript_projection", + "session_id", "user_scope", "storage_scope", "include_memory", @@ -1786,6 +1787,33 @@ mod tool_call_preparation_tests { .expect("object arguments") } + #[test] + fn preflight_preparation_retains_its_publication_session_without_message_bodies() { + let params = json!({ + "name": "tracedecay_lcm_preflight", + "arguments": { + "session_id": "live-session", + "transcript_projection": true, + "storage_scope": "user", + "messages": [{"role": "user", "content": "private transcript"}], + }, + }); + let prepared = + McpServer::prepare_tool_call(&json!(7), ToolCallParams::Raw(Some(¶ms))).unwrap(); + assert_eq!( + prepared.analytics_arguments, + json!({ + "session_id": "live-session", + "transcript_projection": true, + "storage_scope": "user", + }) + ); + assert_eq!( + prepared.arguments["messages"], + params["arguments"]["messages"] + ); + } + /// The typed `rmcp` params and the equivalent raw JSON-RPC params must /// prepare byte-identical dispatch inputs. This is the contract that lets /// the typed transport skip the JSON bridge without forking the catalog. diff --git a/crates/tracedecay/tests/mcp_suite/mcp_handler_test/session_refresh_cancel_test.rs b/crates/tracedecay/tests/mcp_suite/mcp_handler_test/session_refresh_cancel_test.rs index b9bd80e133..03177b9836 100644 --- a/crates/tracedecay/tests/mcp_suite/mcp_handler_test/session_refresh_cancel_test.rs +++ b/crates/tracedecay/tests/mcp_suite/mcp_handler_test/session_refresh_cancel_test.rs @@ -48,6 +48,7 @@ impl SessionTemporalRefreshWakePort for AcceptedIdleSessionRefreshWake { fn wake_and_wait_for_projection( &self, + _session_id: Option, _timeout: Duration, ) -> tracedecay_contracts::SessionTemporalRefreshWakeFuture<'_> { Box::pin(async { true }) diff --git a/crates/tracedecay/tests/session_suite/session_runtime/temporal_refresh.rs b/crates/tracedecay/tests/session_suite/session_runtime/temporal_refresh.rs index 0147039cc9..cf2c554072 100644 --- a/crates/tracedecay/tests/session_suite/session_runtime/temporal_refresh.rs +++ b/crates/tracedecay/tests/session_suite/session_runtime/temporal_refresh.rs @@ -1081,7 +1081,7 @@ async fn hook_ingest_join_runs_the_bound_refresh_worker() { let wake = authority.ensure_profile(®istry).await; let before = registry.profile_pass_count(db.db_path()).await; - tracedecay_mcp::server::join_hook_ingest_refresh(false, Some(&wake), None) + tracedecay_mcp::server::join_hook_ingest_refresh(None, false, Some(&wake), None) .await .expect("hook ingest must join the refresh worker that owns the written store"); @@ -1491,7 +1491,7 @@ async fn project_rekey_retires_old_owner_before_rebinding_wake() { .await; // The replacement worker owns the dirty flag; join its actual pass. assert!( - wake.wake_and_wait_for_projection(Duration::from_secs(1)) + wake.wake_and_wait_for_projection(None, Duration::from_secs(1)) .await ); From d41885d4514702e2784275a7a9c5f528ca1503c7 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Wed, 21 Oct 2026 17:45:37 -0700 Subject: [PATCH 08/15] fix(sessions): join the requesting session publication --- .../src/session_temporal_refresh.rs | 12 ++ .../src/server/live_transcript_refresh.rs | 107 ++++++++++-- crates/tracedecay-mcp/src/server/mod.rs | 3 +- .../registry.rs | 2 + .../wake.rs | 96 ++++++++++- .../worker.rs | 104 ++++++++++-- .../worker_tests.rs | 98 +++++++++++ .../src/lib.rs | 5 +- .../src/projection.rs | 82 +++++++++ .../src/projection/tests.rs | 157 ++++++++++++++++++ crates/tracedecay/src/daemon/profile_owner.rs | 11 +- .../src/mcp/server/graph_tool_owner.rs | 9 +- .../session_runtime/temporal_refresh.rs | 2 +- 13 files changed, 653 insertions(+), 35 deletions(-) diff --git a/crates/tracedecay-contracts/src/session_temporal_refresh.rs b/crates/tracedecay-contracts/src/session_temporal_refresh.rs index e62f8064e8..9a2b1cc10a 100644 --- a/crates/tracedecay-contracts/src/session_temporal_refresh.rs +++ b/crates/tracedecay-contracts/src/session_temporal_refresh.rs @@ -2,6 +2,8 @@ use std::future::Future; use std::pin::Pin; use std::time::Duration; +use tracedecay_domain::SessionId; + pub type SessionTemporalRefreshWakeFuture<'a> = Pin + Send + 'a>>; /// Wake and observe the session-temporal refresh worker bound to one store. @@ -14,6 +16,16 @@ pub trait SessionTemporalRefreshWakePort: Send + Sync { &self, timeout: Duration, ) -> SessionTemporalRefreshWakeFuture<'_>; + + /// Join the named session's committed temporal publication. Owners that + /// cannot prove a session frontier retain the stricter store-wide join. + fn wake_and_wait_for_session_projection<'a>( + &'a self, + _session_id: &'a SessionId, + timeout: Duration, + ) -> SessionTemporalRefreshWakeFuture<'a> { + self.wake_and_wait_for_projection(timeout) + } } /// Typed missing-worker implementation for unmounted servers and tests. diff --git a/crates/tracedecay-mcp/src/server/live_transcript_refresh.rs b/crates/tracedecay-mcp/src/server/live_transcript_refresh.rs index f61c6ff56d..1153cb05f3 100644 --- a/crates/tracedecay-mcp/src/server/live_transcript_refresh.rs +++ b/crates/tracedecay-mcp/src/server/live_transcript_refresh.rs @@ -1,7 +1,8 @@ use std::time::Duration; -use serde_json::Value; +use serde_json::{Map, Value}; +use tracedecay_domain::SessionId; use tracedecay_domain::errors::{Result, TraceDecayError}; use tracedecay_sessions::serving::SessionRefreshWorkerPort; @@ -54,8 +55,10 @@ pub async fn join_required_live_transcript_refresh( let Some(scope) = required_refresh_scope(tool_name, arguments) else { return Ok(LiveTranscriptRefreshJoin::NotRequired); }; - join_scope(scope, project_wake, user_wake, || TraceDecayError::Config { - message: REFRESH_UNAVAILABLE.to_owned(), + join_scope(scope, None, project_wake, user_wake, || { + TraceDecayError::Config { + message: REFRESH_UNAVAILABLE.to_owned(), + } }) .await } @@ -65,6 +68,7 @@ pub async fn join_required_live_transcript_refresh( /// otherwise. pub async fn join_hook_ingest_refresh( user_scope: bool, + session_id: Option<&SessionId>, project_wake: Option<&dyn SessionRefreshWorkerPort>, user_wake: Option<&dyn SessionRefreshWorkerPort>, ) -> Result<()> { @@ -73,7 +77,7 @@ pub async fn join_hook_ingest_refresh( } else { LiveTranscriptRefreshScope::Project }; - join_scope(scope, project_wake, user_wake, || { + join_scope(scope, session_id, project_wake, user_wake, || { TraceDecayError::hook_runtime_with_status( "temporal_refresh_unavailable", true, @@ -85,8 +89,33 @@ pub async fn join_hook_ingest_refresh( .map(|_| ()) } +/// Hermes inline capture preserves this exact public session identity. Source +/// sweeps may write several sessions, and other providers can normalize their +/// native IDs, so those calls keep the aggregate publication contract. +/// Capture still owns request validation and admission before the join runs. +pub fn hook_ingest_refresh_session(arguments: &Map) -> Result> { + if arguments.get("action").and_then(Value::as_str) != Some("ingest_transcript") + || arguments.get("provider").and_then(Value::as_str) != Some("hermes") + || !arguments.get("messages").is_some_and(Value::is_array) + { + return Ok(None); + } + let session_id = arguments + .get("session_id") + .and_then(Value::as_str) + .ok_or_else(|| TraceDecayError::Config { + message: "Hermes turn callback requires a session_id".to_owned(), + })?; + SessionId::new(session_id) + .map(Some) + .map_err(|error| TraceDecayError::Config { + message: format!("invalid Hermes turn session: {error}"), + }) +} + async fn join_scope( scope: LiveTranscriptRefreshScope, + session_id: Option<&SessionId>, project_wake: Option<&dyn SessionRefreshWorkerPort>, user_wake: Option<&dyn SessionRefreshWorkerPort>, unavailable: impl Fn() -> TraceDecayError, @@ -96,10 +125,17 @@ async fn join_scope( LiveTranscriptRefreshScope::User => user_wake, } .ok_or_else(&unavailable)?; - if wake - .wake_and_wait_for_projection(LIVE_TRANSCRIPT_REFRESH_DEADLINE) - .await - { + let published = match session_id { + Some(session_id) => { + wake.wake_and_wait_for_session_projection(session_id, LIVE_TRANSCRIPT_REFRESH_DEADLINE) + .await + } + None => { + wake.wake_and_wait_for_projection(LIVE_TRANSCRIPT_REFRESH_DEADLINE) + .await + } + }; + if published { Ok(LiveTranscriptRefreshJoin::PublicationJoined) } else { Err(unavailable()) @@ -108,11 +144,13 @@ async fn join_scope( #[cfg(test)] mod tests { + use std::sync::Mutex; use std::sync::atomic::{AtomicBool, Ordering}; use std::time::Duration; use serde_json::json; use tracedecay_contracts::{SessionTemporalRefreshWakeFuture, SessionTemporalRefreshWakePort}; + use tracedecay_domain::SessionId; use tracedecay_sessions::serving::{ SessionConvergenceStatus, SessionProjectionServingState, SessionProjectionServingStatus, SessionProjectionServingStatusPort, @@ -125,12 +163,14 @@ mod tests { /// Refresh owner whose publication is the state after `wake_and_wait`. struct PublishingRefresh { published: AtomicBool, + session: Mutex>, } impl PublishingRefresh { fn idle() -> Self { Self { published: AtomicBool::new(false), + session: Mutex::new(None), } } @@ -156,6 +196,16 @@ mod tests { let published = self.wake(); Box::pin(async move { published }) } + + fn wake_and_wait_for_session_projection<'a>( + &'a self, + session_id: &'a SessionId, + _timeout: Duration, + ) -> SessionTemporalRefreshWakeFuture<'a> { + *self.session.lock().expect("record session target") = Some(session_id.clone()); + let published = self.wake(); + Box::pin(async move { published }) + } } impl SessionProjectionServingStatusPort for PublishingRefresh { @@ -175,6 +225,7 @@ mod tests { async fn completed_hook_ingest_fails_when_its_refresh_owner_is_unavailable() { let error = super::join_hook_ingest_refresh( false, + None, Some(&UnavailableSessionTemporalRefreshWake), None, ) @@ -197,7 +248,7 @@ mod tests { #[tokio::test] async fn user_scope_never_falls_back_to_the_project_refresh_owner() { let project = PublishingRefresh::idle(); - let error = super::join_hook_ingest_refresh(true, Some(&project), None) + let error = super::join_hook_ingest_refresh(true, None, Some(&project), None) .await .expect_err("user ingest must require the user refresh owner"); @@ -214,7 +265,7 @@ mod tests { #[tokio::test] async fn project_ingest_does_not_publish_through_the_user_refresh_owner() { let user = PublishingRefresh::idle(); - let error = super::join_hook_ingest_refresh(false, None, Some(&user)) + let error = super::join_hook_ingest_refresh(false, None, None, Some(&user)) .await .expect_err("project ingest must require the project refresh owner"); @@ -232,7 +283,7 @@ mod tests { async fn hook_ingest_joins_the_project_refresh_owner() { let project = PublishingRefresh::idle(); let user = PublishingRefresh::idle(); - super::join_hook_ingest_refresh(false, Some(&project), Some(&user)) + super::join_hook_ingest_refresh(false, None, Some(&project), Some(&user)) .await .expect("project hook ingest must join its refresh owner"); @@ -246,6 +297,40 @@ mod tests { ); } + #[tokio::test] + async fn hermes_inline_join_names_its_session_but_source_scans_remain_aggregate() { + for (arguments, expected) in [ + ( + json!({"action": "ingest_transcript", "provider": "hermes", "session_id": "hermes-live", "messages": [{"role": "user", "content": "live turn"}]}), + Some(SessionId::new("hermes-live").expect("session id")), + ), + ( + json!({"action": "ingest_transcript", "provider": "hermes", "messages": null}), + None, + ), + ( + json!({"action": "ingest_transcript", "provider": "hermes"}), + None, + ), + ( + json!({"action": "ingest_transcript", "provider": "codex", "session_id": "native-provider-id", "messages": []}), + None, + ), + ] { + let target = super::hook_ingest_refresh_session(arguments.as_object().unwrap()) + .expect("derive the canonical target without changing source-scan validation"); + assert_eq!(target, expected); + let project = PublishingRefresh::idle(); + let user = PublishingRefresh::idle(); + super::join_hook_ingest_refresh(true, target.as_ref(), Some(&project), Some(&user)) + .await + .expect("join the owning user store"); + assert!(user.published()); + assert_eq!(*user.session.lock().unwrap(), expected); + assert!(!project.published()); + } + } + #[tokio::test] async fn projected_lcm_preflight_joins_its_user_refresh_owner() { let project = PublishingRefresh::idle(); diff --git a/crates/tracedecay-mcp/src/server/mod.rs b/crates/tracedecay-mcp/src/server/mod.rs index e46070ad4c..615d88ab00 100644 --- a/crates/tracedecay-mcp/src/server/mod.rs +++ b/crates/tracedecay-mcp/src/server/mod.rs @@ -21,7 +21,8 @@ pub use dispatch::{ needs_lazy_sync_before_dispatch, }; pub use live_transcript_refresh::{ - LiveTranscriptRefreshJoin, join_hook_ingest_refresh, join_required_live_transcript_refresh, + LiveTranscriptRefreshJoin, hook_ingest_refresh_session, join_hook_ingest_refresh, + join_required_live_transcript_refresh, }; pub use project_host_admission_replay::{ ProjectHostAdmissionReplayTask, ProjectHostAdmissionReplayWorker, diff --git a/crates/tracedecay-session-runtime/src/session_temporal_refresh_scheduler/registry.rs b/crates/tracedecay-session-runtime/src/session_temporal_refresh_scheduler/registry.rs index 7a58d40ece..6bd60acad6 100644 --- a/crates/tracedecay-session-runtime/src/session_temporal_refresh_scheduler/registry.rs +++ b/crates/tracedecay-session-runtime/src/session_temporal_refresh_scheduler/registry.rs @@ -17,6 +17,7 @@ use super::wake::{ use super::wake::{SessionTemporalRefreshWorkerStatus, enabled_idle_notification}; use super::worker::run_session_temporal_refresh_scheduler; use crate::StoreOwnerKey; +use tracedecay_domain::SessionId; use tracedecay_global_db::RegisteredGlobalDbLeaseV1; use tracedecay_runtime_core::background_cpu::ProcessBackgroundCpuV1; use tracedecay_runtime_core::native_runtime::{self, NativeTask, NativeTaskOutcome}; @@ -35,6 +36,7 @@ macro_rules! define_pass_report { $visibility projected_batches: usize, $visibility completed: usize, $visibility failed: usize, + $visibility failed_sessions: HashSet, $visibility cancelled: usize, $visibility deferred: usize, $visibility retryable_errors: usize, diff --git a/crates/tracedecay-session-runtime/src/session_temporal_refresh_scheduler/wake.rs b/crates/tracedecay-session-runtime/src/session_temporal_refresh_scheduler/wake.rs index 5cf49c106e..5426ddc551 100644 --- a/crates/tracedecay-session-runtime/src/session_temporal_refresh_scheduler/wake.rs +++ b/crates/tracedecay-session-runtime/src/session_temporal_refresh_scheduler/wake.rs @@ -2,6 +2,8 @@ use std::collections::{HashMap, HashSet, VecDeque}; use std::sync::Arc; use std::sync::PoisonError; use std::sync::atomic::{AtomicBool, Ordering}; +use std::time::Instant; +use tracedecay_domain::SessionId; use tracedecay_runtime_core::native_runtime; use tracedecay_store::SessionRefreshBeginOrJoinRequestV1; use tracedecay_temporal_query::execution::ExecutionControl; @@ -18,6 +20,15 @@ use tracedecay_sessions::serving::{ SessionProjectionWorkerRetryClass, }; +/// One caller's publication obligation. Named sessions capture their source +/// frontier before the worker starts the pass; unnamed sweeps retain the +/// aggregate drain contract. +pub(super) struct SessionProjectionJoin { + pub(super) session_id: Option, + pub(super) observed_through: Option, + pub(super) completion: tokio::sync::oneshot::Sender, +} + #[derive(Clone, Copy, Debug, Eq, PartialEq)] pub enum SessionTemporalRefreshRetryClass { Storage, @@ -138,7 +149,7 @@ macro_rules! define_wake_state { /// is still unpublished. The next history window waits. pub(super) history_after_projection: AtomicBool, pub(super) pass_count: std::sync::atomic::AtomicUsize, - projection_joins: std::sync::Mutex>>, + projection_joins: std::sync::Mutex>, pub(super) wake: tokio::sync::Notify, pub(super) idle: tokio::sync::Notify, pub(super) cancelled: AtomicBool, @@ -342,7 +353,10 @@ impl SessionTemporalRefreshWakeState { self.wake.notify_one(); } - fn request_projection_join(&self) -> Option> { + fn request_projection_join( + &self, + session_id: Option, + ) -> Option> { let mut joins = self .projection_joins .lock() @@ -352,16 +366,20 @@ impl SessionTemporalRefreshWakeState { } // Expired callers retain no obligation. The worker owns only live // joins, and each result is delivered once even if a later pass fails. - joins.retain(|join| !join.is_closed()); + joins.retain(|join| !join.completion.is_closed()); let (completion, receiver) = tokio::sync::oneshot::channel(); - joins.push(completion); + joins.push(SessionProjectionJoin { + session_id, + observed_through: None, + completion, + }); self.wake(); Some(receiver) } /// Take the joins covered by the projection about to start. A join that /// arrives during that projection remains queued for the next pass. - pub(super) fn take_projection_joins(&self) -> Vec> { + pub(super) fn take_projection_joins(&self) -> Vec { std::mem::take( &mut *self .projection_joins @@ -698,7 +716,7 @@ impl SessionTemporalRefreshWakeState { self.clear_worker_instrumentation(); self.mark_stopped(); for join in self.take_projection_joins() { - let _ = join.send(false); + let _ = join.completion.send(false); } self.cancellation.notify_waiters(); self.wake.notify_waiters(); @@ -880,7 +898,6 @@ struct SessionTemporalRefreshWakeRoute { target: std::sync::RwLock>, } -#[cfg(any(test, feature = "test-helpers"))] pub(super) fn enabled_idle_notification( state: &SessionTemporalRefreshWakeState, ) -> std::pin::Pin>> { @@ -984,10 +1001,29 @@ impl SessionTemporalRefreshWake { /// may continue after that publication; their idle/admission state is not /// evidence that a temporal generation has committed. pub async fn wake_and_wait_for_projection(&self, timeout: std::time::Duration) -> bool { + self.join_projection(None, timeout).await + } + + /// Join only this session's captured source frontier. Other sessions may + /// remain blocked without invalidating its committed publication. + pub async fn wake_and_wait_for_session_projection( + &self, + session_id: &SessionId, + timeout: std::time::Duration, + ) -> bool { + self.join_projection(Some(session_id.clone()), timeout) + .await + } + + async fn join_projection( + &self, + session_id: Option, + timeout: std::time::Duration, + ) -> bool { let Some(state) = self.target() else { return false; }; - let Some(completion) = state.request_projection_join() else { + let Some(completion) = state.request_projection_join(session_id) else { return false; }; matches!( @@ -1003,6 +1039,38 @@ impl SessionTemporalRefreshWake { ) } + /// Join the worker's idle boundary, including background work. Callers + /// proving a rekey handoff still need that boundary; live transcript + /// publication uses the separate projection joins above. + pub async fn wake_and_wait_until_idle(&self, timeout: std::time::Duration) -> bool { + let Some(state) = self.target() else { + return false; + }; + let before = state.pass_count.load(Ordering::Acquire); + state.wake(); + let deadline = Instant::now() + timeout; + loop { + let idle = tracing::Instrument::instrument( + enabled_idle_notification(&state), + tracing::trace_span!("daemon.scheduler.session_temporal.idle_wait"), + ); + let pass_count = state.pass_count.load(Ordering::Acquire); + let busy = state.busy.load(Ordering::Acquire); + let dirty = state.dirty.load(Ordering::Acquire); + let pending = state.has_requests(); + if pass_count > before && !busy && !dirty && !pending { + return true; + } + if native_runtime::timeout_at(deadline, idle).await.is_err() { + let pass_count = state.pass_count.load(Ordering::Acquire); + return pass_count > before + && !state.busy.load(Ordering::Acquire) + && !state.dirty.load(Ordering::Acquire) + && !state.has_requests(); + } + } + } + /// Marks historical catch-up pending and wakes its worker, unless the /// worker is blocked or gone. /// @@ -1048,6 +1116,18 @@ impl tracedecay_contracts::SessionTemporalRefreshWakePort for SessionTemporalRef self, timeout, )) } + + fn wake_and_wait_for_session_projection<'a>( + &'a self, + session_id: &'a SessionId, + timeout: std::time::Duration, + ) -> tracedecay_contracts::SessionTemporalRefreshWakeFuture<'a> { + Box::pin( + SessionTemporalRefreshWake::wake_and_wait_for_session_projection( + self, session_id, timeout, + ), + ) + } } impl SessionProjectionServingStatusPort for SessionTemporalRefreshWake { diff --git a/crates/tracedecay-session-runtime/src/session_temporal_refresh_scheduler/worker.rs b/crates/tracedecay-session-runtime/src/session_temporal_refresh_scheduler/worker.rs index 013fd9143d..f0c69499b7 100644 --- a/crates/tracedecay-session-runtime/src/session_temporal_refresh_scheduler/worker.rs +++ b/crates/tracedecay-session-runtime/src/session_temporal_refresh_scheduler/worker.rs @@ -25,8 +25,8 @@ use super::projector::{ }; use super::registry::{SessionTemporalRefreshPassReport, session_refresh_retry_delay}; use super::wake::{ - PendingBeginRequestGuard, RecoverySelectionGuard, SessionTemporalRefreshRetryClass, - SessionTemporalRefreshWakeState, TerminalAttemptGuard, + PendingBeginRequestGuard, RecoverySelectionGuard, SessionProjectionJoin, + SessionTemporalRefreshRetryClass, SessionTemporalRefreshWakeState, TerminalAttemptGuard, }; use tracedecay_global_db::{RegisteredGlobalDb, RegisteredGlobalDbLeaseV1}; use tracedecay_runtime_core::db::engine::Error as EngineError; @@ -235,7 +235,7 @@ pub(super) async fn run_session_temporal_refresh_scheduler( let mut history_priority_passes = 0u32; let mut history_release = None; let mut granted_history_admission = None; - let mut projection_joins: Vec> = Vec::new(); + let mut projection_joins: Vec = Vec::new(); let live_projection = LiveProjectionDuringSummary { database: &database, state: &state, @@ -791,10 +791,12 @@ async fn session_projection_refresh( state: &Arc, projector: &dyn SessionTemporalRefreshProjector, policy: SessionTemporalRefreshPolicy, - joins: &mut Vec>, + joins: &mut Vec, ) -> SessionTemporalRefreshPassReport { - joins.retain(|join| !join.is_closed()); + joins.retain(|join| !join.completion.is_closed()); joins.extend(state.take_projection_joins()); + let access = SessionTemporalAccess::new(database.as_ref()); + capture_projection_targets(&access, joins).await; let report = run_session_temporal_refresh_pass(database, state, projector, policy).await; if state.cancelled.load(Ordering::Acquire) { return report; @@ -813,12 +815,93 @@ async fn session_projection_refresh( } else { None }; - if let Some(published) = result { - for join in joins.drain(..) { - let _ = join.send(published); + settle_projection_joins(&access, joins, &report, result).await; + report +} + +async fn capture_projection_targets( + access: &SessionTemporalAccess<'_, RegisteredGlobalDb>, + joins: &mut Vec, +) { + let mut frontiers = HashMap::new(); + for mut join in std::mem::take(joins) { + if let Some(session_id) = &join.session_id + && join.observed_through.is_none() + { + if let std::collections::hash_map::Entry::Vacant(entry) = + frontiers.entry(session_id.clone()) + { + let frontier = access + .session_temporal_publication_result(session_id) + .await + .map(|publication| publication.observed_through()) + .map_err(|error| { + tracing::warn!(%session_id, %error, "session publication target unavailable"); + }) + .ok(); + entry.insert(frontier); + } + let Some(frontier) = frontiers.get(session_id).copied().flatten() else { + let _ = join.completion.send(false); + continue; + }; + join.observed_through = Some(frontier); + } + joins.push(join); + } +} + +async fn settle_projection_joins( + access: &SessionTemporalAccess<'_, RegisteredGlobalDb>, + joins: &mut Vec, + report: &SessionTemporalRefreshPassReport, + aggregate: Option, +) { + let mut publications = HashMap::new(); + for join in std::mem::take(joins) { + if join.completion.is_closed() { + continue; + } + let result = if let Some(session_id) = &join.session_id { + if let std::collections::hash_map::Entry::Vacant(entry) = + publications.entry(session_id.clone()) + { + let publication = access + .session_temporal_publication_result(session_id) + .await + .map_err(|error| { + tracing::warn!(%session_id, %error, "session publication proof unavailable"); + }) + .ok(); + entry.insert(publication); + } + match publications.get(session_id).and_then(Option::as_ref) { + Some(publication) + if join + .observed_through + .is_some_and(|target| publication.covers(target)) => + { + Some(true) + } + Some(_) + if report.failed_sessions.contains(session_id) || aggregate == Some(true) => + { + // The named session must prove its own frontier. A drained + // scope or another session's successful receipt is not proof. + Some(false) + } + Some(_) => None, + None => Some(false), + } + } else { + aggregate + }; + if let Some(published) = result { + let _ = join.completion.send(published); + } else { + joins.push(join); } } - report } /// Keeps one fair summary admission or in-flight summary page alive while the @@ -835,7 +918,7 @@ impl LiveProjectionDuringSummary<'_> { async fn wait_for( &self, stage: F, - joins: &mut Vec>, + joins: &mut Vec, report: &mut SessionTemporalRefreshPassReport, idle_between_projections: bool, ) -> Option @@ -1740,6 +1823,7 @@ async fn apply_fail_effect( "session temporal refresh failed durably" ); report.failed += 1; + report.failed_sessions.insert(recovery.session_id().clone()); state.record_terminal_discovery_failure(recovery); } Err(error) if is_retryable_storage(&error) => { diff --git a/crates/tracedecay-session-runtime/src/session_temporal_refresh_scheduler/worker_tests.rs b/crates/tracedecay-session-runtime/src/session_temporal_refresh_scheduler/worker_tests.rs index 14454612e1..27ea0e08b7 100644 --- a/crates/tracedecay-session-runtime/src/session_temporal_refresh_scheduler/worker_tests.rs +++ b/crates/tracedecay-session-runtime/src/session_temporal_refresh_scheduler/worker_tests.rs @@ -429,6 +429,104 @@ struct ControlledProjector { reject: AtomicBool, } +/// Fails one session while every other session uses the production projector. +struct SessionRefusingProjector { + refused: SessionId, +} + +impl SessionTemporalRefreshProjector for SessionRefusingProjector { + fn project<'a>( + &'a self, + database: &'a RegisteredGlobalDbLeaseV1, + recovery: SessionRefreshRecoveryV1, + ) -> SessionTemporalRefreshProjectionFuture<'a> { + Box::pin(async move { + if recovery.session_id() == &self.refused { + Err(SessionTemporalRefreshProjectorError::terminal( + "fixture_session_projection_refused", + )) + } else { + CanonicalSessionTemporalProjector + .project(database, recovery) + .await + } + }) + } +} + +#[tokio::test] +async fn named_projection_join_survives_an_unrelated_session_failure() { + let harness = RegisteredGlobalDbHarness::open("refresh-join-session-isolation").await; + let store = SessionTemporalStore::new(harness.registered.as_ref()); + let successful = SessionId::new("projection-accepted").expect("successful session"); + let refused = SessionId::new("projection-refused").expect("refused session"); + begin_empty_refreshes(&store, ["projection-accepted", "projection-refused"]).await; + let recoveries = store.running_session_refreshes().await.expect("recoveries"); + assert_eq!(recoveries.len(), 2); + let state = Arc::new(SessionTemporalRefreshWakeState::default()); + let handle = state.handle(); + let successful_join = + handle.wake_and_wait_for_session_projection(&successful, Duration::from_secs(5)); + let refused_join = + handle.wake_and_wait_for_session_projection(&refused, Duration::from_secs(5)); + let aggregate_join = handle.wake_and_wait_for_projection(Duration::from_secs(5)); + tokio::pin!(successful_join, refused_join, aggregate_join); + // Put all three obligations in the same pass before starting the worker. + // The unnamed caller still owns the aggregate failure contract. + let pending = std::future::poll_fn(|cx| { + std::task::Poll::Ready(( + successful_join.as_mut().poll(cx).is_pending(), + refused_join.as_mut().poll(cx).is_pending(), + aggregate_join.as_mut().poll(cx).is_pending(), + )) + }) + .await; + assert_eq!(pending, (true, true, true)); + let worker = tokio::spawn(run_session_temporal_refresh_scheduler( + harness.registered.clone(), + Arc::clone(&state), + Arc::new(SessionRefusingProjector { + refused: refused.clone(), + }), + Arc::new(std::sync::RwLock::new(None)), + Arc::new(tokio::sync::Semaphore::new(0)), + SessionTemporalRefreshPolicy::default(), + )); + + let (successful_result, refused_result, aggregate_result) = + tokio::join!(successful_join, refused_join, aggregate_join); + state.cancel(); + worker.await.expect("worker exits on cancellation"); + assert!( + successful_result, + "another session cannot veto a committed publication" + ); + assert!( + !refused_result, + "the failed session must refuse its own join" + ); + assert!( + !aggregate_result, + "unnamed sweeps must retain aggregate failure" + ); + for recovery in recoveries { + let receipt = store + .session_refresh_receipt(SessionRefreshReceiptRequestV1::new( + recovery.operation_id().clone(), + recovery.session_id().clone(), + )) + .await + .expect("read durable receipt") + .expect("both sessions must settle durably"); + let expected = if recovery.session_id() == &successful { + SessionRefreshTerminalStateV1::Complete + } else { + SessionRefreshTerminalStateV1::Failed + }; + assert_eq!(receipt.state(), expected); + } +} + impl SessionTemporalRefreshProjector for ControlledProjector { fn project<'a>( &'a self, diff --git a/crates/tracedecay-session-temporal-store/src/lib.rs b/crates/tracedecay-session-temporal-store/src/lib.rs index bf57ec23e5..24a1b74b6c 100644 --- a/crates/tracedecay-session-temporal-store/src/lib.rs +++ b/crates/tracedecay-session-temporal-store/src/lib.rs @@ -30,7 +30,10 @@ mod hydration; pub mod operations; mod participant_freeze; mod projection; -pub use projection::{SessionTemporalRefreshDiscoveryCursor, SessionTemporalRefreshDiscoveryPage}; +pub use projection::{ + SessionTemporalPublication, SessionTemporalRefreshDiscoveryCursor, + SessionTemporalRefreshDiscoveryPage, +}; mod query; mod rebuild; mod refresh; diff --git a/crates/tracedecay-session-temporal-store/src/projection.rs b/crates/tracedecay-session-temporal-store/src/projection.rs index 53165af2aa..87863770b8 100644 --- a/crates/tracedecay-session-temporal-store/src/projection.rs +++ b/crates/tracedecay-session-temporal-store/src/projection.rs @@ -75,6 +75,26 @@ pub struct SessionTemporalRefreshDiscoveryPage { has_more: bool, } +/// The canonical source frontier and the active generation's publication. +/// Only an applied relation receipt belonging to that active generation and +/// matching its expected graph watermark proves it is searchable. A pending +/// receipt or retired generation cannot satisfy a new join. +pub struct SessionTemporalPublication { + observed_through: u64, + published_through: Option, +} + +impl SessionTemporalPublication { + pub const fn observed_through(&self) -> u64 { + self.observed_through + } + + pub fn covers(&self, target: u64) -> bool { + self.published_through + .is_some_and(|published| target <= published && published <= self.observed_through) + } +} + impl SessionTemporalRefreshDiscoveryPage { pub fn active_rows_scanned(&self) -> usize { self.active_rows_scanned @@ -402,6 +422,68 @@ async fn skip_trailing_history_effects( } impl SessionTemporalAccess<'_, D> { + /// Read one session's source and publication from the same snapshot. + /// A pending reset invalidates the old publication even when its numeric + /// frontier still covers the source; a future frontier is never proof. + pub async fn session_temporal_publication_result( + &self, + session_id: &SessionId, + ) -> SessionStoreResult { + let snapshot = self + .read_snapshot() + .await + .map_err(|error| storage(DISCOVER_REFRESH, error))?; + let mut rows = snapshot + .query( + "SELECT + COALESCE(( + SELECT effect.observation_sequence + FROM session_temporal_observation_effects AS effect + WHERE effect.session_id = ?1 AND effect.output_count > 0 + ORDER BY effect.observation_sequence DESC LIMIT 1 + ), 0), + (SELECT CAST(json_extract( + active.frozen_watermarks_json, '$.projection_frontier' + ) AS INTEGER) + FROM session_temporal_generations AS active + JOIN session_relation_receipts AS receipt + ON receipt.session_id = active.session_id + AND receipt.generation = active.generation + AND receipt.state = 'applied' + AND receipt.graph_watermark = receipt.expected_graph_watermark + WHERE active.session_id = ?1 AND active.state = 'active' + AND NOT EXISTS ( + SELECT 1 FROM session_temporal_resets AS reset + WHERE reset.session_id = ?1 AND reset.requested_at IS NOT NULL + ))", + params![session_id.as_str()], + ) + .await + .map_err(|error| storage(DISCOVER_REFRESH, error))?; + let row = rows + .next() + .await + .map_err(|error| storage(DISCOVER_REFRESH, error))? + .ok_or_else(|| { + storage_message(DISCOVER_REFRESH, "session publication returned no row") + })?; + let observed_through = u64::try_from( + row.get::(0) + .map_err(|error| storage(DISCOVER_REFRESH, error))?, + ) + .map_err(|error| storage(DISCOVER_REFRESH, error))?; + let published_through = row + .get::>(1) + .map_err(|error| storage(DISCOVER_REFRESH, error))? + .map(u64::try_from) + .transpose() + .map_err(|error| storage(DISCOVER_REFRESH, error))?; + Ok(SessionTemporalPublication { + observed_through, + published_through, + }) + } + /// Discovers sessions that need temporal projection: sessions whose /// output-producing effects committed past `cursor` left them behind their /// projection frontier, plus active generations that lack an applied diff --git a/crates/tracedecay-session-temporal-store/src/projection/tests.rs b/crates/tracedecay-session-temporal-store/src/projection/tests.rs index 3908b37ebb..1ea25043a5 100644 --- a/crates/tracedecay-session-temporal-store/src/projection/tests.rs +++ b/crates/tracedecay-session-temporal-store/src/projection/tests.rs @@ -2505,6 +2505,163 @@ async fn refresh_through( } } +#[asupersync::test] +async fn publication_requires_applied_current_receipt_and_source_coverage() { + let tmp = TempDir::new().unwrap(); + let runtime = HostAdmissionTestRuntimeV1::profile(tmp.path()) + .await + .unwrap(); + let store = temporal_store(&runtime); + let database = runtime + .registered_database(HostAdmissionScope::Profile) + .unwrap(); + let access = SessionTemporalAccess::new(database); + let session_id = fixture_session("session.publication-proof"); + let (first, first_write) = fixture_observation(&session_id, 0, None, false); + Box::pin(persist_fixture(&runtime, first, first_write)).await; + let first_generation = refresh_through(&store, &session_id, 1, 0).await; + let publication = access + .session_temporal_publication_result(&session_id) + .await + .unwrap(); + assert_eq!(publication.observed_through(), 1); + assert!(publication.covers(1)); + assert!( + !publication.covers(2), + "a future source target has no publication proof" + ); + + let (second, second_write) = fixture_observation(&session_id, 1, None, true); + let second_observation = second.observation_id().clone(); + Box::pin(persist_fixture(&runtime, second, second_write)).await; + let publication = access + .session_temporal_publication_result(&session_id) + .await + .unwrap(); + assert_eq!(publication.observed_through(), 2); + assert!(publication.covers(1)); + assert!( + !publication.covers(2), + "the old receipt cannot cover the new observation" + ); + let active_generation = refresh_through(&store, &session_id, 2, 1).await; + assert_ne!(first_generation, active_generation); + let snapshot = database.read_snapshot().await.unwrap(); + let mut older_receipt = snapshot.query( + "SELECT COUNT(*) FROM session_relation_receipts WHERE session_id = ?1 AND generation = ?2 AND state = 'applied'", + params![session_id.as_str(), i64::try_from(first_generation.value()).unwrap()], + ).await.unwrap(); + assert_eq!( + older_receipt + .next() + .await + .unwrap() + .unwrap() + .get::(0) + .unwrap(), + 1 + ); + drop(older_receipt); + drop(snapshot); + assert!( + access + .session_temporal_publication_result(&session_id) + .await + .unwrap() + .covers(2) + ); + + // A pending graph application cannot serve this generation, even though + // an older generation still has its applied receipt. + let transaction = database.begin_write_transaction().await.unwrap(); + assert_eq!(transaction.execute( + "UPDATE session_relation_receipts SET state = 'pending', graph_watermark = NULL, applied_at = NULL WHERE session_id = ?1 AND generation = ?2", + params![session_id.as_str(), i64::try_from(active_generation.value()).unwrap()], + ).await.unwrap(), 1); + transaction.commit().await.unwrap(); + assert!( + !access + .session_temporal_publication_result(&session_id) + .await + .unwrap() + .covers(2), + "a pending graph receipt must not publish" + ); + + let transaction = database.begin_write_transaction().await.unwrap(); + transaction.execute( + "UPDATE session_relation_receipts SET state = 'applied', graph_watermark = expected_graph_watermark || '.mismatch', applied_at = 1 WHERE session_id = ?1 AND generation = ?2", + params![session_id.as_str(), i64::try_from(active_generation.value()).unwrap()], + ).await.expect_err("schema must refuse a mismatched graph publication"); + transaction.execute( + "UPDATE session_relation_receipts SET state = 'applied', graph_watermark = expected_graph_watermark, applied_at = 1 WHERE session_id = ?1 AND generation = ?2", + params![session_id.as_str(), i64::try_from(active_generation.value()).unwrap()], + ).await.unwrap(); + transaction.commit().await.unwrap(); + assert!( + access + .session_temporal_publication_result(&session_id) + .await + .unwrap() + .covers(2) + ); + + let transaction = database.begin_write_transaction().await.unwrap(); + super::request_session_temporal_reset(&transaction, second_observation.as_str()) + .await + .unwrap(); + transaction.commit().await.unwrap(); + assert!( + !access + .session_temporal_publication_result(&session_id) + .await + .unwrap() + .covers(2), + "a pending reset invalidates the old publication" + ); + + let transaction = database.begin_write_transaction().await.unwrap(); + transaction + .execute( + "DELETE FROM session_temporal_resets WHERE session_id = ?1", + params![session_id.as_str()], + ) + .await + .unwrap(); + assert_eq!( + transaction + .execute( + "DELETE FROM session_relation_receipts WHERE session_id = ?1 AND generation = ?2", + params![ + session_id.as_str(), + i64::try_from(active_generation.value()).unwrap() + ] + ) + .await + .unwrap(), + 1 + ); + transaction.commit().await.unwrap(); + assert!( + !access + .session_temporal_publication_result(&session_id) + .await + .unwrap() + .covers(1), + "a retired generation's receipt cannot stand in for the active generation" + ); +} + +#[test] +fn publication_refuses_a_frontier_beyond_the_observed_source() { + let publication = super::SessionTemporalPublication { + observed_through: 2, + published_through: Some(3), + }; + assert!(!publication.covers(2)); + assert!(!publication.covers(3)); +} + /// One observation of a live session: the thread changes every third /// message, each message replies to the previous one, and every fourth /// message supersedes the one before it. diff --git a/crates/tracedecay/src/daemon/profile_owner.rs b/crates/tracedecay/src/daemon/profile_owner.rs index f625653e85..26ea964257 100644 --- a/crates/tracedecay/src/daemon/profile_owner.rs +++ b/crates/tracedecay/src/daemon/profile_owner.rs @@ -40,7 +40,7 @@ use tracedecay_global_db::RegisteredGlobalDbLeaseV1; use tracedecay_mcp::handlers::{ SessionAuthorities, decode_primitive_request, hook_runtime, unknown_tool_error, }; -use tracedecay_mcp::server::join_hook_ingest_refresh; +use tracedecay_mcp::server::{hook_ingest_refresh_session, join_hook_ingest_refresh}; use tracedecay_mcp::tools::dispatch_ceiling::{tool_dispatch_budget, tool_dispatch_deadline_error}; use tracedecay_runtime_core::cancellation::CancellationToken; use tracedecay_runtime_core::native_runtime::{self, TimeoutError}; @@ -185,6 +185,7 @@ async fn profile_hook_runtime( profile_identity: &LocalProfileIdentityAuthorityV1, arguments: Map, ) -> Result { + let publication_session = hook_ingest_refresh_session(&arguments)?; let request = hook_runtime::decode_hook_runtime_request(&Value::Object(arguments))?; let global_db = Box::pin(store_administration.registered_profile_database()).await?; let user_session_db = @@ -208,7 +209,13 @@ async fn profile_hook_runtime( )) .await?; if let HookRuntimeResultV1::IngestTranscript(ingest) = &result { - join_hook_ingest_refresh(ingest.user_scope, None, Some(&refresh_wake)).await?; + join_hook_ingest_refresh( + ingest.user_scope, + publication_session.as_ref(), + None, + Some(&refresh_wake), + ) + .await?; } else { refresh_wake.wake(); } diff --git a/crates/tracedecay/src/mcp/server/graph_tool_owner.rs b/crates/tracedecay/src/mcp/server/graph_tool_owner.rs index 912e89af5a..be4bd8e18f 100644 --- a/crates/tracedecay/src/mcp/server/graph_tool_owner.rs +++ b/crates/tracedecay/src/mcp/server/graph_tool_owner.rs @@ -15,7 +15,7 @@ use tracedecay_daemon_service::{ }; use tracedecay_domain::errors::{Result, TraceDecayError}; use tracedecay_mcp::handlers::graph_tool::graph_tool_owner_stores; -use tracedecay_mcp::server::join_hook_ingest_refresh; +use tracedecay_mcp::server::{hook_ingest_refresh_session, join_hook_ingest_refresh}; use tracedecay_mcp::tools::binding::{BranchSensitivity, tool_branch_sensitivity}; use tracedecay_project::project::TraceDecay; use tracedecay_runtime_core::branch::BranchMemo; @@ -152,6 +152,12 @@ impl McpServer { &self, invocation: GraphToolInvocationV1, ) -> Result { + let publication_session = + if invocation.operation == ApplicationSurfaceOperation::HookRuntime { + hook_ingest_refresh_session(&invocation.arguments)? + } else { + None + }; let (cg, live_branch) = self.reopen_if_branch_drifted_memoized().await; self.probe_before_graph_read(invocation.operation, &cg, &live_branch); let server_stats = match invocation.operation { @@ -191,6 +197,7 @@ impl McpServer { // only once their refresh owner has published it. join_hook_ingest_refresh( ingest.user_scope, + publication_session.as_ref(), self.project_session_refresh_wake.as_deref(), self.user_session_refresh_wake.as_deref(), ) diff --git a/crates/tracedecay/tests/session_suite/session_runtime/temporal_refresh.rs b/crates/tracedecay/tests/session_suite/session_runtime/temporal_refresh.rs index 0147039cc9..68021e1d3b 100644 --- a/crates/tracedecay/tests/session_suite/session_runtime/temporal_refresh.rs +++ b/crates/tracedecay/tests/session_suite/session_runtime/temporal_refresh.rs @@ -1081,7 +1081,7 @@ async fn hook_ingest_join_runs_the_bound_refresh_worker() { let wake = authority.ensure_profile(®istry).await; let before = registry.profile_pass_count(db.db_path()).await; - tracedecay_mcp::server::join_hook_ingest_refresh(false, Some(&wake), None) + tracedecay_mcp::server::join_hook_ingest_refresh(false, None, Some(&wake), None) .await .expect("hook ingest must join the refresh worker that owns the written store"); From a9324cea86374fad7d519dacbe1ec01e691ba66d Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Wed, 21 Oct 2026 17:48:30 -0700 Subject: [PATCH 09/15] test(sessions): preserve rekey idle completion assertions --- .../registry_tests.rs | 94 +++++++++++-------- 1 file changed, 53 insertions(+), 41 deletions(-) diff --git a/crates/tracedecay-session-runtime/src/session_temporal_refresh_scheduler/registry_tests.rs b/crates/tracedecay-session-runtime/src/session_temporal_refresh_scheduler/registry_tests.rs index 2f0d47586b..d52c9c9eb7 100644 --- a/crates/tracedecay-session-runtime/src/session_temporal_refresh_scheduler/registry_tests.rs +++ b/crates/tracedecay-session-runtime/src/session_temporal_refresh_scheduler/registry_tests.rs @@ -4,17 +4,23 @@ use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; use std::time::Duration; use futures_util::FutureExt; +use tracedecay_domain::SessionId; use tracedecay_global_db::tests::harness::RegisteredGlobalDbHarness; use tracedecay_runtime_core::native_runtime; +use tracedecay_session_temporal_store::SessionTemporalStore; use tracedecay_sessions::serving::{ SessionProjectionServingState, SessionProjectionServingStatusPort, SessionProjectionStaleReason, }; +use tracedecay_store::{ + SessionRefreshBeginOrJoinRequestV1, SessionRefreshDispositionV1, SessionRefreshFrontierV1, + SessionRefreshReceiptRequestV1, SessionRefreshStore, SessionRefreshTerminalStateV1, +}; use super::super::history::{ SessionHistoricalCapacityRelease, SessionHistoricalIngestOutcome, SessionHistoricalIngestPass, SessionHistoricalIngestor, SharedSessionHistoricalIngestor, }; -use super::super::wake::SessionTemporalRefreshWakeState; +use super::super::wake::{SessionTemporalRefreshWake, SessionTemporalRefreshWakeState}; use super::SessionTemporalRefreshSchedulerRegistry; use crate::StoreOwnerKey; @@ -137,6 +143,46 @@ async fn mount_held_project( history } +/// A replacement worker may consume a handed-off request before rekey returns. +/// Join its real pass and require the durable completion, rather than racing +/// the consumer for a transient queue length or dirty flag. These empty sessions +/// have no observation history for discovery to reconstruct a lost request from. +async fn assert_rekey_completed_requests( + harness: &RegisteredGlobalDbHarness, + wake: &SessionTemporalRefreshWake, + sessions: &[&str], +) { + assert!( + wake.wake_and_wait_until_idle(Duration::from_secs(5)).await, + "the retained route must join the replacement worker's pass" + ); + let store = SessionTemporalStore::new(harness.registered.as_ref()); + for name in sessions { + let session_id = SessionId::new(*name).expect("session id"); + let joined = store + .begin_or_join_session_refresh(SessionRefreshBeginOrJoinRequestV1::new( + session_id.clone(), + SessionRefreshFrontierV1::new(0, 0).expect("empty frontier"), + )) + .await + .expect("join the handed-off refresh"); + assert_eq!( + joined.disposition(), + SessionRefreshDispositionV1::Joined, + "{name}: the replacement must have begun the handed-off request" + ); + let receipt = store + .session_refresh_receipt(SessionRefreshReceiptRequestV1::new( + joined.operation_id().clone(), + session_id, + )) + .await + .expect("read handed-off refresh receipt") + .expect("the replacement must complete the handed-off request"); + assert_eq!(receipt.state(), SessionRefreshTerminalStateV1::Complete); + } +} + /// Mounts a project whose worker holds its history pass and a profile entry. async fn mount_drain_fixture( harness: &RegisteredGlobalDbHarness, @@ -377,11 +423,10 @@ async fn a_retried_rekey_recovers_the_dropped_attempts_route_and_requests() { .project_state(&old_owner) .await .expect("project entry"); - let request = tracedecay_store::SessionRefreshBeginOrJoinRequestV1::new( + old_state.requeue_request(tracedecay_store::SessionRefreshBeginOrJoinRequestV1::new( tracedecay_domain::SessionId::new("session.dropped-rekey").unwrap(), tracedecay_store::SessionRefreshFrontierV1::new(0, 0).unwrap(), - ); - old_state.requeue_request(request.clone()); + )); assert_eq!(old_state.queued_request_count(), 1); assert!( @@ -396,10 +441,6 @@ async fn a_retried_rekey_recovers_the_dropped_attempts_route_and_requests() { "no worker owns the route of a dropped rekey, so a wake must not be accepted" ); - // Keep the destination in history until the transferred request and route - // are inspected; its worker may otherwise consume them before the test. - let destination_history = - mount_held_project(®istry, &harness, &new_owner, HeldHistory::default()).await; let mut rekey = std::pin::pin!(registry.rekey_project( &old_owner, new_owner.clone(), @@ -415,32 +456,12 @@ async fn a_retried_rekey_recovers_the_dropped_attempts_route_and_requests() { rekey.await; assert!(registry.project_state(&old_owner).await.is_none()); - let new_state = registry + registry .project_state(&new_owner) .await .expect("new owner entry"); - assert_eq!( - new_state.queued_request_count(), - 1, - "the retry must land the retired owner's queued request on the new owner" - ); - let transferred = new_state.take_requests(1); - assert!( - transferred - .iter() - .any(|transferred| transferred.is_equivalent_to(&request)), - "the retry must retain the exact request and frontier" - ); - for request in transferred { - new_state.requeue_request(request); - } - assert!( - old_wake.wake(), - "the old route must reach the new owner once the rekey completes" - ); - assert!(new_state.take_dirty()); + assert_rekey_completed_requests(&harness, &old_wake, &["session.dropped-rekey"]).await; - destination_history.release.notify_one(); registry.shutdown().await; } @@ -803,20 +824,11 @@ async fn a_mapped_rekey_lands_the_workers_queued_requests() { rekey.await; assert!(registry.project_state(&old_owner).await.is_none()); - let new_state = registry + registry .project_state(&new_owner) .await .expect("new owner entry"); - assert_eq!( - new_state.queued_request_count(), - 1, - "the new owner must inherit the mapped worker's queued request" - ); - assert!( - old_wake.wake(), - "the old route must reach the new owner once the rekey completes" - ); - assert!(new_state.take_dirty()); + assert_rekey_completed_requests(&harness, &old_wake, &["session.mapped-rekey"]).await; registry.shutdown().await; } From fba56b950074d26ec7a0fea82f186ad2a9f20df3 Mon Sep 17 00:00:00 2001 From: Zack Jackson Date: Wed, 21 Oct 2026 17:53:04 -0700 Subject: [PATCH 10/15] build(bazel): declare the repository npm configuration --- BUILD.bazel | 1 + MODULE.bazel | 1 + 2 files changed, 2 insertions(+) diff --git a/BUILD.bazel b/BUILD.bazel index 54139007a8..ac204e0138 100644 --- a/BUILD.bazel +++ b/BUILD.bazel @@ -12,6 +12,7 @@ npm_link_all_packages(name = "node_modules") exports_files( [ ".github/release-targets.json", + ".npmrc", "MODULE.bazel", "version.txt", "Cargo.toml", diff --git a/MODULE.bazel b/MODULE.bazel index 966a7e07df..7d77b017fd 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -175,6 +175,7 @@ npm.npm_translate_lock( "//:pnpm-workspace.yaml", "//dashboard:package.json", ], + npmrc = "//:.npmrc", pnpm_lock = "//:pnpm-lock.yaml", # React Router imports React types from the shared npm store, outside # dashboard/node_modules. Make that peer declaration resolvable there. From 810bed1e23ff7d3ec0f651e4d28bc765266d38db Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Thu, 22 Oct 2026 01:57:41 -0700 Subject: [PATCH 11/15] fix(sessions): gate idle notification helper with its callers --- .../src/session_temporal_refresh_scheduler/wake.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/crates/tracedecay-session-runtime/src/session_temporal_refresh_scheduler/wake.rs b/crates/tracedecay-session-runtime/src/session_temporal_refresh_scheduler/wake.rs index 337b22d301..c11b7f5689 100644 --- a/crates/tracedecay-session-runtime/src/session_temporal_refresh_scheduler/wake.rs +++ b/crates/tracedecay-session-runtime/src/session_temporal_refresh_scheduler/wake.rs @@ -923,6 +923,7 @@ struct SessionTemporalRefreshWakeRoute { target: std::sync::RwLock>, } +#[cfg(any(test, feature = "test-helpers"))] pub(super) fn enabled_idle_notification( state: &SessionTemporalRefreshWakeState, ) -> std::pin::Pin>> { From b1ec1e1bc588029eec7b0805d7a52e7096fe682d Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Thu, 22 Oct 2026 11:38:53 -0700 Subject: [PATCH 12/15] test(hosts): accept validated Kiro feedback notices --- .../host_event_fixture_test.rs | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/crates/tracedecay/tests/runtime_acceptance_suite/host_event_fixture_test.rs b/crates/tracedecay/tests/runtime_acceptance_suite/host_event_fixture_test.rs index d00d612c0a..e3fe9e90b1 100644 --- a/crates/tracedecay/tests/runtime_acceptance_suite/host_event_fixture_test.rs +++ b/crates/tracedecay/tests/runtime_acceptance_suite/host_event_fixture_test.rs @@ -1997,6 +1997,17 @@ fn assert_legal_host_response(provider: &str, state: &str, expected: &Value, out *project_root = Value::String("".to_string()); } assert_eq!(document, expected, "{provider}/{state} stdout"); + } else if provider == "kiro" && stdout != expected["stdout"].as_str().unwrap() { + // Kiro delivers its lookup notice as plain stdout, which becomes context. + assert_eq!(expected["stdout"].as_str().unwrap(), "{}"); + let encoded = stdout + .strip_prefix("TraceDecay feedback ready for authorized lookup: ") + .unwrap_or_else(|| panic!("{provider}/{state} emitted unexpected context: {stdout}")); + let notice: tracedecay_application::advisory::AdvisoryHookLookupNoticeV1 = + serde_json::from_str(encoded).expect("feedback notice must match its typed contract"); + notice.validate().expect("feedback notice must be valid"); + assert_eq!(notice.returned_findings, 0); + assert_eq!(notice.omitted_findings, 0); } else { assert_eq!( stdout, From e3a679a80606ed0276c0700cb24a8d8531f81c7a Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Thu, 22 Oct 2026 13:43:59 -0700 Subject: [PATCH 13/15] test(sessions): prove reset recovery before new joins --- .../temporal_projection/activation.rs | 69 ++++++++++++++++++- 1 file changed, 66 insertions(+), 3 deletions(-) diff --git a/crates/tracedecay/tests/session_suite/temporal_projection/activation.rs b/crates/tracedecay/tests/session_suite/temporal_projection/activation.rs index 4dd2d7b146..28a1f8d0d5 100644 --- a/crates/tracedecay/tests/session_suite/temporal_projection/activation.rs +++ b/crates/tracedecay/tests/session_suite/temporal_projection/activation.rs @@ -1,4 +1,12 @@ +use std::future::Future; +use std::sync::Arc; +use std::time::Duration; + use tracedecay_session_runtime::session_temporal_refresh_scheduler::SessionTemporalRefreshSchedulerRegistry; +use tracedecay_session_runtime::session_temporal_refresh_scheduler::history::{ + SessionHistoricalCapacityRelease, SessionHistoricalIngestOutcome, SessionHistoricalIngestPass, + SessionHistoricalIngestor, +}; use super::*; @@ -317,6 +325,30 @@ async fn activation_is_pinned_to_the_generation_the_refresh_extends() { ); } +#[derive(Default)] +struct GatedHistory { + entered: tokio::sync::Notify, + release: tokio::sync::Notify, +} + +impl SessionHistoricalIngestor for GatedHistory { + fn run_pass(&self) -> SessionHistoricalIngestPass<'_> { + Box::pin(async move { + self.entered.notify_one(); + self.release.notified().await; + SessionHistoricalIngestOutcome::Complete.into() + }) + } + + fn capacity_release(&self) -> SessionHistoricalCapacityRelease { + SessionHistoricalCapacityRelease::new(Vec::new()) + } + + fn cancel(&self) { + self.release.notify_one(); + } +} + #[tokio::test] async fn reset_requested_during_a_refresh_supersedes_its_candidate() { let tmp = TempDir::new().unwrap(); @@ -355,16 +387,35 @@ async fn reset_requested_during_a_refresh_supersedes_its_candidate() { assert_eq!(requested, 1, "the retired observation requests one reset"); let registry = SessionTemporalRefreshSchedulerRegistry::default(); - registry - .ensure_profile( + let history = Arc::new(GatedHistory::default()); + let wake = registry + .ensure_profile_with_history( path.clone(), runtime .registered_database_arc(HostAdmissionScope::Profile) .unwrap(), + history.clone(), ) .await; + history.entered.notified().await; + let named_join = wake.wake_and_wait_for_session_projection(&session_id, Duration::from_secs(5)); + let aggregate_join = wake.wake_and_wait_for_projection(Duration::from_secs(5)); + tokio::pin!(named_join, aggregate_join); + assert_eq!( + std::future::poll_fn(|cx| std::task::Poll::Ready(( + named_join.as_mut().poll(cx).is_pending(), + aggregate_join.as_mut().poll(cx).is_pending(), + ))) + .await, + (true, true) + ); + history.release.notify_one(); + assert_eq!( + tokio::join!(named_join, aggregate_join), + (false, false), + "the superseded operation must refuse both waiting publication joins" + ); registry.wait_profile_idle(&path).await; - registry.shutdown().await; assert_eq!( rows( @@ -412,6 +463,18 @@ async fn reset_requested_during_a_refresh_supersedes_its_candidate() { .await, vec![format!("message.temporal.{}.0", session_id.as_str())] ); + + assert!( + wake.wake_and_wait_for_session_projection(&session_id, Duration::from_secs(5)) + .await, + "a new join must observe the automatically rebuilt session publication" + ); + assert!( + wake.wake_and_wait_for_projection(Duration::from_secs(5)) + .await, + "a new aggregate join must observe the completed replacement" + ); + registry.shutdown().await; } #[tokio::test] From dd8467553039bfbc5e4f62487ecd568e43527aad Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Thu, 22 Oct 2026 20:52:11 -0700 Subject: [PATCH 14/15] test(session): synchronize profile wake coalescing --- .../session_runtime/temporal_refresh.rs | 62 +++++++++++++++++-- 1 file changed, 58 insertions(+), 4 deletions(-) diff --git a/crates/tracedecay/tests/session_suite/session_runtime/temporal_refresh.rs b/crates/tracedecay/tests/session_suite/session_runtime/temporal_refresh.rs index 68021e1d3b..9fb34b48f3 100644 --- a/crates/tracedecay/tests/session_suite/session_runtime/temporal_refresh.rs +++ b/crates/tracedecay/tests/session_suite/session_runtime/temporal_refresh.rs @@ -974,9 +974,15 @@ async fn failure_effects_use_typed_terminal_store_operations() { ); } +enum BlockingProjectorOutcome { + CompleteEmptyProjection, + Deferred, +} + struct BlockingProjector { started: Arc, release: Arc, + outcome: BlockingProjectorOutcome, } impl SessionTemporalRefreshProjector for BlockingProjector { @@ -987,10 +993,16 @@ impl SessionTemporalRefreshProjector for BlockingProjector { ) -> SessionTemporalRefreshProjectionFuture<'a> { let started = Arc::clone(&self.started); let release = Arc::clone(&self.release); + let outcome = &self.outcome; Box::pin(async move { started.notify_one(); release.notified().await; - Ok(empty_projection_effect(&recovery)) + match outcome { + BlockingProjectorOutcome::Deferred => Ok(SessionTemporalRefreshEffect::Deferred), + BlockingProjectorOutcome::CompleteEmptyProjection => { + Ok(empty_projection_effect(&recovery)) + } + } }) } } @@ -1011,6 +1023,7 @@ async fn stale_owner_cannot_persist_after_cancellation() { let projector = BlockingProjector { started: Arc::clone(&started), release: Arc::clone(&release), + outcome: BlockingProjectorOutcome::CompleteEmptyProjection, }; let pass = tokio::spawn({ let authority = Arc::clone(&authority); @@ -1142,6 +1155,7 @@ async fn project_retirement_cancels_and_awaits_an_inflight_projector() { Arc::new(BlockingProjector { started: Arc::clone(&started), release: Arc::new(tokio::sync::Notify::new()), + outcome: BlockingProjectorOutcome::CompleteEmptyProjection, }), SessionTemporalRefreshPolicy::default(), ); @@ -1445,9 +1459,30 @@ async fn profile_database_has_one_scheduler_and_equivalent_kicks_coalesce() { let authority = registered_test_database(&temp, "profile-coalescing", HostAdmissionScope::Profile).await; let db = authority.database(); - let registry = SessionTemporalRefreshSchedulerRegistry::default(); + let started = Arc::new(tokio::sync::Notify::new()); + let release = Arc::new(tokio::sync::Notify::new()); + let mut registry = SessionTemporalRefreshSchedulerRegistry::default(); + configure_scheduler( + &mut registry, + Arc::new(BlockingProjector { + started: Arc::clone(&started), + release: Arc::clone(&release), + outcome: BlockingProjectorOutcome::Deferred, + }), + SessionTemporalRefreshPolicy::default(), + ); + let store = tracedecay_session_temporal_store::SessionTemporalStore::new(db); + let started_refresh = store + .begin_or_join_session_refresh(request("session.profile.coalescing", 0)) + .await + .unwrap(); let first = authority.ensure_profile(®istry).await; + tokio::time::timeout(Duration::from_secs(1), started.notified()) + .await + .unwrap(); + assert_eq!(registry.profile_pass_count(db.db_path()).await, 1); + let second = authority.ensure_profile(®istry).await; for _ in 0..32 { second.wake(); @@ -1455,8 +1490,27 @@ async fn profile_database_has_one_scheduler_and_equivalent_kicks_coalesce() { assert!(first.same_route(&second)); assert_eq!(registry.profile_worker_count().await, 1); - registry.wait_profile_idle(db.db_path()).await; - assert!(registry.profile_pass_count(db.db_path()).await <= 2); + assert_eq!(registry.profile_pass_count(db.db_path()).await, 1); + release.notify_one(); + tokio::time::timeout(Duration::from_secs(1), started.notified()) + .await + .unwrap(); + assert_eq!(registry.profile_pass_count(db.db_path()).await, 2); + release.notify_one(); + tokio::time::timeout( + Duration::from_secs(1), + registry.wait_profile_idle(db.db_path()), + ) + .await + .unwrap(); + let pending = store.running_session_refreshes().await.unwrap(); + assert_eq!(pending.len(), 1); + assert_eq!(pending[0].session_id(), started_refresh.session_id()); + let pass_count = registry.profile_pass_count(db.db_path()).await; + assert!( + pass_count <= 2, + "profile scheduler used {pass_count} passes" + ); registry.shutdown().await; assert_eq!(registry.profile_worker_count().await, 0); } From 33a9bde3418fec56d2987460e148fe6a1acb0e3f Mon Sep 17 00:00:00 2001 From: Zack Jackson Date: Fri, 23 Oct 2026 02:43:09 -0700 Subject: [PATCH 15/15] fix(hooks): retain transcript resource refusal details --- .../tracedecay-mcp/src/hook_runtime/errors.rs | 7 +++++ .../src/hook_runtime/errors/tests.rs | 29 +++++++++++++++++++ 2 files changed, 36 insertions(+) diff --git a/crates/tracedecay-mcp/src/hook_runtime/errors.rs b/crates/tracedecay-mcp/src/hook_runtime/errors.rs index 46ec42ef98..b6c71afc83 100644 --- a/crates/tracedecay-mcp/src/hook_runtime/errors.rs +++ b/crates/tracedecay-mcp/src/hook_runtime/errors.rs @@ -27,6 +27,13 @@ pub fn map_transcript_ingest_error( detail: Some(cause), .. } => format!("transcript ingest failed: authority_write_failed: {cause}"), + tracedecay_sessions::runtime::source::TranscriptIngestError::BackgroundResourceUnavailable { + provider, + resource, + } => format!( + "transcript ingest failed: {}: {provider}: {resource}", + disposition.reason_code + ), _ => format!("transcript ingest failed: {}", disposition.reason_code), }; hook_admission_error( diff --git a/crates/tracedecay-mcp/src/hook_runtime/errors/tests.rs b/crates/tracedecay-mcp/src/hook_runtime/errors/tests.rs index 3588e47dd8..db6cd6a255 100644 --- a/crates/tracedecay-mcp/src/hook_runtime/errors/tests.rs +++ b/crates/tracedecay-mcp/src/hook_runtime/errors/tests.rs @@ -251,6 +251,35 @@ fn authority_write_failed_includes_cause_in_hook_detail() { ); } +#[test] +fn transcript_resource_refusal_keeps_the_resource_and_backpressure_status() { + let mapped = map_transcript_ingest_error( + &tracedecay_sessions::runtime::source::TranscriptIngestError::BackgroundResourceUnavailable { + provider: "codex", + resource: "shared JSONL page resident-memory capacity", + }, + ); + let data = structured_hook_error_data(&mapped).unwrap(); + assert_eq!( + data["reason_code"], + "transcript_background_resource_unavailable" + ); + assert_eq!(data["status"], "backpressured"); + assert_eq!(data["retryable"], true); + let detail = data["detail"].as_str().expect("hook detail"); + assert!(detail.contains("codex"), "{detail}"); + assert!( + detail.contains("shared JSONL page resident-memory capacity"), + "{detail}" + ); + assert!( + mapped + .to_string() + .contains("shared JSONL page resident-memory capacity"), + "{mapped}" + ); +} + #[test] fn transcript_hook_errors_keep_bounded_retry_data_without_source_detail() { let error =