From a67ce51260b86cdc8f8b68bfe4be0a613a1f1028 Mon Sep 17 00:00:00 2001 From: ScriptedAlchemy Date: Fri, 16 Oct 2026 16:10:59 +0000 Subject: [PATCH 1/2] refactor(native-integration): make store async, drop actor --- .../src/native_integration/journey_tests.rs | 119 +-- .../src/native_integration/mod.rs | 5 +- .../src/native_integration/registry.rs | 126 ++- .../src/native_integration/stack_runtime.rs | 40 +- .../src/native_integration/store.rs | 842 +++++------------- .../src/native_integration/worktree.rs | 13 +- .../native_integration/worktree_cleanup.rs | 81 +- .../src/native_integration/transaction.rs | 340 ++++--- .../native_integration/transaction_tests.rs | 172 +++- .../src/stack_coordinator.rs | 39 +- .../src/stack_coordinator/transition.rs | 2 +- .../preflight_tests.rs | 113 ++- .../src/git/native_integration.rs | 34 +- .../src/git/stack_signal_expand.rs | 4 +- .../tracedecay-contracts/src/git/worktree.rs | 12 +- .../src/invocation.rs | 2 +- .../src/invocation/dispatch.rs | 21 +- .../src/invocation/github_stack_signal.rs | 22 +- .../src/invocation/native_integration.rs | 76 +- .../src/native_integration.rs | 47 +- 20 files changed, 1027 insertions(+), 1083 deletions(-) diff --git a/crates/tracedecay-agent-hosts/src/native_integration/journey_tests.rs b/crates/tracedecay-agent-hosts/src/native_integration/journey_tests.rs index bf1ee4431d..1f028cbd77 100644 --- a/crates/tracedecay-agent-hosts/src/native_integration/journey_tests.rs +++ b/crates/tracedecay-agent-hosts/src/native_integration/journey_tests.rs @@ -14,7 +14,7 @@ use std::time::{Duration, SystemTime, UNIX_EPOCH}; use super::analysis::DaemonNativeIntegrationAnalysisV1; use super::registry::{DaemonNativeIntegrationServiceRegistry, NativeIntegrationTargetV1}; use super::stack_signals::signal_from_preflight; -use super::store::{DaemonNativeIntegrationStore, SharedDaemonNativeIntegrationStore}; +use super::store::DaemonNativeIntegrationStore; use super::worktree::DaemonNativeWorktreeAuthority; use tracedecay_application::native_integration::{ GixNativeIntegrationAdapter, NativeApplyEffectV1, NativeIntegrationAnalysisPort, @@ -533,11 +533,12 @@ async fn preflight( request: NativeIntegrationPreflightRequestV1, ) -> tracedecay_domain::NativeIntegrationPreviewV1 { let signal = CancellationSignal::active("cancel.native.journey.preflight").expect("signal"); - let outcome = - native_runtime::spawn_blocking(move || owner.service().preflight(request, &signal)) - .await - .expect("preflight join") - .expect("preflight result"); + let outcome = native_runtime::on_blocking_pool(async move { + owner.service().preflight(request, &signal).await + }) + .await + .expect("preflight join") + .expect("preflight result"); let NativeIntegrationPreflightOutcomeV1::Preview(preview) = outcome else { panic!("fixture must produce an eligible native preview: {outcome:?}"); }; @@ -818,10 +819,12 @@ async fn checked_out_declared_stack_conflict_enqueues_without_moving_refs() { owner .store() .save_preview(preview.clone()) + .await .expect("save canonical conflict preview"); owner .store() .save_preview(occupied_preview.clone()) + .await .expect("save canonical occupied preview"); let stack_runtime = mount_stack_runtime_off_tokio( owner.clone(), @@ -878,14 +881,16 @@ async fn checked_out_declared_stack_conflict_enqueues_without_moving_refs() { let cancellation = CancellationSignal::active("cancel.native.journey.declared-conflict.expand") .expect("expand cancellation"); - let denied = stack_runtime.expand( - GitHubStackSignalExpandSurfaceRequest { - signal_id: Some(signal.signal_id.clone()), - expected_watermark_id: Some(signal.watermark_id.clone()), - } - .into_application_request(other_context), - &cancellation, - ); + let denied = stack_runtime + .expand( + GitHubStackSignalExpandSurfaceRequest { + signal_id: Some(signal.signal_id.clone()), + expected_watermark_id: Some(signal.watermark_id.clone()), + } + .into_application_request(other_context), + &cancellation, + ) + .await; assert_eq!(denied, Err(GitHubStackSignalExpandPortError::Concealed)); let expanded = stack_runtime .expand( @@ -896,6 +901,7 @@ async fn checked_out_declared_stack_conflict_enqueues_without_moving_refs() { .into_application_request(live_context.clone()), &cancellation, ) + .await .expect("expand exact authorized conflict signal"); let GitHubStackSignalExpandSurfaceResultV1::Expanded { evidence } = expanded else { panic!("expected expanded stack conflict evidence"); @@ -931,6 +937,7 @@ async fn checked_out_declared_stack_conflict_enqueues_without_moving_refs() { .into_application_request(live_context), &cancellation, ) + .await .expect("expand destination-occupied dependency-ready signal"); let GitHubStackSignalExpandSurfaceResultV1::Expanded { evidence } = occupied else { panic!("expected destination-occupied stack evidence"); @@ -1158,6 +1165,7 @@ async fn independent_pair_applies_supported_modes_and_survives_daemon_restart() owner .store() .save_approval(approval.clone()) + .await .expect("durably issue approval"); let transaction_id = NativeIntegrationTransactionId::new(format!("transaction.native.journey.{index}")) @@ -1166,17 +1174,20 @@ async fn independent_pair_applies_supported_modes_and_survives_daemon_restart() let apply_owner = owner.clone(); let apply_signal = CancellationSignal::active("cancel.native.journey.apply").expect("signal"); - let receipt = native_runtime::spawn_blocking(move || { - apply_owner.service().apply( - NativeIntegrationApplyRequestV1 { - context, - transaction_id: transaction_id.clone(), - preview, - approval, - observed_at: OBSERVED_AT, - }, - &apply_signal, - ) + let receipt = native_runtime::on_blocking_pool(async move { + apply_owner + .service() + .apply( + NativeIntegrationApplyRequestV1 { + context, + transaction_id: transaction_id.clone(), + preview, + approval, + observed_at: OBSERVED_AT, + }, + &apply_signal, + ) + .await }) .await .expect("apply join") @@ -1318,6 +1329,7 @@ async fn independent_pair_applies_supported_modes_and_survives_daemon_restart() &CancellationSignal::active("cancel.native.journey.terminal-consumer") .expect("cancellation"), ) + .await .expect("expand terminal signal"); let GitHubStackSignalExpandSurfaceResultV1::Expanded { evidence } = expanded else { panic!("expected terminal stack evidence"); @@ -1355,17 +1367,14 @@ async fn independent_pair_applies_supported_modes_and_survives_daemon_restart() registry.shutdown().await.expect("shutdown owner registry"); let (restarted_registry, restarted_owner, _analysis) = mount(database, repository_root).await; - let durable = native_runtime::spawn_blocking(move || { - restarted_owner.service().status( - tracedecay_contracts::NativeIntegrationStatusRequestV1 { - transaction_id: status_transaction_id, - }, - ) - }) - .await - .expect("status join") - .expect("durable status") - .expect("durable transaction status"); + let durable = restarted_owner + .service() + .status(tracedecay_contracts::NativeIntegrationStatusRequestV1 { + transaction_id: status_transaction_id, + }) + .await + .expect("durable status") + .expect("durable transaction status"); assert_eq!( durable.terminal_outcome, Some(NativeIntegrationTerminalOutcomeV1::Committed) @@ -1450,6 +1459,7 @@ async fn foreign_destination_ref_drift_terminates_without_mutating_the_foreign_t owner .store() .save_approval(approval.clone()) + .await .expect("durably issue approval"); git(&repository_root, &["checkout", "-b", "foreign"]); @@ -1468,20 +1478,23 @@ async fn foreign_destination_ref_drift_terminates_without_mutating_the_foreign_t let apply_owner = owner.clone(); let signal = CancellationSignal::active("cancel.native.journey.drift").expect("signal"); - let receipt = native_runtime::spawn_blocking(move || { - apply_owner.service().apply( - NativeIntegrationApplyRequestV1 { - context, - transaction_id: NativeIntegrationTransactionId::new( - "transaction.native.journey.drift", - ) - .expect("transaction id"), - preview, - approval, - observed_at: OBSERVED_AT, - }, - &signal, - ) + let receipt = native_runtime::on_blocking_pool(async move { + apply_owner + .service() + .apply( + NativeIntegrationApplyRequestV1 { + context, + transaction_id: NativeIntegrationTransactionId::new( + "transaction.native.journey.drift", + ) + .expect("transaction id"), + preview, + approval, + observed_at: OBSERVED_AT, + }, + &signal, + ) + .await }) .await .expect("apply join") @@ -1566,9 +1579,7 @@ async fn owned_worktree_inspection_shares_the_discovery_walk_another_thread_owns ) .expect("registered authorized scope set"), ); - let store = SharedDaemonNativeIntegrationStore::from_arc(Arc::new( - DaemonNativeIntegrationStore::open(database).expect("native integration store"), - )); + let store = DaemonNativeIntegrationStore::open(database); let authority = Arc::new( DaemonNativeWorktreeAuthority::open( project_id.clone(), @@ -1620,5 +1631,5 @@ async fn owned_worktree_inspection_shares_the_discovery_walk_another_thread_owns 1 ); tracedecay_runtime_core::git_repository::reset_repository_discovery_for_test(&repository_root); - store.shutdown().expect("shutdown native integration store"); + assert!(store.shutdown(), "the store closes once"); } diff --git a/crates/tracedecay-agent-hosts/src/native_integration/mod.rs b/crates/tracedecay-agent-hosts/src/native_integration/mod.rs index c749c00393..ab021fe506 100644 --- a/crates/tracedecay-agent-hosts/src/native_integration/mod.rs +++ b/crates/tracedecay-agent-hosts/src/native_integration/mod.rs @@ -1,9 +1,8 @@ //! Daemon composition of the Plan 36 native-integration authority. //! //! `registry` retains one composed transaction coordinator per exact -//! project/repository identity; `store` bridges the synchronous -//! `tracedecay-store` contract onto the async registered session database -//! through one bounded actor per database. +//! project/repository identity; `store` implements the async +//! `tracedecay-store` contract directly over the registered session database. mod analysis; mod registry; diff --git a/crates/tracedecay-agent-hosts/src/native_integration/registry.rs b/crates/tracedecay-agent-hosts/src/native_integration/registry.rs index 23cdbe07f6..9032a41e6d 100644 --- a/crates/tracedecay-agent-hosts/src/native_integration/registry.rs +++ b/crates/tracedecay-agent-hosts/src/native_integration/registry.rs @@ -1,8 +1,8 @@ //! One retained native-integration transaction authority per daemon-owned //! project store. //! -//! The registry composes the four coordinator inputs, the durable store -//! actor, the exact-pair topology resolver, the native Gix mechanics, and the +//! The registry composes the four coordinator inputs, the durable store, +//! the exact-pair topology resolver, the native Gix mechanics, and the //! pinned-policy authorization, completes durable startup recovery, and only //! then exposes the owner to invocation routing. A project without a mounted //! owner keeps answering the typed unavailable result; nothing here guesses @@ -43,14 +43,14 @@ use tracedecay_global_db::{RegisteredGlobalDbLeaseV1, VerifiedGraphRuntimePortV1 use tracedecay_rusqlite_runtime::repository::AuthorizedScopeSetSqliteStorage; use super::stack_runtime::DaemonGitHubStackRuntimeV1; -use super::store::{DaemonNativeIntegrationStore, SharedDaemonNativeIntegrationStore}; +use super::store::DaemonNativeIntegrationStore; use super::worktree::{DaemonAuthorizedScopeSetReader, DaemonNativeWorktreeAuthority}; const MAX_PENDING_WORKTREE_CLEANUPS: u32 = 4_096; /// The one exact composition served to invocation routing. pub type DaemonProjectNativeIntegrationCoordinator = NativeIntegrationTransactionCoordinator< - SharedDaemonNativeIntegrationStore, + DaemonNativeIntegrationStore, ExactPairNativeIntegrationTopology, GixNativeIntegrationAdapter, DaemonNativeIntegrationAuthorization, @@ -79,12 +79,11 @@ fn worktree_recovery_error(error: WorktreeContractError) -> NativeIntegrationPor } } -/// Retains the one `DaemonNativeIntegrationStore` actor for each daemon-owned -/// project database. Dropping the registry closes every actor when the daemon -/// store administration shuts down. +/// Retains the one `DaemonNativeIntegrationStore` for each daemon-owned +/// project database. Shutdown releases every retained lease. #[derive(Default)] struct NativeIntegrationStoreRegistry { - stores: Mutex>, + stores: Mutex>, closed: AtomicBool, } @@ -92,7 +91,7 @@ impl NativeIntegrationStoreRegistry { fn ensure( &self, database: RegisteredGlobalDbLeaseV1, - ) -> NativeIntegrationStoreResult { + ) -> NativeIntegrationStoreResult { // The registered runtime authority already supplies the canonical // database identity; a fresh SQLite shard may not have materialized // its path yet, so no filesystem lookup happens here. @@ -104,9 +103,9 @@ impl NativeIntegrationStoreRegistry { &self, path: PathBuf, open: F, - ) -> NativeIntegrationStoreResult + ) -> NativeIntegrationStoreResult where - F: FnOnce() -> NativeIntegrationStoreResult, + F: FnOnce() -> DaemonNativeIntegrationStore, { if self.closed.load(Ordering::SeqCst) { return Err(tracedecay_store::NativeIntegrationStoreError::unavailable( @@ -125,29 +124,21 @@ impl NativeIntegrationStoreRegistry { if let Some(existing) = stores.get(&path) { return Ok(existing.clone()); } - let store = SharedDaemonNativeIntegrationStore::from_arc(Arc::new(open()?)); + let store = open(); stores.insert(path, store.clone()); Ok(store) } - async fn shutdown_all(&self) -> NativeIntegrationStoreResult { + fn shutdown_all(&self) -> NativeIntegrationStoreResult { self.closed.store(true, Ordering::SeqCst); - let stores = { - let mut retained = self - .stores - .lock() - .map_err(tracedecay_store::NativeIntegrationStoreError::unavailable)?; - retained.drain().map(|(_, store)| store).collect::>() - }; - tracedecay_runtime_core::native_runtime::spawn_blocking(move || { - let mut joined = 0usize; - for store in stores { - joined = joined.saturating_add(usize::from(store.shutdown()?)); - } - Ok(joined) - }) - .await - .map_err(tracedecay_store::NativeIntegrationStoreError::unavailable)? + let mut retained = self + .stores + .lock() + .map_err(tracedecay_store::NativeIntegrationStoreError::unavailable)?; + Ok(retained + .drain() + .filter(|(_, store)| store.shutdown()) + .count()) } } @@ -159,7 +150,7 @@ pub struct DaemonNativeIntegrationOwner { service: Arc, snapshots: Arc>>, worktrees: Option>, - store: SharedDaemonNativeIntegrationStore, + store: DaemonNativeIntegrationStore, scope_sets: Option, stack_runtimes: Arc>>>, } @@ -212,14 +203,15 @@ impl DaemonNativeIntegrationOwner { self.snapshots.snapshot(request, cancellation) } - pub fn store(&self) -> &SharedDaemonNativeIntegrationStore { + pub fn store(&self) -> &DaemonNativeIntegrationStore { &self.store } - pub fn cleanup_recovery_roots(&self) -> Result, NativeIntegrationPortError> { + pub async fn cleanup_recovery_roots(&self) -> Result, NativeIntegrationPortError> { let mut roots = self .store .pending_worktree_cleanups(&self.repository_id, MAX_PENDING_WORKTREE_CLEANUPS) + .await .map_err(|_| NativeIntegrationPortError::Unavailable)? .into_iter() .map(|transaction| transaction.command.worktree_root) @@ -243,9 +235,11 @@ impl DaemonNativeIntegrationOwner { .ok_or(NativeIntegrationPortError::Unavailable)?; let store = self.store.clone(); let repository_id = self.repository_id.clone(); - tracedecay_runtime_core::native_runtime::spawn_blocking(move || { + // Reconciliation interleaves store awaits with synchronous native Git. + tracedecay_runtime_core::native_runtime::on_blocking_pool(async move { let pending = store .pending_worktree_cleanups(&repository_id, MAX_PENDING_WORKTREE_CLEANUPS) + .await .map_err(|_| NativeIntegrationPortError::Unavailable)?; let mut reconciled = 0usize; for (index, transaction) in pending.into_iter().enumerate() { @@ -268,6 +262,7 @@ impl DaemonNativeIntegrationOwner { }, &signal, ) + .await .map_err(worktree_recovery_error)?; match outcome { WorktreeCleanupReconciliationV1::Removed { .. } @@ -385,7 +380,7 @@ pub struct NativeIntegrationTargetV1 { pub policy_digest: ManifestDigest, } -/// Owns the store actor, topology resolver, native mechanics, authorization, +/// Owns the store, topology resolver, native mechanics, authorization, /// and coordinator for each exact project/repository identity. #[derive(Default)] pub struct DaemonNativeIntegrationServiceRegistry { @@ -397,7 +392,7 @@ pub struct DaemonNativeIntegrationServiceRegistry { impl DaemonNativeIntegrationServiceRegistry { /// Returns the retained owner for this exact identity, or composes exactly - /// one: store actor, topology, mechanics, pinned-policy authorization, + /// one: store, topology, mechanics, pinned-policy authorization, /// then durable startup recovery. A failed recovery mounts nothing. #[tracing::instrument( name = "daemon.native_integration.native_ensure", @@ -485,7 +480,7 @@ impl DaemonNativeIntegrationServiceRegistry { open_store: F, ) -> Result where - F: FnOnce() -> NativeIntegrationStoreResult, + F: FnOnce() -> NativeIntegrationStoreResult, { if self.shutdown_fenced.load(Ordering::SeqCst) { return Err(NativeIntegrationPortError::Unavailable); @@ -511,8 +506,8 @@ impl DaemonNativeIntegrationServiceRegistry { return Ok(owner); } - // Open/retain the store actor under the creation gate before native - // recovery runs on a blocking thread. Later ensures reuse this actor. + // Open/retain the store under the creation gate before native + // recovery runs on a blocking thread. Later ensures reuse this store. let store = open_store().map_err(|_| NativeIntegrationPortError::Unavailable)?; let recovery_store = store.clone(); let native_root = repository_root.clone(); @@ -522,7 +517,7 @@ impl DaemonNativeIntegrationServiceRegistry { let owner_project_id = project_id.clone(); let owner_repository_id = repository_id.clone(); let (owner_project_id, owner_repository_id, service, snapshots, worktrees) = - tracedecay_runtime_core::native_runtime::spawn_blocking(move || { + tracedecay_runtime_core::native_runtime::on_blocking_pool(async move { let topology = Arc::new(match (topology_shard, topology_runtime) { (Some(expected_shard), Some(runtime)) => { ExactPairNativeIntegrationTopology::open_with_graph_runtime_provider( @@ -576,12 +571,15 @@ impl DaemonNativeIntegrationServiceRegistry { // serves a single request. Failing closed here mounts nothing. let pending = recovery_store .pending_transactions(None) + .await .map_err(|_| NativeIntegrationPortError::Unavailable)?; for record in pending { - coordinator.recover(&NativeIntegrationRecoveryRequestV1 { - transaction_id: record.status.transaction_id.clone(), - observed_at, - })?; + coordinator + .recover(&NativeIntegrationRecoveryRequestV1 { + transaction_id: record.status.transaction_id.clone(), + observed_at, + }) + .await?; } Ok::<_, NativeIntegrationPortError>(( owner_project_id, @@ -687,7 +685,6 @@ impl DaemonNativeIntegrationServiceRegistry { self.owners.lock().await.clear(); self.stores .shutdown_all() - .await .map_err(|_| NativeIntegrationPortError::Unavailable) } } @@ -1075,33 +1072,22 @@ mod tests { // or a fabricated status. let unknown = NativeIntegrationTransactionId::new("transaction.native.unknown").expect("id"); - let status_owner = owner.clone(); - let status = tokio::task::spawn_blocking(move || { - status_owner - .service() - .status(NativeIntegrationStatusRequestV1 { - transaction_id: NativeIntegrationTransactionId::new( - "transaction.native.unknown", - ) - .expect("id"), - }) - }) - .await - .expect("status join") - .expect("status read"); + let status = owner + .service() + .status(NativeIntegrationStatusRequestV1 { + transaction_id: unknown.clone(), + }) + .await + .expect("status read"); assert_eq!(status, None); - let cancel_owner = owner.clone(); - let disposition = tokio::task::spawn_blocking(move || { - cancel_owner - .service() - .cancel(NativeIntegrationCancelRequestV1 { - transaction_id: unknown, - requested_at: UtcMicros(2), - }) - }) - .await - .expect("cancel join") - .expect("cancel disposition"); + let disposition = owner + .service() + .cancel(NativeIntegrationCancelRequestV1 { + transaction_id: unknown, + requested_at: UtcMicros(2), + }) + .await + .expect("cancel disposition"); assert_eq!( disposition, NativeIntegrationCancelDispositionV1::UnknownTransaction diff --git a/crates/tracedecay-agent-hosts/src/native_integration/stack_runtime.rs b/crates/tracedecay-agent-hosts/src/native_integration/stack_runtime.rs index 38d0f33ffb..c8dd814eeb 100644 --- a/crates/tracedecay-agent-hosts/src/native_integration/stack_runtime.rs +++ b/crates/tracedecay-agent-hosts/src/native_integration/stack_runtime.rs @@ -43,7 +43,7 @@ use tracedecay_global_db::{ use tracedecay_runtime_core::native_runtime::{self, NativeTask}; use super::registry::DaemonProjectNativeIntegrationService; -use super::store::SharedDaemonNativeIntegrationStore; +use super::store::DaemonNativeIntegrationStore; const STACK_DELIVERY_STORE_ACTOR_CAPACITY: usize = 64; const STACK_DELIVERY_STORE_ACTOR_TIMEOUT: Duration = Duration::from_secs(5); @@ -332,7 +332,7 @@ struct StackRuntimePortsV1 { access: Arc>, store: Arc, native_service: Arc, - native_store: SharedDaemonNativeIntegrationStore, + native_store: DaemonNativeIntegrationStore, authorizations: Arc>>, preflight_outcomes: Arc>>, } @@ -734,7 +734,7 @@ fn map_native_preflight_error(error: NativeIntegrationContractError) -> StackCoo } impl OptionalStackPreflightPort for StackRuntimePortsV1 { - fn preflight( + async fn preflight( &self, request: &NativeIntegrationPreflightRequestV1, cancellation: &CancellationSignal, @@ -744,6 +744,7 @@ impl OptionalStackPreflightPort for StackRuntimePortsV1 { let outcome = self .native_service .preflight(request.clone(), cancellation) + .await .map_err(map_native_preflight_error)?; self.preflight_outcomes .lock() @@ -825,7 +826,7 @@ impl DaemonGitHubStackRuntimeV1 { database: RegisteredGlobalDbLeaseV1, coordinator: Arc, native_service: Arc, - native_store: SharedDaemonNativeIntegrationStore, + native_store: DaemonNativeIntegrationStore, ) -> Result, StackCoordinatorErrorV1> { if project_id != scope.project_id || access.scope != scope { return Err(StackCoordinatorErrorV1::Stale); @@ -894,15 +895,15 @@ impl DaemonGitHubStackRuntimeV1 { /// Executes preflight through the coordinator's circuit and bounded /// admission path. The port stores the exact outcome produced by that /// invocation so callers do not bypass the circuit with a second Git run. - pub fn preflight( + pub async fn preflight( &self, request: &NativeIntegrationPreflightRequestV1, cancellation: &CancellationSignal, ) -> Result { - self.preflight_via_circuit(request, cancellation) + self.preflight_via_circuit(request, cancellation).await } - fn preflight_via_circuit( + async fn preflight_via_circuit( &self, request: &NativeIntegrationPreflightRequestV1, cancellation: &CancellationSignal, @@ -915,16 +916,20 @@ impl DaemonGitHubStackRuntimeV1 { .ports .native_service .preflight(request.clone(), cancellation) + .await .map_err(map_native_preflight_error); } let request_digest = canonical_sha256(request) .map_err(|error| StackCoordinatorErrorV1::Invalid(error.to_string()))?; - let disposition = self.coordinator.optional_preflight( - &self.ports, - request, - cancellation, - StackRuntimePortsV1::now()?, - )?; + let disposition = self + .coordinator + .optional_preflight( + &self.ports, + request, + cancellation, + StackRuntimePortsV1::now()?, + ) + .await?; let outcome = self .ports .preflight_outcomes @@ -959,7 +964,7 @@ impl DaemonGitHubStackRuntimeV1 { self.ports.pending_host_for_scope() } - fn expand_request( + async fn expand_request( &self, request: GitHubStackSignalExpandRequestV1, cancellation: &CancellationSignal, @@ -1021,6 +1026,7 @@ impl DaemonGitHubStackRuntimeV1 { .ports .native_store .read_preview_by_digest(&signal.state_digest) + .await .map_err(|_| GitHubStackSignalExpandPortError::Unavailable)?; let (preview, terminal) = match preview_by_state { Some(preview) => (preview, None), @@ -1029,12 +1035,14 @@ impl DaemonGitHubStackRuntimeV1 { .ports .native_store .read_receipt_by_digest(&signal.state_digest) + .await .map_err(|_| GitHubStackSignalExpandPortError::Unavailable)? .ok_or(GitHubStackSignalExpandPortError::NativeEvidenceUnavailable)?; let preview = self .ports .native_store .read_preview_by_digest(&receipt.status.preview_digest) + .await .map_err(|_| GitHubStackSignalExpandPortError::Unavailable)? .ok_or(GitHubStackSignalExpandPortError::NativeEvidenceUnavailable)?; if preview.preview_id != receipt.status.preview_id @@ -1149,12 +1157,12 @@ impl DaemonGitHubStackRuntimeV1 { } impl GitHubStackSignalExpandPort for DaemonGitHubStackRuntimeV1 { - fn expand( + async fn expand( &self, request: GitHubStackSignalExpandRequestV1, cancellation: &CancellationSignal, ) -> Result { - self.expand_request(request, cancellation) + self.expand_request(request, cancellation).await } } diff --git a/crates/tracedecay-agent-hosts/src/native_integration/store.rs b/crates/tracedecay-agent-hosts/src/native_integration/store.rs index d36638555f..308e4179b9 100644 --- a/crates/tracedecay-agent-hosts/src/native_integration/store.rs +++ b/crates/tracedecay-agent-hosts/src/native_integration/store.rs @@ -1,16 +1,14 @@ -//! Bounded synchronous bridge to the registered session-store runtime. +//! Native integration transaction store over the registered project session +//! database. //! -//! The native-integration transaction coordinator is deliberately synchronous -//! because its native Git mechanics are synchronous. Calling an async database -//! through `block_on` on a Tokio worker would pin that worker while an -//! `IMMEDIATE` writer waits, so this adapter owns one bounded actor thread; -//! the actor owns the async rusqlite-runtime calls and every synchronous port -//! call receives exactly one reply. It has no filesystem path and cannot -//! create a side-file authority. - -use std::sync::mpsc::{Receiver, RecvTimeoutError, SyncSender, TrySendError, sync_channel}; -use std::sync::{Arc, Mutex}; -use std::time::Duration; +//! Every call awaits the async global-db store directly. Each mutation there +//! is one `IMMEDIATE` SQLite transaction (one-use approval consumption and +//! every phase compare-and-swap included), so concurrent callers need no actor +//! to serialize them; per-transaction ordering comes from the coordinator +//! awaiting each step in turn under its repository lock. It has no filesystem +//! path and cannot create a side-file authority. + +use std::sync::{Arc, Mutex, PoisonError}; use tracedecay_domain::{ CodeGenerationId, ManifestDigest, NativeIntegrationApprovalId, NativeIntegrationApprovalV1, @@ -24,164 +22,46 @@ use tracedecay_store::{ }; use tracedecay_global_db::{GlobalDbNativeIntegrationStore, RegisteredGlobalDbLeaseV1}; -use tracedecay_runtime_core::native_runtime; - -/// The actor queue is intentionally finite: saturation fails closed instead of -/// accumulating unbounded mutation work while a durable writer is stalled. -const NATIVE_INTEGRATION_STORE_ACTOR_CAPACITY: usize = 64; -// Keep the sync port bounded by the same five-second writer wait used by -// `RegisteredGlobalDb`; callers can reconcile durable state after an -// unavailable result instead of pinning a daemon worker forever. -const NATIVE_INTEGRATION_STORE_ACTOR_TIMEOUT: Duration = Duration::from_secs(5); - -type Reply = SyncSender>; - -enum StoreCommand { - SavePreview(Box, Reply<()>), - ReadPreview( - NativeIntegrationPreviewId, - Reply>, - ), - ReadPreviewByDigest(ManifestDigest, Reply>), - SaveApproval(Box, Reply<()>), - ReadApproval( - NativeIntegrationApprovalId, - Reply>, - ), - BeginOrReplay( - Box, - Reply, - ), - ReadStatus( - NativeIntegrationTransactionId, - Reply>, - ), - ReadRecord( - NativeIntegrationTransactionId, - Reply>, - ), - ReadReceipt( - NativeIntegrationTransactionId, - Reply>, - ), - ReadReceiptByDigest(ManifestDigest, Reply>), - CompareAndSwapStatus( - NativeIntegrationTransactionId, - u64, - Box, - Reply, - ), - WriteTerminal( - NativeIntegrationTransactionId, - u64, - Box, - Reply, - ), - PendingTransactions(Option, Reply>), - LiveCandidateGenerationBindings(RepositoryId, UtcMicros, Reply>), - ApprovalConsumed(NativeIntegrationApprovalId, Reply), - QuarantineRepository(RepositoryId, NativeIntegrationTransactionId, Reply<()>), - BeginWorktreeCleanup( - Box, - Reply, - ), - ReadWorktreeCleanup( - ManifestDigest, - Reply>, - ), - PendingWorktreeCleanups( - RepositoryId, - u32, - Reply>, - ), - CompareAndSwapWorktreeCleanup( - ManifestDigest, - u64, - Box, - Reply, - ), - WriteWorktreeCleanupTerminal( - ManifestDigest, - u64, - Box, - Reply, - ), -} - -enum ActorDatabase { - Registered(RegisteredGlobalDbLeaseV1), -} -impl ActorDatabase { - fn native_integration_store(&self) -> GlobalDbNativeIntegrationStore<'_> { - match self { - Self::Registered(database) => GlobalDbNativeIntegrationStore::new(database), - } - } -} - -/// Synchronous `tracedecay-store` contract adapter over one already-open, -/// canonical registered project session database. +/// Shared handle to the one daemon-owned store for a project database. /// -/// Dropping the last adapter closes the command channel and lets the dedicated -/// actor exit. It intentionally has no `Clone` implementation: one daemon -/// owner holds one bounded queue and actor for its transaction authority; -/// sharing goes through [`SharedDaemonNativeIntegrationStore`]. -pub(crate) struct DaemonNativeIntegrationStore { - commands: Mutex>>, - worker: Mutex>>, +/// Clones share the lease. `shutdown`, or dropping the last clone, releases +/// it; calls after shutdown fail closed as unavailable, while a call already +/// in flight finishes on the lease it holds. +#[derive(Clone)] +pub struct DaemonNativeIntegrationStore { + database: Arc>>, } impl DaemonNativeIntegrationStore { - pub(crate) fn open(database: RegisteredGlobalDbLeaseV1) -> NativeIntegrationStoreResult { - Self::open_actor(ActorDatabase::Registered(database)) - } - #[tracing::instrument( name = "daemon.native_integration.store_open", level = "trace", skip_all )] - fn open_actor(database: ActorDatabase) -> NativeIntegrationStoreResult { - let (commands, receiver) = sync_channel(NATIVE_INTEGRATION_STORE_ACTOR_CAPACITY); - let worker = std::thread::Builder::new() - .name("tracedecay-native-integration-store".to_owned()) - .spawn(move || run_store_actor(&database, &receiver)) - .map_err(NativeIntegrationStoreError::unavailable)?; - Ok(Self { - commands: Mutex::new(Some(commands)), - worker: Mutex::new(Some(worker)), - }) + pub(crate) fn open(database: RegisteredGlobalDbLeaseV1) -> Self { + Self { + database: Arc::new(Mutex::new(Some(database))), + } } - fn submit(&self, command: StoreCommand) -> NativeIntegrationStoreResult<()> { - self.commands + /// Releases the lease. Returns whether this call closed the store. + pub(crate) fn shutdown(&self) -> bool { + self.database .lock() - .map_err(NativeIntegrationStoreError::unavailable)? - .as_ref() - .ok_or_else(|| { - NativeIntegrationStoreError::unavailable( - "native integration store actor is shut down", - ) - })? - .try_send(command) - .map_err(|error| match error { - TrySendError::Full(_) | TrySendError::Disconnected(_) => { - NativeIntegrationStoreError::unavailable(error) - } - }) + .unwrap_or_else(PoisonError::into_inner) + .take() + .is_some() } - fn await_reply( - receiver: &Receiver>, - ) -> NativeIntegrationStoreResult { - receiver - .recv_timeout(NATIVE_INTEGRATION_STORE_ACTOR_TIMEOUT) - .map_err(|error| match error { - RecvTimeoutError::Timeout | RecvTimeoutError::Disconnected => { - NativeIntegrationStoreError::unavailable(error) - } - })? + fn database(&self) -> NativeIntegrationStoreResult { + self.database + .lock() + .unwrap_or_else(PoisonError::into_inner) + .clone() + .ok_or_else(|| { + NativeIntegrationStoreError::unavailable("native integration store is shut down") + }) } /// Persists one issued approval commitment (approval issuance operation). @@ -190,13 +70,14 @@ impl DaemonNativeIntegrationStore { level = "trace", skip_all )] - pub(crate) fn save_approval( + pub async fn save_approval( &self, approval: NativeIntegrationApprovalV1, ) -> NativeIntegrationStoreResult<()> { - let (reply, receiver) = sync_channel(1); - self.submit(StoreCommand::SaveApproval(Box::new(approval), reply))?; - Self::await_reply(&receiver) + let database = self.database()?; + GlobalDbNativeIntegrationStore::new(&database) + .save_approval(approval) + .await } /// Reads one issued approval commitment for apply resolution. @@ -205,13 +86,14 @@ impl DaemonNativeIntegrationStore { level = "trace", skip_all )] - pub(crate) fn read_approval( + pub async fn read_approval( &self, approval_id: &NativeIntegrationApprovalId, ) -> NativeIntegrationStoreResult> { - let (reply, receiver) = sync_channel(1); - self.submit(StoreCommand::ReadApproval(approval_id.clone(), reply))?; - Self::await_reply(&receiver) + let database = self.database()?; + GlobalDbNativeIntegrationStore::new(&database) + .read_approval(approval_id) + .await } #[tracing::instrument( @@ -219,16 +101,14 @@ impl DaemonNativeIntegrationStore { level = "trace", skip_all )] - pub(crate) fn read_preview_by_digest( + pub async fn read_preview_by_digest( &self, preview_digest: &ManifestDigest, ) -> NativeIntegrationStoreResult> { - let (reply, receiver) = sync_channel(1); - self.submit(StoreCommand::ReadPreviewByDigest( - preview_digest.clone(), - reply, - ))?; - Self::await_reply(&receiver) + let database = self.database()?; + GlobalDbNativeIntegrationStore::new(&database) + .read_preview_by_digest(preview_digest) + .await } #[tracing::instrument( @@ -236,41 +116,14 @@ impl DaemonNativeIntegrationStore { level = "trace", skip_all )] - pub(crate) fn read_receipt_by_digest( + pub async fn read_receipt_by_digest( &self, receipt_digest: &ManifestDigest, ) -> NativeIntegrationStoreResult> { - let (reply, receiver) = sync_channel(1); - self.submit(StoreCommand::ReadReceiptByDigest( - receipt_digest.clone(), - reply, - ))?; - Self::await_reply(&receiver) - } - - pub(crate) fn shutdown(&self) -> NativeIntegrationStoreResult { - self.commands - .lock() - .map_err(NativeIntegrationStoreError::unavailable)? - .take(); - let worker = self - .worker - .lock() - .map_err(NativeIntegrationStoreError::unavailable)? - .take(); - let Some(worker) = worker else { - return Ok(false); - }; - worker.join().map_err(|_| { - NativeIntegrationStoreError::unavailable("native integration store worker panicked") - })?; - Ok(true) - } -} - -impl Drop for DaemonNativeIntegrationStore { - fn drop(&mut self) { - let _ = self.shutdown(); + let database = self.database()?; + GlobalDbNativeIntegrationStore::new(&database) + .read_receipt_by_digest(receipt_digest) + .await } } @@ -280,13 +133,14 @@ impl NativeIntegrationStore for DaemonNativeIntegrationStore { level = "trace", skip_all )] - fn save_preview( + async fn save_preview( &self, preview: NativeIntegrationPreviewV1, ) -> NativeIntegrationStoreResult<()> { - let (reply, receiver) = sync_channel(1); - self.submit(StoreCommand::SavePreview(Box::new(preview), reply))?; - Self::await_reply(&receiver) + let database = self.database()?; + GlobalDbNativeIntegrationStore::new(&database) + .save_preview(preview) + .await } #[tracing::instrument( @@ -294,13 +148,14 @@ impl NativeIntegrationStore for DaemonNativeIntegrationStore { level = "trace", skip_all )] - fn read_preview( + async fn read_preview( &self, preview_id: &NativeIntegrationPreviewId, ) -> NativeIntegrationStoreResult> { - let (reply, receiver) = sync_channel(1); - self.submit(StoreCommand::ReadPreview(preview_id.clone(), reply))?; - Self::await_reply(&receiver) + let database = self.database()?; + GlobalDbNativeIntegrationStore::new(&database) + .read_preview(preview_id) + .await } #[tracing::instrument( @@ -308,13 +163,14 @@ impl NativeIntegrationStore for DaemonNativeIntegrationStore { level = "trace", skip_all )] - fn begin_or_replay( + async fn begin_or_replay( &self, record: NativeIntegrationRecordV1, ) -> NativeIntegrationStoreResult { - let (reply, receiver) = sync_channel(1); - self.submit(StoreCommand::BeginOrReplay(Box::new(record), reply))?; - Self::await_reply(&receiver) + let database = self.database()?; + GlobalDbNativeIntegrationStore::new(&database) + .begin_or_replay(record) + .await } #[tracing::instrument( @@ -322,13 +178,14 @@ impl NativeIntegrationStore for DaemonNativeIntegrationStore { level = "trace", skip_all )] - fn read_status( + async fn read_status( &self, transaction_id: &NativeIntegrationTransactionId, ) -> NativeIntegrationStoreResult> { - let (reply, receiver) = sync_channel(1); - self.submit(StoreCommand::ReadStatus(transaction_id.clone(), reply))?; - Self::await_reply(&receiver) + let database = self.database()?; + GlobalDbNativeIntegrationStore::new(&database) + .read_status(transaction_id) + .await } #[tracing::instrument( @@ -336,13 +193,14 @@ impl NativeIntegrationStore for DaemonNativeIntegrationStore { level = "trace", skip_all )] - fn read_record( + async fn read_record( &self, transaction_id: &NativeIntegrationTransactionId, ) -> NativeIntegrationStoreResult> { - let (reply, receiver) = sync_channel(1); - self.submit(StoreCommand::ReadRecord(transaction_id.clone(), reply))?; - Self::await_reply(&receiver) + let database = self.database()?; + GlobalDbNativeIntegrationStore::new(&database) + .read_record(transaction_id) + .await } #[tracing::instrument( @@ -350,13 +208,14 @@ impl NativeIntegrationStore for DaemonNativeIntegrationStore { level = "trace", skip_all )] - fn read_receipt( + async fn read_receipt( &self, transaction_id: &NativeIntegrationTransactionId, ) -> NativeIntegrationStoreResult> { - let (reply, receiver) = sync_channel(1); - self.submit(StoreCommand::ReadReceipt(transaction_id.clone(), reply))?; - Self::await_reply(&receiver) + let database = self.database()?; + GlobalDbNativeIntegrationStore::new(&database) + .read_receipt(transaction_id) + .await } #[tracing::instrument( @@ -364,20 +223,16 @@ impl NativeIntegrationStore for DaemonNativeIntegrationStore { level = "trace", skip_all )] - fn compare_and_swap_status( + async fn compare_and_swap_status( &self, transaction_id: &NativeIntegrationTransactionId, expected_phase_revision: u64, replacement: NativeIntegrationTransactionStatusV1, ) -> NativeIntegrationStoreResult { - let (reply, receiver) = sync_channel(1); - self.submit(StoreCommand::CompareAndSwapStatus( - transaction_id.clone(), - expected_phase_revision, - Box::new(replacement), - reply, - ))?; - Self::await_reply(&receiver) + let database = self.database()?; + GlobalDbNativeIntegrationStore::new(&database) + .compare_and_swap_status(transaction_id, expected_phase_revision, replacement) + .await } #[tracing::instrument( @@ -385,20 +240,16 @@ impl NativeIntegrationStore for DaemonNativeIntegrationStore { level = "trace", skip_all )] - fn write_terminal( + async fn write_terminal( &self, transaction_id: &NativeIntegrationTransactionId, expected_phase_revision: u64, receipt: NativeIntegrationReceiptV1, ) -> NativeIntegrationStoreResult { - let (reply, receiver) = sync_channel(1); - self.submit(StoreCommand::WriteTerminal( - transaction_id.clone(), - expected_phase_revision, - Box::new(receipt), - reply, - ))?; - Self::await_reply(&receiver) + let database = self.database()?; + GlobalDbNativeIntegrationStore::new(&database) + .write_terminal(transaction_id, expected_phase_revision, receipt) + .await } #[tracing::instrument( @@ -406,30 +257,25 @@ impl NativeIntegrationStore for DaemonNativeIntegrationStore { level = "trace", skip_all )] - fn pending_transactions( + async fn pending_transactions( &self, repository_id: Option<&RepositoryId>, ) -> NativeIntegrationStoreResult> { - let (reply, receiver) = sync_channel(1); - self.submit(StoreCommand::PendingTransactions( - repository_id.cloned(), - reply, - ))?; - Self::await_reply(&receiver) + let database = self.database()?; + GlobalDbNativeIntegrationStore::new(&database) + .pending_transactions(repository_id) + .await } - fn live_candidate_generation_bindings( + async fn live_candidate_generation_bindings( &self, repository_id: &RepositoryId, observed_at: UtcMicros, ) -> NativeIntegrationStoreResult> { - let (reply, receiver) = sync_channel(1); - self.submit(StoreCommand::LiveCandidateGenerationBindings( - repository_id.clone(), - observed_at, - reply, - ))?; - Self::await_reply(&receiver) + let database = self.database()?; + GlobalDbNativeIntegrationStore::new(&database) + .live_candidate_generation_bindings(repository_id, observed_at) + .await } #[tracing::instrument( @@ -437,13 +283,14 @@ impl NativeIntegrationStore for DaemonNativeIntegrationStore { level = "trace", skip_all )] - fn approval_consumed( + async fn approval_consumed( &self, approval_id: &NativeIntegrationApprovalId, ) -> NativeIntegrationStoreResult { - let (reply, receiver) = sync_channel(1); - self.submit(StoreCommand::ApprovalConsumed(approval_id.clone(), reply))?; - Self::await_reply(&receiver) + let database = self.database()?; + GlobalDbNativeIntegrationStore::new(&database) + .approval_consumed(approval_id) + .await } #[tracing::instrument( @@ -451,18 +298,15 @@ impl NativeIntegrationStore for DaemonNativeIntegrationStore { level = "trace", skip_all )] - fn quarantine_repository( + async fn quarantine_repository( &self, repository_id: &RepositoryId, transaction_id: &NativeIntegrationTransactionId, ) -> NativeIntegrationStoreResult<()> { - let (reply, receiver) = sync_channel(1); - self.submit(StoreCommand::QuarantineRepository( - repository_id.clone(), - transaction_id.clone(), - reply, - ))?; - Self::await_reply(&receiver) + let database = self.database()?; + GlobalDbNativeIntegrationStore::new(&database) + .quarantine_repository(repository_id, transaction_id) + .await } #[tracing::instrument( @@ -470,16 +314,14 @@ impl NativeIntegrationStore for DaemonNativeIntegrationStore { level = "trace", skip_all )] - fn begin_worktree_cleanup( + async fn begin_worktree_cleanup( &self, transaction: NativeWorktreeCleanupTransactionV1, ) -> NativeIntegrationStoreResult { - let (reply, receiver) = sync_channel(1); - self.submit(StoreCommand::BeginWorktreeCleanup( - Box::new(transaction), - reply, - ))?; - Self::await_reply(&receiver) + let database = self.database()?; + GlobalDbNativeIntegrationStore::new(&database) + .begin_worktree_cleanup(transaction) + .await } #[tracing::instrument( @@ -487,16 +329,14 @@ impl NativeIntegrationStore for DaemonNativeIntegrationStore { level = "trace", skip_all )] - fn read_worktree_cleanup( + async fn read_worktree_cleanup( &self, confirmation_digest: &ManifestDigest, ) -> NativeIntegrationStoreResult> { - let (reply, receiver) = sync_channel(1); - self.submit(StoreCommand::ReadWorktreeCleanup( - confirmation_digest.clone(), - reply, - ))?; - Self::await_reply(&receiver) + let database = self.database()?; + GlobalDbNativeIntegrationStore::new(&database) + .read_worktree_cleanup(confirmation_digest) + .await } #[tracing::instrument( @@ -504,18 +344,15 @@ impl NativeIntegrationStore for DaemonNativeIntegrationStore { level = "trace", skip_all )] - fn pending_worktree_cleanups( + async fn pending_worktree_cleanups( &self, repository_id: &RepositoryId, limit: u32, ) -> NativeIntegrationStoreResult> { - let (reply, receiver) = sync_channel(1); - self.submit(StoreCommand::PendingWorktreeCleanups( - repository_id.clone(), - limit, - reply, - ))?; - Self::await_reply(&receiver) + let database = self.database()?; + GlobalDbNativeIntegrationStore::new(&database) + .pending_worktree_cleanups(repository_id, limit) + .await } #[tracing::instrument( @@ -523,20 +360,20 @@ impl NativeIntegrationStore for DaemonNativeIntegrationStore { level = "trace", skip_all )] - fn compare_and_swap_worktree_cleanup( + async fn compare_and_swap_worktree_cleanup( &self, confirmation_digest: &ManifestDigest, expected_phase_revision: u64, replacement: NativeWorktreeCleanupTransactionV1, ) -> NativeIntegrationStoreResult { - let (reply, receiver) = sync_channel(1); - self.submit(StoreCommand::CompareAndSwapWorktreeCleanup( - confirmation_digest.clone(), - expected_phase_revision, - Box::new(replacement), - reply, - ))?; - Self::await_reply(&receiver) + let database = self.database()?; + GlobalDbNativeIntegrationStore::new(&database) + .compare_and_swap_worktree_cleanup( + confirmation_digest, + expected_phase_revision, + replacement, + ) + .await } #[tracing::instrument( @@ -544,327 +381,104 @@ impl NativeIntegrationStore for DaemonNativeIntegrationStore { level = "trace", skip_all )] - fn write_worktree_cleanup_terminal( + async fn write_worktree_cleanup_terminal( &self, confirmation_digest: &ManifestDigest, expected_phase_revision: u64, receipt: NativeWorktreeCleanupReceiptV1, ) -> NativeIntegrationStoreResult { - let (reply, receiver) = sync_channel(1); - self.submit(StoreCommand::WriteWorktreeCleanupTerminal( - confirmation_digest.clone(), - expected_phase_revision, - Box::new(receipt), - reply, - ))?; - Self::await_reply(&receiver) + let database = self.database()?; + GlobalDbNativeIntegrationStore::new(&database) + .write_worktree_cleanup_terminal(confirmation_digest, expected_phase_revision, receipt) + .await } } -fn settle( - operation: impl Future>, -) -> NativeIntegrationStoreResult { - native_runtime::block_on(operation) - .unwrap_or_else(|error| Err(NativeIntegrationStoreError::unavailable(error))) -} - -fn run_store_actor(database: &ActorDatabase, receiver: &Receiver) { - while let Ok(command) = receiver.recv() { - let store = database.native_integration_store(); - match command { - StoreCommand::SavePreview(preview, reply) => { - let _ = reply.send(settle(store.save_preview(*preview))); - } - StoreCommand::ReadPreview(preview_id, reply) => { - let _ = reply.send(settle(store.read_preview(&preview_id))); - } - StoreCommand::ReadPreviewByDigest(preview_digest, reply) => { - let _ = reply.send(settle(store.read_preview_by_digest(&preview_digest))); - } - StoreCommand::SaveApproval(approval, reply) => { - let _ = reply.send(settle(store.save_approval(*approval))); - } - StoreCommand::ReadApproval(approval_id, reply) => { - let _ = reply.send(settle(store.read_approval(&approval_id))); - } - StoreCommand::BeginOrReplay(record, reply) => { - let _ = reply.send(settle(store.begin_or_replay(*record))); - } - StoreCommand::ReadStatus(transaction_id, reply) => { - let _ = reply.send(settle(store.read_status(&transaction_id))); - } - StoreCommand::ReadRecord(transaction_id, reply) => { - let _ = reply.send(settle(store.read_record(&transaction_id))); - } - StoreCommand::ReadReceipt(transaction_id, reply) => { - let _ = reply.send(settle(store.read_receipt(&transaction_id))); - } - StoreCommand::ReadReceiptByDigest(receipt_digest, reply) => { - let _ = reply.send(settle(store.read_receipt_by_digest(&receipt_digest))); - } - StoreCommand::CompareAndSwapStatus( - transaction_id, - expected_phase_revision, - replacement, - reply, - ) => { - let _ = reply.send(settle(store.compare_and_swap_status( - &transaction_id, - expected_phase_revision, - *replacement, - ))); - } - StoreCommand::WriteTerminal( - transaction_id, - expected_phase_revision, - receipt, - reply, - ) => { - let _ = reply.send(settle(store.write_terminal( - &transaction_id, - expected_phase_revision, - *receipt, - ))); - } - StoreCommand::PendingTransactions(repository_id, reply) => { - let _ = reply.send(settle(store.pending_transactions(repository_id.as_ref()))); - } - StoreCommand::LiveCandidateGenerationBindings(repository_id, observed_at, reply) => { - let _ = reply.send(settle( - store.live_candidate_generation_bindings(&repository_id, observed_at), - )); - } - StoreCommand::ApprovalConsumed(approval_id, reply) => { - let _ = reply.send(settle(store.approval_consumed(&approval_id))); - } - StoreCommand::QuarantineRepository(repository_id, transaction_id, reply) => { - let _ = reply.send(settle( - store.quarantine_repository(&repository_id, &transaction_id), - )); - } - StoreCommand::BeginWorktreeCleanup(transaction, reply) => { - let _ = reply.send(settle(store.begin_worktree_cleanup(*transaction))); - } - StoreCommand::ReadWorktreeCleanup(confirmation_digest, reply) => { - let _ = reply.send(settle(store.read_worktree_cleanup(&confirmation_digest))); - } - StoreCommand::PendingWorktreeCleanups(repository_id, limit, reply) => { - let _ = reply.send(settle( - store.pending_worktree_cleanups(&repository_id, limit), - )); - } - StoreCommand::CompareAndSwapWorktreeCleanup( - confirmation_digest, - expected_phase_revision, - replacement, - reply, - ) => { - let _ = reply.send(settle(store.compare_and_swap_worktree_cleanup( - &confirmation_digest, - expected_phase_revision, - *replacement, - ))); - } - StoreCommand::WriteWorktreeCleanupTerminal( - confirmation_digest, - expected_phase_revision, - receipt, - reply, - ) => { - let _ = reply.send(settle(store.write_worktree_cleanup_terminal( - &confirmation_digest, - expected_phase_revision, - *receipt, - ))); - } +#[cfg(test)] +mod tests { + use tracedecay_domain::test_fixtures::digest; + use tracedecay_domain::{ActorId, CapabilityId}; + use tracedecay_global_db::tests::harness::RegisteredGlobalDbHarness; + use tracedecay_runtime_core::native_runtime::{self, NativeTask, NativeTaskOutcome}; + + use super::*; + + fn approval(approval_id: &str) -> NativeIntegrationApprovalV1 { + NativeIntegrationApprovalV1 { + approval_id: NativeIntegrationApprovalId::new(approval_id).expect("approval id"), + preview_id: NativeIntegrationPreviewId::new("preview.native-store").expect("preview"), + preview_digest: digest('1'), + principal: ActorId::new("actor.native-store").expect("principal"), + delegated_agent: None, + capability: CapabilityId::new("capability.git.native-integration-apply") + .expect("capability"), + grant_digest: digest('2'), + issued_at: UtcMicros(1), + expires_at: UtcMicros(2_000_000), + approval_digest: digest('3'), } - } -} - -/// Cloneable handle over the one retained store actor for a project database. -#[derive(Clone)] -pub struct SharedDaemonNativeIntegrationStore { - inner: Arc, -} - -impl SharedDaemonNativeIntegrationStore { - pub(crate) fn from_arc(inner: Arc) -> Self { - Self { inner } - } - - pub(crate) fn shutdown(&self) -> NativeIntegrationStoreResult { - self.inner.shutdown() - } - - pub fn save_approval( - &self, - approval: NativeIntegrationApprovalV1, - ) -> NativeIntegrationStoreResult<()> { - self.inner.save_approval(approval) - } - - pub fn read_approval( - &self, - approval_id: &NativeIntegrationApprovalId, - ) -> NativeIntegrationStoreResult> { - self.inner.read_approval(approval_id) - } - - pub fn read_preview_by_digest( - &self, - preview_digest: &ManifestDigest, - ) -> NativeIntegrationStoreResult> { - self.inner.read_preview_by_digest(preview_digest) - } - - pub fn read_receipt_by_digest( - &self, - receipt_digest: &ManifestDigest, - ) -> NativeIntegrationStoreResult> { - self.inner.read_receipt_by_digest(receipt_digest) - } -} - -impl NativeIntegrationStore for SharedDaemonNativeIntegrationStore { - fn save_preview( - &self, - preview: NativeIntegrationPreviewV1, - ) -> NativeIntegrationStoreResult<()> { - self.inner.save_preview(preview) - } - - fn read_preview( - &self, - preview_id: &NativeIntegrationPreviewId, - ) -> NativeIntegrationStoreResult> { - self.inner.read_preview(preview_id) - } - - fn begin_or_replay( - &self, - record: NativeIntegrationRecordV1, - ) -> NativeIntegrationStoreResult { - self.inner.begin_or_replay(record) - } - - fn read_status( - &self, - transaction_id: &NativeIntegrationTransactionId, - ) -> NativeIntegrationStoreResult> { - self.inner.read_status(transaction_id) - } - - fn read_record( - &self, - transaction_id: &NativeIntegrationTransactionId, - ) -> NativeIntegrationStoreResult> { - self.inner.read_record(transaction_id) - } - - fn read_receipt( - &self, - transaction_id: &NativeIntegrationTransactionId, - ) -> NativeIntegrationStoreResult> { - self.inner.read_receipt(transaction_id) - } - - fn compare_and_swap_status( - &self, - transaction_id: &NativeIntegrationTransactionId, - expected_phase_revision: u64, - replacement: NativeIntegrationTransactionStatusV1, - ) -> NativeIntegrationStoreResult { - self.inner - .compare_and_swap_status(transaction_id, expected_phase_revision, replacement) - } - - fn write_terminal( - &self, - transaction_id: &NativeIntegrationTransactionId, - expected_phase_revision: u64, - receipt: NativeIntegrationReceiptV1, - ) -> NativeIntegrationStoreResult { - self.inner - .write_terminal(transaction_id, expected_phase_revision, receipt) - } - - fn pending_transactions( - &self, - repository_id: Option<&RepositoryId>, - ) -> NativeIntegrationStoreResult> { - self.inner.pending_transactions(repository_id) - } - - fn live_candidate_generation_bindings( - &self, - repository_id: &RepositoryId, - observed_at: UtcMicros, - ) -> NativeIntegrationStoreResult> { - self.inner - .live_candidate_generation_bindings(repository_id, observed_at) - } - - fn approval_consumed( - &self, - approval_id: &NativeIntegrationApprovalId, - ) -> NativeIntegrationStoreResult { - self.inner.approval_consumed(approval_id) - } - - fn quarantine_repository( - &self, - repository_id: &RepositoryId, - transaction_id: &NativeIntegrationTransactionId, - ) -> NativeIntegrationStoreResult<()> { - self.inner - .quarantine_repository(repository_id, transaction_id) - } - - fn begin_worktree_cleanup( - &self, - transaction: NativeWorktreeCleanupTransactionV1, - ) -> NativeIntegrationStoreResult { - self.inner.begin_worktree_cleanup(transaction) - } - - fn read_worktree_cleanup( - &self, - confirmation_digest: &ManifestDigest, - ) -> NativeIntegrationStoreResult> { - self.inner.read_worktree_cleanup(confirmation_digest) - } - - fn pending_worktree_cleanups( - &self, - repository_id: &RepositoryId, - limit: u32, - ) -> NativeIntegrationStoreResult> { - self.inner.pending_worktree_cleanups(repository_id, limit) - } - - fn compare_and_swap_worktree_cleanup( - &self, - confirmation_digest: &ManifestDigest, - expected_phase_revision: u64, - replacement: NativeWorktreeCleanupTransactionV1, - ) -> NativeIntegrationStoreResult { - self.inner.compare_and_swap_worktree_cleanup( - confirmation_digest, - expected_phase_revision, - replacement, - ) - } + .seal() + .expect("sealed approval") + } + + #[tokio::test] + async fn store_awaited_on_a_native_worker_never_parks_it() { + let database = RegisteredGlobalDbHarness::open("native-integration-native-worker").await; + let registered = database.registered.clone(); + let writer_waits = native_runtime::native_thread_writer_waits(); + let reader_waits = native_runtime::native_thread_reader_waits(); + + let task = NativeTask::spawn(async move { + assert!( + native_runtime::Runtime::current_handle().is_some(), + "the store must be awaited on a native worker" + ); + let store = DaemonNativeIntegrationStore::open(registered); + let approval = approval("approval.native-store.worker"); + store.save_approval(approval.clone()).await?; + let read = store.read_approval(&approval.approval_id).await; + store.shutdown(); + read.map(|read| (read, approval)) + }) + .expect("native task admitted"); - fn write_worktree_cleanup_terminal( - &self, - confirmation_digest: &ManifestDigest, - expected_phase_revision: u64, - receipt: NativeWorktreeCleanupReceiptV1, - ) -> NativeIntegrationStoreResult { - self.inner.write_worktree_cleanup_terminal( - confirmation_digest, - expected_phase_revision, - receipt, - ) + let NativeTaskOutcome::Completed(read) = task.wait().await else { + panic!("store task did not complete"); + }; + let (read, approval) = read.expect("store calls settle on a native worker"); + assert_eq!(read, Some(approval)); + assert_eq!( + native_runtime::native_thread_writer_waits(), + writer_waits, + "no writer wait parked a native worker" + ); + assert_eq!( + native_runtime::native_thread_reader_waits(), + reader_waits, + "no reader wait parked a native worker" + ); + } + + #[tokio::test] + async fn shutdown_closes_every_clone_once_and_fails_later_calls_closed() { + let database = RegisteredGlobalDbHarness::open("native-integration-shutdown").await; + let store = DaemonNativeIntegrationStore::open(database.registered.clone()); + let clone = store.clone(); + let approval = approval("approval.native-store.shutdown"); + clone + .save_approval(approval.clone()) + .await + .expect("open store accepts writes"); + + assert!(store.shutdown(), "the first shutdown closes the store"); + assert!(!clone.shutdown(), "a clone shares the one closed lease"); + assert!(matches!( + clone.read_approval(&approval.approval_id).await, + Err(NativeIntegrationStoreError::Unavailable(reason)) if reason.contains("shut down") + )); + assert!(matches!( + clone.pending_transactions(None).await, + Err(NativeIntegrationStoreError::Unavailable(_)) + )); } } diff --git a/crates/tracedecay-agent-hosts/src/native_integration/worktree.rs b/crates/tracedecay-agent-hosts/src/native_integration/worktree.rs index 73d76670ed..3e1c568cc1 100644 --- a/crates/tracedecay-agent-hosts/src/native_integration/worktree.rs +++ b/crates/tracedecay-agent-hosts/src/native_integration/worktree.rs @@ -30,7 +30,7 @@ use tracedecay_domain::{ use tracedecay_runtime_core::git_repository::{GitRepositoryAuthority, GitRepositoryStatus}; use tracedecay_rusqlite_runtime::repository::AuthorizedScopeSetSqliteStorage; -use super::store::SharedDaemonNativeIntegrationStore; +use super::store::DaemonNativeIntegrationStore; #[derive(Clone)] pub struct DaemonAuthorizedScopeSetReader { @@ -261,7 +261,7 @@ pub struct DaemonNativeWorktreeAuthority { pub(super) repository_id: RepositoryId, pub(super) repository_root: PathBuf, pub(super) repository_common_dir: PathBuf, - pub(super) store: SharedDaemonNativeIntegrationStore, + pub(super) store: DaemonNativeIntegrationStore, pub(super) holder_fence: WorktreeHolderAdmissionFenceV1, inventory_epoch: AtomicU64, } @@ -272,7 +272,7 @@ impl DaemonNativeWorktreeAuthority { project_id: ProjectId, repository_id: RepositoryId, repository_root: &Path, - store: SharedDaemonNativeIntegrationStore, + store: DaemonNativeIntegrationStore, ) -> Result { project_id.validate()?; repository_id.validate()?; @@ -677,22 +677,23 @@ impl NativeWorktreePort for DaemonNativeWorktreeAuthority { ))) } - fn remove( + async fn remove( &self, request: &WorktreeCleanupRemoveRequestV1, scope_set: &AuthorizedScopeSet, cancellation: &CancellationSignal, ) -> Result { - self.remove_cleanup(request, scope_set, cancellation) + self.remove_cleanup(request, scope_set, cancellation).await } - fn reconcile( + async fn reconcile( &self, request: &WorktreeCleanupReconcileRequestV1, scope_set: &AuthorizedScopeSet, cancellation: &CancellationSignal, ) -> Result { self.reconcile_cleanup(request, scope_set, cancellation) + .await } } diff --git a/crates/tracedecay-agent-hosts/src/native_integration/worktree_cleanup.rs b/crates/tracedecay-agent-hosts/src/native_integration/worktree_cleanup.rs index 7b963c1cd8..f301926ffd 100644 --- a/crates/tracedecay-agent-hosts/src/native_integration/worktree_cleanup.rs +++ b/crates/tracedecay-agent-hosts/src/native_integration/worktree_cleanup.rs @@ -29,7 +29,7 @@ enum CleanupNativeStateV1 { impl DaemonNativeWorktreeAuthority { #[tracing::instrument(name = "daemon.git.worktree.remove", level = "trace", skip_all)] - pub(super) fn remove_cleanup( + pub(super) async fn remove_cleanup( &self, request: &WorktreeCleanupRemoveRequestV1, scope_set: &AuthorizedScopeSet, @@ -126,7 +126,7 @@ impl DaemonNativeWorktreeAuthority { } .seal()?; - let transaction = match self.store.begin_worktree_cleanup(prepared) { + let transaction = match self.store.begin_worktree_cleanup(prepared).await { Ok(NativeWorktreeCleanupBeginResultV1::Started(transaction)) => *transaction, Ok(NativeWorktreeCleanupBeginResultV1::Replay(receipt)) => { return Ok(removal_from_outcome( @@ -136,7 +136,9 @@ impl DaemonNativeWorktreeAuthority { )); } Ok(NativeWorktreeCleanupBeginResultV1::RecoveryRequired(transaction)) => { - return self.reconcile_for_remove(*transaction, scope_set, admission, cancellation); + return self + .reconcile_for_remove(*transaction, scope_set, admission, cancellation) + .await; } Err(error) => return Ok(removal_store_error(error)), }; @@ -144,6 +146,7 @@ impl DaemonNativeWorktreeAuthority { if !cancellation.try_begin_commit() { return match self .write_terminal(transaction, NativeWorktreeCleanupOutcomeV1::AbortedNoChange) + .await { Ok(_) => Ok(WorktreeCleanupRemovalV1::Unavailable), Err(outcome) => { @@ -152,11 +155,11 @@ impl DaemonNativeWorktreeAuthority { } }; } - self.execute_remove(transaction, scope_set, admission) + self.execute_remove(transaction, scope_set, admission).await } #[tracing::instrument(name = "daemon.git.worktree.reconcile", level = "trace", skip_all)] - pub(super) fn reconcile_cleanup( + pub(super) async fn reconcile_cleanup( &self, request: &WorktreeCleanupReconcileRequestV1, scope_set: &AuthorizedScopeSet, @@ -168,6 +171,7 @@ impl DaemonNativeWorktreeAuthority { let transaction = match self .store .read_worktree_cleanup(&request.confirmation_digest) + .await { Ok(Some(transaction)) => transaction, Ok(None) => return Ok(WorktreeCleanupReconciliationV1::Stale), @@ -204,22 +208,25 @@ impl DaemonNativeWorktreeAuthority { return Ok(WorktreeCleanupReconciliationV1::Unavailable); } self.reconcile_transaction(transaction, scope_set, admission) + .await } - fn execute_remove( + async fn execute_remove( &self, transaction: NativeWorktreeCleanupTransactionV1, scope_set: &AuthorizedScopeSet, admission: WorktreeCleanupAdmissionV1, ) -> Result { - let started = - match self.advance_phase(transaction, NativeWorktreeCleanupPhaseV1::MutationStarted) { - Ok(transaction) => transaction, - Err(outcome) => { - admission.retain_recovery(); - return Ok(outcome); - } - }; + let started = match self + .advance_phase(transaction, NativeWorktreeCleanupPhaseV1::MutationStarted) + .await + { + Ok(transaction) => transaction, + Err(outcome) => { + admission.retain_recovery(); + return Ok(outcome); + } + }; let command = run_worktree_remove( &started.command.repository_root, &started.command.worktree_root, @@ -228,7 +235,10 @@ impl DaemonNativeWorktreeAuthority { match native_state { CleanupNativeStateV1::Removed => { let observed_at = now_at_least(started.updated_at); - match self.write_terminal(started, NativeWorktreeCleanupOutcomeV1::Removed) { + match self + .write_terminal(started, NativeWorktreeCleanupOutcomeV1::Removed) + .await + { Ok(receipt) => Ok(WorktreeCleanupRemovalV1::Removed { confirmation_digest: receipt.transaction.confirmation_digest, observed_at: receipt.completed_at, @@ -252,7 +262,9 @@ impl DaemonNativeWorktreeAuthority { CleanupNativeStateV1::Present if command.is_ok_and(|output| !output.status.success()) => { - match self.write_terminal(started, NativeWorktreeCleanupOutcomeV1::AbortedNoChange) + match self + .write_terminal(started, NativeWorktreeCleanupOutcomeV1::AbortedNoChange) + .await { Ok(_) => Ok(WorktreeCleanupRemovalV1::Denied), Err(outcome) => { @@ -264,6 +276,7 @@ impl DaemonNativeWorktreeAuthority { CleanupNativeStateV1::Foreign => { match self .write_terminal(started, NativeWorktreeCleanupOutcomeV1::RefusedForeignDrift) + .await { Ok(_) => Ok(WorktreeCleanupRemovalV1::Denied), Err(outcome) => { @@ -273,7 +286,9 @@ impl DaemonNativeWorktreeAuthority { } } CleanupNativeStateV1::Present | CleanupNativeStateV1::Uncertain => { - match self.advance_phase(started, NativeWorktreeCleanupPhaseV1::NeedsReconciliation) + match self + .advance_phase(started, NativeWorktreeCleanupPhaseV1::NeedsReconciliation) + .await { Ok(_) | Err(WorktreeCleanupRemovalV1::DurabilityUncertain) => { admission.retain_recovery(); @@ -288,7 +303,7 @@ impl DaemonNativeWorktreeAuthority { } } - fn reconcile_for_remove( + async fn reconcile_for_remove( &self, transaction: NativeWorktreeCleanupTransactionV1, scope_set: &AuthorizedScopeSet, @@ -300,7 +315,10 @@ impl DaemonNativeWorktreeAuthority { return Ok(WorktreeCleanupRemovalV1::Unavailable); } Ok( - match self.reconcile_transaction(transaction, scope_set, admission)? { + match self + .reconcile_transaction(transaction, scope_set, admission) + .await? + { WorktreeCleanupReconciliationV1::Removed { confirmation_digest, observed_at, @@ -321,7 +339,7 @@ impl DaemonNativeWorktreeAuthority { ) } - fn reconcile_transaction( + async fn reconcile_transaction( &self, transaction: NativeWorktreeCleanupTransactionV1, scope_set: &AuthorizedScopeSet, @@ -329,7 +347,10 @@ impl DaemonNativeWorktreeAuthority { ) -> Result { match self.cleanup_native_state(&transaction, scope_set)? { CleanupNativeStateV1::Removed => { - match self.write_terminal(transaction, NativeWorktreeCleanupOutcomeV1::Removed) { + match self + .write_terminal(transaction, NativeWorktreeCleanupOutcomeV1::Removed) + .await + { Ok(receipt) => Ok(WorktreeCleanupReconciliationV1::Removed { confirmation_digest: receipt.transaction.confirmation_digest, observed_at: receipt.completed_at, @@ -343,6 +364,7 @@ impl DaemonNativeWorktreeAuthority { CleanupNativeStateV1::Present => { match self .write_terminal(transaction, NativeWorktreeCleanupOutcomeV1::AbortedNoChange) + .await { Ok(_) => Ok(WorktreeCleanupReconciliationV1::StillPresent), Err(_) => { @@ -352,10 +374,13 @@ impl DaemonNativeWorktreeAuthority { } } CleanupNativeStateV1::Foreign => { - match self.write_terminal( - transaction, - NativeWorktreeCleanupOutcomeV1::RefusedForeignDrift, - ) { + match self + .write_terminal( + transaction, + NativeWorktreeCleanupOutcomeV1::RefusedForeignDrift, + ) + .await + { Ok(_) => Ok(WorktreeCleanupReconciliationV1::Denied), Err(_) => { admission.retain_recovery(); @@ -412,7 +437,7 @@ impl DaemonNativeWorktreeAuthority { })) } - fn advance_phase( + async fn advance_phase( &self, current: NativeWorktreeCleanupTransactionV1, phase: NativeWorktreeCleanupPhaseV1, @@ -431,10 +456,11 @@ impl DaemonNativeWorktreeAuthority { current.phase_revision, replacement, ) + .await .map_err(removal_store_error) } - fn write_terminal( + async fn write_terminal( &self, current: NativeWorktreeCleanupTransactionV1, outcome: NativeWorktreeCleanupOutcomeV1, @@ -460,6 +486,7 @@ impl DaemonNativeWorktreeAuthority { current.phase_revision, receipt, ) + .await .map_err(removal_store_error) } } diff --git a/crates/tracedecay-application/src/native_integration/transaction.rs b/crates/tracedecay-application/src/native_integration/transaction.rs index e82768cf66..1d246e025b 100644 --- a/crates/tracedecay-application/src/native_integration/transaction.rs +++ b/crates/tracedecay-application/src/native_integration/transaction.rs @@ -137,7 +137,7 @@ pub struct NativeIntegrationTransactionCoordinator { topology: Arc, native: Arc, authorization: Arc, - repository_locks: Mutex>>>, + repository_locks: Mutex>>>, cancellations: Mutex>, } @@ -161,7 +161,7 @@ where N: NativeIntegrationMechanics, A: NativeIntegrationAuthorizationPort, { - fn preflight( + async fn preflight( &self, request: &NativeIntegrationPreflightRequestV1, cancellation: &CancellationSignal, @@ -212,13 +212,14 @@ where .preflight(&selection, request, cancellation, &native_cancellation)?; self.store .save_preview(preview.clone()) + .await .map_err(map_store_error)?; Ok(NativeIntegrationPreflightOutcomeV1::Preview(Box::new( preview, ))) } - fn apply( + async fn apply( &self, request: &NativeIntegrationApplyRequestV1, external_cancellation: &CancellationSignal, @@ -228,13 +229,14 @@ where let mut locks = self.repository_locks.lock().map_err(lock_error)?; locks .entry(repository_id.clone()) - .or_insert_with(|| Arc::new(Mutex::new(()))) + .or_insert_with(|| Arc::new(tokio::sync::Mutex::new(()))) .clone() }; - let _guard = repository_lock.lock().map_err(lock_error)?; + let _guard = repository_lock.lock().await; if let Some(receipt) = self .store .read_receipt(&request.transaction_id) + .await .map_err(map_store_error)? { return Ok(receipt); @@ -287,7 +289,11 @@ where status, terminal_receipt: None, }; - let begin = self.store.begin_or_replay(record).map_err(map_store_error); + let begin = self + .store + .begin_or_replay(record) + .await + .map_err(map_store_error); let record = match begin { Ok(NativeIntegrationBeginResultV1::Started(record)) => *record, Ok(NativeIntegrationBeginResultV1::Replay(receipt)) => { @@ -305,7 +311,9 @@ where }; if cancellation.is_cancelled() { - let receipt = self.finish_from_live_precommit_probe(&record, request.observed_at); + let receipt = self + .finish_from_live_precommit_probe(&record, request.observed_at) + .await; self.clear_cancellation(&request.transaction_id)?; return receipt; } @@ -315,7 +323,9 @@ where NativeIntegrationPhaseV1::CandidateVerified, false, request.observed_at, - ) { + ) + .await + { Ok(status) => status, Err(error) => { tracing::warn!( @@ -323,7 +333,9 @@ where transaction_id = %record.status.transaction_id, "candidate verification status advance failed; settling from live probe" ); - let receipt = self.finish_from_live_precommit_probe(&record, request.observed_at); + let receipt = self + .finish_from_live_precommit_probe(&record, request.observed_at) + .await; self.clear_cancellation(&request.transaction_id)?; return receipt; } @@ -335,12 +347,16 @@ where if self.authorization.authorize_apply(request, true) != NativeIntegrationAuthorizationOutcomeV1::Authorized { - let receipt = self.finish_from_live_precommit_probe(&record, request.observed_at); + let receipt = self + .finish_from_live_precommit_probe(&record, request.observed_at) + .await; self.clear_cancellation(&request.transaction_id)?; return receipt; } if cancellation.is_cancelled() { - let receipt = self.finish_from_live_precommit_probe(&record, request.observed_at); + let receipt = self + .finish_from_live_precommit_probe(&record, request.observed_at) + .await; self.clear_cancellation(&request.transaction_id)?; return receipt; } @@ -354,7 +370,9 @@ where NativeIntegrationAnalysisRevalidationV1::Stale | NativeIntegrationAnalysisRevalidationV1::Unavailable, ) => { - let receipt = self.finish_from_live_precommit_probe(&record, request.observed_at); + let receipt = self + .finish_from_live_precommit_probe(&record, request.observed_at) + .await; self.clear_cancellation(&request.transaction_id)?; return receipt; } @@ -364,7 +382,9 @@ where transaction_id = %record.status.transaction_id, "semantic revalidation failed after durable begin; settling from live probe" ); - let receipt = self.finish_from_live_precommit_probe(&record, request.observed_at); + let receipt = self + .finish_from_live_precommit_probe(&record, request.observed_at) + .await; self.clear_cancellation(&request.transaction_id)?; return receipt; } @@ -382,15 +402,17 @@ where index_digest, worktree_digest, }) => { - let receipt = self.write_terminal( - &record, - NativeIntegrationTerminalOutcomeV1::AbortedNoChange, - tip, - tree, - index_digest, - worktree_digest, - request.observed_at, - ); + let receipt = self + .write_terminal( + &record, + NativeIntegrationTerminalOutcomeV1::AbortedNoChange, + tip, + tree, + index_digest, + worktree_digest, + request.observed_at, + ) + .await; self.clear_cancellation(&request.transaction_id)?; return receipt; } @@ -399,17 +421,19 @@ where // the existing fail-closed inspection path for that ambiguity. Ok(NativeIntegrationProbeV1::CommittedState { .. }) | Ok(NativeIntegrationProbeV1::Unavailable) => { - let receipt = self.needs_inspection(&record, request.observed_at); + let receipt = self.needs_inspection(&record, request.observed_at).await; self.clear_cancellation(&request.transaction_id)?; return receipt; } Err(error) => { - let receipt = self.needs_inspection_after_error( - &record, - request.observed_at, - "pre-commit probe", - error, - ); + let receipt = self + .needs_inspection_after_error( + &record, + request.observed_at, + "pre-commit probe", + error, + ) + .await; self.clear_cancellation(&request.transaction_id)?; return receipt; } @@ -420,7 +444,9 @@ where NativeIntegrationPhaseV1::RefCommitStarted, false, request.observed_at, - ) { + ) + .await + { Ok(status) => status, Err(error) => { tracing::warn!( @@ -428,7 +454,9 @@ where transaction_id = %record.status.transaction_id, "commit-start status advance failed; settling from live probe" ); - let receipt = self.finish_from_live_precommit_probe(&record, request.observed_at); + let receipt = self + .finish_from_live_precommit_probe(&record, request.observed_at) + .await; self.clear_cancellation(&request.transaction_id)?; return receipt; } @@ -439,54 +467,73 @@ where }; let effect = self.native.apply(&record.preview, &cancellation); let receipt = match effect { - Err(error) => self.needs_inspection_after_error( - &record, - request.observed_at, - "native apply", - error, - ), + Err(error) => { + self.needs_inspection_after_error( + &record, + request.observed_at, + "native apply", + error, + ) + .await + } Ok(NativeApplyEffectV1::Committed { new_tip, final_tree, final_index_digest, final_worktree_digest, - }) => self.finish_from_probe( - &record, - NativeIntegrationProbeV1::CommittedState { - tip: new_tip, - tree: final_tree, - index_digest: final_index_digest, - worktree_digest: final_worktree_digest, - }, - request.observed_at, - ), + }) => { + self.finish_from_probe( + &record, + NativeIntegrationProbeV1::CommittedState { + tip: new_tip, + tree: final_tree, + index_digest: final_index_digest, + worktree_digest: final_worktree_digest, + }, + request.observed_at, + ) + .await + } Ok(NativeApplyEffectV1::FailedNoChange) => { self.finish_from_live_probe(&record, request.observed_at) + .await } Ok(NativeApplyEffectV1::UnknownAfterCommitPoint { candidate_tip }) => { match (self.native.probe(&record), candidate_tip) { (Ok(NativeIntegrationProbeV1::Diverged { .. }), Some(candidate_tip)) => { match self.native.rollback(&record, &candidate_tip) { - Ok(rolled_back) => self.finish_rolled_back_or_inspect( - &record, - rolled_back, - request.observed_at, - ), - Err(error) => self.needs_inspection_after_error( - &record, - request.observed_at, - "native rollback", - error, - ), + Ok(rolled_back) => { + self.finish_rolled_back_or_inspect( + &record, + rolled_back, + request.observed_at, + ) + .await + } + Err(error) => { + self.needs_inspection_after_error( + &record, + request.observed_at, + "native rollback", + error, + ) + .await + } } } - (Ok(probe), _) => self.finish_from_probe(&record, probe, request.observed_at), - (Err(error), _) => self.needs_inspection_after_error( - &record, - request.observed_at, - "post-commit probe", - error, - ), + (Ok(probe), _) => { + self.finish_from_probe(&record, probe, request.observed_at) + .await + } + (Err(error), _) => { + self.needs_inspection_after_error( + &record, + request.observed_at, + "post-commit probe", + error, + ) + .await + } } } }; @@ -494,22 +541,24 @@ where receipt } - fn status( + async fn status( &self, request: &NativeIntegrationStatusRequestV1, ) -> Result, NativeIntegrationPortError> { self.store .read_status(&request.transaction_id) + .await .map_err(map_store_error) } - fn cancel( + async fn cancel( &self, request: &NativeIntegrationCancelRequestV1, ) -> Result { let Some(status) = self .store .read_status(&request.transaction_id) + .await .map_err(map_store_error)? else { return Ok(NativeIntegrationCancelDispositionV1::UnknownTransaction); @@ -528,6 +577,7 @@ where replacement.updated_at = request.requested_at; self.store .compare_and_swap_status(&request.transaction_id, status.phase_revision, replacement) + .await .map_err(map_store_error)?; // Publish the durable cancellation revision before waking the live // apply. Once the token is visible, its terminal settlement can reload @@ -543,13 +593,14 @@ where Ok(NativeIntegrationCancelDispositionV1::CancellationRequested) } - fn recover( + async fn recover( &self, request: &NativeIntegrationRecoveryRequestV1, ) -> Result { if let Some(receipt) = self .store .read_receipt(&request.transaction_id) + .await .map_err(map_store_error)? { return Ok(receipt); @@ -557,10 +608,12 @@ where let record = self .store .read_record(&request.transaction_id) + .await .map_err(map_store_error)? .ok_or(NativeIntegrationPortError::Stale)?; let probe = self.native.probe(&record)?; self.finish_from_probe(&record, probe, request.observed_at) + .await } } @@ -580,7 +633,7 @@ where Ok(()) } - fn finish_rolled_back_or_inspect( + async fn finish_rolled_back_or_inspect( &self, record: &NativeIntegrationRecordV1, probe: NativeIntegrationProbeV1, @@ -592,20 +645,23 @@ where tree, index_digest, worktree_digest, - } => self.write_terminal( - record, - NativeIntegrationTerminalOutcomeV1::RolledBack, - tip, - tree, - index_digest, - worktree_digest, - observed_at, - ), - _ => self.needs_inspection(record, observed_at), + } => { + self.write_terminal( + record, + NativeIntegrationTerminalOutcomeV1::RolledBack, + tip, + tree, + index_digest, + worktree_digest, + observed_at, + ) + .await + } + _ => self.needs_inspection(record, observed_at).await, } } - fn finish_from_live_precommit_probe( + async fn finish_from_live_precommit_probe( &self, record: &NativeIntegrationRecordV1, observed_at: UtcMicros, @@ -613,15 +669,17 @@ where let current = self .store .read_record(&record.status.transaction_id) + .await .map_err(map_store_error)? .ok_or(NativeIntegrationPortError::Stale)?; if current.preview != record.preview || current.approval != record.approval { return Err(NativeIntegrationPortError::TransactionConflict); } self.finish_current_from_live_precommit_probe(¤t, observed_at) + .await } - fn finish_current_from_live_precommit_probe( + async fn finish_current_from_live_precommit_probe( &self, record: &NativeIntegrationRecordV1, observed_at: UtcMicros, @@ -640,45 +698,54 @@ where index_digest, worktree_digest, }, - ) => self.write_terminal( - record, - NativeIntegrationTerminalOutcomeV1::AbortedNoChange, - tip, - tree, - index_digest, - worktree_digest, - observed_at, - ), + ) => { + self.write_terminal( + record, + NativeIntegrationTerminalOutcomeV1::AbortedNoChange, + tip, + tree, + index_digest, + worktree_digest, + observed_at, + ) + .await + } Ok(NativeIntegrationProbeV1::CommittedState { .. }) | Ok(NativeIntegrationProbeV1::Unavailable) => { - self.needs_inspection(record, observed_at) + self.needs_inspection(record, observed_at).await + } + Err(error) => { + self.needs_inspection_after_error( + record, + observed_at, + "pre-commit settlement probe", + error, + ) + .await } - Err(error) => self.needs_inspection_after_error( - record, - observed_at, - "pre-commit settlement probe", - error, - ), } } - fn finish_from_live_probe( + async fn finish_from_live_probe( &self, record: &NativeIntegrationRecordV1, observed_at: UtcMicros, ) -> Result { match self.native.probe(record) { - Ok(probe) => self.finish_from_probe(record, probe, observed_at), - Err(error) => self.needs_inspection_after_error( - record, - observed_at, - "terminal settlement probe", - error, - ), + Ok(probe) => self.finish_from_probe(record, probe, observed_at).await, + Err(error) => { + self.needs_inspection_after_error( + record, + observed_at, + "terminal settlement probe", + error, + ) + .await + } } } - fn needs_inspection_after_error( + async fn needs_inspection_after_error( &self, record: &NativeIntegrationRecordV1, observed_at: UtcMicros, @@ -691,10 +758,10 @@ where operation, "native transaction state is uncertain after an operation error" ); - self.needs_inspection(record, observed_at) + self.needs_inspection(record, observed_at).await } - fn finish_from_probe( + async fn finish_from_probe( &self, record: &NativeIntegrationRecordV1, probe: NativeIntegrationProbeV1, @@ -706,37 +773,43 @@ where tree, index_digest, worktree_digest, - } => self.write_terminal( - record, - NativeIntegrationTerminalOutcomeV1::AbortedNoChange, - tip, - tree, - index_digest, - worktree_digest, - observed_at, - ), + } => { + self.write_terminal( + record, + NativeIntegrationTerminalOutcomeV1::AbortedNoChange, + tip, + tree, + index_digest, + worktree_digest, + observed_at, + ) + .await + } NativeIntegrationProbeV1::CommittedState { tip, tree, index_digest, worktree_digest, - } => self.write_terminal( - record, - NativeIntegrationTerminalOutcomeV1::Committed, - tip, - tree, - index_digest, - worktree_digest, - observed_at, - ), + } => { + self.write_terminal( + record, + NativeIntegrationTerminalOutcomeV1::Committed, + tip, + tree, + index_digest, + worktree_digest, + observed_at, + ) + .await + } NativeIntegrationProbeV1::Diverged { .. } | NativeIntegrationProbeV1::Unavailable => { - self.needs_inspection(record, observed_at) + self.needs_inspection(record, observed_at).await } } } #[allow(clippy::too_many_arguments)] - fn write_terminal( + async fn write_terminal( &self, record: &NativeIntegrationRecordV1, outcome: NativeIntegrationTerminalOutcomeV1, @@ -755,7 +828,8 @@ where NativeIntegrationPhaseV1::FinalStateVerification, record.status.cancellation_requested, observed_at, - )? + ) + .await? } else { record.status.clone() }; @@ -781,16 +855,18 @@ where expected_phase_revision, receipt, ) + .await .map_err(map_store_error) } - fn needs_inspection( + async fn needs_inspection( &self, record: &NativeIntegrationRecordV1, observed_at: UtcMicros, ) -> Result { self.store .quarantine_repository(&record.status.repository_id, &record.status.transaction_id) + .await .map_err(map_store_error)?; self.write_terminal( record, @@ -801,10 +877,11 @@ where record.preview.repository_snapshot.worktree_digest.clone(), observed_at, ) + .await } } -fn advance_status( +async fn advance_status( store: &S, current: &NativeIntegrationTransactionStatusV1, phase: NativeIntegrationPhaseV1, @@ -818,6 +895,7 @@ fn advance_status( replacement.updated_at = observed_at; store .compare_and_swap_status(¤t.transaction_id, current.phase_revision, replacement) + .await .map_err(map_store_error) } diff --git a/crates/tracedecay-application/src/native_integration/transaction_tests.rs b/crates/tracedecay-application/src/native_integration/transaction_tests.rs index 781a0c191b..83d53d4b63 100644 --- a/crates/tracedecay-application/src/native_integration/transaction_tests.rs +++ b/crates/tracedecay-application/src/native_integration/transaction_tests.rs @@ -4,6 +4,8 @@ use std::collections::BTreeMap; use std::sync::atomic::{AtomicUsize, Ordering}; use std::sync::{Arc, Mutex}; +use futures_util::FutureExt; + use tracedecay_contracts::{ CancellationContext, CancellationSignal, CapabilityGrantId, CapabilityGrantSnapshot, Deadline, DisclosureClass, NativeIntegrationApplyRequestV1, NativeIntegrationCancelDispositionV1, @@ -48,7 +50,7 @@ struct StatusStore { } impl NativeIntegrationStore for StatusStore { - fn save_preview( + async fn save_preview( &self, _preview: tracedecay_domain::NativeIntegrationPreviewV1, ) -> NativeIntegrationStoreResult<()> { @@ -57,14 +59,14 @@ impl NativeIntegrationStore for StatusStore { )) } - fn read_preview( + async fn read_preview( &self, _preview_id: &NativeIntegrationPreviewId, ) -> NativeIntegrationStoreResult> { Ok(None) } - fn begin_or_replay( + async fn begin_or_replay( &self, record: NativeIntegrationRecordV1, ) -> NativeIntegrationStoreResult { @@ -80,28 +82,28 @@ impl NativeIntegrationStore for StatusStore { Ok(NativeIntegrationBeginResultV1::Started(Box::new(record))) } - fn read_status( + async fn read_status( &self, transaction_id: &NativeIntegrationTransactionId, ) -> NativeIntegrationStoreResult> { Ok(self.statuses.lock().unwrap().get(transaction_id).cloned()) } - fn read_record( + async fn read_record( &self, transaction_id: &NativeIntegrationTransactionId, ) -> NativeIntegrationStoreResult> { Ok(self.records.lock().unwrap().get(transaction_id).cloned()) } - fn read_receipt( + async fn read_receipt( &self, transaction_id: &NativeIntegrationTransactionId, ) -> NativeIntegrationStoreResult> { Ok(self.receipts.lock().unwrap().get(transaction_id).cloned()) } - fn compare_and_swap_status( + async fn compare_and_swap_status( &self, transaction_id: &NativeIntegrationTransactionId, expected_phase_revision: u64, @@ -125,7 +127,7 @@ impl NativeIntegrationStore for StatusStore { Ok(replacement) } - fn write_terminal( + async fn write_terminal( &self, transaction_id: &NativeIntegrationTransactionId, expected_phase_revision: u64, @@ -152,14 +154,14 @@ impl NativeIntegrationStore for StatusStore { Ok(receipt) } - fn pending_transactions( + async fn pending_transactions( &self, _repository_id: Option<&RepositoryId>, ) -> NativeIntegrationStoreResult> { Ok(Vec::new()) } - fn live_candidate_generation_bindings( + async fn live_candidate_generation_bindings( &self, _repository_id: &RepositoryId, _observed_at: UtcMicros, @@ -167,14 +169,14 @@ impl NativeIntegrationStore for StatusStore { Ok(Vec::new()) } - fn approval_consumed( + async fn approval_consumed( &self, _approval_id: &NativeIntegrationApprovalId, ) -> NativeIntegrationStoreResult { Ok(false) } - fn quarantine_repository( + async fn quarantine_repository( &self, _repository_id: &RepositoryId, transaction_id: &NativeIntegrationTransactionId, @@ -186,7 +188,7 @@ impl NativeIntegrationStore for StatusStore { Ok(()) } - fn begin_worktree_cleanup( + async fn begin_worktree_cleanup( &self, _transaction: tracedecay_domain::NativeWorktreeCleanupTransactionV1, ) -> NativeIntegrationStoreResult { @@ -195,7 +197,7 @@ impl NativeIntegrationStore for StatusStore { )) } - fn read_worktree_cleanup( + async fn read_worktree_cleanup( &self, _confirmation_digest: &ManifestDigest, ) -> NativeIntegrationStoreResult> @@ -203,7 +205,7 @@ impl NativeIntegrationStore for StatusStore { Ok(None) } - fn compare_and_swap_worktree_cleanup( + async fn compare_and_swap_worktree_cleanup( &self, _confirmation_digest: &ManifestDigest, _expected_phase_revision: u64, @@ -214,7 +216,7 @@ impl NativeIntegrationStore for StatusStore { )) } - fn write_worktree_cleanup_terminal( + async fn write_worktree_cleanup_terminal( &self, _confirmation_digest: &ManifestDigest, _expected_phase_revision: u64, @@ -371,6 +373,8 @@ impl NativeIntegrationMechanics for DurableCancelDuringRevalidation { let status = self .store .read_status(&transaction_id) + .now_or_never() + .expect("fixture store is synchronous") .unwrap() .expect("durable status"); let mut cancelled = status.clone(); @@ -378,6 +382,8 @@ impl NativeIntegrationMechanics for DurableCancelDuringRevalidation { cancelled.cancellation_requested = true; self.store .compare_and_swap_status(&transaction_id, status.phase_revision, cancelled) + .now_or_never() + .expect("fixture store is synchronous") .expect("concurrent durable cancellation"); return Ok(NativeIntegrationAnalysisRevalidationV1::Stale); } @@ -649,8 +655,8 @@ fn status( } } -#[test] -fn cancellation_is_durable_only_before_the_ref_commit_boundary() { +#[tokio::test] +async fn cancellation_is_durable_only_before_the_ref_commit_boundary() { let store = Arc::new(StatusStore::default()); for status in [ status( @@ -689,6 +695,7 @@ fn cancellation_is_durable_only_before_the_ref_commit_boundary() { transaction_id: prepared.clone(), requested_at: UtcMicros(2), }) + .await .unwrap(), NativeIntegrationCancelDispositionV1::CancellationRequested ); @@ -696,6 +703,7 @@ fn cancellation_is_durable_only_before_the_ref_commit_boundary() { .status(&NativeIntegrationStatusRequestV1 { transaction_id: prepared, }) + .await .unwrap() .unwrap(); assert!(durable.cancellation_requested); @@ -708,6 +716,7 @@ fn cancellation_is_durable_only_before_the_ref_commit_boundary() { .unwrap(), requested_at: UtcMicros(2), }) + .await .unwrap(), NativeIntegrationCancelDispositionV1::CommitPointPassed ); @@ -718,6 +727,7 @@ fn cancellation_is_durable_only_before_the_ref_commit_boundary() { .unwrap(), requested_at: UtcMicros(2), }) + .await .unwrap(), NativeIntegrationCancelDispositionV1::AlreadyTerminal( NativeIntegrationTerminalOutcomeV1::Committed @@ -729,13 +739,14 @@ fn cancellation_is_durable_only_before_the_ref_commit_boundary() { transaction_id: NativeIntegrationTransactionId::new("transaction.unknown").unwrap(), requested_at: UtcMicros(2), }) + .await .unwrap(), NativeIntegrationCancelDispositionV1::UnknownTransaction ); } -#[test] -fn cancellation_after_begin_records_live_diverged_state() { +#[tokio::test] +async fn cancellation_after_begin_records_live_diverged_state() { let store = Arc::new(StatusStore::default()); let diverged = NativeIntegrationProbeV1::Diverged { tip: oid('a'), @@ -759,6 +770,7 @@ fn cancellation_after_begin_records_live_diverged_state() { let receipt = coordinator .apply(&request, &cancellation) + .await .expect("cancelled transaction must terminate from live state"); assert_eq!( @@ -771,8 +783,8 @@ fn cancellation_after_begin_records_live_diverged_state() { assert_eq!(receipt.final_worktree_digest, digest('d')); } -#[test] -fn durable_cancellation_revision_is_settled_after_candidate_verification() { +#[tokio::test] +async fn durable_cancellation_revision_is_settled_after_candidate_verification() { let store = Arc::new(StatusStore::default()); let coordinator = NativeIntegrationTransactionCoordinator::new( store.clone(), @@ -795,6 +807,7 @@ fn durable_cancellation_revision_is_settled_after_candidate_verification() { let receipt = coordinator .apply(&request, &cancellation) + .await .expect("durably cancelled apply must settle"); assert_eq!( @@ -806,14 +819,15 @@ fn durable_cancellation_revision_is_settled_after_candidate_verification() { assert_eq!( store .read_receipt(&transaction_id) + .await .unwrap() .expect("terminal receipt"), receipt ); } -#[test] -fn apply_error_after_commit_start_writes_inspection_receipt() { +#[tokio::test] +async fn apply_error_after_commit_start_writes_inspection_receipt() { let store = Arc::new(StatusStore::default()); let coordinator = NativeIntegrationTransactionCoordinator::new( store.clone(), @@ -837,6 +851,7 @@ fn apply_error_after_commit_start_writes_inspection_receipt() { let receipt = coordinator .apply(&request, &cancellation) + .await .expect("post-begin apply error must produce an inspection receipt"); assert_eq!( @@ -846,6 +861,7 @@ fn apply_error_after_commit_start_writes_inspection_receipt() { assert_eq!( store .read_receipt(&transaction_id) + .await .expect("receipt read") .as_ref(), Some(&receipt) @@ -856,8 +872,8 @@ fn apply_error_after_commit_start_writes_inspection_receipt() { ); } -#[test] -fn probe_error_after_begin_writes_inspection_receipt() { +#[tokio::test] +async fn probe_error_after_begin_writes_inspection_receipt() { let store = Arc::new(StatusStore::default()); let coordinator = NativeIntegrationTransactionCoordinator::new( store.clone(), @@ -876,6 +892,7 @@ fn probe_error_after_begin_writes_inspection_receipt() { let receipt = coordinator .apply(&request, &cancellation) + .await .expect("post-begin probe error must produce an inspection receipt"); assert_eq!( @@ -885,6 +902,7 @@ fn probe_error_after_begin_writes_inspection_receipt() { assert!( store .read_receipt(&transaction_id) + .await .expect("receipt read") .is_some() ); @@ -893,3 +911,105 @@ fn probe_error_after_begin_writes_inspection_receipt() { &[transaction_id] ); } + +/// Counts applies in flight. Each apply waits briefly for a second one to +/// arrive, so two unserialized applies always overlap. +#[derive(Default)] +struct OverlapProbe { + active: AtomicUsize, + peak: AtomicUsize, +} + +impl NativeIntegrationMechanics for OverlapProbe { + fn preflight( + &self, + _selection: &NativeIntegrationSelectionV1, + _request: &NativeIntegrationPreflightRequestV1, + _cancellation_signal: &CancellationSignal, + _cancellation: &CancellationToken, + ) -> Result { + Err(NativeIntegrationPortError::Unavailable) + } + + fn apply( + &self, + _preview: &NativeIntegrationPreviewV1, + _cancellation: &CancellationToken, + ) -> Result { + let active = self.active.fetch_add(1, Ordering::SeqCst) + 1; + self.peak.fetch_max(active, Ordering::SeqCst); + let deadline = std::time::Instant::now() + std::time::Duration::from_millis(300); + while self.active.load(Ordering::SeqCst) < 2 && std::time::Instant::now() < deadline { + std::thread::sleep(std::time::Duration::from_millis(5)); + } + self.peak + .fetch_max(self.active.load(Ordering::SeqCst), Ordering::SeqCst); + self.active.fetch_sub(1, Ordering::SeqCst); + Ok(NativeApplyEffectV1::FailedNoChange) + } + + fn revalidate_analysis( + &self, + _preview: &NativeIntegrationPreviewV1, + _deadline: &Deadline, + _cancellation: &CancellationSignal, + ) -> Result { + Ok(NativeIntegrationAnalysisRevalidationV1::Current) + } + + fn probe( + &self, + _record: &NativeIntegrationRecordV1, + ) -> Result { + Ok(NativeIntegrationProbeV1::OldState { + tip: oid('2'), + tree: oid('5'), + index_digest: digest('8'), + worktree_digest: digest('9'), + }) + } + + fn rollback( + &self, + _record: &NativeIntegrationRecordV1, + _committed_tip: &GitOidV1, + ) -> Result { + Err(NativeIntegrationPortError::Unavailable) + } +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn concurrent_applies_to_one_repository_never_overlap() { + let mechanics = Arc::new(OverlapProbe::default()); + let coordinator = Arc::new(NativeIntegrationTransactionCoordinator::new( + Arc::new(StatusStore::default()), + Arc::new(UnusedTopology), + Arc::clone(&mechanics), + Arc::new(Authorized), + )); + let applies = ["transaction.serial-first", "transaction.serial-second"].map(|transaction| { + let coordinator = Arc::clone(&coordinator); + let (request, _) = apply_fixture(transaction); + tokio::spawn(async move { + let cancellation = + CancellationSignal::active(format!("cancel.{transaction}")).expect("cancellation"); + coordinator.apply(&request, &cancellation).await + }) + }); + + for apply in applies { + let receipt = apply + .await + .expect("apply task") + .expect("apply settles from the live probe"); + assert_eq!( + receipt.status.terminal_outcome, + Some(NativeIntegrationTerminalOutcomeV1::AbortedNoChange) + ); + } + assert_eq!( + mechanics.peak.load(Ordering::SeqCst), + 1, + "applies to one repository must hold its lock across the native commit" + ); +} diff --git a/crates/tracedecay-application/src/stack_coordinator.rs b/crates/tracedecay-application/src/stack_coordinator.rs index 8565b3e078..f9fec8dc40 100644 --- a/crates/tracedecay-application/src/stack_coordinator.rs +++ b/crates/tracedecay-application/src/stack_coordinator.rs @@ -1,8 +1,9 @@ use std::collections::{BTreeMap, BTreeSet}; -use std::sync::{Arc, Condvar, Mutex}; +use std::sync::{Arc, Mutex}; use std::time::{Duration, Instant}; use serde::{Deserialize, Serialize}; +use tokio::sync::Notify; use tracedecay_contracts::{ CancellationSignal, NativeIntegrationPreflightOutcomeV1, NativeIntegrationPreflightRequestV1, ResolvedScope, @@ -16,6 +17,7 @@ use tracedecay_domain::{ RetrievalAnchorId, StackSignalId, StackSignalKindV1, UtcMicros, WorkStackDriftObservedV1, canonical_sha256, derive_git_topology_anchor_id, }; +use tracedecay_runtime_core::native_runtime; mod transition; pub use transition::*; @@ -53,7 +55,7 @@ struct PreflightKey { struct InFlight { result: Mutex>>, - complete: Condvar, + complete: Notify, } #[derive(Default)] @@ -498,7 +500,7 @@ impl DaemonGitHubStackCoordinatorV1 { } } - pub fn optional_preflight( + pub async fn optional_preflight( &self, preflight: &P, request: &NativeIntegrationPreflightRequestV1, @@ -558,43 +560,42 @@ impl DaemonGitHubStackCoordinatorV1 { .or_default() += 1; let flight = Arc::new(InFlight { result: Mutex::new(None), - complete: Condvar::new(), + complete: Notify::new(), }); state.in_flight.insert(key.clone(), Arc::clone(&flight)); (flight, true) } }; if !owner { - let mut result = flight.result.lock().map_err(lock_error)?; let deadline = Instant::now() + PREFLIGHT_JOIN_TIMEOUT; - while result.is_none() { + loop { + // Registered before the result is read, so a publication + // between the read and the wait still wakes this joiner. + let completed = flight.complete.notified(); + if let Some(result) = flight.result.lock().map_err(lock_error)?.clone() { + return result; + } if cancellation.is_cancelled() { return Err(StackCoordinatorErrorV1::Cancelled); } let now_instant = Instant::now(); if now_instant >= deadline { - drop(result); self.fail_circuit(&circuit_key, &policy, now)?; return Err(StackCoordinatorErrorV1::Unavailable); } - result = flight - .complete - .wait_timeout( - result, - PREFLIGHT_JOIN_WAIT_SLICE.min(deadline - now_instant), - ) - .map_err(lock_error)? - .0; + let _ = native_runtime::timeout( + PREFLIGHT_JOIN_WAIT_SLICE.min(deadline - now_instant), + completed, + ) + .await; } - return result.clone().ok_or_else(|| { - StackCoordinatorErrorV1::Invalid("missing joined preflight result".to_owned()) - })?; } let result = preflight .preflight(request, cancellation) + .await .map(classify_preflight); *flight.result.lock().map_err(lock_error)? = Some(result.clone()); - flight.complete.notify_all(); + flight.complete.notify_waiters(); let mut state = self.preflights.lock().map_err(lock_error)?; state.in_flight.remove(&key); state.daemon_active = state.daemon_active.saturating_sub(1); diff --git a/crates/tracedecay-application/src/stack_coordinator/transition.rs b/crates/tracedecay-application/src/stack_coordinator/transition.rs index e712279ba9..6c84ed7ef0 100644 --- a/crates/tracedecay-application/src/stack_coordinator/transition.rs +++ b/crates/tracedecay-application/src/stack_coordinator/transition.rs @@ -229,7 +229,7 @@ pub trait OptionalStackPreflightPort: Send + Sync { &self, request: &NativeIntegrationPreflightRequestV1, cancellation: &CancellationSignal, - ) -> Result; + ) -> impl Future> + Send; } #[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] diff --git a/crates/tracedecay-application/tests/application_suite/github_stack_coordinator/preflight_tests.rs b/crates/tracedecay-application/tests/application_suite/github_stack_coordinator/preflight_tests.rs index 3ec51d2b4a..2812b593ba 100644 --- a/crates/tracedecay-application/tests/application_suite/github_stack_coordinator/preflight_tests.rs +++ b/crates/tracedecay-application/tests/application_suite/github_stack_coordinator/preflight_tests.rs @@ -26,6 +26,12 @@ use tracedecay_tool_catalog::{CapabilityId, UseCaseId}; use super::{actor, digest}; +/// Drives one preflight on this dedicated test thread; the blocking fixture +/// port parks the thread that polls it. +fn run(future: F) -> F::Output { + tracedecay_runtime_core::native_runtime::block_on(future).expect("native block_on") +} + fn oid(seed: char) -> GitOidV1 { GitOidV1::new(seed.to_string().repeat(40)).unwrap() } @@ -155,7 +161,7 @@ struct PreflightOutcomes { } impl OptionalStackPreflightPort for PreflightOutcomes { - fn preflight( + async fn preflight( &self, _request: &NativeIntegrationPreflightRequestV1, _cancellation: &CancellationSignal, @@ -190,7 +196,7 @@ impl BlockingPreflight { } impl OptionalStackPreflightPort for BlockingPreflight { - fn preflight( + async fn preflight( &self, _request: &NativeIntegrationPreflightRequestV1, _cancellation: &CancellationSignal, @@ -325,8 +331,8 @@ fn preflight_request(index: usize) -> NativeIntegrationPreflightRequestV1 { } } -#[test] -fn half_open_circuit_rejects_stale_and_denied_but_success_closes() { +#[tokio::test] +async fn half_open_circuit_rejects_stale_and_denied_but_success_closes() { let coordinator = DaemonGitHubStackCoordinatorV1::default(); let request = preflight_request(1); let policy = StackCircuitPolicyV1 { @@ -356,48 +362,56 @@ fn half_open_circuit_rejects_stale_and_denied_but_success_closes() { assert_eq!( coordinator .optional_preflight(&preflight, &request, &cancellation, UtcMicros(10)) + .await .unwrap(), OptionalPreflightDispositionV1::Unavailable ); assert_eq!( coordinator .optional_preflight(&preflight, &request, &cancellation, UtcMicros(11)) + .await .unwrap(), OptionalPreflightDispositionV1::SuppressedOpenCircuit ); assert_eq!( coordinator .optional_preflight(&preflight, &request, &cancellation, UtcMicros(110)) + .await .unwrap(), OptionalPreflightDispositionV1::Stale ); assert_eq!( coordinator .optional_preflight(&preflight, &request, &cancellation, UtcMicros(111)) + .await .unwrap(), OptionalPreflightDispositionV1::SuppressedOpenCircuit ); assert_eq!( coordinator .optional_preflight(&preflight, &request, &cancellation, UtcMicros(210)) + .await .unwrap(), OptionalPreflightDispositionV1::Denied ); assert_eq!( coordinator .optional_preflight(&preflight, &request, &cancellation, UtcMicros(211)) + .await .unwrap(), OptionalPreflightDispositionV1::SuppressedOpenCircuit ); assert_eq!( coordinator .optional_preflight(&preflight, &request, &cancellation, UtcMicros(310)) + .await .unwrap(), OptionalPreflightDispositionV1::Complete ); assert_eq!( coordinator .optional_preflight(&preflight, &request, &cancellation, UtcMicros(311)) + .await .unwrap(), OptionalPreflightDispositionV1::Unavailable ); @@ -437,26 +451,25 @@ fn preflight_admission_enforces_repository_and_daemon_bounds() { let coordinator = Arc::clone(&coordinator); let preflight = Arc::clone(&preflight); handles.push(std::thread::spawn(move || { - coordinator.optional_preflight( + run(coordinator.optional_preflight( preflight.as_ref(), &admitted, &CancellationSignal::active(format!("cancel.stack.repo-bound.{index}")).unwrap(), UtcMicros(10), - ) + )) })); } preflight.wait_for_active(MAX_REPOSITORY_PREFLIGHTS); let mut saturated = request.clone(); saturated.preview_id = NativeIntegrationPreviewId::new("preview.stack.repo-saturated").unwrap(); assert_eq!( - coordinator - .optional_preflight( - preflight.as_ref(), - &saturated, - &CancellationSignal::active("cancel.stack.repo-saturated").unwrap(), - UtcMicros(10), - ) - .unwrap(), + run(coordinator.optional_preflight( + preflight.as_ref(), + &saturated, + &CancellationSignal::active("cancel.stack.repo-saturated").unwrap(), + UtcMicros(10), + )) + .unwrap(), OptionalPreflightDispositionV1::Saturated ); preflight.release(); @@ -476,26 +489,25 @@ fn preflight_admission_enforces_repository_and_daemon_bounds() { let coordinator = Arc::clone(&coordinator); let preflight = Arc::clone(&preflight); handles.push(std::thread::spawn(move || { - coordinator.optional_preflight( + run(coordinator.optional_preflight( preflight.as_ref(), &request, &CancellationSignal::active(format!("cancel.stack.daemon-bound.{index}")).unwrap(), UtcMicros(20), - ) + )) })); } preflight.wait_for_active(MAX_DAEMON_PREFLIGHTS); let saturated = preflight_request(100); register_policy(&coordinator, &saturated); assert_eq!( - coordinator - .optional_preflight( - preflight.as_ref(), - &saturated, - &CancellationSignal::active("cancel.stack.daemon-saturated").unwrap(), - UtcMicros(20), - ) - .unwrap(), + run(coordinator.optional_preflight( + preflight.as_ref(), + &saturated, + &CancellationSignal::active("cancel.stack.daemon-saturated").unwrap(), + UtcMicros(20), + )) + .unwrap(), OptionalPreflightDispositionV1::Saturated ); preflight.release(); @@ -517,18 +529,23 @@ fn joined_preflight_observes_cancellation_without_waiting_for_owner() { let owner_preflight = Arc::clone(&preflight); let owner_request = request.clone(); let owner = std::thread::spawn(move || { - owner_coordinator.optional_preflight( + run(owner_coordinator.optional_preflight( owner_preflight.as_ref(), &owner_request, &CancellationSignal::active("cancel.stack.join-owner").unwrap(), UtcMicros(30), - ) + )) }); preflight.wait_for_active(1); let cancelled = CancellationSignal::active("cancel.stack.joiner").unwrap(); cancelled.cancel(UtcMicros(31)); assert_eq!( - coordinator.optional_preflight(preflight.as_ref(), &request, &cancelled, UtcMicros(30)), + run(coordinator.optional_preflight( + preflight.as_ref(), + &request, + &cancelled, + UtcMicros(30) + )), Err(StackCoordinatorErrorV1::Cancelled) ); preflight.release(); @@ -537,3 +554,45 @@ fn joined_preflight_observes_cancellation_without_waiting_for_owner() { OptionalPreflightDispositionV1::Unavailable ); } + +#[test] +fn joined_preflight_returns_the_single_owner_result() { + let coordinator = Arc::new(DaemonGitHubStackCoordinatorV1::default()); + let preflight = Arc::new(BlockingPreflight::default()); + let request = preflight_request(300); + register_policy(&coordinator, &request); + let spawn = |suffix: &'static str| { + let coordinator = Arc::clone(&coordinator); + let preflight = Arc::clone(&preflight); + let request = request.clone(); + std::thread::spawn(move || { + run(coordinator.optional_preflight( + preflight.as_ref(), + &request, + &CancellationSignal::active(format!("cancel.stack.single-flight.{suffix}")) + .unwrap(), + UtcMicros(40), + )) + }) + }; + let owner = spawn("owner"); + preflight.wait_for_active(1); + let joiner = spawn("joiner"); + std::thread::sleep(std::time::Duration::from_millis(200)); + preflight.release(); + + assert_eq!( + owner.join().unwrap(), + Ok(OptionalPreflightDispositionV1::Unavailable) + ); + assert_eq!( + joiner.join().unwrap(), + Ok(OptionalPreflightDispositionV1::Unavailable), + "a joiner returns the owner's published result" + ); + assert_eq!( + preflight.state.lock().unwrap().0, + 1, + "the joined request runs native preflight once" + ); +} diff --git a/crates/tracedecay-contracts/src/git/native_integration.rs b/crates/tracedecay-contracts/src/git/native_integration.rs index 10b41cd60c..af2b543505 100644 --- a/crates/tracedecay-contracts/src/git/native_integration.rs +++ b/crates/tracedecay-contracts/src/git/native_integration.rs @@ -369,33 +369,39 @@ pub struct NativeIntegrationRecoveryRequestV1 { /// Closed runtime boundary. Mutation, journal fsync, repository queues, /// one-use approval CAS, native object/ref writes, rollback, and restart /// recovery remain behind this port. -pub trait NativeIntegrationPort { +pub trait NativeIntegrationPort: Sync { fn preflight( &self, request: &NativeIntegrationPreflightRequestV1, cancellation: &CancellationSignal, - ) -> Result; + ) -> impl Future< + Output = Result, + > + Send; fn apply( &self, request: &NativeIntegrationApplyRequestV1, cancellation: &CancellationSignal, - ) -> Result; + ) -> impl Future> + Send; fn status( &self, request: &NativeIntegrationStatusRequestV1, - ) -> Result, NativeIntegrationPortError>; + ) -> impl Future< + Output = Result, NativeIntegrationPortError>, + > + Send; fn cancel( &self, request: &NativeIntegrationCancelRequestV1, - ) -> Result; + ) -> impl Future< + Output = Result, + > + Send; fn recover( &self, request: &NativeIntegrationRecoveryRequestV1, - ) -> Result; + ) -> impl Future> + Send; } /// Stable failure taxonomy. Read-only partial/unavailable is represented by a @@ -450,13 +456,13 @@ impl NativeIntegrationService

{ level = "trace", skip_all )] - pub fn preflight( + pub async fn preflight( &self, request: NativeIntegrationPreflightRequestV1, cancellation: &CancellationSignal, ) -> Result { request.validate()?; - let outcome = self.port.preflight(&request, cancellation)?; + let outcome = self.port.preflight(&request, cancellation).await?; if let NativeIntegrationPreflightOutcomeV1::Preview(preview) = &outcome { preview.validate().map_err(ApplicationContractError::from)?; } @@ -468,13 +474,13 @@ impl NativeIntegrationService

{ level = "trace", skip_all )] - pub fn apply( + pub async fn apply( &self, request: NativeIntegrationApplyRequestV1, cancellation: &CancellationSignal, ) -> Result { request.validate()?; - let receipt = self.port.apply(&request, cancellation)?; + let receipt = self.port.apply(&request, cancellation).await?; receipt.validate().map_err(ApplicationContractError::from)?; Ok(receipt) } @@ -484,12 +490,12 @@ impl NativeIntegrationService

{ level = "trace", skip_all )] - pub fn status( + pub async fn status( &self, request: NativeIntegrationStatusRequestV1, ) -> Result, NativeIntegrationContractError> { request.validate()?; - let status = self.port.status(&request)?; + let status = self.port.status(&request).await?; if let Some(status) = &status { status.validate().map_err(ApplicationContractError::from)?; } @@ -501,11 +507,11 @@ impl NativeIntegrationService

{ level = "trace", skip_all )] - pub fn cancel( + pub async fn cancel( &self, request: NativeIntegrationCancelRequestV1, ) -> Result { request.validate()?; - Ok(self.port.cancel(&request)?) + Ok(self.port.cancel(&request).await?) } } diff --git a/crates/tracedecay-contracts/src/git/stack_signal_expand.rs b/crates/tracedecay-contracts/src/git/stack_signal_expand.rs index 798a97d219..994bd4f0da 100644 --- a/crates/tracedecay-contracts/src/git/stack_signal_expand.rs +++ b/crates/tracedecay-contracts/src/git/stack_signal_expand.rs @@ -236,7 +236,9 @@ pub trait GitHubStackSignalExpandPort: Send + Sync { &self, request: GitHubStackSignalExpandRequestV1, cancellation: &CancellationSignal, - ) -> Result; + ) -> impl Future< + Output = Result, + > + Send; } /// Typed adapter failures that never disclose whether a signal exists. diff --git a/crates/tracedecay-contracts/src/git/worktree.rs b/crates/tracedecay-contracts/src/git/worktree.rs index adc60729d3..f29fe3b8f7 100644 --- a/crates/tracedecay-contracts/src/git/worktree.rs +++ b/crates/tracedecay-contracts/src/git/worktree.rs @@ -521,14 +521,14 @@ pub trait NativeWorktreePort: Send + Sync { request: &WorktreeCleanupRemoveRequestV1, scope_set: &AuthorizedScopeSet, cancellation: &CancellationSignal, - ) -> Result; + ) -> impl Future> + Send; fn reconcile( &self, request: &WorktreeCleanupReconcileRequestV1, scope_set: &AuthorizedScopeSet, cancellation: &CancellationSignal, - ) -> Result; + ) -> impl Future> + Send; } /// Application service that makes persisted scope-set identity the sole @@ -612,25 +612,25 @@ where } #[tracing::instrument(name = "application.git.worktree.remove", level = "trace", skip_all)] - pub fn remove( + pub async fn remove( &self, request: &WorktreeCleanupRemoveRequestV1, cancellation: &CancellationSignal, ) -> Result { request.validate()?; let scope_set = self.authorize(&request.binding())?; - self.port.remove(request, &scope_set, cancellation) + self.port.remove(request, &scope_set, cancellation).await } #[tracing::instrument(name = "application.git.worktree.reconcile", level = "trace", skip_all)] - pub fn reconcile( + pub async fn reconcile( &self, request: &WorktreeCleanupReconcileRequestV1, cancellation: &CancellationSignal, ) -> Result { request.validate()?; let scope_set = self.authorize(&request.binding())?; - self.port.reconcile(request, &scope_set, cancellation) + self.port.reconcile(request, &scope_set, cancellation).await } } diff --git a/crates/tracedecay-daemon-service/src/invocation.rs b/crates/tracedecay-daemon-service/src/invocation.rs index f13be9fc0c..0119748912 100644 --- a/crates/tracedecay-daemon-service/src/invocation.rs +++ b/crates/tracedecay-daemon-service/src/invocation.rs @@ -631,7 +631,7 @@ impl DaemonInvocationService { &self, owner: &DaemonNativeIntegrationOwner, ) -> Result<(), tracedecay_contracts::NativeIntegrationPortError> { - let roots = owner.cleanup_recovery_roots()?; + let roots = owner.cleanup_recovery_roots().await?; self.worktree_holder_admission .mark_recovery_required(roots) .await; diff --git a/crates/tracedecay-daemon-service/src/invocation/dispatch.rs b/crates/tracedecay-daemon-service/src/invocation/dispatch.rs index 7b86b9c3c8..02d486f67a 100644 --- a/crates/tracedecay-daemon-service/src/invocation/dispatch.rs +++ b/crates/tracedecay-daemon-service/src/invocation/dispatch.rs @@ -333,15 +333,18 @@ impl DaemonInvocationService { observed_at, deadline, cancellation, - } => execute_github_stack_signal_expand( - request_id, - configuration_runtime.clone(), - native_integration_service, - request, - observed_at, - deadline, - cancellation, - ), + } => { + Box::pin(execute_github_stack_signal_expand( + request_id, + configuration_runtime.clone(), + native_integration_service, + request, + observed_at, + deadline, + cancellation, + )) + .await + } DaemonInvocationPayload::NativeIntegration { surface_operation, request, diff --git a/crates/tracedecay-daemon-service/src/invocation/github_stack_signal.rs b/crates/tracedecay-daemon-service/src/invocation/github_stack_signal.rs index 97d4bbe72f..959d812580 100644 --- a/crates/tracedecay-daemon-service/src/invocation/github_stack_signal.rs +++ b/crates/tracedecay-daemon-service/src/invocation/github_stack_signal.rs @@ -16,7 +16,7 @@ use tracedecay_agent_hosts::native_integration::DaemonNativeIntegrationOwner; /// payload: the daemon mints a context from the admitted project route and /// the runtime settles only that context actor's durable host-pending row. #[tracing::instrument(name = "daemon.service.github_stack.expand", level = "trace", skip_all)] -pub(super) fn execute_github_stack_signal_expand( +pub(super) async fn execute_github_stack_signal_expand( wire_request_id: String, registered: Option, owner: Option, @@ -63,9 +63,23 @@ pub(super) fn execute_github_stack_signal_expand( Ok(signal) => signal, Err(problem) => return application_problem(wire_request_id, problem), }; - let result = match runtime.expand(request.into_application_request(context), &signal) { - Ok(result) => result, - Err(error) => error.into_surface_result(), + // Expansion interleaves native-store awaits with synchronous delivery-store + // waits, so it runs on the blocking pool. + let expansion = tracedecay_runtime_core::native_runtime::on_blocking_pool(async move { + runtime + .expand(request.into_application_request(context), &signal) + .await + }) + .await; + let result = match expansion { + Ok(Ok(result)) => result, + Ok(Err(error)) => error.into_surface_result(), + Err(_) => { + return DaemonInvocationResponse::problem( + wire_request_id, + DaemonInvocationProblem::Unavailable, + ); + } }; let Ok(payload) = serde_json::to_value(result) else { return DaemonInvocationResponse::problem( diff --git a/crates/tracedecay-daemon-service/src/invocation/native_integration.rs b/crates/tracedecay-daemon-service/src/invocation/native_integration.rs index dae48baa7e..78e1df3be8 100644 --- a/crates/tracedecay-daemon-service/src/invocation/native_integration.rs +++ b/crates/tracedecay-daemon-service/src/invocation/native_integration.rs @@ -297,9 +297,8 @@ fn publish_transaction_status( } } -/// Reads and publishes the durable status a mutation just advanced, from the -/// same blocking context that ran the mutation. -fn publish_current_transaction_status( +/// Reads and publishes the durable status a mutation just advanced. +async fn publish_current_transaction_status( broadcast: Option<&Arc>, owner: &DaemonNativeIntegrationOwner, transaction_id: &tracedecay_domain::NativeIntegrationTransactionId, @@ -307,19 +306,23 @@ fn publish_current_transaction_status( if broadcast.is_none() { return; } - if let Ok(Some(status)) = owner.service().status(NativeIntegrationStatusRequestV1 { - transaction_id: transaction_id.clone(), - }) { + if let Ok(Some(status)) = owner + .service() + .status(NativeIntegrationStatusRequestV1 { + transaction_id: transaction_id.clone(), + }) + .await + { publish_transaction_status(broadcast, &status); } } /// Runs one operation against the mounted per-project owner. /// -/// The kernel and its store bridge are synchronous (native Git plus a bounded -/// store actor), so every owner call crosses to a blocking thread; the -/// coordinator's own cancellation map keeps a running apply cancellable -/// through the separate cancel operation. +/// Store-only operations await the async store in place. Operations that +/// interleave store awaits with synchronous native Git run on the blocking +/// pool; the coordinator's own cancellation map keeps a running apply +/// cancellable through the separate cancel operation. #[allow(clippy::too_many_arguments)] #[tracing::instrument( name = "daemon.service.native_integration.owner_execute", @@ -379,7 +382,7 @@ async fn execute_with_owner( let signal_scope = context.scope().clone(); let signal_context = context.clone(); let outcome = tracing::Instrument::instrument( - tracedecay_runtime_core::native_runtime::spawn_blocking(move || { + tracedecay_runtime_core::native_runtime::on_blocking_pool(async move { let stack_runtime = owner.github_stack_runtime(context.scope())?; let topology = registered_topology_request(&owner, preflight.snapshot, observed_at)?; @@ -394,8 +397,14 @@ async fn execute_with_owner( let outcome = match stack_runtime.as_ref() { Some(runtime) => runtime .preflight(&application_request, &signal) + .await .map_err(stack_coordinator_contract_error), - None => owner.service().preflight(application_request, &signal), + None => { + owner + .service() + .preflight(application_request, &signal) + .await + } }; if let ( Some(runtime), @@ -456,9 +465,9 @@ async fn execute_with_owner( ) .map_err(|_| invalid())?; tracing::Instrument::instrument( - tracedecay_runtime_core::native_runtime::spawn_blocking(move || { + async move { let store = owner.store(); - let preview = match store.read_preview(&approve.preview_id) { + let preview = match store.read_preview(&approve.preview_id).await { Ok(Some(preview)) if preview.preview_digest == approve.preview_digest => { preview } @@ -509,7 +518,7 @@ async fn execute_with_owner( } .seal() .map_err(|_| invalid_native_integration_request())?; - match store.save_approval(approval.clone()) { + match store.save_approval(approval.clone()).await { Ok(()) => Ok(NativeIntegrationSurfaceResultV1::Approval( NativeIntegrationApprovalProjectionV1::project( &approval, @@ -523,11 +532,10 @@ async fn execute_with_owner( } Err(_) => Err(unavailable_native_integration()), } - }), + }, tracing::trace_span!("daemon.service.native_integration.approve"), ) .await - .map_err(|_| unavailable_native_integration())? .map(NativeIntegrationExecutionV1::without_preview) } NativeIntegrationSurfaceRequest::Apply(apply) => { @@ -537,13 +545,13 @@ async fn execute_with_owner( let signal_scope = context.scope().clone(); let signal_context = context.clone(); tracing::Instrument::instrument( - tracedecay_runtime_core::native_runtime::spawn_blocking(move || { + tracedecay_runtime_core::native_runtime::on_blocking_pool(async move { // The caller names its preview and one-use approval by exact // identity and digest; both must already be durable. A // missing or mismatched fact is denied without disclosing // whether the target was absent or denied. let store = owner.store(); - let preview = match store.read_preview(&apply.preview_id) { + let preview = match store.read_preview(&apply.preview_id).await { Ok(Some(preview)) if preview.preview_digest == apply.preview_digest => { preview } @@ -556,7 +564,7 @@ async fn execute_with_owner( } Err(_) => return Err(unavailable_native_integration()), }; - let approval = match store.read_approval(&apply.approval_id) { + let approval = match store.read_approval(&apply.approval_id).await { Ok(Some(approval)) if approval.approval_digest == apply.approval_digest => { approval } @@ -579,12 +587,13 @@ async fn execute_with_owner( approval, observed_at, }; - let apply_outcome = owner.service().apply(application_request, &signal); + let apply_outcome = owner.service().apply(application_request, &signal).await; publish_current_transaction_status( status_broadcast.as_ref(), &owner, &applied_transaction_id, - ); + ) + .await; match apply_outcome { Ok(receipt) => { if let Some(runtime) = stack_runtime.as_ref() @@ -624,13 +633,10 @@ async fn execute_with_owner( transaction_id: status.transaction_id, }; let outcome = tracing::Instrument::instrument( - tracedecay_runtime_core::native_runtime::spawn_blocking(move || { - owner.service().status(application_request) - }), + async move { owner.service().status(application_request).await }, tracing::trace_span!("daemon.service.native_integration.status"), ) - .await - .map_err(|_| unavailable_native_integration())?; + .await; match outcome { Ok(Some(status)) => { publish_transaction_status(status_broadcast.as_ref(), &status); @@ -656,19 +662,19 @@ async fn execute_with_owner( requested_at: observed_at, }; let outcome = tracing::Instrument::instrument( - tracedecay_runtime_core::native_runtime::spawn_blocking(move || { - let disposition = owner.service().cancel(application_request); + async move { + let disposition = owner.service().cancel(application_request).await; publish_current_transaction_status( status_broadcast.as_ref(), &owner, &cancelled_transaction_id, - ); + ) + .await; disposition - }), + }, tracing::trace_span!("daemon.service.native_integration.cancel"), ) - .await - .map_err(|_| unavailable_native_integration())?; + .await; match outcome { Ok(disposition) => Ok(NativeIntegrationExecutionV1::without_preview( NativeIntegrationSurfaceResultV1::from_cancel(disposition), @@ -723,7 +729,7 @@ async fn execute_worktree_with_owner( WorktreeUnavailableReasonV1::Unavailable, )); }; - tracedecay_runtime_core::native_runtime::spawn_blocking(move || { + tracedecay_runtime_core::native_runtime::on_blocking_pool(async move { let outcome = match request { NativeWorktreeSurfaceRequest::Inventory(request) => service .inventory(&request, &signal) @@ -736,9 +742,11 @@ async fn execute_worktree_with_owner( .map(NativeWorktreeSurfaceResultV1::Confirmation), NativeWorktreeSurfaceRequest::Remove(request) => service .remove(&request, &signal) + .await .map(NativeWorktreeSurfaceResultV1::Removal), NativeWorktreeSurfaceRequest::Reconcile(request) => service .reconcile(&request, &signal) + .await .map(NativeWorktreeSurfaceResultV1::Reconciliation), }; outcome.or_else(|error| worktree_result_from_error(operation, error)) diff --git a/crates/tracedecay-store/src/native_integration.rs b/crates/tracedecay-store/src/native_integration.rs index ec1d837867..915404eda3 100644 --- a/crates/tracedecay-store/src/native_integration.rs +++ b/crates/tracedecay-store/src/native_integration.rs @@ -120,41 +120,45 @@ pub enum NativeWorktreeCleanupBeginResultV1 { /// Durable transaction authority. pub trait NativeIntegrationStore: Send + Sync { - fn save_preview(&self, preview: NativeIntegrationPreviewV1) - -> NativeIntegrationStoreResult<()>; + fn save_preview( + &self, + preview: NativeIntegrationPreviewV1, + ) -> impl Future> + Send; fn read_preview( &self, preview_id: &NativeIntegrationPreviewId, - ) -> NativeIntegrationStoreResult>; + ) -> impl Future>> + Send; /// Atomically consumes the approval and inserts the `Prepared` record. fn begin_or_replay( &self, record: NativeIntegrationRecordV1, - ) -> NativeIntegrationStoreResult; + ) -> impl Future> + Send; fn read_status( &self, transaction_id: &NativeIntegrationTransactionId, - ) -> NativeIntegrationStoreResult>; + ) -> impl Future< + Output = NativeIntegrationStoreResult>, + > + Send; fn read_record( &self, transaction_id: &NativeIntegrationTransactionId, - ) -> NativeIntegrationStoreResult>; + ) -> impl Future>> + Send; fn read_receipt( &self, transaction_id: &NativeIntegrationTransactionId, - ) -> NativeIntegrationStoreResult>; + ) -> impl Future>> + Send; fn compare_and_swap_status( &self, transaction_id: &NativeIntegrationTransactionId, expected_phase_revision: u64, replacement: NativeIntegrationTransactionStatusV1, - ) -> NativeIntegrationStoreResult; + ) -> impl Future> + Send; /// Atomically publishes the terminal status and its receipt. fn write_terminal( @@ -162,12 +166,12 @@ pub trait NativeIntegrationStore: Send + Sync { transaction_id: &NativeIntegrationTransactionId, expected_phase_revision: u64, receipt: NativeIntegrationReceiptV1, - ) -> NativeIntegrationStoreResult; + ) -> impl Future> + Send; fn pending_transactions( &self, repository_id: Option<&RepositoryId>, - ) -> NativeIntegrationStoreResult>; + ) -> impl Future>> + Send; /// Candidate generations protected by an unexpired preview or unfinished /// transaction. Retention consumes this exact durable census; generation @@ -176,18 +180,18 @@ pub trait NativeIntegrationStore: Send + Sync { &self, repository_id: &RepositoryId, observed_at: UtcMicros, - ) -> NativeIntegrationStoreResult>; + ) -> impl Future>> + Send; fn approval_consumed( &self, approval_id: &NativeIntegrationApprovalId, - ) -> NativeIntegrationStoreResult; + ) -> impl Future> + Send; fn quarantine_repository( &self, repository_id: &RepositoryId, transaction_id: &NativeIntegrationTransactionId, - ) -> NativeIntegrationStoreResult<()>; + ) -> impl Future> + Send; /// Durably records exact cleanup intent before native Git may mutate the /// registered worktree. Exact replay returns the terminal receipt or the @@ -196,12 +200,14 @@ pub trait NativeIntegrationStore: Send + Sync { fn begin_worktree_cleanup( &self, transaction: NativeWorktreeCleanupTransactionV1, - ) -> NativeIntegrationStoreResult; + ) -> impl Future> + Send; fn read_worktree_cleanup( &self, confirmation_digest: &ManifestDigest, - ) -> NativeIntegrationStoreResult>; + ) -> impl Future< + Output = NativeIntegrationStoreResult>, + > + Send; /// Bounded startup recovery census for unfinished cleanup journals in one /// exact repository. Implementations must fail rather than truncate. @@ -209,10 +215,11 @@ pub trait NativeIntegrationStore: Send + Sync { &self, _repository_id: &RepositoryId, _limit: u32, - ) -> NativeIntegrationStoreResult> { - Err(NativeIntegrationStoreError::unavailable( + ) -> impl Future>> + Send + { + std::future::ready(Err(NativeIntegrationStoreError::unavailable( "pending worktree cleanups are not implemented", - )) + ))) } fn compare_and_swap_worktree_cleanup( @@ -220,12 +227,12 @@ pub trait NativeIntegrationStore: Send + Sync { confirmation_digest: &ManifestDigest, expected_phase_revision: u64, replacement: NativeWorktreeCleanupTransactionV1, - ) -> NativeIntegrationStoreResult; + ) -> impl Future> + Send; fn write_worktree_cleanup_terminal( &self, confirmation_digest: &ManifestDigest, expected_phase_revision: u64, receipt: NativeWorktreeCleanupReceiptV1, - ) -> NativeIntegrationStoreResult; + ) -> impl Future> + Send; } From 87bcf02c0ef75247bbe91d3db685f70a43236618 Mon Sep 17 00:00:00 2001 From: ScriptedAlchemy Date: Fri, 16 Oct 2026 16:30:18 +0000 Subject: [PATCH 2/2] refactor(stack-delivery): await delivery store, drop actor --- .../src/native_integration/journey_tests.rs | 6 + .../src/native_integration/registry.rs | 2 +- .../native_integration/stack_hook_wakeup.rs | 10 +- .../src/native_integration/stack_runtime.rs | 405 +++++++----------- .../src/stack_coordinator.rs | 36 +- .../src/stack_coordinator/transition.rs | 19 +- .../github_stack_coordinator.rs | 214 ++++++--- .../src/invocation/github_stack_signal.rs | 23 +- .../src/invocation/native_integration.rs | 2 + .../src/handlers/hook_runtime/admission.rs | 12 +- 10 files changed, 383 insertions(+), 346 deletions(-) diff --git a/crates/tracedecay-agent-hosts/src/native_integration/journey_tests.rs b/crates/tracedecay-agent-hosts/src/native_integration/journey_tests.rs index 1f028cbd77..7a624dba01 100644 --- a/crates/tracedecay-agent-hosts/src/native_integration/journey_tests.rs +++ b/crates/tracedecay-agent-hosts/src/native_integration/journey_tests.rs @@ -858,17 +858,21 @@ async fn checked_out_declared_stack_conflict_enqueues_without_moving_refs() { .expect("other recipient signal"); stack_runtime .enqueue_from_preflight(other_signal.clone(), &other_context) + .await .expect("enqueue older other-recipient signal"); stack_runtime .enqueue_from_preflight(signal.clone(), &live_context) + .await .expect("enqueue actual conflict"); stack_runtime .enqueue_from_preflight(occupied_signal.clone(), &live_context) + .await .expect("enqueue destination-occupied dependency-ready signal"); tokio::time::timeout(Duration::from_secs(3), async { loop { if stack_runtime .pending_host_deliveries() + .await .is_ok_and(|pending| pending.len() == 3) { break; @@ -1305,11 +1309,13 @@ async fn independent_pair_applies_supported_modes_and_survives_daemon_restart() .await; stack_runtime .enqueue_from_preflight(stack_signal.clone(), &live_context) + .await .expect("enqueue terminal signal"); tokio::time::timeout(Duration::from_secs(3), async { loop { if stack_runtime .pending_host_deliveries() + .await .is_ok_and(|pending| pending.len() == 1) { break; diff --git a/crates/tracedecay-agent-hosts/src/native_integration/registry.rs b/crates/tracedecay-agent-hosts/src/native_integration/registry.rs index 9032a41e6d..5e20203228 100644 --- a/crates/tracedecay-agent-hosts/src/native_integration/registry.rs +++ b/crates/tracedecay-agent-hosts/src/native_integration/registry.rs @@ -309,7 +309,7 @@ impl DaemonNativeIntegrationOwner { /// Mounts exactly one stack-delivery runtime for one exact project scope. /// Re-opening the project refreshes its source-access expiry but never - /// creates a second queue actor or a second background drain task. + /// creates a second delivery store or a second background drain task. pub fn mount_github_stack_runtime( &self, database: RegisteredGlobalDbLeaseV1, diff --git a/crates/tracedecay-agent-hosts/src/native_integration/stack_hook_wakeup.rs b/crates/tracedecay-agent-hosts/src/native_integration/stack_hook_wakeup.rs index 865f8c55a8..f72cd6470f 100644 --- a/crates/tracedecay-agent-hosts/src/native_integration/stack_hook_wakeup.rs +++ b/crates/tracedecay-agent-hosts/src/native_integration/stack_hook_wakeup.rs @@ -34,7 +34,7 @@ pub fn register_github_stack_hook_runtime( /// Returns only whether an authenticated `CursorDesktop` hook should wake its /// user. Store failures suppress a wakeup; durable `host_pending` rows remain /// available for a later hook admission and MCP expansion. -pub fn github_stack_hook_available(project_id: [u8; 16], worktree_id: [u8; 16]) -> bool { +pub async fn github_stack_hook_available(project_id: [u8; 16], worktree_id: [u8; 16]) -> bool { let runtime = runtime_registry().lock().ok().and_then(|mut registry| { let runtime = registry .get(&(project_id, worktree_id)) @@ -44,7 +44,11 @@ pub fn github_stack_hook_available(project_id: [u8; 16], worktree_id: [u8; 16]) } runtime }); + let Some(runtime) = runtime else { + return false; + }; runtime - .and_then(|runtime| runtime.pending_host_deliveries().ok()) - .is_some_and(|deliveries| !deliveries.is_empty()) + .pending_host_deliveries() + .await + .is_ok_and(|deliveries| !deliveries.is_empty()) } diff --git a/crates/tracedecay-agent-hosts/src/native_integration/stack_runtime.rs b/crates/tracedecay-agent-hosts/src/native_integration/stack_runtime.rs index c8dd814eeb..e649e506e3 100644 --- a/crates/tracedecay-agent-hosts/src/native_integration/stack_runtime.rs +++ b/crates/tracedecay-agent-hosts/src/native_integration/stack_runtime.rs @@ -1,12 +1,10 @@ //! Daemon-local runtime for durable GitHub stack transition delivery. //! -//! The synchronous coordinator accesses the asynchronous registered database -//! through one bounded actor owned by the native-integration project owner. -//! Publication advances a durable row only to `host_pending`; authenticated +//! The coordinator awaits the registered database directly through the +//! native-integration project owner's delivery store. Publication advances a durable row only to `host_pending`; authenticated //! MCP expansion is the sole settlement path. use std::collections::{BTreeMap, BTreeSet}; -use std::sync::mpsc::{Receiver, RecvTimeoutError, SyncSender, TrySendError, sync_channel}; use std::sync::{Arc, Mutex}; use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; @@ -38,284 +36,138 @@ use tracedecay_domain::{ use tracedecay_global_db::{ GitHubStackDeliveryKeyV1, GitHubStackDeliveryRecordV1, GitHubStackDeliveryStateV1, GitHubStackSignalAppendOutcomeV1, GitHubStackSignalRecordV1, - MAX_GITHUB_STACK_DELIVERY_BATCH_V1, RegisteredGlobalDb, RegisteredGlobalDbLeaseV1, + MAX_GITHUB_STACK_DELIVERY_BATCH_V1, RegisteredGlobalDbLeaseV1, }; use tracedecay_runtime_core::native_runtime::{self, NativeTask}; use super::registry::DaemonProjectNativeIntegrationService; use super::store::DaemonNativeIntegrationStore; -const STACK_DELIVERY_STORE_ACTOR_CAPACITY: usize = 64; -const STACK_DELIVERY_STORE_ACTOR_TIMEOUT: Duration = Duration::from_secs(5); const STACK_DELIVERY_TICK: Duration = Duration::from_millis(250); const STACK_CIRCUIT_FAILURE_THRESHOLD: u32 = 3; const STACK_CIRCUIT_OPEN_MICROS: i64 = 30_000_000; -type StoreReply = SyncSender>; - -enum StoreCommand { - Append( - Box, - Vec, - StoreReply, - ), - Pending(StoreReply>), - Publish(String, Vec, StoreReply<()>), - Acknowledge(String, Vec, StoreReply<()>), - Signal(String, StoreReply>), - OldestPending( - String, - String, - StoreReply>, - ), - RecipientState( - String, - String, - StoreReply>, - ), - AuthorizationLost(String, String, StoreReply<()>), - HostAcknowledge(String, String, StoreReply<()>), - PendingHost(String, StoreReply>), -} - -/// Bounded synchronous view of the canonical registered-project delivery -/// tables. This actor is not an authority: every command delegates to the -/// single global-db schema/API owner. +/// One project's view of the canonical registered-project delivery tables. +/// It is not an authority: every call awaits the single global-db +/// schema/API owner, where each mutation is one write transaction. struct DaemonStackDeliveryStoreV1 { - commands: Mutex>>, - worker: Mutex>>, + database: RegisteredGlobalDbLeaseV1, + project_id: String, } impl DaemonStackDeliveryStoreV1 { - #[tracing::instrument( - name = "daemon.native_integration.stack_open", - level = "trace", - skip_all - )] - fn open(database: RegisteredGlobalDbLeaseV1, project_id: &ProjectId) -> Result { - let project_id = project_id.as_str().to_owned(); - let (commands, receiver) = sync_channel(STACK_DELIVERY_STORE_ACTOR_CAPACITY); - let worker = std::thread::Builder::new() - .name("tracedecay-github-stack-delivery-store".to_owned()) - .spawn(move || run_delivery_store_actor(&database, &project_id, &receiver)) - .map_err(|error| format!("spawn stack delivery actor: {error}"))?; - Ok(Self { - commands: Mutex::new(Some(commands)), - worker: Mutex::new(Some(worker)), - }) - } - - fn submit(&self, make: impl FnOnce(StoreReply) -> StoreCommand) -> Result { - let (reply, receiver) = sync_channel(1); - self.commands - .lock() - .map_err(|_| "stack delivery actor lock is poisoned".to_owned())? - .as_ref() - .ok_or_else(|| "stack delivery actor is closed".to_owned())? - .try_send(make(reply)) - .map_err(|error| match error { - TrySendError::Full(_) => "stack delivery actor is saturated".to_owned(), - TrySendError::Disconnected(_) => "stack delivery actor is unavailable".to_owned(), - })?; - receiver - .recv_timeout(STACK_DELIVERY_STORE_ACTOR_TIMEOUT) - .map_err(|error| match error { - RecvTimeoutError::Timeout => "stack delivery actor timed out".to_owned(), - RecvTimeoutError::Disconnected => "stack delivery actor is unavailable".to_owned(), - })? - } - - fn append( + async fn append( &self, record: GitHubStackSignalRecordV1, recipients: Vec, ) -> Result { - self.submit(|reply| StoreCommand::Append(Box::new(record), recipients, reply)) + self.database + .append_github_stack_signal(record, recipients) + .await } - fn pending(&self) -> Result, String> { - self.submit(StoreCommand::Pending) + async fn pending(&self) -> Result, String> { + self.database + .pending_github_stack_deliveries(&self.project_id, MAX_GITHUB_STACK_DELIVERY_BATCH_V1) + .await } - fn publish( + async fn publish( &self, watermark_id: &StackDeliveryWatermarkId, deliveries: Vec, ) -> Result<(), String> { - self.submit(|reply| { - StoreCommand::Publish(watermark_id.as_str().to_owned(), deliveries, reply) - }) + self.database + .publish_github_stack_deliveries(&self.project_id, watermark_id.as_str(), &deliveries) + .await } - fn acknowledge( + async fn acknowledge( &self, watermark_id: &StackDeliveryWatermarkId, deliveries: Vec, ) -> Result<(), String> { - self.submit(|reply| { - StoreCommand::Acknowledge(watermark_id.as_str().to_owned(), deliveries, reply) - }) + self.database + .acknowledge_github_stack_deliveries( + &self.project_id, + watermark_id.as_str(), + &deliveries, + ) + .await } - fn signal( + async fn signal( &self, signal_id: &StackSignalId, ) -> Result, String> { - self.submit(|reply| StoreCommand::Signal(signal_id.as_str().to_owned(), reply)) + self.database + .github_stack_signal(&self.project_id, signal_id.as_str()) + .await } - fn oldest_pending( + async fn oldest_pending( &self, scope: &ResolvedScope, recipient: &ActorId, ) -> Result, String> { - self.submit(|reply| { - StoreCommand::OldestPending( - scope.scope_digest.as_str().to_owned(), - recipient.as_str().to_owned(), - reply, + self.database + .oldest_host_pending_github_stack_delivery( + &self.project_id, + scope.scope_digest.as_str(), + recipient.as_str(), ) - }) + .await } - fn recipient_state( + async fn recipient_state( &self, signal_id: &StackSignalId, recipient: &ActorId, ) -> Result, String> { - self.submit(|reply| { - StoreCommand::RecipientState( - signal_id.as_str().to_owned(), - recipient.as_str().to_owned(), - reply, - ) - }) + self.database + .github_stack_recipient_state(&self.project_id, signal_id.as_str(), recipient.as_str()) + .await } - fn authorization_lost( + async fn authorization_lost( &self, signal_id: &StackSignalId, recipient: &ActorId, ) -> Result<(), String> { - self.submit(|reply| { - StoreCommand::AuthorizationLost( - signal_id.as_str().to_owned(), - recipient.as_str().to_owned(), - reply, + self.database + .record_github_stack_authorization_loss( + &self.project_id, + signal_id.as_str(), + recipient.as_str(), ) - }) + .await } - fn host_acknowledge( + async fn host_acknowledge( &self, signal_id: &StackSignalId, recipient: &ActorId, ) -> Result<(), String> { - self.submit(|reply| { - StoreCommand::HostAcknowledge( - signal_id.as_str().to_owned(), - recipient.as_str().to_owned(), - reply, + self.database + .acknowledge_github_stack_host_delivery( + &self.project_id, + signal_id.as_str(), + recipient.as_str(), ) - }) + .await } - fn pending_host( + async fn pending_host( &self, scope: &ResolvedScope, ) -> Result, String> { - self.submit(|reply| { - StoreCommand::PendingHost(scope.scope_digest.as_str().to_owned(), reply) - }) - } - - fn shutdown(&self) { - if let Ok(mut commands) = self.commands.lock() { - commands.take(); - } - if let Ok(mut worker) = self.worker.lock() - && let Some(worker) = worker.take() - { - let _ = worker.join(); - } - } -} - -impl Drop for DaemonStackDeliveryStoreV1 { - fn drop(&mut self) { - self.shutdown(); - } -} - -fn settle(operation: impl Future>) -> Result { - native_runtime::block_on(operation)? -} - -fn run_delivery_store_actor( - database: &RegisteredGlobalDb, - project_id: &str, - receiver: &Receiver, -) { - while let Ok(command) = receiver.recv() { - match command { - StoreCommand::Append(record, recipients, reply) => { - let _ = reply.send(settle( - database.append_github_stack_signal(*record, recipients), - )); - } - StoreCommand::Pending(reply) => { - let _ = reply.send(settle(database.pending_github_stack_deliveries( - project_id, - MAX_GITHUB_STACK_DELIVERY_BATCH_V1, - ))); - } - StoreCommand::Publish(watermark, deliveries, reply) => { - let _ = reply.send(settle(database.publish_github_stack_deliveries( - project_id, - &watermark, - &deliveries, - ))); - } - StoreCommand::Acknowledge(watermark, deliveries, reply) => { - let _ = reply.send(settle(database.acknowledge_github_stack_deliveries( - project_id, - &watermark, - &deliveries, - ))); - } - StoreCommand::Signal(signal_id, reply) => { - let _ = reply.send(settle(database.github_stack_signal(project_id, &signal_id))); - } - StoreCommand::OldestPending(scope_digest, recipient, reply) => { - let _ = reply.send(settle(database.oldest_host_pending_github_stack_delivery( - project_id, - &scope_digest, - &recipient, - ))); - } - StoreCommand::RecipientState(signal_id, recipient, reply) => { - let _ = reply.send(settle( - database.github_stack_recipient_state(project_id, &signal_id, &recipient), - )); - } - StoreCommand::AuthorizationLost(signal_id, recipient, reply) => { - let _ = reply.send(settle(database.record_github_stack_authorization_loss( - project_id, &signal_id, &recipient, - ))); - } - StoreCommand::HostAcknowledge(signal_id, recipient, reply) => { - let _ = reply.send(settle(database.acknowledge_github_stack_host_delivery( - project_id, &signal_id, &recipient, - ))); - } - StoreCommand::PendingHost(scope_digest, reply) => { - let _ = reply.send(settle(database.pending_host_github_stack_deliveries( - project_id, - &scope_digest, - MAX_GITHUB_STACK_DELIVERY_BATCH_V1, - ))); - } - } + self.database + .pending_host_github_stack_deliveries( + &self.project_id, + scope.scope_digest.as_str(), + MAX_GITHUB_STACK_DELIVERY_BATCH_V1, + ) + .await } } @@ -507,17 +359,18 @@ impl StackRuntimePortsV1 { Ok(signal) } - fn pending_host_for_scope( + async fn pending_host_for_scope( &self, ) -> Result, StackCoordinatorErrorV1> { self.store .pending_host(&self.scope) + .await .map_err(Self::database_error) } } impl StackCoordinatorStore for StackRuntimePortsV1 { - fn append_signal( + async fn append_signal( &self, signal: StackSignalV1, recipients: Vec, @@ -530,6 +383,7 @@ impl StackCoordinatorStore for StackRuntimePortsV1 { match self .store .append(record, recipient_ids) + .await .map_err(Self::database_error)? { GitHubStackSignalAppendOutcomeV1::Saturated { .. } => { @@ -540,11 +394,12 @@ impl StackCoordinatorStore for StackRuntimePortsV1 { } } - fn pending_deliveries( + async fn pending_deliveries( &self, ) -> Result, StackCoordinatorErrorV1> { self.store .pending() + .await .map_err(Self::database_error)? .into_iter() .map(|delivery| { @@ -563,7 +418,7 @@ impl StackCoordinatorStore for StackRuntimePortsV1 { .collect() } - fn acknowledge( + async fn acknowledge( &self, watermark_id: &StackDeliveryWatermarkId, deliveries: &[StackPendingDeliveryV1], @@ -577,21 +432,23 @@ impl StackCoordinatorStore for StackRuntimePortsV1 { .collect(); self.store .acknowledge(watermark_id, keys) + .await .map_err(Self::database_error) } - fn signal( + async fn signal( &self, signal_id: &StackSignalId, ) -> Result, StackCoordinatorErrorV1> { self.store .signal(signal_id) + .await .map_err(Self::database_error)? .map(|record| self.record_to_signal(&record)) .transpose() } - fn record_authorization_loss( + async fn record_authorization_loss( &self, signal: &StackSignalV1, recipient: &ActorId, @@ -608,6 +465,7 @@ impl StackCoordinatorStore for StackRuntimePortsV1 { self.validate_scope(signal)?; self.store .authorization_lost(&signal.signal_id, recipient) + .await .map_err(Self::database_error) } } @@ -639,7 +497,7 @@ impl StackDeliveryAuthorizationPort for StackRuntimePortsV1 { Ok(evidence.recipients.into_iter().collect()) } - fn authorize( + async fn authorize( &self, recipient: &ActorId, signal: &StackSignalV1, @@ -669,7 +527,11 @@ impl StackDeliveryAuthorizationPort for StackRuntimePortsV1 { } Err(_) => return StackDeliveryAuthorizationV1::Unavailable, } - match self.store.recipient_state(&signal.signal_id, recipient) { + match self + .store + .recipient_state(&signal.signal_id, recipient) + .await + { Ok(Some(GitHubStackDeliveryStateV1::AuthorizationLost) | None) => { StackDeliveryAuthorizationV1::Denied } @@ -685,7 +547,7 @@ impl StackDeliveryPort for StackRuntimePortsV1 { level = "trace", skip_all )] - fn deliver(&self, batch: &StackDeliveryBatchV1) -> Result<(), StackCoordinatorErrorV1> { + async fn deliver(&self, batch: &StackDeliveryBatchV1) -> Result<(), StackCoordinatorErrorV1> { if batch.deliveries.is_empty() || batch.deliveries.len() > MAX_GITHUB_STACK_DELIVERY_BATCH_V1 { @@ -711,6 +573,7 @@ impl StackDeliveryPort for StackRuntimePortsV1 { .collect(); self.store .publish(&batch.watermark_id, keys) + .await .map_err(Self::database_error) } } @@ -768,22 +631,18 @@ pub struct DaemonGitHubStackRuntimeV1 { /// its task is aborted or no native timer starts. async fn drain_loop(coordinator: Arc, ports: StackRuntimePortsV1) { loop { - let drain_coordinator = Arc::clone(&coordinator); - let drain_ports = ports.clone(); - let drained = native_runtime::spawn_blocking(move || { - let now = StackRuntimePortsV1::now()?; - drain_coordinator.drain_due(&drain_ports, &drain_ports, &drain_ports, now) - }) - .await; + let drained = match StackRuntimePortsV1::now() { + Ok(now) => coordinator.drain_due(&ports, &ports, &ports, now).await, + Err(error) => Err(error), + }; match drained { - Ok(Ok(_)) => {} - Ok(Err(StackCoordinatorErrorV1::Unavailable)) => { + Ok(_) => {} + Err(StackCoordinatorErrorV1::Unavailable) => { tracing::debug!( "GitHub stack drain deferred because its durable store is unavailable" ); } - Ok(Err(error)) => tracing::warn!(?error, "GitHub stack delivery drain failed"), - Err(error) => tracing::warn!(%error, "GitHub stack delivery drain task failed"), + Err(error) => tracing::warn!(?error, "GitHub stack delivery drain failed"), } if let Err(error) = native_runtime::sleep_until(Instant::now() + STACK_DELIVERY_TICK).await { @@ -840,10 +699,10 @@ impl DaemonGitHubStackRuntimeV1 { } .seal()?; coordinator.register_circuit_policy(&scope, policy)?; - let store = Arc::new( - DaemonStackDeliveryStoreV1::open(database, &project_id) - .map_err(StackRuntimePortsV1::database_error)?, - ); + let store = Arc::new(DaemonStackDeliveryStoreV1 { + database, + project_id: project_id.as_str().to_owned(), + }); let ports = StackRuntimePortsV1 { scope: scope.clone(), access: Arc::new(Mutex::new(access)), @@ -863,7 +722,7 @@ impl DaemonGitHubStackRuntimeV1 { })) } - pub fn enqueue_from_preflight( + pub async fn enqueue_from_preflight( &self, signal: StackSignalV1, context: &RequestContext, @@ -871,6 +730,7 @@ impl DaemonGitHubStackRuntimeV1 { self.ports.authorization_from_context(&signal, context)?; self.coordinator .enqueue_transition(&self.ports, &self.ports, &self.scope, signal) + .await } pub fn refresh_access( @@ -880,7 +740,7 @@ impl DaemonGitHubStackRuntimeV1 { self.ports.replace_access(access) } - pub fn enqueue_from_approval( + pub async fn enqueue_from_approval( &self, signal: StackSignalV1, approval: &NativeIntegrationApprovalV1, @@ -890,6 +750,7 @@ impl DaemonGitHubStackRuntimeV1 { .authorization_from_approval(&signal, approval, context)?; self.coordinator .enqueue_transition(&self.ports, &self.ports, &self.scope, signal) + .await } /// Executes preflight through the coordinator's circuit and bounded @@ -958,10 +819,10 @@ impl DaemonGitHubStackRuntimeV1 { } } - pub fn pending_host_deliveries( + pub async fn pending_host_deliveries( &self, ) -> Result, StackCoordinatorErrorV1> { - self.ports.pending_host_for_scope() + self.ports.pending_host_for_scope().await } async fn expand_request( @@ -1000,6 +861,7 @@ impl DaemonGitHubStackRuntimeV1 { .ports .store .oldest_pending(&self.scope, request.context().actor()) + .await .map_err(|_| GitHubStackSignalExpandPortError::Unavailable)? .map(|delivery| StackSignalId::new(delivery.signal.signal_id)) .transpose() @@ -1014,6 +876,7 @@ impl DaemonGitHubStackRuntimeV1 { request.context().actor(), &selected_signal_id, ) + .await .map_err(map_expand_error)? .ok_or(GitHubStackSignalExpandPortError::Concealed)?; if request @@ -1149,6 +1012,7 @@ impl DaemonGitHubStackRuntimeV1 { self.ports .store .host_acknowledge(&signal.signal_id, request.context().actor()) + .await .map_err(|_| GitHubStackSignalExpandPortError::Unavailable)?; Ok(GitHubStackSignalExpandSurfaceResultV1::Expanded { evidence: Box::new(evidence), @@ -1182,3 +1046,62 @@ fn map_expand_error(error: StackCoordinatorErrorV1) -> GitHubStackSignalExpandPo | StackCoordinatorErrorV1::Invalid(_) => GitHubStackSignalExpandPortError::Unavailable, } } + +#[cfg(test)] +mod tests { + use tracedecay_global_db::tests::harness::RegisteredGlobalDbHarness; + use tracedecay_runtime_core::native_runtime::NativeTaskOutcome; + + use super::*; + + #[tokio::test] + async fn delivery_store_awaited_on_a_native_worker_never_parks_it() { + let database = RegisteredGlobalDbHarness::open("github-stack-native-worker").await; + let store = DaemonStackDeliveryStoreV1 { + database: database.registered.clone(), + project_id: "project.stack-native-worker".to_owned(), + }; + let writer_waits = native_runtime::native_thread_writer_waits(); + let reader_waits = native_runtime::native_thread_reader_waits(); + + let task = NativeTask::spawn(async move { + assert!( + native_runtime::Runtime::current_handle().is_some(), + "the store must be awaited on a native worker" + ); + store + .append( + GitHubStackSignalRecordV1 { + project_id: store.project_id.clone(), + signal_id: "signal.stack-native-worker".to_owned(), + scope_digest: "scope.stack-native-worker".to_owned(), + repository_id: "repository.stack-native-worker".to_owned(), + watermark_id: "watermark.stack-native-worker".to_owned(), + observed_at_micros: 1, + signal_json: "{}".to_owned(), + }, + vec!["actor.stack-native-worker".to_owned()], + ) + .await?; + store.pending().await + }) + .expect("native task admitted"); + + let NativeTaskOutcome::Completed(pending) = task.wait().await else { + panic!("store task did not complete"); + }; + let pending = pending.expect("store calls settle on a native worker"); + assert_eq!(pending.len(), 1); + assert_eq!(pending[0].recipient, "actor.stack-native-worker"); + assert_eq!( + native_runtime::native_thread_writer_waits(), + writer_waits, + "no writer wait parked a native worker" + ); + assert_eq!( + native_runtime::native_thread_reader_waits(), + reader_waits, + "no reader wait parked a native worker" + ); + } +} diff --git a/crates/tracedecay-application/src/stack_coordinator.rs b/crates/tracedecay-application/src/stack_coordinator.rs index f9fec8dc40..97c4589955 100644 --- a/crates/tracedecay-application/src/stack_coordinator.rs +++ b/crates/tracedecay-application/src/stack_coordinator.rs @@ -196,7 +196,11 @@ pub struct DaemonGitHubStackCoordinatorV1 { policies: Mutex>, observations: Mutex>, circuit_policies: Mutex>, - dedupe: Mutex>, + /// Held across the append so a duplicate debounced signal racing the + /// first one cannot append twice. + dedupe: tokio::sync::Mutex< + BTreeMap<(ManifestDigest, StackSignalKindV1, ManifestDigest), UtcMicros>, + >, circuits: Mutex>, preflights: Mutex, drift_intervals: Mutex>>, @@ -342,7 +346,7 @@ impl DaemonGitHubStackCoordinatorV1 { } #[tracing::instrument(name = "usecases.stack.enqueue", level = "trace", skip_all)] - pub fn enqueue_transition( + pub async fn enqueue_transition( &self, store: &S, authorization: &A, @@ -359,7 +363,7 @@ impl DaemonGitHubStackCoordinatorV1 { let mut stale = false; for recipient in recipients { recipient.validate().map_err(invalid)?; - match authorization.authorize(&recipient, &signal) { + match authorization.authorize(&recipient, &signal).await { StackDeliveryAuthorizationV1::Authorized => authorized.push(recipient), StackDeliveryAuthorizationV1::Denied => denied = true, StackDeliveryAuthorizationV1::Stale => stale = true, @@ -377,25 +381,25 @@ impl DaemonGitHubStackCoordinatorV1 { } } if signal.kind.is_material() { - return store.append_signal(signal, authorized); + return store.append_signal(signal, authorized).await; } let key = ( signal.scope_digest.clone(), signal.kind, signal.state_digest.clone(), ); - let mut dedupe = self.dedupe.lock().map_err(lock_error)?; + let mut dedupe = self.dedupe.lock().await; dedupe.retain(|_, seen| signal.observed_at.0.saturating_sub(seen.0) <= DEDUPE_TTL_MICROS); if dedupe.contains_key(&key) { return Ok(()); } - store.append_signal(signal.clone(), authorized)?; + store.append_signal(signal.clone(), authorized).await?; dedupe.insert(key, signal.observed_at); Ok(()) } #[tracing::instrument(name = "usecases.stack.drain", level = "trace", skip_all)] - pub fn drain_due< + pub async fn drain_due< S: StackCoordinatorStore, A: StackDeliveryAuthorizationPort, D: StackDeliveryPort, @@ -406,7 +410,7 @@ impl DaemonGitHubStackCoordinatorV1 { delivery: &D, now: UtcMicros, ) -> Result { - let mut pending = store.pending_deliveries()?; + let mut pending = store.pending_deliveries().await?; pending.retain(|(_, signal)| { signal.kind.is_material() || signal @@ -441,7 +445,7 @@ impl DaemonGitHubStackCoordinatorV1 { break; } consumed += 1; - match authorization.authorize(&item.recipient, signal) { + match authorization.authorize(&item.recipient, signal).await { StackDeliveryAuthorizationV1::Authorized => { recipients.insert(item.recipient.clone()); signal_ids.insert(item.signal_id.clone()); @@ -453,7 +457,9 @@ impl DaemonGitHubStackCoordinatorV1 { } outcome @ (StackDeliveryAuthorizationV1::Denied | StackDeliveryAuthorizationV1::Stale) => { - store.record_authorization_loss(signal, &item.recipient, outcome)?; + store + .record_authorization_loss(signal, &item.recipient, outcome) + .await?; acknowledged.push(item.clone()) } StackDeliveryAuthorizationV1::Unavailable => {} @@ -472,27 +478,27 @@ impl DaemonGitHubStackCoordinatorV1 { partial: consumed < pending.len(), }; if !batch.recipients.is_empty() { - delivery.deliver(&batch)?; + delivery.deliver(&batch).await?; delivered += batch.deliveries.len(); } - store.acknowledge(&watermark, &acknowledged)?; + store.acknowledge(&watermark, &acknowledged).await?; pending.drain(..consumed); } Ok(delivered) } #[tracing::instrument(name = "usecases.stack.expand", level = "trace", skip_all)] - pub fn expand_transition( + pub async fn expand_transition( &self, store: &S, authorization: &A, recipient: &ActorId, signal_id: &StackSignalId, ) -> Result, StackCoordinatorErrorV1> { - let Some(signal) = store.signal(signal_id)? else { + let Some(signal) = store.signal(signal_id).await? else { return Ok(None); }; - match authorization.authorize(recipient, &signal) { + match authorization.authorize(recipient, &signal).await { StackDeliveryAuthorizationV1::Authorized => Ok(Some(signal)), StackDeliveryAuthorizationV1::Denied => Err(StackCoordinatorErrorV1::Denied), StackDeliveryAuthorizationV1::Stale => Err(StackCoordinatorErrorV1::Stale), diff --git a/crates/tracedecay-application/src/stack_coordinator/transition.rs b/crates/tracedecay-application/src/stack_coordinator/transition.rs index 6c84ed7ef0..75271e79ed 100644 --- a/crates/tracedecay-application/src/stack_coordinator/transition.rs +++ b/crates/tracedecay-application/src/stack_coordinator/transition.rs @@ -185,25 +185,27 @@ pub trait StackCoordinatorStore: Send + Sync { &self, signal: StackSignalV1, recipients: Vec, - ) -> Result<(), StackCoordinatorErrorV1>; + ) -> impl Future> + Send; fn pending_deliveries( &self, - ) -> Result, StackCoordinatorErrorV1>; + ) -> impl Future< + Output = Result, StackCoordinatorErrorV1>, + > + Send; fn acknowledge( &self, watermark_id: &StackDeliveryWatermarkId, deliveries: &[StackPendingDeliveryV1], - ) -> Result<(), StackCoordinatorErrorV1>; + ) -> impl Future> + Send; fn signal( &self, signal_id: &StackSignalId, - ) -> Result, StackCoordinatorErrorV1>; + ) -> impl Future, StackCoordinatorErrorV1>> + Send; fn record_authorization_loss( &self, signal: &StackSignalV1, recipient: &ActorId, outcome: StackDeliveryAuthorizationV1, - ) -> Result<(), StackCoordinatorErrorV1>; + ) -> impl Future> + Send; } pub trait StackDeliveryAuthorizationPort: Send + Sync { @@ -217,11 +219,14 @@ pub trait StackDeliveryAuthorizationPort: Send + Sync { &self, recipient: &ActorId, signal: &StackSignalV1, - ) -> StackDeliveryAuthorizationV1; + ) -> impl Future + Send; } pub trait StackDeliveryPort: Send + Sync { - fn deliver(&self, batch: &StackDeliveryBatchV1) -> Result<(), StackCoordinatorErrorV1>; + fn deliver( + &self, + batch: &StackDeliveryBatchV1, + ) -> impl Future> + Send; } pub trait OptionalStackPreflightPort: Send + Sync { diff --git a/crates/tracedecay-application/tests/application_suite/github_stack_coordinator.rs b/crates/tracedecay-application/tests/application_suite/github_stack_coordinator.rs index 4967dc4469..bc66162ce7 100644 --- a/crates/tracedecay-application/tests/application_suite/github_stack_coordinator.rs +++ b/crates/tracedecay-application/tests/application_suite/github_stack_coordinator.rs @@ -61,7 +61,7 @@ struct MemoryStore { } impl StackCoordinatorStore for MemoryStore { - fn append_signal( + async fn append_signal( &self, signal: StackSignalV1, recipients: Vec, @@ -92,13 +92,13 @@ impl StackCoordinatorStore for MemoryStore { } } - fn pending_deliveries( + async fn pending_deliveries( &self, ) -> Result, StackCoordinatorErrorV1> { Ok(self.pending.lock().unwrap().clone()) } - fn acknowledge( + async fn acknowledge( &self, watermark_id: &tracedecay_domain::StackDeliveryWatermarkId, deliveries: &[StackPendingDeliveryV1], @@ -110,14 +110,14 @@ impl StackCoordinatorStore for MemoryStore { Ok(()) } - fn signal( + async fn signal( &self, signal_id: &tracedecay_domain::StackSignalId, ) -> Result, StackCoordinatorErrorV1> { Ok(self.signals.lock().unwrap().get(signal_id).cloned()) } - fn record_authorization_loss( + async fn record_authorization_loss( &self, signal: &StackSignalV1, recipient: &tracedecay_domain::ActorId, @@ -146,7 +146,7 @@ impl StackDeliveryAuthorizationPort for Authorization { Ok(self.recipients.lock().unwrap().clone()) } - fn authorize( + async fn authorize( &self, recipient: &tracedecay_domain::ActorId, _signal: &StackSignalV1, @@ -166,7 +166,7 @@ struct RecordingDelivery { } impl StackDeliveryPort for RecordingDelivery { - fn deliver(&self, batch: &StackDeliveryBatchV1) -> Result<(), StackCoordinatorErrorV1> { + async fn deliver(&self, batch: &StackDeliveryBatchV1) -> Result<(), StackCoordinatorErrorV1> { let mut fail_next = self.fail_next.lock().unwrap(); if *fail_next { *fail_next = false; @@ -454,8 +454,8 @@ fn mounted_provider_observation_reports_pr_tip_merge_base_and_ci_drift() { })); } -#[test] -fn mounted_owner_dedupes_debounced_signals_but_drains_all_material_overflow() { +#[tokio::test] +async fn mounted_owner_dedupes_debounced_signals_but_drains_all_material_overflow() { let coordinator = DaemonGitHubStackCoordinatorV1::default(); let exact_scope = scope(91); let store = MemoryStore::default(); @@ -478,9 +478,11 @@ fn mounted_owner_dedupes_debounced_signals_but_drains_all_material_overflow() { ); coordinator .enqueue_transition(&store, &authorization, &exact_scope, first) + .await .unwrap(); coordinator .enqueue_transition(&store, &authorization, &exact_scope, duplicate) + .await .unwrap(); let mut expanded_id = None; for index in 3..132 { @@ -492,71 +494,85 @@ fn mounted_owner_dedupes_debounced_signals_but_drains_all_material_overflow() { expanded_id.get_or_insert_with(|| material.signal_id.clone()); coordinator .enqueue_transition(&store, &authorization, &exact_scope, material) + .await .unwrap(); } authorization.denied.lock().unwrap().insert(actor(64)); let delivered = coordinator .drain_due(&store, &authorization, &delivery, UtcMicros(2_000_000)) + .await .unwrap(); assert_eq!(delivered, 64 * 130); assert!(store.pending.lock().unwrap().is_empty()); assert_eq!(store.authorization_losses.lock().unwrap().len(), 130); - let batches = delivery.batches.lock().unwrap(); - assert!(batches.len() > 1); - assert!(batches.iter().all(|batch| { - batch.recipients.len() <= MAX_BATCH_RECIPIENTS && batch.signals.len() <= MAX_BATCH_SIGNALS - })); - drop(batches); + { + let batches = delivery.batches.lock().unwrap(); + assert!(batches.len() > 1); + assert!(batches.iter().all(|batch| { + batch.recipients.len() <= MAX_BATCH_RECIPIENTS + && batch.signals.len() <= MAX_BATCH_SIGNALS + })); + } assert_eq!( - coordinator.expand_transition( - &store, - &authorization, - &actor(64), - expanded_id.as_ref().unwrap(), - ), + coordinator + .expand_transition( + &store, + &authorization, + &actor(64), + expanded_id.as_ref().unwrap(), + ) + .await, Err(StackCoordinatorErrorV1::Denied) ); *authorization.recipients.lock().unwrap() = vec![actor(64)]; assert_eq!( - coordinator.enqueue_transition( - &store, - &authorization, - &exact_scope, - signal(&exact_scope, 200, StackSignalKindV1::ActualConflict), - ), + coordinator + .enqueue_transition( + &store, + &authorization, + &exact_scope, + signal(&exact_scope, 200, StackSignalKindV1::ActualConflict), + ) + .await, Err(StackCoordinatorErrorV1::Denied) ); } -#[test] -fn transition_signal_and_circuit_policy_reject_stale_or_tampered_identity() { +#[tokio::test] +async fn transition_signal_and_circuit_policy_reject_stale_or_tampered_identity() { let coordinator = DaemonGitHubStackCoordinatorV1::default(); let exact_scope = scope(92); let store = MemoryStore::default(); let authorization = Authorization::default(); *authorization.recipients.lock().unwrap() = vec![actor(1)]; assert_eq!( - coordinator.enqueue_transition( - &store, - &authorization, - &exact_scope, - signal(&scope(93), 1, StackSignalKindV1::ActualConflict), - ), + coordinator + .enqueue_transition( + &store, + &authorization, + &exact_scope, + signal(&scope(93), 1, StackSignalKindV1::ActualConflict), + ) + .await, Err(StackCoordinatorErrorV1::Stale) ); let mut tampered_signal = signal(&exact_scope, 2, StackSignalKindV1::ActualConflict); tampered_signal.signal_id = tracedecay_domain::StackSignalId::new("signal.stack.tampered").unwrap(); assert!(matches!( - coordinator.enqueue_transition(&store, &authorization, &exact_scope, tampered_signal,), + coordinator + .enqueue_transition(&store, &authorization, &exact_scope, tampered_signal,) + .await, Err(StackCoordinatorErrorV1::Invalid(_)) )); let mut tampered_watermark = signal(&exact_scope, 3, StackSignalKindV1::ActualConflict); tampered_watermark.watermark_id = tracedecay_domain::StackDeliveryWatermarkId::new("watermark.stack.tampered").unwrap(); assert!(matches!( - coordinator.enqueue_transition(&store, &authorization, &exact_scope, tampered_watermark,), + coordinator + .enqueue_transition(&store, &authorization, &exact_scope, tampered_watermark,) + .await, Err(StackCoordinatorErrorV1::Invalid(_)) )); @@ -580,8 +596,8 @@ fn transition_signal_and_circuit_policy_reject_stale_or_tampered_identity() { .unwrap(); } -#[test] -fn durable_saturation_and_restart_delivery_preserve_material_work() { +#[tokio::test] +async fn durable_saturation_and_restart_delivery_preserve_material_work() { let exact_scope = scope(93); let store = MemoryStore::default(); let authorization = Authorization::default(); @@ -589,27 +605,21 @@ fn durable_saturation_and_restart_delivery_preserve_material_work() { *store.saturate_after_persist.lock().unwrap() = true; let signal = signal(&exact_scope, 1, StackSignalKindV1::ActualConflict); assert_eq!( - DaemonGitHubStackCoordinatorV1::default().enqueue_transition( - &store, - &authorization, - &exact_scope, - signal.clone(), - ), + DaemonGitHubStackCoordinatorV1::default() + .enqueue_transition(&store, &authorization, &exact_scope, signal.clone(),) + .await, Err(StackCoordinatorErrorV1::Saturated) ); - assert_eq!(store.signal(&signal.signal_id).unwrap(), Some(signal)); + assert_eq!(store.signal(&signal.signal_id).await.unwrap(), Some(signal)); assert_eq!(store.pending.lock().unwrap().len(), 1); *store.saturate_after_persist.lock().unwrap() = false; let failed_delivery = RecordingDelivery::default(); *failed_delivery.fail_next.lock().unwrap() = true; assert_eq!( - DaemonGitHubStackCoordinatorV1::default().drain_due( - &store, - &authorization, - &failed_delivery, - UtcMicros(2), - ), + DaemonGitHubStackCoordinatorV1::default() + .drain_due(&store, &authorization, &failed_delivery, UtcMicros(2),) + .await, Err(StackCoordinatorErrorV1::Unavailable) ); assert_eq!(store.pending.lock().unwrap().len(), 1); @@ -621,14 +631,15 @@ fn durable_saturation_and_restart_delivery_preserve_material_work() { &RecordingDelivery::default(), UtcMicros(2), ) + .await .unwrap(), 1 ); assert!(store.pending.lock().unwrap().is_empty()); } -#[test] -fn debounce_classes_and_five_minute_dedupe_ttl_advance_independently() { +#[tokio::test] +async fn debounce_classes_and_five_minute_dedupe_ttl_advance_independently() { let coordinator = DaemonGitHubStackCoordinatorV1::default(); let exact_scope = scope(94); let store = MemoryStore::default(); @@ -649,31 +660,37 @@ fn debounce_classes_and_five_minute_dedupe_ttl_advance_independently() { ); coordinator .enqueue_transition(&store, &authorization, &exact_scope, ready.clone()) + .await .unwrap(); coordinator .enqueue_transition(&store, &authorization, &exact_scope, drift) + .await .unwrap(); assert_eq!( coordinator .drain_due(&store, &authorization, &delivery, UtcMicros(249_999)) + .await .unwrap(), 0 ); assert_eq!( coordinator .drain_due(&store, &authorization, &delivery, UtcMicros(250_000)) + .await .unwrap(), 1 ); assert_eq!( coordinator .drain_due(&store, &authorization, &delivery, UtcMicros(999_999)) + .await .unwrap(), 0 ); assert_eq!( coordinator .drain_due(&store, &authorization, &delivery, UtcMicros(1_000_000)) + .await .unwrap(), 1 ); @@ -686,6 +703,7 @@ fn debounce_classes_and_five_minute_dedupe_ttl_advance_independently() { ); coordinator .enqueue_transition(&store, &authorization, &exact_scope, duplicate) + .await .unwrap(); assert!(store.pending.lock().unwrap().is_empty()); let after_ttl = signal_at( @@ -696,6 +714,7 @@ fn debounce_classes_and_five_minute_dedupe_ttl_advance_independently() { ); coordinator .enqueue_transition(&store, &authorization, &exact_scope, after_ttl) + .await .unwrap(); assert_eq!(store.pending.lock().unwrap().len(), 1); } @@ -750,3 +769,90 @@ fn daemon_restart_returns_unavailable_until_a_delayed_exact_probe_upgrades_state "refs/heads/feature" ); } + +/// Counts appends and yields inside each one, so two enqueues that both pass +/// the dedupe check before either records its key would both append. +#[derive(Default)] +struct YieldingAppendStore { + inner: MemoryStore, + appends: std::sync::atomic::AtomicUsize, +} + +impl StackCoordinatorStore for YieldingAppendStore { + async fn append_signal( + &self, + signal: StackSignalV1, + recipients: Vec, + ) -> Result<(), StackCoordinatorErrorV1> { + self.appends + .fetch_add(1, std::sync::atomic::Ordering::SeqCst); + tokio::time::sleep(std::time::Duration::from_millis(20)).await; + self.inner.append_signal(signal, recipients).await + } + + async fn pending_deliveries( + &self, + ) -> Result, StackCoordinatorErrorV1> { + self.inner.pending_deliveries().await + } + + async fn acknowledge( + &self, + watermark_id: &tracedecay_domain::StackDeliveryWatermarkId, + deliveries: &[StackPendingDeliveryV1], + ) -> Result<(), StackCoordinatorErrorV1> { + self.inner.acknowledge(watermark_id, deliveries).await + } + + async fn signal( + &self, + signal_id: &tracedecay_domain::StackSignalId, + ) -> Result, StackCoordinatorErrorV1> { + self.inner.signal(signal_id).await + } + + async fn record_authorization_loss( + &self, + signal: &StackSignalV1, + recipient: &tracedecay_domain::ActorId, + outcome: StackDeliveryAuthorizationV1, + ) -> Result<(), StackCoordinatorErrorV1> { + self.inner + .record_authorization_loss(signal, recipient, outcome) + .await + } +} + +#[tokio::test] +async fn concurrent_duplicate_debounced_signals_append_once() { + let coordinator = DaemonGitHubStackCoordinatorV1::default(); + let exact_scope = scope(95); + let store = YieldingAppendStore::default(); + let authorization = Authorization::default(); + *authorization.recipients.lock().unwrap() = vec![actor(1)]; + let first = signal_at( + &exact_scope, + StackSignalKindV1::DependencyReady, + digest('e'), + UtcMicros(1), + ); + let duplicate = signal_at( + &exact_scope, + StackSignalKindV1::DependencyReady, + digest('e'), + UtcMicros(2), + ); + + let (first, duplicate) = tokio::join!( + coordinator.enqueue_transition(&store, &authorization, &exact_scope, first), + coordinator.enqueue_transition(&store, &authorization, &exact_scope, duplicate), + ); + + first.unwrap(); + duplicate.unwrap(); + assert_eq!( + store.appends.load(std::sync::atomic::Ordering::SeqCst), + 1, + "the dedupe check and the append it guards are one critical section" + ); +} diff --git a/crates/tracedecay-daemon-service/src/invocation/github_stack_signal.rs b/crates/tracedecay-daemon-service/src/invocation/github_stack_signal.rs index 959d812580..9b063c52a8 100644 --- a/crates/tracedecay-daemon-service/src/invocation/github_stack_signal.rs +++ b/crates/tracedecay-daemon-service/src/invocation/github_stack_signal.rs @@ -63,23 +63,12 @@ pub(super) async fn execute_github_stack_signal_expand( Ok(signal) => signal, Err(problem) => return application_problem(wire_request_id, problem), }; - // Expansion interleaves native-store awaits with synchronous delivery-store - // waits, so it runs on the blocking pool. - let expansion = tracedecay_runtime_core::native_runtime::on_blocking_pool(async move { - runtime - .expand(request.into_application_request(context), &signal) - .await - }) - .await; - let result = match expansion { - Ok(Ok(result)) => result, - Ok(Err(error)) => error.into_surface_result(), - Err(_) => { - return DaemonInvocationResponse::problem( - wire_request_id, - DaemonInvocationProblem::Unavailable, - ); - } + let result = match runtime + .expand(request.into_application_request(context), &signal) + .await + { + Ok(result) => result, + Err(error) => error.into_surface_result(), }; let Ok(payload) = serde_json::to_value(result) else { return DaemonInvocationResponse::problem( diff --git a/crates/tracedecay-daemon-service/src/invocation/native_integration.rs b/crates/tracedecay-daemon-service/src/invocation/native_integration.rs index 78e1df3be8..34ad1ef6fe 100644 --- a/crates/tracedecay-daemon-service/src/invocation/native_integration.rs +++ b/crates/tracedecay-daemon-service/src/invocation/native_integration.rs @@ -415,6 +415,7 @@ async fn execute_with_owner( { runtime .enqueue_from_preflight(stack_signal, &signal_context) + .await .map_err(stack_coordinator_contract_error)?; } outcome @@ -607,6 +608,7 @@ async fn execute_with_owner( &signal_approval, &signal_context, ) + .await .map_err(|_| unavailable_native_integration())?; } NativeIntegrationReceiptProjectionV1::project(&receipt) diff --git a/crates/tracedecay-mcp/src/handlers/hook_runtime/admission.rs b/crates/tracedecay-mcp/src/handlers/hook_runtime/admission.rs index 4d1ff6002a..240cb96d4f 100644 --- a/crates/tracedecay-mcp/src/handlers/hook_runtime/admission.rs +++ b/crates/tracedecay-mcp/src/handlers/hook_runtime/admission.rs @@ -795,16 +795,12 @@ async fn admit_hook_v2_envelope_with_lifecycle_inner( let worktree_id = envelope.worktree_id; tracedecay_runtime_core::native_runtime::timeout( std::time::Duration::from_millis(50), - tracedecay_runtime_core::native_runtime::spawn_blocking(move || { - tracedecay_agent_hosts::native_integration::github_stack_hook_available( - project_id, - worktree_id, - ) - }), + tracedecay_agent_hosts::native_integration::github_stack_hook_available( + project_id, + worktree_id, + ), ) .await - .ok() - .and_then(std::result::Result::ok) .unwrap_or(false) } else { false