diff --git a/crates/tracedecay-application/src/work/registered.rs b/crates/tracedecay-application/src/work/registered.rs index b5ebb2ef3b..b2e89ee0b0 100644 --- a/crates/tracedecay-application/src/work/registered.rs +++ b/crates/tracedecay-application/src/work/registered.rs @@ -60,9 +60,6 @@ pub struct RegisteredWorkApplicationServicesV1 { attempts: tracedecay_contracts::WorkAttemptService< tracedecay_rusqlite_runtime::work::WorkSqliteStorage, >, - attempt_effects: tracedecay_contracts::WorkAttemptEffectServiceV1< - tracedecay_rusqlite_runtime::work::WorkSqliteStorage, - >, run_control: tracedecay_contracts::WorkRunControlService< tracedecay_rusqlite_runtime::work::WorkSqliteStorage, >, @@ -98,13 +95,7 @@ pub struct RegisteredWorkProductServicesV1 { attempts: tracedecay_contracts::WorkProductAttemptServiceV1< tracedecay_rusqlite_runtime::work::WorkSqliteStorage, >, - synthesis: tracedecay_contracts::WorkProductSynthesisAttemptServiceV1< - tracedecay_rusqlite_runtime::work::WorkSqliteStorage, - >, - retry: tracedecay_contracts::WorkProductRetryServiceV1< - tracedecay_rusqlite_runtime::work::WorkSqliteStorage, - tracedecay_contracts::RuntimeWorkRetryEvidenceV1, - >, + storage: tracedecay_rusqlite_runtime::work::WorkSqliteStorage, } impl RegisteredWorkProductServicesV1 { @@ -137,13 +128,7 @@ impl RegisteredWorkProductServicesV1 { storage.clone(), ), attempts: tracedecay_contracts::WorkProductAttemptServiceV1::new(storage.clone()), - synthesis: tracedecay_contracts::WorkProductSynthesisAttemptServiceV1::new( - storage.clone(), - ), - retry: tracedecay_contracts::WorkProductRetryServiceV1::new( - storage, - tracedecay_contracts::RuntimeWorkRetryEvidenceV1, - ), + storage, }) } @@ -165,6 +150,198 @@ impl RegisteredWorkProductServicesV1 { }) } + /// Starts one attempt as a single writer transaction: the run's pause + /// fence, the replay check, the lease epoch, the product link and the + /// attempt row commit together, so a pause cannot land between the fence + /// and the reservation. The current product graph is read before the job, + /// on a reader; the link's graph-version compare-and-swap fences it. + pub async fn start_attempt( + &self, + context: tracedecay_contracts::RequestContext, + binding: tracedecay_contracts::WorkProductBindingV1, + revisions: tracedecay_contracts::WorkProductRevisionPinsV1, + topology: tracedecay_domain::WorkTopologyPolicyV1, + command: tracedecay_contracts::StartWorkAttemptCommand, + ) -> Result { + let prefetch_context = context.clone(); + let observed_at = command.occurred_at; + let graph = self + .storage + .prefetch_graph(move |storage| { + tracedecay_contracts::PrefetchedWorkGraphReadV1::for_attempt_admission( + storage, + &prefetch_context, + observed_at, + ) + }) + .await; + self.storage + .transact_with_graph(graph, move |storage| { + tracedecay_contracts::WorkRunControlService::new(storage).admit_reservation( + &context, + &command.task_id, + &command.run_id, + )?; + tracedecay_contracts::WorkProductAttemptServiceV1::new(storage) + .start_against_registered_topology( + &context, + &binding, + &revisions, + &topology, + command.clone(), + ) + }) + .await + .unwrap_or_else(|_| Err(attempt_unavailable())) + } + + /// Commits one product graph mutation as a single writer transaction: the + /// revision check, the command replay check and the event append with its + /// verified version commit together. The current graph the mutation + /// checks its expected version against is read before the job, on a + /// reader; the append's graph-version compare-and-swap fences it. + pub async fn mutate( + &self, + context: tracedecay_contracts::RequestContext, + binding: tracedecay_contracts::WorkProductBindingV1, + request: tracedecay_contracts::WorkProductMutationRequestV1, + revisions: tracedecay_contracts::WorkProductRevisionPinsV1, + ) -> Result< + tracedecay_contracts::WorkProductMutationReceiptV1, + tracedecay_contracts::WorkProductApplicationErrorV1, + > { + let (prefetch_context, prefetch_request) = (context.clone(), request.clone()); + let graph = self + .storage + .prefetch_graph(move |storage| { + tracedecay_contracts::PrefetchedWorkGraphReadV1::for_mutation( + storage, + &prefetch_context, + &prefetch_request, + ) + }) + .await; + self.storage + .transact_with_graph(graph, move |storage| { + tracedecay_contracts::WorkProductMutationServiceV1::new(storage, storage, storage) + .mutate(&context, &binding, request.clone(), &revisions) + }) + .await + .unwrap_or(Err( + tracedecay_contracts::WorkProductApplicationErrorV1::EventAuthorityUnavailable, + )) + } + + /// Admits one synthesis attempt as a single writer transaction: the + /// replay check, the source reads, the run's pause fence, the source + /// payload hydration `prepare_instructions` does through the job storage + /// and the admission commit together. + #[allow(clippy::too_many_arguments)] + pub async fn synthesize( + &self, + context: tracedecay_contracts::RequestContext, + binding: tracedecay_contracts::WorkProductBindingV1, + revisions: tracedecay_contracts::WorkProductRevisionPinsV1, + topology: tracedecay_domain::WorkTopologyPolicyV1, + command: tracedecay_contracts::AdmitWorkSynthesisCommand, + prepare_instructions: F, + ) -> Result< + tracedecay_contracts::WorkSynthesisAttemptV1, + tracedecay_contracts::ApplicationProblem, + > + where + F: Fn( + &tracedecay_rusqlite_runtime::work_run_control::WorkSqliteJobStorage<'_>, + &tracedecay_contracts::StartWorkAttemptCommand, + &[tracedecay_domain::WorkAttemptV1], + ) -> Result + + Send + + Sync + + 'static, + { + let prefetch_context = context.clone(); + let observed_at = command.start.occurred_at; + let graph = self + .storage + .prefetch_graph(move |storage| { + tracedecay_contracts::PrefetchedWorkGraphReadV1::for_attempt_admission( + storage, + &prefetch_context, + observed_at, + ) + }) + .await; + self.storage + .transact_with_graph(graph, move |storage| { + tracedecay_contracts::admit_work_synthesis_against_registered_topology( + &tracedecay_contracts::WorkProductSynthesisAttemptServiceV1::new(storage), + &context, + &binding, + &revisions, + &topology, + command.clone(), + |start, sources| { + tracedecay_contracts::WorkRunControlService::new(storage) + .admit_reservation(&context, &start.task_id, &start.run_id)?; + prepare_instructions(&storage, start, sources) + }, + ) + }) + .await + .unwrap_or_else(|_| Err(attempt_unavailable())) + } + + /// Admits one retry as a single writer transaction: the replay check, the + /// effect-safety read, the lease epoch, the product link, the retry + /// receipt, the attempt row and the workflow child rebind commit together. + /// `workflow_rebind` is read before the job; its journal append is + /// sequence-checked inside it. + #[allow(clippy::too_many_arguments)] + pub async fn retry_attempt( + &self, + context: tracedecay_contracts::RequestContext, + binding: tracedecay_contracts::WorkProductBindingV1, + revisions: tracedecay_contracts::WorkProductRevisionPinsV1, + topology: tracedecay_domain::WorkTopologyPolicyV1, + command: tracedecay_contracts::RetryWorkAttemptCommandV1, + restarted_at: tracedecay_domain::UtcMicros, + workflow_rebind: Option, + ) -> Result< + tracedecay_contracts::WorkRetryAttemptOutcomeV1, + tracedecay_contracts::ApplicationProblem, + > { + let prefetch_context = context.clone(); + let observed_at = restarted_at; + let graph = self + .storage + .prefetch_graph(move |storage| { + tracedecay_contracts::PrefetchedWorkGraphReadV1::for_attempt_admission( + storage, + &prefetch_context, + observed_at, + ) + }) + .await; + self.storage + .transact_with_graph(graph, move |storage| { + tracedecay_contracts::WorkProductRetryServiceV1::new( + storage, + tracedecay_contracts::RuntimeWorkRetryEvidenceV1, + ) + .retry( + &context, + &binding, + &revisions, + &topology, + command.clone(), + restarted_at, + workflow_rebind.clone(), + ) + }) + .await + .unwrap_or_else(|_| Err(attempt_unavailable())) + } + pub const fn reads( &self, ) -> &tracedecay_contracts::WorkProductReadServiceV1< @@ -191,23 +368,6 @@ impl RegisteredWorkProductServicesV1 { > { &self.attempts } - - pub const fn synthesis( - &self, - ) -> &tracedecay_contracts::WorkProductSynthesisAttemptServiceV1< - tracedecay_rusqlite_runtime::work::WorkSqliteStorage, - > { - &self.synthesis - } - - pub const fn retry( - &self, - ) -> &tracedecay_contracts::WorkProductRetryServiceV1< - tracedecay_rusqlite_runtime::work::WorkSqliteStorage, - tracedecay_contracts::RuntimeWorkRetryEvidenceV1, - > { - &self.retry - } } pub struct RegisteredAuthorizedWorkProductServicesV1 { @@ -233,7 +393,6 @@ impl RegisteredWorkApplicationServicesV1 { let storage = db.work_storage()?; Ok(Self { attempts: tracedecay_contracts::WorkAttemptService::new(storage.clone()), - attempt_effects: tracedecay_contracts::WorkAttemptEffectServiceV1::new(storage.clone()), run_control: tracedecay_contracts::WorkRunControlService::new(storage.clone()), placement: tracedecay_contracts::WorkPlacementService::new(storage.clone()), artifact_hydration: tracedecay_contracts::WorkArtifactHydrationService::new( @@ -283,6 +442,188 @@ impl RegisteredWorkApplicationServicesV1 { .unwrap_or_else(|_| Err(run_control_unavailable())) } + /// Fences every open attempt after a restart as a single writer + /// transaction: the census and every fence commit together or not at all. + pub async fn resume_attempts( + &self, + context: tracedecay_contracts::RequestContext, + command: tracedecay_contracts::ResumeWorkAttemptsCommand, + ) -> Result< + tracedecay_contracts::WorkAttemptRecoveryReportV1, + tracedecay_contracts::ApplicationProblem, + > { + self.storage + .transact(move |storage| { + tracedecay_contracts::WorkAttemptService::new(storage).resume(&context, &command) + }) + .await + .unwrap_or_else(|_| Err(attempt_unavailable())) + } + + /// Runs one provider-driver step of the attempt authority as a single + /// writer transaction: its load, its decision and its fenced update + /// commit together. + async fn attempt_step( + &self, + step: impl Fn( + &tracedecay_contracts::WorkAttemptService< + tracedecay_rusqlite_runtime::work_run_control::WorkSqliteJobStorage<'_>, + >, + ) -> Result + + Send + + Sync + + 'static, + ) -> Result + where + R: Send + 'static, + { + self.storage + .transact(move |storage| step(&tracedecay_contracts::WorkAttemptService::new(storage))) + .await + .unwrap_or_else(|_| Err(attempt_unavailable())) + } + + /// Records a cancellation request against an open attempt as a single + /// writer transaction: the load, the conflict check and the fenced update + /// commit together. + pub async fn cancel_attempt( + &self, + context: tracedecay_contracts::RequestContext, + command: tracedecay_contracts::CancelWorkAttemptCommand, + ) -> Result { + self.attempt_step(move |attempts| attempts.request_cancellation(&context, command.clone())) + .await + } + + /// Marks the provider running under `actual_route`. + pub async fn mark_running( + &self, + context: tracedecay_contracts::RequestContext, + identity: tracedecay_domain::WorkAttemptIdentityV1, + actual_route: tracedecay_domain::WorkProviderRouteV1, + ) -> Result { + self.attempt_step(move |attempts| { + attempts.mark_running(&context, &identity, actual_route.clone()) + }) + .await + } + + /// Fences an attempt whose provider could not start for recovery. + pub async fn mark_provider_unavailable( + &self, + context: tracedecay_contracts::RequestContext, + identity: tracedecay_domain::WorkAttemptIdentityV1, + observed_at: tracedecay_domain::UtcMicros, + ) -> Result { + self.attempt_step(move |attempts| { + attempts.mark_provider_unavailable(&context, &identity, observed_at) + }) + .await + } + + /// Fails a recovery-required attempt with sealed denial evidence. + pub async fn fail_recovery( + &self, + context: tracedecay_contracts::RequestContext, + identity: tracedecay_domain::WorkAttemptIdentityV1, + evidence: tracedecay_contracts::WorkAttemptEvidenceRecordV1, + ) -> Result { + self.attempt_step(move |attempts| attempts.fail_recovery(&context, &identity, &evidence)) + .await + } + + /// Acknowledges a durable cancellation request. + pub async fn acknowledge_cancellation( + &self, + context: tracedecay_contracts::RequestContext, + identity: tracedecay_domain::WorkAttemptIdentityV1, + acknowledged_at: tracedecay_domain::UtcMicros, + ) -> Result { + self.attempt_step(move |attempts| { + attempts.acknowledge_cancellation(&context, &identity, acknowledged_at) + }) + .await + } + + /// Escalates an acknowledged cancellation to forced termination. + pub async fn escalate_cancellation( + &self, + context: tracedecay_contracts::RequestContext, + identity: tracedecay_domain::WorkAttemptIdentityV1, + escalated_at: tracedecay_domain::UtcMicros, + ) -> Result { + self.attempt_step(move |attempts| { + attempts.escalate_cancellation(&context, &identity, escalated_at) + }) + .await + } + + /// Seals an attempt with terminal evidence and its artifacts. + pub async fn settle_with_artifacts( + &self, + context: tracedecay_contracts::RequestContext, + identity: tracedecay_domain::WorkAttemptIdentityV1, + evidence: tracedecay_contracts::WorkAttemptEvidenceRecordV1, + artifacts: Vec, + ) -> Result { + self.attempt_step(move |attempts| { + attempts.settle_with_artifacts(&context, &identity, &evidence, artifacts.clone()) + }) + .await + } + + /// Records the immutable dispatch receipt before a provider launch, as a + /// single writer transaction. + pub async fn record_effect_dispatch( + &self, + context: tracedecay_contracts::RequestContext, + attempt: tracedecay_domain::WorkAttemptIdentityV1, + effect_state: tracedecay_domain::WorkEffectStateV1, + dispatched_at: tracedecay_domain::UtcMicros, + deadline: tracedecay_domain::UtcMicros, + ) -> Result< + tracedecay_contracts::WorkAttemptEffectDispatchOutcomeV1, + tracedecay_contracts::ApplicationProblem, + > { + self.storage + .transact(move |storage| { + tracedecay_contracts::WorkAttemptEffectServiceV1::new(storage).record_dispatch( + &context, + attempt.clone(), + effect_state, + dispatched_at, + deadline, + ) + }) + .await + .unwrap_or_else(|_| Err(attempt_unavailable())) + } + + /// Settles the dispatch receipt's effect resolution, as a single writer + /// transaction. + pub async fn settle_effect_dispatch( + &self, + context: tracedecay_contracts::RequestContext, + attempt: tracedecay_domain::WorkAttemptIdentityV1, + resolution: tracedecay_contracts::WorkAttemptEffectResolutionV1, + resolved_at: tracedecay_domain::UtcMicros, + ) -> Result< + tracedecay_contracts::WorkAttemptEffectHolderV1, + tracedecay_contracts::ApplicationProblem, + > { + self.storage + .transact(move |storage| { + tracedecay_contracts::WorkAttemptEffectServiceV1::new(storage).settle( + &context, + &attempt, + resolution, + resolved_at, + ) + }) + .await + .unwrap_or_else(|_| Err(attempt_unavailable())) + } + /// Records one duplicate adjudication as a single writer transaction: the /// replay check, the revision compare and the append commit together. /// `current_evidence` is read before the job, outside the writer. @@ -427,14 +768,6 @@ impl RegisteredWorkApplicationServicesV1 { &self.attempts } - pub const fn attempt_effects( - &self, - ) -> &tracedecay_contracts::WorkAttemptEffectServiceV1< - tracedecay_rusqlite_runtime::work::WorkSqliteStorage, - > { - &self.attempt_effects - } - /// The run-level pause/resume authority. /// /// It is a separate service from [`Self::attempts`] because the aggregate @@ -566,6 +899,10 @@ fn run_control_unavailable() -> tracedecay_contracts::ApplicationProblem { tracedecay_contracts::WorkRunControlStorageError::Unavailable.into() } +fn attempt_unavailable() -> tracedecay_contracts::ApplicationProblem { + tracedecay_contracts::WorkAttemptStorageError::Unavailable.into() +} + fn attach_error(operation: &str, error: impl std::fmt::Display) -> TraceDecayError { TraceDecayError::Database { operation: operation.to_owned(), diff --git a/crates/tracedecay-application/tests/application_suite/main.rs b/crates/tracedecay-application/tests/application_suite/main.rs index 24b64fcf06..981a37b39b 100644 --- a/crates/tracedecay-application/tests/application_suite/main.rs +++ b/crates/tracedecay-application/tests/application_suite/main.rs @@ -17,6 +17,7 @@ mod observability_rollup_convergence; // root: the work_rollup bench mounts it, and the suite itself uses a subset. pub mod observability_runtime_contract; mod registered_scope_route; +mod work_attempt_jobs; mod work_leaf_jobs; mod work_run_control_jobs; mod work_service_composition; diff --git a/crates/tracedecay-application/tests/application_suite/work_attempt_jobs.rs b/crates/tracedecay-application/tests/application_suite/work_attempt_jobs.rs new file mode 100644 index 0000000000..f39e5d33e1 --- /dev/null +++ b/crates/tracedecay-application/tests/application_suite/work_attempt_jobs.rs @@ -0,0 +1,898 @@ +use std::collections::BTreeSet; +use std::time::Duration; + +use tracedecay_application::work::{ + RegisteredWorkApplicationServicesV1, RegisteredWorkProductServicesV1, +}; +use tracedecay_contracts::{ + AdmitWorkSynthesisCommand, ApplicationProblem, PauseWorkRunCommand, + PrepareWorkProductMutationRequestV1, RequestContext, ResumeWorkAttemptsCommand, + RetryWorkAttemptCommandV1, StartWorkAttemptCommand, WorkAttemptEvidenceRecordV1, + WorkAttemptProviderOutcomeV1, WorkAttemptStoragePort, WorkProductAuthorizedRelationScopeV1, + WorkProductBindingV1, WorkProductChangeDraftV1, WorkProductMutationReceiptV1, + WorkProductRevisionPinsV1, WorkProductSelectionScopeV1, WorkRetryAttemptOutcomeV1, + WorkRetryCauseV1, WorkRetryFailureSelectorV1, WorkRetrySourceV1, WorkSynthesisAttemptV1, + WorkflowArtifactPayload, WorkflowArtifactStorePort, workflow_artifact_payload_digest, +}; +use tracedecay_domain::test_fixtures::{digest, fixture_abs_root, id}; +use tracedecay_domain::{ + AttemptId, CatalogGenerationId, CommitId, ConfigurationRevisionId, InitiativeId, MilestoneId, + PolicyRevisionId, ProjectId, ProposalId, RepositoryId, RunId, UtcMicros, WorkArtifactId, + WorkArtifactRefV1, WorkAttemptIdentityV1, WorkAttemptStateV1, WorkCommandId, WorkEffectStateV1, + WorkHierarchyV1, WorkInitiativeV1, WorkItemInputV1, WorkItemV1, WorkMilestoneV1, WorkPlanId, + WorkPlanV1, WorkProposalDispositionV1, WorkProposalV1, WorkRouteDecisionV1, + WorkRunControlReasonV1, WorkScoreKindV1, WorkShapeAssessmentV1, WorkSizingV1, + WorkflowOperationRef, WorkflowOutputName, +}; +use tracedecay_global_db::RegisteredGlobalDb; +use tracedecay_global_db::tests::harness::RegisteredGlobalDbTestRuntime; +use tracedecay_rusqlite_runtime::native_runtime::{NativeTask, NativeTaskOutcome}; +use tracedecay_tool_catalog::{CapabilityId, UseCaseId}; + +use crate::work_leaf_jobs::on_native_worker; +use crate::work_run_control_jobs::{ + authority, context, context_for, execution_snapshot, route, task, +}; + +const PROJECT: &str = "project.run-control-jobs"; +const REPOSITORY: &str = "repository.run-control-jobs"; + +async fn project_runtime( + profile: &std::path::Path, + project: &std::path::Path, +) -> RegisteredGlobalDbTestRuntime { + tracedecay_global_db::register_registered_schema_installer(); + RegisteredGlobalDbTestRuntime::project(profile, project, id::(PROJECT)) + .await + .expect("registered project store opens") +} + +fn product_context() -> RequestContext { + context_for("start_attempt") +} + +fn binding() -> WorkProductBindingV1 { + WorkProductBindingV1::new( + CapabilityId::new("capability.work.start_attempt").unwrap(), + UseCaseId::new("use-case.work.start_attempt").unwrap(), + ) +} + +fn revisions() -> WorkProductRevisionPinsV1 { + WorkProductRevisionPinsV1 { + policy_revision_id: id::("policy-revision.attempt-jobs"), + configuration_revision_id: id::( + "configuration-revision.attempt-jobs", + ), + catalog_generation_id: id::("catalog-generation.attempt-jobs"), + } +} + +fn selection() -> WorkProductSelectionScopeV1 { + WorkProductSelectionScopeV1::relations(BTreeSet::from([ + WorkProductAuthorizedRelationScopeV1::Repository { + project_id: id::(PROJECT), + repository_id: id::(REPOSITORY), + }, + ])) + .unwrap() +} + +/// Prepares one mutation from the current head, commits it as a writer job +/// on a native worker, and adds the native waits the commit parked to +/// `waits`. +async fn mutate( + database: &RegisteredGlobalDb, + change: WorkProductChangeDraftV1, + command_id: &str, + occurred_at: UtcMicros, + waits: &mut (u64, u64), +) -> WorkProductMutationReceiptV1 { + let product = + RegisteredWorkProductServicesV1::attach(database, binding()).expect("product services"); + let request = product + .mutations() + .prepare_mutation( + &product_context(), + &binding(), + PrepareWorkProductMutationRequestV1 { + selection: selection(), + change, + causation_event_id: None, + evidence: Vec::new(), + }, + id::(command_id), + occurred_at, + revisions(), + ) + .expect("the mutation prepares"); + let (receipt, writer_waits, reader_waits) = on_native_worker(async move { + product + .mutate(product_context(), binding(), request, revisions()) + .await + }) + .await; + waits.0 += writer_waits; + waits.1 += reader_waits; + receipt.expect("the mutation commits") +} + +/// Publishes `task()` with an accepted proposal and admitted execution, the +/// product graph every attempt admission is bound to, and returns the native +/// writer and reader waits its mutation jobs parked. +async fn admit_task(database: &RegisteredGlobalDb) -> (u64, u64) { + let mut waits = (0, 0); + let created_at = UtcMicros(10); + let initiative_id = id::("initiative.attempt-jobs"); + let plan_id = id::("plan.attempt-jobs"); + let milestone_id = id::("milestone.attempt-jobs"); + let item = WorkItemV1::new(WorkItemInputV1 { + task_id: task(), + hierarchy: WorkHierarchyV1::new( + initiative_id.clone(), + plan_id.clone(), + milestone_id.clone(), + ), + title: "Run the admitted provider step".to_owned(), + dependencies: BTreeSet::new(), + informational_relations: BTreeSet::new(), + causal_candidates: BTreeSet::new(), + acceptance_criteria: Vec::new(), + effort: 1, + scheduled_at: None, + deadline: None, + created_at, + updated_at: created_at, + }) + .unwrap(); + let created = mutate( + database, + WorkProductChangeDraftV1::CreateTask { + initiative: WorkInitiativeV1::new( + initiative_id.clone(), + "Attempt jobs".to_owned(), + created_at, + ) + .unwrap(), + plan: WorkPlanV1::new( + plan_id.clone(), + initiative_id, + "Attempt jobs plan".to_owned(), + created_at, + ) + .unwrap(), + milestone: WorkMilestoneV1::new( + milestone_id, + plan_id, + "Attempt jobs milestone".to_owned(), + created_at, + ) + .unwrap(), + item: Box::new(item), + }, + "command.attempt-jobs.create", + UtcMicros(10_000), + &mut waits, + ) + .await; + let proposal = WorkProposalV1::new( + id::("proposal.attempt-jobs"), + task(), + created.verified_graph_version().graph_version(), + WorkShapeAssessmentV1::new(WorkScoreKindV1::Ordinal, 1, 1, 1, 1).unwrap(), + WorkSizingV1::new(WorkScoreKindV1::Ordinal, 1, 1, 1, "attempt jobs").unwrap(), + Vec::new(), + WorkRouteDecisionV1::selected(route(), Vec::new(), BTreeSet::new(), "none".to_owned()) + .unwrap(), + "Run the admitted provider step.".to_owned(), + digest('4'), + digest('5'), + ) + .unwrap(); + mutate( + database, + WorkProductChangeDraftV1::DecideProposal { + proposal, + disposition: WorkProposalDispositionV1::Accepted, + }, + "command.attempt-jobs.accept", + UtcMicros(11_000), + &mut waits, + ) + .await; + mutate( + database, + WorkProductChangeDraftV1::AdmitExecution { task_id: task() }, + "command.attempt-jobs.admit", + UtcMicros(12_000), + &mut waits, + ) + .await; + waits +} + +fn start_command( + run_id: &str, + attempt_id: &str, + occurred_at: UtcMicros, +) -> StartWorkAttemptCommand { + StartWorkAttemptCommand { + task_id: task(), + run_id: id::(run_id), + attempt_id: id::(attempt_id), + operation: id::("operation.attempt-jobs.execute-provider"), + execution_snapshot: execution_snapshot(), + worktree_root: fixture_abs_root("/tmp/attempt-jobs"), + reference: None, + commit: id::("0123456789abcdef0123456789abcdef01234567"), + instructions: "Execute the admitted provider step.".to_owned(), + effect_state: WorkEffectStateV1::Observational, + occurred_at, + } +} + +async fn start( + product: &RegisteredWorkProductServicesV1, + command: StartWorkAttemptCommand, +) -> Result { + product + .start_attempt( + product_context(), + binding(), + revisions(), + execution_snapshot().topology().clone(), + command, + ) + .await +} + +fn identity(run_id: &str, attempt_id: &str) -> WorkAttemptIdentityV1 { + WorkAttemptIdentityV1::new(task(), id::(run_id), id::(attempt_id)).unwrap() +} + +#[tokio::test] +async fn attempt_admissions_awaited_on_a_native_worker_park_no_native_thread() { + let profile = tempfile::tempdir().expect("profile"); + let project = tempfile::tempdir().expect("project"); + let runtime = project_runtime(profile.path(), project.path()).await; + let database = runtime.project_database().expect("project database"); + admit_task(database).await; + let product = + RegisteredWorkProductServicesV1::attach(database, binding()).expect("product services"); + let work = RegisteredWorkApplicationServicesV1::attach(database).expect("Work services"); + let original = identity("run.attempt-jobs.native", "attempt.attempt-jobs.native"); + let retry_command = RetryWorkAttemptCommandV1 { + original_attempt: original.clone(), + new_attempt_id: id::("attempt.attempt-jobs.native.retry"), + failure: WorkRetryFailureSelectorV1 { + source: WorkRetrySourceV1::Runtime, + cause: WorkRetryCauseV1::RestartRecoveryRequired, + evidence_ref: "recovery-required".to_owned(), + }, + command_id: id::("command.attempt-jobs.native.retry"), + }; + let (outcome, writer_waits, reader_waits) = on_native_worker(async move { + let started = start( + &product, + start_command( + "run.attempt-jobs.native", + "attempt.attempt-jobs.native", + UtcMicros(20_000), + ), + ) + .await; + let replayed = start( + &product, + start_command( + "run.attempt-jobs.native", + "attempt.attempt-jobs.native", + UtcMicros(20_000), + ), + ) + .await; + let resumed = work + .resume_attempts( + product_context(), + ResumeWorkAttemptsCommand { + occurred_at: UtcMicros(30_000), + }, + ) + .await; + let retried = product + .retry_attempt( + product_context(), + binding(), + revisions(), + execution_snapshot().topology().clone(), + retry_command, + UtcMicros(40_000), + None, + ) + .await; + (started, replayed, resumed, retried) + }) + .await; + assert_eq!( + writer_waits, 0, + "the attempt admissions parked no native worker on the writer" + ); + assert_eq!( + reader_waits, 0, + "the attempt admissions parked no native worker on a reader" + ); + let (started, replayed, resumed, retried) = outcome; + let started = started.expect("the start commits"); + assert_eq!(started.state(), WorkAttemptStateV1::Leased); + assert_eq!( + replayed.expect("the same start replays"), + started, + "the replayed start returns the committed attempt" + ); + let resumed = resumed.expect("the resume commits"); + assert_eq!( + resumed + .recovery_required + .iter() + .map(|attempt| attempt.identity().clone()) + .collect::>(), + vec![original.clone()], + "the resume fenced the started attempt" + ); + let WorkRetryAttemptOutcomeV1::Created { attempt, .. } = retried.expect("the retry commits") + else { + panic!("the retry creates a replacement attempt"); + }; + let storage = database.work_storage().expect("Work storage"); + assert_eq!( + storage + .load(&authority(), &original) + .expect("the original reads") + .state(), + WorkAttemptStateV1::RecoveryRequired + ); + assert_eq!( + storage + .load(&authority(), attempt.identity()) + .expect("the replacement reads"), + attempt, + "the retry job committed its replacement attempt" + ); +} + +#[tokio::test] +async fn synthesis_awaited_on_a_native_worker_hydrates_sources_in_its_job() { + let profile = tempfile::tempdir().expect("profile"); + let project = tempfile::tempdir().expect("project"); + let runtime = project_runtime(profile.path(), project.path()).await; + let database = runtime.project_database().expect("project database"); + admit_task(database).await; + let product = + RegisteredWorkProductServicesV1::attach(database, binding()).expect("product services"); + let source = identity("run.attempt-jobs.source", "attempt.attempt-jobs.source"); + let source_product = + RegisteredWorkProductServicesV1::attach(database, binding()).expect("product services"); + let (started, _, _) = on_native_worker(async move { + start( + &source_product, + start_command( + "run.attempt-jobs.source", + "attempt.attempt-jobs.source", + UtcMicros(20_000), + ), + ) + .await + }) + .await; + started.expect("the source attempt starts"); + let bytes = b"source answer".to_vec(); + let artifact = WorkArtifactRefV1::new( + id::("artifact.provider.stdout"), + workflow_artifact_payload_digest(&bytes).unwrap(), + bytes.len() as u64, + ) + .unwrap(); + database + .workflow_storage() + .expect("workflow storage") + .persist(&WorkflowArtifactPayload::new(artifact.clone(), bytes).unwrap()) + .expect("the source payload persists"); + let attempts = RegisteredWorkApplicationServicesV1::attach(database).expect("Work services"); + attempts + .attempts() + .mark_running(&product_context(), &source, route()) + .expect("the source runs"); + attempts + .attempts() + .settle_with_artifacts( + &product_context(), + &source, + &WorkAttemptEvidenceRecordV1 { + identity: source.clone(), + requested_route: route(), + actual_route: Some(route()), + outcome: WorkAttemptProviderOutcomeV1::Exited { code: 0 }, + stdout: None, + stderr: None, + provider_session: None, + provider_fallback: None, + observed_at: UtcMicros(21_000), + }, + vec![artifact.clone()], + ) + .expect("the source succeeds"); + let command = AdmitWorkSynthesisCommand { + start: start_command( + "run.attempt-jobs.synthesis", + "attempt.attempt-jobs.synthesis", + UtcMicros(30_000), + ), + output_name: id::("output.attempt-jobs"), + sources: vec![source], + }; + let (outcome, writer_waits, reader_waits) = on_native_worker(async move { + product + .synthesize( + product_context(), + binding(), + revisions(), + execution_snapshot().topology().clone(), + command, + move |storage, start, sources| { + let payload = storage + .load_workflow_artifact(&sources[0].artifacts()[0]) + .map_err(|_| { + ApplicationProblem::from( + tracedecay_contracts::WorkAttemptStorageError::Unavailable, + ) + })?; + Ok(format!( + "{}\n{}", + start.instructions, + String::from_utf8_lossy(payload.bytes()) + )) + }, + ) + .await + }) + .await; + assert_eq!( + writer_waits, 0, + "synthesis parked no native worker on the writer" + ); + assert_eq!( + reader_waits, 0, + "synthesis parked no native worker on a reader" + ); + let WorkSynthesisAttemptV1::Admitted(admission) = outcome.expect("the synthesis commits") + else { + panic!("the cited source admits a synthesis"); + }; + assert_eq!( + admission.attempt.execution().instructions(), + "Execute the admitted provider step.\nsource answer", + "the job hydrated the source payload into the provider instructions" + ); + assert_eq!( + database + .work_storage() + .expect("Work storage") + .load(&authority(), admission.attempt.identity()) + .expect("the synthesis attempt reads"), + admission.attempt, + "the synthesis job committed its attempt" + ); +} + +/// A pause racing a start lands either before the start's job, which then +/// refuses the reservation as paused, or after it. It can no longer land +/// between the reservation fence and the attempt insert. +#[tokio::test] +async fn a_pause_racing_a_start_never_lands_between_its_fence_and_its_reservation() { + let profile = tempfile::tempdir().expect("profile"); + let project = tempfile::tempdir().expect("project"); + let runtime = project_runtime(profile.path(), project.path()).await; + let database = runtime.project_database().expect("project database"); + admit_task(database).await; + let run = "run.run-control-jobs"; + let storage = database.work_storage().expect("Work storage"); + let seed = crate::work_run_control_jobs::leased_attempt_of(id::(run), "seed"); + storage + .insert(&authority(), &seed) + .expect("the run holds a seed attempt"); + let attempts = RegisteredWorkApplicationServicesV1::attach(database).expect("Work services"); + attempts + .attempts() + .mark_running(&product_context(), seed.identity(), route()) + .expect("the seed runs"); + attempts + .attempts() + .settle( + &product_context(), + seed.identity(), + &WorkAttemptEvidenceRecordV1 { + identity: seed.identity().clone(), + requested_route: route(), + actual_route: Some(route()), + outcome: WorkAttemptProviderOutcomeV1::Exited { code: 1 }, + stdout: None, + stderr: None, + provider_session: None, + provider_fallback: None, + observed_at: UtcMicros(15_000), + }, + ) + .expect("the seed settles, so the run frees its capacity"); + let product = + RegisteredWorkProductServicesV1::attach(database, binding()).expect("product services"); + let work = RegisteredWorkApplicationServicesV1::attach(database).expect("Work services"); + // Hold the writer so the start's job and the pause's job queue behind it + // in the order they were submitted. + let blocker = NativeTask::spawn(async move { + storage + .transact(|_| { + std::thread::sleep(Duration::from_millis(400)); + Ok::<_, ()>(()) + }) + .await + .is_ok() + }) + .expect("the blocker spawns"); + tokio::time::sleep(Duration::from_millis(50)).await; + let starter = NativeTask::spawn(async move { + start( + &product, + start_command(run, "attempt.attempt-jobs.raced", UtcMicros(20_000)), + ) + .await + }) + .expect("the start spawns"); + tokio::time::sleep(Duration::from_millis(150)).await; + let pauser = NativeTask::spawn(async move { + work.pause_run( + context(), + PauseWorkRunCommand { + task_id: task(), + run_id: id::(run), + reason: WorkRunControlReasonV1::OperatorRequest, + expected_authority_version: None, + occurred_at: UtcMicros(25_000), + }, + ) + .await + .map(|transition| transition.control.state()) + }) + .expect("the pause spawns"); + let NativeTaskOutcome::Completed(blocked) = blocker.wait().await else { + panic!("the blocker completes"); + }; + assert!(blocked, "the blocker held the writer"); + let NativeTaskOutcome::Completed(started) = starter.wait().await else { + panic!("the start completes"); + }; + let NativeTaskOutcome::Completed(paused) = pauser.wait().await else { + panic!("the pause completes"); + }; + paused.expect("the pause commits"); + let raced = identity(run, "attempt.attempt-jobs.raced"); + let stored = database + .work_storage() + .expect("Work storage") + .load(&authority(), &raced); + match started { + Ok(attempt) => assert_eq!( + stored.expect("a committed start stored its attempt"), + attempt, + "the start committed before the pause" + ), + Err(problem) => { + assert_eq!( + problem + .diagnostic() + .map(|diagnostic| diagnostic.code.as_str()), + Some("application.work-run-control.paused"), + "a start the pause reached first is refused by the reservation fence, \ + never by a pause committed between the fence and the insert: {problem:?}" + ); + assert!(stored.is_err(), "a refused start stored no attempt"); + } + } +} + +fn provider_evidence( + attempt: &WorkAttemptIdentityV1, + outcome: WorkAttemptProviderOutcomeV1, + observed_at: UtcMicros, +) -> WorkAttemptEvidenceRecordV1 { + WorkAttemptEvidenceRecordV1 { + identity: attempt.clone(), + requested_route: route(), + actual_route: Some(route()), + outcome, + stdout: None, + stderr: None, + provider_session: None, + provider_fallback: None, + observed_at, + } +} + +#[tokio::test] +async fn provider_driver_steps_awaited_on_a_native_worker_park_no_native_thread() { + let profile = tempfile::tempdir().expect("profile"); + let project = tempfile::tempdir().expect("project"); + let runtime = project_runtime(profile.path(), project.path()).await; + let database = runtime.project_database().expect("project database"); + let storage = database.work_storage().expect("Work storage"); + let run = id::("run.attempt-jobs.driver"); + let [succeeded, denied, cancelled] = ["succeeded", "denied", "cancelled"].map(|suffix| { + let attempt = crate::work_run_control_jobs::leased_attempt_of( + run.clone(), + &format!("attempt-jobs.driver.{suffix}"), + ); + storage + .insert(&authority(), &attempt) + .expect("the attempt is stored"); + attempt + }); + let (effect_state, deadline) = ( + succeeded.execution().effect_state(), + succeeded.execution().deadline(), + ); + let [succeeded, denied, cancelled] = + [succeeded, denied, cancelled].map(|attempt| attempt.identity().clone()); + let setup = RegisteredWorkApplicationServicesV1::attach(database).expect("Work services"); + setup + .attempts() + .mark_running(&product_context(), &cancelled, route()) + .expect("the cancelled attempt runs"); + setup + .attempts() + .request_cancellation( + &product_context(), + tracedecay_contracts::CancelWorkAttemptCommand { + task_id: task(), + run_id: run.clone(), + attempt_id: cancelled.attempt_id().clone(), + request_id: id("cancellation.attempt-jobs.driver"), + occurred_at: UtcMicros(20_000), + }, + ) + .expect("the cancellation is requested"); + let work = RegisteredWorkApplicationServicesV1::attach(database).expect("Work services"); + let identities = (succeeded.clone(), denied.clone(), cancelled.clone()); + let (outcome, writer_waits, reader_waits) = on_native_worker(async move { + let (succeeded, denied, cancelled) = identities; + let context = product_context(); + let dispatched = work + .record_effect_dispatch( + context.clone(), + succeeded.clone(), + effect_state, + UtcMicros(21_000), + deadline, + ) + .await + .map(|_| ()); + let running = work + .mark_running(context.clone(), succeeded.clone(), route()) + .await + .map(|attempt| attempt.state()); + let effect = work + .settle_effect_dispatch( + context.clone(), + succeeded.clone(), + tracedecay_contracts::WorkAttemptEffectResolutionV1::NoEffect, + UtcMicros(22_000), + ) + .await + .map(|holder| holder.resolution()); + let settled = work + .settle_with_artifacts( + context.clone(), + succeeded.clone(), + provider_evidence( + &succeeded, + WorkAttemptProviderOutcomeV1::Exited { code: 0 }, + UtcMicros(23_000), + ), + Vec::new(), + ) + .await + .map(|attempt| attempt.state()); + let fenced = work + .mark_provider_unavailable(context.clone(), denied.clone(), UtcMicros(24_000)) + .await + .map(|attempt| attempt.state()); + let failed = work + .fail_recovery( + context.clone(), + denied.clone(), + WorkAttemptEvidenceRecordV1 { + actual_route: None, + ..provider_evidence( + &denied, + WorkAttemptProviderOutcomeV1::LaunchFailed, + UtcMicros(25_000), + ) + }, + ) + .await + .map(|attempt| attempt.state()); + let acknowledged = work + .acknowledge_cancellation(context.clone(), cancelled.clone(), UtcMicros(26_000)) + .await + .map(|attempt| attempt.state()); + let escalated = work + .escalate_cancellation(context.clone(), cancelled.clone(), UtcMicros(27_000)) + .await + .map(|attempt| attempt.state()); + let sealed = work + .settle_with_artifacts( + context, + cancelled.clone(), + provider_evidence( + &cancelled, + WorkAttemptProviderOutcomeV1::Cancelled, + UtcMicros(28_000), + ), + Vec::new(), + ) + .await + .map(|attempt| attempt.state()); + vec![ + dispatched.map(|()| None), + running.map(Some), + effect.map(|_| None), + settled.map(Some), + fenced.map(Some), + failed.map(Some), + acknowledged.map(Some), + escalated.map(Some), + sealed.map(Some), + ] + }) + .await; + assert_eq!( + writer_waits, 0, + "the provider driver's steps parked no native worker on the writer" + ); + assert_eq!( + reader_waits, 0, + "the provider driver's steps parked no native worker on a reader" + ); + assert_eq!( + outcome, + vec![ + Ok(None), + Ok(Some(WorkAttemptStateV1::Running)), + Ok(None), + Ok(Some(WorkAttemptStateV1::Succeeded)), + Ok(Some(WorkAttemptStateV1::RecoveryRequired)), + Ok(Some(WorkAttemptStateV1::Failed)), + Ok(Some(WorkAttemptStateV1::CancellationAcknowledged)), + Ok(Some(WorkAttemptStateV1::CancellationEscalated)), + Ok(Some(WorkAttemptStateV1::Cancelled)), + ], + "every driver step committed its transition" + ); + for (identity, state) in [ + (&succeeded, WorkAttemptStateV1::Succeeded), + (&denied, WorkAttemptStateV1::Failed), + (&cancelled, WorkAttemptStateV1::Cancelled), + ] { + assert_eq!( + storage + .load(&authority(), identity) + .expect("the attempt reads") + .state(), + state, + "the driver's jobs committed the terminal state" + ); + } +} + +#[tokio::test] +async fn cancellation_awaited_on_a_native_worker_parks_no_native_thread() { + let profile = tempfile::tempdir().expect("profile"); + let project = tempfile::tempdir().expect("project"); + let runtime = project_runtime(profile.path(), project.path()).await; + let database = runtime.project_database().expect("project database"); + let storage = database.work_storage().expect("Work storage"); + let attempt = crate::work_run_control_jobs::leased_attempt_of( + id::("run.attempt-jobs.cancel"), + "attempt-jobs.cancel", + ); + storage + .insert(&authority(), &attempt) + .expect("the attempt is stored"); + let identity = attempt.identity().clone(); + let command = |request_id: &str| tracedecay_contracts::CancelWorkAttemptCommand { + task_id: task(), + run_id: identity.run_id().clone(), + attempt_id: identity.attempt_id().clone(), + request_id: id(request_id), + occurred_at: UtcMicros(20_000), + }; + let (first, replay, other) = ( + command("cancellation.attempt-jobs.1"), + command("cancellation.attempt-jobs.1"), + command("cancellation.attempt-jobs.2"), + ); + let work = RegisteredWorkApplicationServicesV1::attach(database).expect("Work services"); + let (outcome, writer_waits, reader_waits) = on_native_worker(async move { + ( + work.cancel_attempt(product_context(), first).await, + work.cancel_attempt(product_context(), replay).await, + work.cancel_attempt(product_context(), other).await, + ) + }) + .await; + assert_eq!( + writer_waits, 0, + "the cancellation parked no native worker on the writer" + ); + assert_eq!( + reader_waits, 0, + "the cancellation parked no native worker on a reader" + ); + let (requested, replayed, conflicting) = outcome; + let requested = requested.expect("the cancellation commits"); + assert_eq!(requested.state(), WorkAttemptStateV1::CancellationRequested); + assert_eq!( + replayed.expect("the same request replays"), + requested, + "the same request replays the committed attempt" + ); + assert_eq!( + conflicting + .expect_err("a different request conflicts") + .diagnostic() + .map(|diagnostic| diagnostic.code.clone()), + Some("application.work-attempt.cancellation-conflict".to_owned()) + ); + assert_eq!( + storage + .load(&authority(), &identity) + .expect("the attempt reads"), + requested, + "the job committed the cancellation request" + ); +} + +#[tokio::test] +async fn product_mutations_awaited_on_a_native_worker_park_no_native_thread() { + let profile = tempfile::tempdir().expect("profile"); + let project = tempfile::tempdir().expect("project"); + let runtime = project_runtime(profile.path(), project.path()).await; + let database = runtime.project_database().expect("project database"); + let (writer_waits, reader_waits) = admit_task(database).await; + assert_eq!( + writer_waits, 0, + "the create, proposal and admission jobs parked no native worker on the writer" + ); + assert_eq!( + reader_waits, 0, + "the create, proposal and admission jobs parked no native worker on a reader" + ); + let product = + RegisteredWorkProductServicesV1::attach(database, binding()).expect("product services"); + let read = product + .reads() + .read_graph( + &product_context(), + tracedecay_contracts::WorkGraphReadRequestV1::current(selection(), UtcMicros(13_000)), + ) + .expect("the graph reads") + .into_current_snapshot() + .expect("the graph has a head"); + let item = read.graph().item(&task()).expect("the task is published"); + assert!( + item.is_execution_admitted(), + "the admission job committed the execution admission" + ); + assert_eq!( + item.accepted_proposal(), + Some(&id::("proposal.attempt-jobs")), + "the proposal job committed the acceptance" + ); +} diff --git a/crates/tracedecay-application/tests/application_suite/work_leaf_jobs.rs b/crates/tracedecay-application/tests/application_suite/work_leaf_jobs.rs index 4c931d8eba..074996fd65 100644 --- a/crates/tracedecay-application/tests/application_suite/work_leaf_jobs.rs +++ b/crates/tracedecay-application/tests/application_suite/work_leaf_jobs.rs @@ -41,7 +41,7 @@ async fn project_runtime( /// Runs `job` on a native runtime worker and returns its output with the /// writer and reader waits it parked that worker on. -async fn on_native_worker( +pub(crate) async fn on_native_worker( job: impl Future + Send + 'static, ) -> (T, u64, u64) { let writer_waits = native_thread_writer_waits(); diff --git a/crates/tracedecay-application/tests/application_suite/work_run_control_jobs.rs b/crates/tracedecay-application/tests/application_suite/work_run_control_jobs.rs index e5e284df64..a428bfe762 100644 --- a/crates/tracedecay-application/tests/application_suite/work_run_control_jobs.rs +++ b/crates/tracedecay-application/tests/application_suite/work_run_control_jobs.rs @@ -40,6 +40,12 @@ fn run() -> RunId { } pub(crate) fn context() -> RequestContext { + context_for("pause_run") +} + +/// A request context granted the `capability.work.{operation}` capability +/// and use case, under the same authority as [`context`]. +pub(crate) fn context_for(operation: &str) -> RequestContext { let scope = ResolvedScope::new( id::(PROJECT), id::(REPOSITORY), @@ -55,8 +61,8 @@ pub(crate) fn context() -> RequestContext { UtcMicros(1), UtcMicros(100_000), scope.clone(), - BTreeSet::from([CapabilityId::new("capability.work.pause_run").unwrap()]), - BTreeSet::from([UseCaseId::new("use-case.work.pause_run").unwrap()]), + BTreeSet::from([CapabilityId::new(format!("capability.work.{operation}")).unwrap()]), + BTreeSet::from([UseCaseId::new(format!("use-case.work.{operation}")).unwrap()]), DisclosureClass::Sensitive, ) .unwrap(); @@ -82,7 +88,7 @@ pub(crate) fn authority() -> WorkAuthority { .unwrap() } -fn route() -> WorkProviderRouteV1 { +pub(crate) fn route() -> WorkProviderRouteV1 { WorkProviderRouteV1::new( id::("provider.work.claude-code-cli"), id::("route.run-control-jobs.claude-code.v1"), @@ -111,37 +117,11 @@ pub(crate) fn leased_attempt_of(run_id: RunId, suffix: &str) -> WorkAttemptV1 { id::("proposal.run-control-jobs"), ) .unwrap(); - let snapshot = WorkExecutionSnapshot::new(WorkExecutionSnapshotInput { - configuration_revision_id: id::("configuration-revision.rcj.1"), - configuration_snapshot_id: id::("configuration-snapshot.rcj.1"), - effective_behavior_digest: digest('c'), - resolution_provenance_digest: digest('d'), - route: route(), - backend: WorkProviderBackendV1::ClaudeCodeCli, - protocol: WorkProviderProtocol::ClaudeStreamJson, - model: "claude-test".to_owned(), - executable: WorkExecutableReference::new( - "executable.claude.code-cli".to_owned(), - digest('e'), - ) - .unwrap(), - sandbox: WorkSandboxPolicy::Required, - approval: WorkApprovalPolicy::Never, - filesystem: WorkFilesystemPolicy::WorkspaceWrite, - egress: WorkEgressPolicy::Deny, - environment_allowlist: BTreeSet::new(), - credential_references: BTreeSet::new(), - limits: WorkExecutionLimits::new(128_000, 8_192, 16_384, 16_384, 65_536, 1).unwrap(), - deadline: UtcMicros(1_000_000), - fallback: WorkFallbackTopology::Disabled, - topology: tracedecay_domain::safe_work_topology_policy_v1(), - }) - .unwrap(); let envelope = WorkExecutionEnvelopeV1::new( identity.clone(), binding.clone(), id::("operation.run-control-jobs.execute-provider"), - snapshot, + execution_snapshot(), id::(PROJECT), id::(REPOSITORY), id::(WORKTREE), @@ -174,6 +154,36 @@ pub(crate) fn leased_attempt_of(run_id: RunId, suffix: &str) -> WorkAttemptV1 { .unwrap() } +/// The pinned execution snapshot every fixture attempt runs under. +pub(crate) fn execution_snapshot() -> WorkExecutionSnapshot { + WorkExecutionSnapshot::new(WorkExecutionSnapshotInput { + configuration_revision_id: id::("configuration-revision.rcj.1"), + configuration_snapshot_id: id::("configuration-snapshot.rcj.1"), + effective_behavior_digest: digest('c'), + resolution_provenance_digest: digest('d'), + route: route(), + backend: WorkProviderBackendV1::ClaudeCodeCli, + protocol: WorkProviderProtocol::ClaudeStreamJson, + model: "claude-test".to_owned(), + executable: WorkExecutableReference::new( + "executable.claude.code-cli".to_owned(), + digest('e'), + ) + .unwrap(), + sandbox: WorkSandboxPolicy::Required, + approval: WorkApprovalPolicy::Never, + filesystem: WorkFilesystemPolicy::WorkspaceWrite, + egress: WorkEgressPolicy::Deny, + environment_allowlist: BTreeSet::new(), + credential_references: BTreeSet::new(), + limits: WorkExecutionLimits::new(128_000, 8_192, 16_384, 16_384, 65_536, 1).unwrap(), + deadline: UtcMicros(1_000_000), + fallback: WorkFallbackTopology::Disabled, + topology: tracedecay_domain::safe_work_topology_policy_v1(), + }) + .unwrap() +} + #[tokio::test] async fn pause_and_resume_awaited_on_a_native_worker_park_no_native_thread() { tracedecay_global_db::register_registered_schema_installer(); diff --git a/crates/tracedecay-contracts/src/lib.rs b/crates/tracedecay-contracts/src/lib.rs index 55c7782800..ea8c5c64be 100644 --- a/crates/tracedecay-contracts/src/lib.rs +++ b/crates/tracedecay-contracts/src/lib.rs @@ -458,8 +458,8 @@ pub use work_attempt::{ }; pub use work_attempt_effect::{ WorkAttemptEffectDispatchOutcomeV1, WorkAttemptEffectHolderErrorV1, WorkAttemptEffectHolderV1, - WorkAttemptEffectResolutionV1, WorkAttemptEffectServiceV1, WorkAttemptEffectStorageErrorV1, - WorkAttemptEffectStoragePortV1, + WorkAttemptEffectReadPortV1, WorkAttemptEffectResolutionV1, WorkAttemptEffectServiceV1, + WorkAttemptEffectStorageErrorV1, WorkAttemptEffectStoragePortV1, }; pub use work_catalog::{ WORK_APPLICATION_OPERATION_IDS_V1, work_executable_binding, work_executable_binding_registry, @@ -537,7 +537,7 @@ pub use work_product::{ AddWorkTaskRequestV1, AdmitWorkExecutionRequestV1, AdmittedWorkExecutionV1, AuthorizedWorkProductScopeV1, CreateWorkProductRequestV1, CreateWorkTaskRequestV1, DecideWorkProposalRequestV1, MAX_WORK_EVIDENCE_SELECTION_V1, - MAX_WORK_GRAPH_TEMPORAL_ENTRIES_V1, MAX_WORK_HISTORY_EVENTS_V1, + MAX_WORK_GRAPH_TEMPORAL_ENTRIES_V1, MAX_WORK_HISTORY_EVENTS_V1, PrefetchedWorkGraphReadV1, PrepareWorkProductMutationRequestV1, ReviewWorkProposalDispositionV1, ReviewWorkProposalRequestV1, SelectedWorkEvidenceV1, VerifiedWorkEvidenceExpansionV1, VerifiedWorkGraphVersionV1, WorkEvidenceExpandRequestV1, WorkEvidenceExpansionV1, @@ -569,7 +569,7 @@ pub use work_run_control::{ PauseWorkRunCommand, ResumeWorkRunCommand, WorkRunAdmissionV1, WorkRunControlFrontierV1, WorkRunControlReadingV1, WorkRunControlRequestV1, WorkRunControlService, WorkRunControlStorageError, WorkRunControlStoragePort, WorkRunControlTransitionPort, - WorkRunControlTransitionReceiptV1, WorkRunLiveAttemptV1, + WorkRunControlTransitionReceiptV1, WorkRunLiveAttemptV1, WorkRunReservationPort, }; pub use work_synthesis::{ AdmitWorkSynthesisCommand, WorkSynthesisAdmissionRecordV1, WorkSynthesisAdmissionV1, diff --git a/crates/tracedecay-contracts/src/work_attempt.rs b/crates/tracedecay-contracts/src/work_attempt.rs index ef23d19a7f..d3d2897eff 100644 --- a/crates/tracedecay-contracts/src/work_attempt.rs +++ b/crates/tracedecay-contracts/src/work_attempt.rs @@ -82,7 +82,7 @@ pub enum WorkAttemptInsertOutcome { /// Durable attempt persistence. Every transition is a compare-and-swap on the /// exact prior lease fence and state, so a fenced-out writer cannot advance a /// row it no longer owns. -pub trait WorkAttemptStoragePort: Send + Sync { +pub trait WorkAttemptStoragePort { /// Mints the next monotonic fence epoch for this authority scope. fn next_fence_epoch(&self, authority: &WorkAuthority) -> Result; diff --git a/crates/tracedecay-contracts/src/work_attempt/problem.rs b/crates/tracedecay-contracts/src/work_attempt/problem.rs index 9ba10a9031..e84d3b48c0 100644 --- a/crates/tracedecay-contracts/src/work_attempt/problem.rs +++ b/crates/tracedecay-contracts/src/work_attempt/problem.rs @@ -85,3 +85,9 @@ pub(super) fn denied_problem(code: &str, message: &str) -> ApplicationProblem { legal_actions: vec![LegalAction::Refresh], } } + +impl From for ApplicationProblem { + fn from(error: WorkAttemptStorageError) -> Self { + storage_problem(error) + } +} diff --git a/crates/tracedecay-contracts/src/work_attempt/product_admission.rs b/crates/tracedecay-contracts/src/work_attempt/product_admission.rs index 58a62debca..da8afbdb99 100644 --- a/crates/tracedecay-contracts/src/work_attempt/product_admission.rs +++ b/crates/tracedecay-contracts/src/work_attempt/product_admission.rs @@ -78,6 +78,37 @@ pub(crate) fn replayed_attempt_matches_command( Ok(attempt.identity() == identity && attempt.execution() == &expected) } +fn attempt_admission_selection( + context: &RequestContext, +) -> Result { + WorkProductSelectionScopeV1::relations(BTreeSet::from([ + WorkProductAuthorizedRelationScopeV1::Repository { + project_id: context.scope().project_id.clone(), + repository_id: context.scope().repository_id.clone(), + }, + ])) + .map_err(|_| invalid_start_problem()) +} + +impl crate::PrefetchedWorkGraphReadV1 { + /// The graph read an attempt admission (start, synthesis or retry) + /// observed at `observed_at` makes, read before its writer job. A context + /// with no valid selection prefetches nothing; the service refuses it. + pub fn for_attempt_admission( + storage: &S, + context: &RequestContext, + observed_at: UtcMicros, + ) -> Self + where + S: WorkGraphReadPortV1 + WorkProductOwnerAuthorizationPortV1, + { + attempt_admission_selection(context).map_or_else( + |_| Self::default(), + |selection| Self::current(storage, context, &selection, observed_at), + ) + } +} + /// Reads the current verified product graph under the exact relation scope /// resolved for this request. The caller cannot select a different profile or /// repository relation for attempt admission. @@ -91,13 +122,7 @@ where S: WorkGraphReadPortV1 + WorkProductOwnerAuthorizationPortV1, { admit_product_attempt_request(context, binding, observed_at)?; - let selection = WorkProductSelectionScopeV1::relations(BTreeSet::from([ - WorkProductAuthorizedRelationScopeV1::Repository { - project_id: context.scope().project_id.clone(), - repository_id: context.scope().repository_id.clone(), - }, - ])) - .map_err(|_| invalid_start_problem())?; + let selection = attempt_admission_selection(context)?; let authorized_scope = storage .authorize_scope(context, &selection, observed_at) .map_err(owner_problem)?; diff --git a/crates/tracedecay-contracts/src/work_attempt/product_synthesis_admission.rs b/crates/tracedecay-contracts/src/work_attempt/product_synthesis_admission.rs index ce24020bce..14a4784ea8 100644 --- a/crates/tracedecay-contracts/src/work_attempt/product_synthesis_admission.rs +++ b/crates/tracedecay-contracts/src/work_attempt/product_synthesis_admission.rs @@ -7,10 +7,9 @@ use tracedecay_domain::{ }; use crate::{ - ApplicationProblem, RequestContext, WorkGraphReadPortV1, WorkProductAttemptAdmissionPortV1, - WorkProductAttemptAdmissionV1, WorkProductBindingV1, WorkProductOwnerAuthorizationPortV1, - WorkProductRevisionPinsV1, WorkProductSynthesisAdmissionV1, WorkSynthesisAdmissionRecordV1, - WorkSynthesisAdmissionV1, + ApplicationProblem, RequestContext, WorkGraphReadPortV1, WorkProductAttemptAdmissionV1, + WorkProductBindingV1, WorkProductOwnerAuthorizationPortV1, WorkProductRevisionPinsV1, + WorkProductSynthesisAdmissionV1, WorkSynthesisAdmissionRecordV1, WorkSynthesisAdmissionV1, }; use super::{ @@ -39,8 +38,7 @@ impl WorkProductSynthesisAttemptServiceV1 where S: WorkSynthesisAdmissionStoragePort + WorkGraphReadPortV1 - + WorkProductOwnerAuthorizationPortV1 - + WorkProductAttemptAdmissionPortV1, + + WorkProductOwnerAuthorizationPortV1, { pub const fn new(storage: S) -> Self { Self { storage } diff --git a/crates/tracedecay-contracts/src/work_attempt/synthesis_admission.rs b/crates/tracedecay-contracts/src/work_attempt/synthesis_admission.rs index e8c3e5028c..4f122949c5 100644 --- a/crates/tracedecay-contracts/src/work_attempt/synthesis_admission.rs +++ b/crates/tracedecay-contracts/src/work_attempt/synthesis_admission.rs @@ -43,4 +43,14 @@ pub trait WorkSynthesisAdmissionStoragePort: WorkAttemptStoragePort { authority: &WorkAuthority, identity: &WorkAttemptIdentityV1, ) -> Result; + + /// Commits the accepted-attempt link, the attempt row and its synthesis + /// record as one transaction. + fn admit_synthesis( + &self, + admission: &crate::WorkProductSynthesisAdmissionV1, + ) -> Result< + (crate::WorkProductEventCommitV1, WorkSynthesisInsertOutcome), + crate::WorkProductAttemptAdmissionErrorV1, + >; } diff --git a/crates/tracedecay-contracts/src/work_attempt_effect.rs b/crates/tracedecay-contracts/src/work_attempt_effect.rs index 87120f17f0..f819e172d1 100644 --- a/crates/tracedecay-contracts/src/work_attempt_effect.rs +++ b/crates/tracedecay-contracts/src/work_attempt_effect.rs @@ -155,9 +155,18 @@ pub enum WorkAttemptEffectStorageErrorV1 { Unavailable, } +/// The effect-holder read: retry safety and execution history consult it. +pub trait WorkAttemptEffectReadPortV1 { + fn load_effect_dispatch( + &self, + authority: &WorkAuthority, + attempt: &WorkAttemptIdentityV1, + ) -> Result, WorkAttemptEffectStorageErrorV1>; +} + /// Persistence owned by the exact Work-attempt authority. Implementations /// must make the immutable dispatch identity replay-safe and settlement CASed. -pub trait WorkAttemptEffectStoragePortV1: Send + Sync { +pub trait WorkAttemptEffectStoragePortV1: WorkAttemptEffectReadPortV1 { fn begin_effect_dispatch( &self, authority: &WorkAuthority, @@ -171,12 +180,6 @@ pub trait WorkAttemptEffectStoragePortV1: Send + Sync { resolution: WorkAttemptEffectResolutionV1, resolved_at: UtcMicros, ) -> Result; - - fn load_effect_dispatch( - &self, - authority: &WorkAuthority, - attempt: &WorkAttemptIdentityV1, - ) -> Result, WorkAttemptEffectStorageErrorV1>; } /// Application boundary used by the daemon provider runtime and leak source. @@ -184,14 +187,16 @@ pub struct WorkAttemptEffectServiceV1 { storage: S, } -impl WorkAttemptEffectServiceV1 -where - S: WorkAttemptEffectStoragePortV1, -{ +impl WorkAttemptEffectServiceV1 { pub const fn new(storage: S) -> Self { Self { storage } } +} +impl WorkAttemptEffectServiceV1 +where + S: WorkAttemptEffectStoragePortV1, +{ #[tracing::instrument( name = "application.work.attempt_effect.record_dispatch", level = "trace", @@ -231,7 +236,12 @@ where .settle_effect_dispatch(&authority, attempt, resolution, resolved_at) .map_err(effect_problem) } +} +impl WorkAttemptEffectServiceV1 +where + S: WorkAttemptEffectReadPortV1, +{ #[tracing::instrument( name = "application.work.attempt_effect.load", level = "trace", diff --git a/crates/tracedecay-contracts/src/work_evidence.rs b/crates/tracedecay-contracts/src/work_evidence.rs index 060e8514b8..346071252e 100644 --- a/crates/tracedecay-contracts/src/work_evidence.rs +++ b/crates/tracedecay-contracts/src/work_evidence.rs @@ -519,7 +519,7 @@ pub struct WorkEvidenceRetrievalServiceV1 { impl WorkEvidenceRetrievalServiceV1 where R: WorkEvidenceRootReadPortV1, - A: WorkProductOwnerAuthorizationPortV1, + A: WorkProductOwnerAuthorizationPortV1 + Send + Sync, T: WorkAttemptReceiptReadPortV1, S: WorkTaskSessionPortV1, H: WorkAnchorHydrationPortV1, @@ -801,7 +801,7 @@ impl WorkTaskSessionReauthorizationPortV1 for WorkEvidenceRetrievalServiceV1 where R: WorkEvidenceRootReadPortV1, - A: WorkProductOwnerAuthorizationPortV1, + A: WorkProductOwnerAuthorizationPortV1 + Send + Sync, T: WorkAttemptReceiptReadPortV1, S: WorkTaskSessionPortV1, H: WorkAnchorHydrationPortV1, diff --git a/crates/tracedecay-contracts/src/work_execution_history.rs b/crates/tracedecay-contracts/src/work_execution_history.rs index e43e185a92..3dafa747fc 100644 --- a/crates/tracedecay-contracts/src/work_execution_history.rs +++ b/crates/tracedecay-contracts/src/work_execution_history.rs @@ -14,8 +14,8 @@ use tracedecay_domain::{ use crate::work::work_authority; use crate::{ - ApplicationProblem, RequestContext, SafeDiagnostic, WorkAttemptEffectStorageErrorV1, - WorkAttemptEffectStoragePortV1, WorkAttemptListCoverageV1, WorkAttemptListV1, + ApplicationProblem, RequestContext, SafeDiagnostic, WorkAttemptEffectReadPortV1, + WorkAttemptEffectStorageErrorV1, WorkAttemptListCoverageV1, WorkAttemptListV1, }; #[derive(Clone, Debug, Deserialize, Serialize, JsonSchema, PartialEq, Eq)] @@ -85,7 +85,7 @@ pub fn project_work_execution_history( attempts: WorkAttemptListV1, ) -> Result where - S: WorkAttemptEffectStoragePortV1, + S: WorkAttemptEffectReadPortV1, { let authority = work_authority(context)?; let WorkAttemptListV1::Listed { diff --git a/crates/tracedecay-contracts/src/work_product/attempt_admission.rs b/crates/tracedecay-contracts/src/work_product/attempt_admission.rs index 9c62cda2eb..505dfa4e0a 100644 --- a/crates/tracedecay-contracts/src/work_product/attempt_admission.rs +++ b/crates/tracedecay-contracts/src/work_product/attempt_admission.rs @@ -6,10 +6,7 @@ use tracedecay_domain::{ WorkProductEventPayloadV1, WorkflowRunEventKind, configuration::TopologyConcurrencyPolicyV1, }; -use crate::{ - WorkRetryAttemptOutcomeV1, WorkRetryWriteV1, WorkSynthesisAdmissionRecordV1, - WorkSynthesisInsertOutcome, WorkflowRunAppendRequest, -}; +use crate::{WorkRetryWriteV1, WorkSynthesisAdmissionRecordV1, WorkflowRunAppendRequest}; use super::{WorkProductEventCommitV1, WorkProductEventDraftV1, WorkProductPortContextV1}; @@ -183,25 +180,9 @@ fn selection_covers_authority( } /// Canonical one-transaction authority for product linkage and runtime rows. -pub trait WorkProductAttemptAdmissionPortV1: Send + Sync { +pub trait WorkProductAttemptAdmissionPortV1 { fn admit_attempt( &self, admission: &WorkProductAttemptAdmissionV1, ) -> Result; - - fn admit_retry( - &self, - admission: &WorkProductRetryAdmissionV1, - ) -> Result< - (WorkProductEventCommitV1, WorkRetryAttemptOutcomeV1), - WorkProductAttemptAdmissionErrorV1, - >; - - fn admit_synthesis( - &self, - admission: &WorkProductSynthesisAdmissionV1, - ) -> Result< - (WorkProductEventCommitV1, WorkSynthesisInsertOutcome), - WorkProductAttemptAdmissionErrorV1, - >; } diff --git a/crates/tracedecay-contracts/src/work_product/mutation.rs b/crates/tracedecay-contracts/src/work_product/mutation.rs index de274b335f..292a7bb444 100644 --- a/crates/tracedecay-contracts/src/work_product/mutation.rs +++ b/crates/tracedecay-contracts/src/work_product/mutation.rs @@ -22,6 +22,26 @@ pub use contracts::*; const WORK_PRODUCT_MUTATION_DIGEST_DOMAIN: &str = "tracedecay.application.work-product-mutation.final-v2"; +impl super::PrefetchedWorkGraphReadV1 { + /// The current graph read a submitted mutation checks its expected version + /// against, read before its writer job. + pub fn for_mutation( + storage: &S, + context: &RequestContext, + request: &WorkProductMutationRequestV1, + ) -> Self + where + S: WorkGraphReadPortV1 + WorkProductOwnerAuthorizationPortV1, + { + Self::current( + storage, + context, + request.selection(), + request.mutation_identity().occurred_at, + ) + } +} + pub struct WorkProductMutationServiceV1 { graph: G, owner_authority: A, diff --git a/crates/tracedecay-contracts/src/work_product/mutation/contracts.rs b/crates/tracedecay-contracts/src/work_product/mutation/contracts.rs index 8010566849..0427c926da 100644 --- a/crates/tracedecay-contracts/src/work_product/mutation/contracts.rs +++ b/crates/tracedecay-contracts/src/work_product/mutation/contracts.rs @@ -144,7 +144,10 @@ impl WorkProductEventCommitV1 { /// /// A successful call commits both records in one transaction. There is no /// intermediate appended-but-unpublished state for a restart to reconcile. -pub trait WorkProductEventPortV1: Send + Sync { +/// +/// It carries no `Send + Sync` bound so a storage scoped to one writer +/// transaction can implement it. +pub trait WorkProductEventPortV1 { fn replay( &self, context: &WorkProductPortContextV1, @@ -424,4 +427,19 @@ impl WorkProductMutationRequestV1 { Self::RecordHandoff(request) => &request.mutation, } } + + pub const fn selection(&self) -> &WorkProductSelectionScopeV1 { + match self { + Self::Create(request) => &request.selection, + Self::AddTask(request) => &request.selection, + Self::CreateTask(request) => &request.selection, + Self::DecideProposal(request) => &request.selection, + Self::DecideRelationReplan(request) => &request.selection, + Self::ApplyRelationReplan(request) => &request.selection, + Self::AcceptTask(request) => &request.selection, + Self::AdmitExecution(request) => &request.selection, + Self::LinkAcceptedAttempt(request) => &request.selection, + Self::RecordHandoff(request) => &request.selection, + } + } } diff --git a/crates/tracedecay-contracts/src/work_product/read.rs b/crates/tracedecay-contracts/src/work_product/read.rs index 97e4b01b0f..acd2e56626 100644 --- a/crates/tracedecay-contracts/src/work_product/read.rs +++ b/crates/tracedecay-contracts/src/work_product/read.rs @@ -657,7 +657,9 @@ impl From for WorkProductApplicationErrorV1 { } } -pub trait WorkGraphReadPortV1: Send + Sync { +/// It carries no `Send + Sync` bound so a storage scoped to one writer +/// transaction can implement it. +pub trait WorkGraphReadPortV1 { fn read_graph( &self, context: &WorkProductPortContextV1, @@ -678,6 +680,64 @@ where } } +/// One current graph read made on a reader before a write job, answered to +/// the job's service as an owned value, so the journal replay behind it never +/// holds the writer. The job's event compare-and-swap on the graph version is +/// its staleness fence. +/// +/// It answers only the exact scope and request it was read for; any other +/// read is refused as unavailable rather than answered from the wrong slice. +#[derive(Clone, Debug, Default)] +pub struct PrefetchedWorkGraphReadV1 { + read: Option<( + AuthorizedWorkProductScopeV1, + WorkGraphReadRequestV1, + Result, + )>, +} + +impl PrefetchedWorkGraphReadV1 { + /// Reads the current graph under `selection` as `context` observed it at + /// `observed_at`. A selection the owner refuses prefetches nothing: the + /// service refuses it again inside the job. + pub fn current( + storage: &S, + context: &RequestContext, + selection: &WorkProductSelectionScopeV1, + observed_at: UtcMicros, + ) -> Self + where + S: WorkGraphReadPortV1 + WorkProductOwnerAuthorizationPortV1, + { + let Ok(scope) = storage.authorize_scope(context, selection, observed_at) else { + return Self { read: None }; + }; + let port_context = WorkProductPortContextV1::from_request(context, scope, observed_at); + let request = WorkGraphReadRequestV1::current(selection.clone(), observed_at); + let read = storage.read_graph(&port_context, &request); + Self { + read: Some((port_context.authorized_scope().clone(), request, read)), + } + } +} + +impl WorkGraphReadPortV1 for PrefetchedWorkGraphReadV1 { + fn read_graph( + &self, + context: &WorkProductPortContextV1, + request: &WorkGraphReadRequestV1, + ) -> Result { + match &self.read { + Some((scope, prefetched, read)) + if scope == context.authorized_scope() && prefetched == request => + { + read.clone() + } + _ => Err(WorkGraphReadPortErrorV1::Unavailable), + } + } +} + pub struct WorkProductReadServiceV1 { graph: G, owner_authority: A, diff --git a/crates/tracedecay-contracts/src/work_product/types.rs b/crates/tracedecay-contracts/src/work_product/types.rs index 4b72b67043..d68165d145 100644 --- a/crates/tracedecay-contracts/src/work_product/types.rs +++ b/crates/tracedecay-contracts/src/work_product/types.rs @@ -118,7 +118,7 @@ pub enum WorkProductOwnerAuthorizationErrorV1 { /// Resolves the registered profile owner and authorizes every selected /// project/repository relation against the request context. -pub trait WorkProductOwnerAuthorizationPortV1: Send + Sync { +pub trait WorkProductOwnerAuthorizationPortV1 { fn authorize_scope( &self, context: &RequestContext, diff --git a/crates/tracedecay-contracts/src/work_retry.rs b/crates/tracedecay-contracts/src/work_retry.rs index f0042cfe16..c4996db110 100644 --- a/crates/tracedecay-contracts/src/work_retry.rs +++ b/crates/tracedecay-contracts/src/work_retry.rs @@ -23,13 +23,13 @@ use crate::work_attempt::{ product_attempt_projection_binding, }; use crate::work_attempt_effect::{ - WorkAttemptEffectResolutionV1, WorkAttemptEffectStorageErrorV1, WorkAttemptEffectStoragePortV1, + WorkAttemptEffectReadPortV1, WorkAttemptEffectResolutionV1, WorkAttemptEffectStorageErrorV1, }; use crate::{ ApplicationContractError, ApplicationProblem, RequestContext, RetryDirective, SafeDiagnostic, - WorkGraphReadPortV1, WorkProductAttemptAdmissionPortV1, WorkProductAttemptAdmissionV1, - WorkProductBindingV1, WorkProductOwnerAuthorizationPortV1, WorkProductRetryAdmissionV1, - WorkProductRevisionPinsV1, WorkflowFanOutAttemptBindingV1, WorkflowRunAppendRequest, + WorkGraphReadPortV1, WorkProductAttemptAdmissionV1, WorkProductBindingV1, + WorkProductOwnerAuthorizationPortV1, WorkProductRetryAdmissionV1, WorkProductRevisionPinsV1, + WorkflowFanOutAttemptBindingV1, WorkflowRunAppendRequest, }; const RETRY_INPUT_DIGEST_DOMAIN: &str = "tracedecay.application.work-retry-input.v1"; @@ -329,6 +329,16 @@ pub trait WorkRetryStoragePortV1: WorkAttemptStoragePort { write: &WorkRetryWriteV1, concurrency: &TopologyConcurrencyPolicyV1, ) -> Result; + + /// Commits the accepted-attempt link, the retry receipt, the attempt row + /// and any workflow child rebind as one transaction. + fn admit_retry( + &self, + admission: &WorkProductRetryAdmissionV1, + ) -> Result< + (crate::WorkProductEventCommitV1, WorkRetryAttemptOutcomeV1), + crate::WorkProductAttemptAdmissionErrorV1, + >; } /// Public retry admission over the verified Work product graph. The legacy @@ -350,10 +360,9 @@ pub struct WorkflowFanOutRetryRebindV1 { impl WorkProductRetryServiceV1 where S: WorkRetryStoragePortV1 - + WorkAttemptEffectStoragePortV1 + + WorkAttemptEffectReadPortV1 + WorkGraphReadPortV1 - + WorkProductOwnerAuthorizationPortV1 - + WorkProductAttemptAdmissionPortV1, + + WorkProductOwnerAuthorizationPortV1, E: WorkRetryEvidencePortV1, { pub const fn new(storage: S, evidence: E) -> Self { @@ -610,7 +619,7 @@ fn require_retry_effect_safe( original: &WorkAttemptV1, ) -> Result<(), ApplicationProblem> where - S: WorkAttemptEffectStoragePortV1, + S: WorkAttemptEffectReadPortV1, { if original.execution().effect_state() != WorkEffectStateV1::CompoundNonRepeatable { return Ok(()); diff --git a/crates/tracedecay-contracts/src/work_run_control.rs b/crates/tracedecay-contracts/src/work_run_control.rs index 17b357a44c..6bc15bfe0f 100644 --- a/crates/tracedecay-contracts/src/work_run_control.rs +++ b/crates/tracedecay-contracts/src/work_run_control.rs @@ -140,9 +140,23 @@ pub trait WorkRunControlTransitionPort { ) -> Result<(), WorkRunControlStorageError>; } +/// The control read a reservation fence makes. It carries no `Send + Sync` +/// bound so the reservation can be fenced inside the writer job that admits +/// the attempt. +pub trait WorkRunReservationPort { + /// The published control row for one run, or `None` when the run has never + /// been controlled. + fn load_run_control( + &self, + authority: &WorkAuthority, + task_id: &TaskId, + run_id: &RunId, + ) -> Result, WorkRunControlStorageError>; +} + /// The durable run-control rows and the attempt evidence they are derived /// from. -pub trait WorkRunControlStoragePort: Send + Sync { +pub trait WorkRunControlStoragePort: WorkRunReservationPort + Send + Sync { /// The admitted deadline and live attempt frontier for one run, or `None` /// when the run holds no durable attempt at all. Retry-superseded recovery /// records still count toward history, but are absent from the frontier. @@ -153,15 +167,6 @@ pub trait WorkRunControlStoragePort: Send + Sync { run_id: &RunId, ) -> Result, WorkRunControlStorageError>; - /// The published control row for one run, or `None` when the run has never - /// been controlled. - fn load_run_control( - &self, - authority: &WorkAuthority, - task_id: &TaskId, - run_id: &RunId, - ) -> Result, WorkRunControlStorageError>; - /// Publishes `next` under a compare-and-swap on the authority version the /// caller read. `expected` is `None` only for the first publication. fn publish_run_control( @@ -446,35 +451,8 @@ where impl WorkRunControlService where - S: WorkRunControlStoragePort, + S: WorkRunReservationPort, { - /// Reads the published control state for one run. - #[tracing::instrument(name = "application.work.run_control.read", level = "trace", skip_all)] - pub fn read( - &self, - context: &RequestContext, - request: &WorkRunControlRequestV1, - ) -> Result { - let authority = work_authority(context)?; - let admission = self.require_admission(&authority, &request.task_id, &request.run_id)?; - let control = self - .storage - .load_run_control(&authority, &request.task_id, &request.run_id) - .map_err(ApplicationProblem::from)?; - Ok(match control { - Some(control) => WorkRunControlReadingV1::Controlled { - control, - live_attempts: admission.live_attempts, - total_attempts: admission.total_attempts, - }, - None => WorkRunControlReadingV1::Uncontrolled { - deadline: admission.deadline, - live_attempts: admission.live_attempts, - total_attempts: admission.total_attempts, - }, - }) - } - /// Refuses a new attempt reservation while the run is paused. /// /// This is the fence Plan 32 requires: "pause and cancellation fence new @@ -504,6 +482,38 @@ where Some(_) | None => Ok(()), } } +} + +impl WorkRunControlService +where + S: WorkRunControlStoragePort, +{ + /// Reads the published control state for one run. + #[tracing::instrument(name = "application.work.run_control.read", level = "trace", skip_all)] + pub fn read( + &self, + context: &RequestContext, + request: &WorkRunControlRequestV1, + ) -> Result { + let authority = work_authority(context)?; + let admission = self.require_admission(&authority, &request.task_id, &request.run_id)?; + let control = self + .storage + .load_run_control(&authority, &request.task_id, &request.run_id) + .map_err(ApplicationProblem::from)?; + Ok(match control { + Some(control) => WorkRunControlReadingV1::Controlled { + control, + live_attempts: admission.live_attempts, + total_attempts: admission.total_attempts, + }, + None => WorkRunControlReadingV1::Uncontrolled { + deadline: admission.deadline, + live_attempts: admission.live_attempts, + total_attempts: admission.total_attempts, + }, + }) + } /// Reads the next bounded, cyclic recovery page of settled interval /// receipts. It skips only receipts whose exact owner fact was durably diff --git a/crates/tracedecay-contracts/src/work_synthesis.rs b/crates/tracedecay-contracts/src/work_synthesis.rs index b2ce1afc45..a6be265c74 100644 --- a/crates/tracedecay-contracts/src/work_synthesis.rs +++ b/crates/tracedecay-contracts/src/work_synthesis.rs @@ -33,9 +33,8 @@ use crate::work_attempt::{ }; use crate::workflow_synthesis::WorkflowSynthesisDraft; use crate::{ - ApplicationProblem, RequestContext, SafeDiagnostic, WorkGraphReadPortV1, - WorkProductAttemptAdmissionPortV1, WorkProductBindingV1, WorkProductOwnerAuthorizationPortV1, - WorkProductRevisionPinsV1, + ApplicationProblem, RequestContext, SafeDiagnostic, WorkGraphReadPortV1, WorkProductBindingV1, + WorkProductOwnerAuthorizationPortV1, WorkProductRevisionPinsV1, }; const WORK_SYNTHESIS_SOURCE_SET_DOMAIN: &str = @@ -208,8 +207,7 @@ pub fn admit_work_synthesis_against_registered_topology( where S: WorkSynthesisAdmissionStoragePort + WorkGraphReadPortV1 - + WorkProductOwnerAuthorizationPortV1 - + WorkProductAttemptAdmissionPortV1, + + WorkProductOwnerAuthorizationPortV1, F: FnOnce(&StartWorkAttemptCommand, &[WorkAttemptV1]) -> Result, { crate::require_registered_work_topology( diff --git a/crates/tracedecay-contracts/tests/contracts_suite/common/mod.rs b/crates/tracedecay-contracts/tests/contracts_suite/common/mod.rs index 172f08b826..9da9812aaa 100644 --- a/crates/tracedecay-contracts/tests/contracts_suite/common/mod.rs +++ b/crates/tracedecay-contracts/tests/contracts_suite/common/mod.rs @@ -604,89 +604,6 @@ impl WorkProductAttemptAdmissionPortV1 for WorkProductAttemptStore { attempt: admission.attempt.clone(), }) } - - fn admit_retry( - &self, - _admission: &tracedecay_contracts::WorkProductRetryAdmissionV1, - ) -> Result< - ( - WorkProductEventCommitV1, - tracedecay_contracts::WorkRetryAttemptOutcomeV1, - ), - WorkProductAttemptAdmissionErrorV1, - > { - Err(WorkProductAttemptAdmissionErrorV1::Unavailable) - } - - fn admit_synthesis( - &self, - admission: &tracedecay_contracts::WorkProductSynthesisAdmissionV1, - ) -> Result< - ( - WorkProductEventCommitV1, - tracedecay_contracts::WorkSynthesisInsertOutcome, - ), - WorkProductAttemptAdmissionErrorV1, - > { - admission.validate()?; - let attempt = &admission.admission.attempt; - let payload = serde_json::to_string(attempt) - .map_err(|_| WorkProductAttemptAdmissionErrorV1::Unavailable)?; - let key = attempt_key(&admission.admission.authority, attempt.identity()); - let mut rows = self - .inner - .lock() - .map_err(|_| WorkProductAttemptAdmissionErrorV1::Unavailable)?; - let product = rows - .events - .iter() - .find(|event| { - event.commit.event().command_id() == &admission.admission.product_draft.command_id - }) - .map(|event| event.commit.clone()); - let synthesis = rows.syntheses.get(&key).cloned(); - match (product, synthesis, rows.attempts.get(&key)) { - (Some(product), Some(existing), Some(existing_attempt)) - if existing.request_digest == admission.synthesis.request_digest - && existing.result == admission.synthesis.result - && existing_attempt == &payload => - { - return Ok(( - product, - WorkSynthesisInsertOutcome::Replayed(Box::new(existing.result)), - )); - } - (Some(_), Some(_), Some(_)) => { - return Err(WorkProductAttemptAdmissionErrorV1::IdentityConflict); - } - (None, None, None) => {} - _ => return Err(WorkProductAttemptAdmissionErrorV1::IdentityConflict), - } - let (next_graph, product) = product_commit_for(&rows, &admission.admission.product_draft)?; - attempt - .validate_graph_admission(&next_graph) - .map_err(|_| WorkProductAttemptAdmissionErrorV1::InvalidAdmission)?; - if matches!( - attempt_capacity( - &rows, - &admission.admission.authority, - attempt.identity().task_id(), - &admission.admission.concurrency, - ) - .map_err(|_| WorkProductAttemptAdmissionErrorV1::Unavailable)? - .verdict(), - WorkAttemptCapacityVerdictV1::Exhausted(_) - ) { - return Err(WorkProductAttemptAdmissionErrorV1::CapacityExceeded); - } - rows.graph = Some(next_graph); - rows.events.push(StoredProductEvent { - commit: product.clone(), - }); - rows.attempts.insert(key.clone(), payload); - rows.syntheses.insert(key, admission.synthesis.clone()); - Ok((product, WorkSynthesisInsertOutcome::Inserted)) - } } impl WorkAttemptStoragePort for WorkProductAttemptStore { @@ -913,4 +830,74 @@ impl WorkSynthesisAdmissionStoragePort for WorkProductAttemptStore { .cloned() .ok_or(WorkAttemptStorageError::AttemptConflict) } + + fn admit_synthesis( + &self, + admission: &tracedecay_contracts::WorkProductSynthesisAdmissionV1, + ) -> Result< + ( + WorkProductEventCommitV1, + tracedecay_contracts::WorkSynthesisInsertOutcome, + ), + WorkProductAttemptAdmissionErrorV1, + > { + admission.validate()?; + let attempt = &admission.admission.attempt; + let payload = serde_json::to_string(attempt) + .map_err(|_| WorkProductAttemptAdmissionErrorV1::Unavailable)?; + let key = attempt_key(&admission.admission.authority, attempt.identity()); + let mut rows = self + .inner + .lock() + .map_err(|_| WorkProductAttemptAdmissionErrorV1::Unavailable)?; + let product = rows + .events + .iter() + .find(|event| { + event.commit.event().command_id() == &admission.admission.product_draft.command_id + }) + .map(|event| event.commit.clone()); + let synthesis = rows.syntheses.get(&key).cloned(); + match (product, synthesis, rows.attempts.get(&key)) { + (Some(product), Some(existing), Some(existing_attempt)) + if existing.request_digest == admission.synthesis.request_digest + && existing.result == admission.synthesis.result + && existing_attempt == &payload => + { + return Ok(( + product, + WorkSynthesisInsertOutcome::Replayed(Box::new(existing.result)), + )); + } + (Some(_), Some(_), Some(_)) => { + return Err(WorkProductAttemptAdmissionErrorV1::IdentityConflict); + } + (None, None, None) => {} + _ => return Err(WorkProductAttemptAdmissionErrorV1::IdentityConflict), + } + let (next_graph, product) = product_commit_for(&rows, &admission.admission.product_draft)?; + attempt + .validate_graph_admission(&next_graph) + .map_err(|_| WorkProductAttemptAdmissionErrorV1::InvalidAdmission)?; + if matches!( + attempt_capacity( + &rows, + &admission.admission.authority, + attempt.identity().task_id(), + &admission.admission.concurrency, + ) + .map_err(|_| WorkProductAttemptAdmissionErrorV1::Unavailable)? + .verdict(), + WorkAttemptCapacityVerdictV1::Exhausted(_) + ) { + return Err(WorkProductAttemptAdmissionErrorV1::CapacityExceeded); + } + rows.graph = Some(next_graph); + rows.events.push(StoredProductEvent { + commit: product.clone(), + }); + rows.attempts.insert(key.clone(), payload); + rows.syntheses.insert(key, admission.synthesis.clone()); + Ok((product, WorkSynthesisInsertOutcome::Inserted)) + } } diff --git a/crates/tracedecay-contracts/tests/contracts_suite/work_run_control_service.rs b/crates/tracedecay-contracts/tests/contracts_suite/work_run_control_service.rs index 6dd7867053..ecc93bca97 100644 --- a/crates/tracedecay-contracts/tests/contracts_suite/work_run_control_service.rs +++ b/crates/tracedecay-contracts/tests/contracts_suite/work_run_control_service.rs @@ -22,6 +22,7 @@ use tracedecay_contracts::{ ResumeWorkRunCommand, WorkRunAdmissionV1, WorkRunControlFrontierV1, WorkRunControlReadingV1, WorkRunControlRequestV1, WorkRunControlService, WorkRunControlStorageError, WorkRunControlStoragePort, WorkRunControlTransitionPort, WorkRunLiveAttemptV1, + WorkRunReservationPort, }; use tracedecay_domain::{ ActorId, AttemptId, ProjectId, RepositoryId, RunId, TaskId, UtcMicros, WorkAuthority, @@ -215,29 +216,31 @@ impl WorkRunControlTransitionPort for TestStore { } } -impl WorkRunControlStoragePort for TestStore { - fn run_admission( +impl WorkRunReservationPort for TestStore { + fn load_run_control( &self, authority: &WorkAuthority, task_id: &TaskId, run_id: &RunId, - ) -> Result, WorkRunControlStorageError> { + ) -> Result, WorkRunControlStorageError> { Ok(self - .admissions + .controls .lock() .unwrap() .get(&(authority.clone(), task_id.clone(), run_id.clone())) .cloned()) } +} - fn load_run_control( +impl WorkRunControlStoragePort for TestStore { + fn run_admission( &self, authority: &WorkAuthority, task_id: &TaskId, run_id: &RunId, - ) -> Result, WorkRunControlStorageError> { + ) -> Result, WorkRunControlStorageError> { Ok(self - .controls + .admissions .lock() .unwrap() .get(&(authority.clone(), task_id.clone(), run_id.clone())) diff --git a/crates/tracedecay-daemon-service/src/invocation/work/attempt_operations.rs b/crates/tracedecay-daemon-service/src/invocation/work/attempt_operations.rs index 15672aa7d6..bf2b9591dd 100644 --- a/crates/tracedecay-daemon-service/src/invocation/work/attempt_operations.rs +++ b/crates/tracedecay-daemon-service/src/invocation/work/attempt_operations.rs @@ -8,9 +8,9 @@ use tracedecay_contracts::{ AdmitWorkSynthesisCommand, ApplicationProblem, CancelWorkAttemptCommand, Deadline, RequestContext, RequestId, ResumeWorkAttemptsCommand, RetryWorkAttemptCommandV1, SafeDiagnostic, StartWorkAttemptCommand, WorkAttemptStatusRequestV1, WorkSynthesisAttemptV1, - WorkflowArtifactStorePort, }; use tracedecay_domain::{ManifestDigest, UtcMicros, WorkAttemptStateV1, WorkAttemptV1}; +use tracedecay_rusqlite_runtime::work_run_control::WorkSqliteJobStorage; use tracedecay_tool_catalog::UseCaseId; use tracedecay_application::work::{ @@ -31,16 +31,10 @@ fn consume_synthesis_bytes(remaining: &mut u64, bytes: u64) -> Result<(), Applic } fn synthesis_source_context( - registered: &RegisteredWorkRuntime, + artifacts: &WorkSqliteJobStorage<'_>, start: &StartWorkAttemptCommand, attempts: &[WorkAttemptV1], ) -> Result { - let artifacts = registered.database.workflow_storage().map_err(|_| { - ApplicationProblem::unavailable(SafeDiagnostic { - code: "application.work-synthesis.artifact-body-unavailable".to_owned(), - message: "The admitted synthesis source payload authority is unavailable.".to_owned(), - }) - })?; let mut remaining = start.execution_snapshot.limits().max_protocol_bytes(); consume_synthesis_bytes(&mut remaining, start.instructions.len() as u64)?; consume_synthesis_bytes(&mut remaining, 2)?; @@ -50,7 +44,7 @@ fn synthesis_source_context( let mut payloads = Vec::with_capacity(attempt.artifacts().len()); for artifact in attempt.artifacts() { consume_synthesis_bytes(&mut remaining, artifact.byte_length())?; - let payload = artifacts.load(artifact).map_err(|_| { + let payload = artifacts.load_workflow_artifact(artifact).map_err(|_| { ApplicationProblem::unavailable(SafeDiagnostic { code: "application.work-synthesis.artifact-body-unavailable".to_owned(), message: "An admitted synthesis source payload is absent or invalid." @@ -103,10 +97,8 @@ fn synthesis_source_context( } #[allow(clippy::too_many_arguments)] -#[tracing::instrument(name = "daemon.service.work.start_attempt", level = "trace", skip_all)] -pub(super) fn start_attempt( +pub(super) async fn start_attempt( registered: &RegisteredWorkRuntime, - services: &RegisteredWorkApplicationServicesV1, binding: tracedecay_contracts::WorkProductBindingV1, attempt_processes: &Arc, observability_producer: Option<&Arc>, @@ -121,30 +113,25 @@ pub(super) fn start_attempt( deadline: Deadline, command: StartWorkAttemptCommand, ) -> DaemonInvocationResponse { - let started = services - .run_control() - .admit_reservation(context, &command.task_id, &command.run_id) - .and_then(|()| { - RegisteredWorkProductServicesV1::attach(®istered.database, binding.clone()) - .map_err(|_| { - work_product_problem( - tracedecay_contracts::WorkProductApplicationErrorV1::GraphAuthorityUnavailable, - ) - }) - .and_then(|product| { - preparation::current_work_product_revision_pins(registered).and_then( - |revisions| { - product.attempts().start_against_registered_topology( - context, - &binding, - &revisions, - ®istered.work_topology_policy, - command, - ) - }, - ) - }) - }); + let started = match ( + attach_product(registered, &binding), + preparation::current_work_product_revision_pins(registered), + ) { + (Ok(product), Ok(revisions)) => { + tracing::Instrument::instrument( + product.start_attempt( + context.clone(), + binding, + revisions, + registered.work_topology_policy.clone(), + command, + ), + tracing::trace_span!("daemon.service.work.start_attempt"), + ) + .await + } + (Err(problem), _) | (_, Err(problem)) => Err(problem), + }; if let (Ok(attempt), Some(project_root)) = (&started, project_root) && attempt.state() == WorkAttemptStateV1::Leased { @@ -171,11 +158,20 @@ pub(super) fn start_attempt( ) } +fn attach_product( + registered: &RegisteredWorkRuntime, + binding: &tracedecay_contracts::WorkProductBindingV1, +) -> Result { + RegisteredWorkProductServicesV1::attach(®istered.database, binding.clone()).map_err(|_| { + work_product_problem( + tracedecay_contracts::WorkProductApplicationErrorV1::GraphAuthorityUnavailable, + ) + }) +} + #[allow(clippy::too_many_arguments)] -#[tracing::instrument(name = "daemon.service.work.synthesize", level = "trace", skip_all)] -pub(super) fn synthesize( +pub(super) async fn synthesize( registered: &RegisteredWorkRuntime, - services: &RegisteredWorkApplicationServicesV1, binding: tracedecay_contracts::WorkProductBindingV1, attempt_processes: &Arc, observability_producer: Option<&Arc>, @@ -190,31 +186,26 @@ pub(super) fn synthesize( deadline: Deadline, command: AdmitWorkSynthesisCommand, ) -> DaemonInvocationResponse { - let admitted = RegisteredWorkProductServicesV1::attach(®istered.database, binding.clone()) - .map_err(|_| { - work_product_problem( - tracedecay_contracts::WorkProductApplicationErrorV1::GraphAuthorityUnavailable, - ) - }) - .and_then(|product| { - let revisions = preparation::current_work_product_revision_pins(registered)?; - tracedecay_contracts::admit_work_synthesis_against_registered_topology( - product.synthesis(), - context, - &binding, - &revisions, - ®istered.work_topology_policy, - command, - |start, sources| { - services.run_control().admit_reservation( - context, - &start.task_id, - &start.run_id, - )?; - synthesis_source_context(registered, start, sources) - }, + let admitted = match ( + attach_product(registered, &binding), + preparation::current_work_product_revision_pins(registered), + ) { + (Ok(product), Ok(revisions)) => { + tracing::Instrument::instrument( + product.synthesize( + context.clone(), + binding, + revisions, + registered.work_topology_policy.clone(), + command, + synthesis_source_context, + ), + tracing::trace_span!("daemon.service.work.synthesize"), ) - }); + .await + } + (Err(problem), _) | (_, Err(problem)) => Err(problem), + }; if let (Ok(WorkSynthesisAttemptV1::Admitted(admission)), Some(project_root)) = (&admitted, project_root) && admission.attempt.state() == WorkAttemptStateV1::Leased @@ -273,8 +264,7 @@ pub(super) fn attempt_status( } #[allow(clippy::too_many_arguments)] -#[tracing::instrument(name = "daemon.service.work.cancel_attempt", level = "trace", skip_all)] -pub(super) fn cancel_attempt( +pub(super) async fn cancel_attempt( registered: &RegisteredWorkRuntime, services: &RegisteredWorkApplicationServicesV1, attempt_processes: &Arc, @@ -288,7 +278,11 @@ pub(super) fn cancel_attempt( deadline: Deadline, command: CancelWorkAttemptCommand, ) -> DaemonInvocationResponse { - let cancelled = services.attempts().request_cancellation(context, command); + let cancelled = tracing::Instrument::instrument( + services.cancel_attempt(context.clone(), command), + tracing::trace_span!("daemon.service.work.cancel_attempt"), + ) + .await; if let Ok(attempt) = &cancelled { attempt_processes.signal_cancellation(&context.scope().worktree_id, attempt.identity()); } @@ -308,10 +302,8 @@ pub(super) fn cancel_attempt( } #[allow(clippy::too_many_arguments)] -#[tracing::instrument(name = "daemon.service.work.retry_attempt", level = "trace", skip_all)] -pub(super) fn retry_attempt( +pub(super) async fn retry_attempt( registered: &RegisteredWorkRuntime, - _services: &RegisteredWorkApplicationServicesV1, binding: tracedecay_contracts::WorkProductBindingV1, attempt_processes: &Arc, observability_producer: Option<&Arc>, @@ -326,33 +318,35 @@ pub(super) fn retry_attempt( deadline: Deadline, command: RetryWorkAttemptCommandV1, ) -> DaemonInvocationResponse { - let retried = if project_root.is_none() { + let prepared = if project_root.is_none() { Err(ApplicationProblem::unavailable(SafeDiagnostic { code: "application.work-retry.project-root-unavailable".to_owned(), message: "The Work retry runtime owner is unavailable.".to_owned(), })) } else { - RegisteredWorkProductServicesV1::attach(®istered.database, binding.clone()) - .map_err(|_| { - work_product_problem( - tracedecay_contracts::WorkProductApplicationErrorV1::GraphAuthorityUnavailable, - ) - }) - .and_then(|product| { - preparation::current_work_product_revision_pins(registered).and_then(|revisions| { - let workflow_rebind = - workflow_retry_rebind(registered, &command.original_attempt)?; - product.retry().retry( - context, - &binding, - &revisions, - ®istered.work_topology_policy, - command, - observed_at, - workflow_rebind, - ) - }) - }) + attach_product(registered, &binding).and_then(|product| { + let revisions = preparation::current_work_product_revision_pins(registered)?; + let workflow_rebind = workflow_retry_rebind(registered, &command.original_attempt)?; + Ok((product, revisions, workflow_rebind)) + }) + }; + let retried = match prepared { + Ok((product, revisions, workflow_rebind)) => { + tracing::Instrument::instrument( + product.retry_attempt( + context.clone(), + binding, + revisions, + registered.work_topology_policy.clone(), + command, + observed_at, + workflow_rebind, + ), + tracing::trace_span!("daemon.service.work.retry_attempt"), + ) + .await + } + Err(problem) => Err(problem), }; if let Ok(outcome) = &retried { let _ = tracedecay_application::observability::record_work_retry_observation( @@ -432,12 +426,7 @@ fn workflow_retry_rebind( } #[allow(clippy::too_many_arguments)] -#[tracing::instrument( - name = "daemon.service.work.resume_attempts", - level = "trace", - skip_all -)] -pub(super) fn resume_attempts( +pub(super) async fn resume_attempts( registered: &RegisteredWorkRuntime, services: &RegisteredWorkApplicationServicesV1, attempt_processes: &Arc, @@ -463,7 +452,11 @@ pub(super) fn resume_attempts( "Work attempt recovery requires the current worktree to have no live provider holder.", )) } else { - services.attempts().resume(context, &command) + tracing::Instrument::instrument( + services.resume_attempts(context.clone(), command), + tracing::trace_span!("daemon.service.work.resume_attempts"), + ) + .await }; if report.is_ok() && let Some(project_root) = project_root diff --git a/crates/tracedecay-daemon-service/src/invocation/work/preparation.rs b/crates/tracedecay-daemon-service/src/invocation/work/preparation.rs index f3f1784d0f..b8465cb7c3 100644 --- a/crates/tracedecay-daemon-service/src/invocation/work/preparation.rs +++ b/crates/tracedecay-daemon-service/src/invocation/work/preparation.rs @@ -318,13 +318,65 @@ pub(super) fn repository_selection( .map_err(|_| work_product_authority_unavailable()) } -pub(super) fn decide_product_proposal( +/// Commits one product graph mutation as a writer job. +pub(super) async fn mutate_product( registered: &RegisteredWorkRuntime, context: &RequestContext, capability: &str, use_case: &UseCaseId, - request: tracedecay_contracts::DecideWorkProposalRequestV1, + request: tracedecay_contracts::WorkProductMutationRequestV1, ) -> Result { + let (services, binding) = product_services(registered, capability, use_case)?; + let revisions = current_work_product_revision_pins(registered)?; + services + .mutate(context.clone(), binding, request, revisions) + .await + .map_err(work_product_problem) +} + +/// Admits execution of one task: the execution snapshot is prepared from a +/// reader before the admission event commits as a writer job. +pub(super) async fn admit_execution( + registered: &RegisteredWorkRuntime, + context: &RequestContext, + capability: &str, + use_case: &UseCaseId, + command: tracedecay_contracts::AdmitWorkExecutionRequestV1, +) -> Result { + let (services, binding) = product_services(registered, capability, use_case)?; + let revisions = current_work_product_revision_pins(registered)?; + if command.mutation.revisions != revisions { + return Err(work_product_problem( + tracedecay_contracts::WorkProductApplicationErrorV1::RevisionConflict, + )); + } + let execution_snapshot = prepare_execution_snapshot(registered, context, &binding, &command)?; + let mutation = services + .mutate( + context.clone(), + binding, + tracedecay_contracts::WorkProductMutationRequestV1::AdmitExecution(command), + revisions, + ) + .await + .map_err(work_product_problem)?; + Ok(tracedecay_contracts::AdmittedWorkExecutionV1 { + mutation, + execution_snapshot, + }) +} + +fn product_services( + registered: &RegisteredWorkRuntime, + capability: &str, + use_case: &UseCaseId, +) -> Result< + ( + tracedecay_application::work::RegisteredWorkProductServicesV1, + tracedecay_contracts::WorkProductBindingV1, + ), + ApplicationProblem, +> { let capability = CapabilityId::new(capability).map_err(|_| work_product_authority_unavailable())?; let binding = tracedecay_contracts::WorkProductBindingV1::new(capability, use_case.clone()); @@ -333,15 +385,7 @@ pub(super) fn decide_product_proposal( binding.clone(), ) .map_err(|_| work_product_authority_unavailable())?; - if request.mutation.revisions != current_work_product_revision_pins(registered)? { - return Err(work_product_problem( - tracedecay_contracts::WorkProductApplicationErrorV1::RevisionConflict, - )); - } - services - .mutations() - .decide_proposal(context, &binding, request) - .map_err(work_product_problem) + Ok((services, binding)) } fn work_product_authority_unavailable() -> ApplicationProblem { diff --git a/crates/tracedecay-daemon-service/src/invocation/work/request_dispatch.rs b/crates/tracedecay-daemon-service/src/invocation/work/request_dispatch.rs index f41a54f89c..fdf1cb4bb5 100644 --- a/crates/tracedecay-daemon-service/src/invocation/work/request_dispatch.rs +++ b/crates/tracedecay-daemon-service/src/invocation/work/request_dispatch.rs @@ -118,193 +118,133 @@ pub(super) async fn dispatch_work_application( ) } WorkApplicationInvocationV1::Create(command) => { - let _span = tracing::trace_span!("daemon.service.work.create").entered(); - { - let Ok(capability_id) = CapabilityId::new(*capability) else { - return DaemonInvocationResponse::problem( - request_id, - DaemonInvocationProblem::Unavailable, - ); - }; - let binding = tracedecay_contracts::WorkProductBindingV1::new( - capability_id, - use_case.clone(), - ); - let created = tracedecay_application::work::RegisteredWorkProductServicesV1::attach( - ®istered.database, - binding.clone(), - ) - .map_err(|_| { - tracedecay_contracts::WorkProductApplicationErrorV1::GraphAuthorityUnavailable - }) - .and_then(|product| { - preparation::current_work_product_revision_pins(®istered) - .map_err(|_| { - tracedecay_contracts::WorkProductApplicationErrorV1::GraphAuthorityUnavailable - }) - .and_then(|revisions| { - if command.mutation.revisions != revisions { - return Err( - tracedecay_contracts::WorkProductApplicationErrorV1::RevisionConflict, - ); - } - product - .mutations() - .create_task(&context, &binding, command) - }) - }) - .map_err(work_product_problem); - complete_work_effect( + let created = tracing::Instrument::instrument( + preparation::mutate_product( ®istered, - request_id, &context, - canonical_request_id, - operation_key, - use_case, - input_digest, - created, - observed_at, - deadline, - WorkApplicationOutcomeV1::Create, - ) - } + capability, + &use_case, + tracedecay_contracts::WorkProductMutationRequestV1::CreateTask(Box::new( + command, + )), + ), + tracing::trace_span!("daemon.service.work.create"), + ) + .await; + complete_work_effect( + ®istered, + request_id, + &context, + canonical_request_id, + operation_key, + use_case, + input_digest, + created, + observed_at, + deadline, + WorkApplicationOutcomeV1::Create, + ) } WorkApplicationInvocationV1::ReviewProposal(request) => { - let _span = tracing::trace_span!("daemon.service.work.review_proposal").entered(); - { - let proposal_ref = request.proposal.proposal_id().as_str().to_owned(); - let command_ref = request.mutation.command_id.as_str().to_owned(); - let disposition = request.disposition; - let occurred_at = request.mutation.occurred_at; - let result = preparation::decide_product_proposal( + let proposal_ref = request.proposal.proposal_id().as_str().to_owned(); + let command_ref = request.mutation.command_id.as_str().to_owned(); + let disposition = request.disposition; + let occurred_at = request.mutation.occurred_at; + let result = tracing::Instrument::instrument( + preparation::mutate_product( ®istered, &context, capability, &use_case, - request.into(), + tracedecay_contracts::WorkProductMutationRequestV1::DecideProposal( + request.into(), + ), + ), + tracing::trace_span!("daemon.service.work.review_proposal"), + ) + .await; + if result.is_ok() { + let _ = tracedecay_application::observability::record_reliance_decision( + observability_producer.as_deref(), + &proposal_ref, + &command_ref, + Some(disposition), + occurred_at, ); - if result.is_ok() { - let _ = tracedecay_application::observability::record_reliance_decision( - observability_producer.as_deref(), - &proposal_ref, - &command_ref, - Some(disposition), - occurred_at, - ); - } - complete_work_effect( - ®istered, - request_id, - &context, - canonical_request_id, - operation_key, - use_case, - input_digest, - result, - observed_at, - deadline, - WorkApplicationOutcomeV1::ReviewProposal, - ) } + complete_work_effect( + ®istered, + request_id, + &context, + canonical_request_id, + operation_key, + use_case, + input_digest, + result, + observed_at, + deadline, + WorkApplicationOutcomeV1::ReviewProposal, + ) } WorkApplicationInvocationV1::AcceptProposal(command) => { - let _span = tracing::trace_span!("daemon.service.work.accept_proposal").entered(); - { - let proposal_ref = command.proposal.proposal_id().as_str().to_owned(); - let command_ref = command.mutation.command_id.as_str().to_owned(); - let occurred_at = command.mutation.occurred_at; - let result = preparation::decide_product_proposal( + let proposal_ref = command.proposal.proposal_id().as_str().to_owned(); + let command_ref = command.mutation.command_id.as_str().to_owned(); + let occurred_at = command.mutation.occurred_at; + let result = tracing::Instrument::instrument( + preparation::mutate_product( ®istered, &context, capability, &use_case, - command.into(), + tracedecay_contracts::WorkProductMutationRequestV1::DecideProposal( + command.into(), + ), + ), + tracing::trace_span!("daemon.service.work.accept_proposal"), + ) + .await; + if result.is_ok() { + let _ = tracedecay_application::observability::record_reliance_decision( + observability_producer.as_deref(), + &proposal_ref, + &command_ref, + None, + occurred_at, ); - if result.is_ok() { - let _ = tracedecay_application::observability::record_reliance_decision( - observability_producer.as_deref(), - &proposal_ref, - &command_ref, - None, - occurred_at, - ); - } - complete_work_effect( - ®istered, - request_id, - &context, - canonical_request_id, - operation_key, - use_case, - input_digest, - result, - observed_at, - deadline, - WorkApplicationOutcomeV1::AcceptProposal, - ) } + complete_work_effect( + ®istered, + request_id, + &context, + canonical_request_id, + operation_key, + use_case, + input_digest, + result, + observed_at, + deadline, + WorkApplicationOutcomeV1::AcceptProposal, + ) } WorkApplicationInvocationV1::AdmitExecution(command) => { - let _span = tracing::trace_span!("daemon.service.work.admit_execution").entered(); - { - let result = CapabilityId::new(*capability) - .map_err(|_| { - work_product_problem( - tracedecay_contracts::WorkProductApplicationErrorV1::GraphAuthorityUnavailable, - ) - }) - .and_then(|capability| { - let binding = tracedecay_contracts::WorkProductBindingV1::new( - capability, - use_case.clone(), - ); - tracedecay_application::work::RegisteredWorkProductServicesV1::attach( - ®istered.database, - binding.clone(), - ) - .map_err(|_| { - work_product_problem( - tracedecay_contracts::WorkProductApplicationErrorV1::GraphAuthorityUnavailable, - ) - }) - .and_then(|product| { - let revisions = - preparation::current_work_product_revision_pins(®istered)?; - if command.mutation.revisions != revisions { - return Err(work_product_problem( - tracedecay_contracts::WorkProductApplicationErrorV1::RevisionConflict, - )); - } - let execution_snapshot = preparation::prepare_execution_snapshot( - ®istered, - &context, - &binding, - &command, - )?; - let mutation = product - .mutations() - .admit_execution(&context, &binding, command) - .map_err(work_product_problem)?; - Ok(tracedecay_contracts::AdmittedWorkExecutionV1 { - mutation, - execution_snapshot, - }) - }) - }); - complete_work_effect( - ®istered, - request_id, - &context, - canonical_request_id, - operation_key, - use_case, - input_digest, - result, - observed_at, - deadline, - WorkApplicationOutcomeV1::AdmitExecution, - ) - } + let result = tracing::Instrument::instrument( + preparation::admit_execution(®istered, &context, capability, &use_case, command), + tracing::trace_span!("daemon.service.work.admit_execution"), + ) + .await; + complete_work_effect( + ®istered, + request_id, + &context, + canonical_request_id, + operation_key, + use_case, + input_digest, + result, + observed_at, + deadline, + WorkApplicationOutcomeV1::AdmitExecution, + ) } WorkApplicationInvocationV1::StartAttempt(command) => { let Ok(capability) = CapabilityId::new(*capability) else { @@ -317,7 +257,6 @@ pub(super) async fn dispatch_work_application( tracedecay_contracts::WorkProductBindingV1::new(capability, use_case.clone()); attempt_operations::start_attempt( ®istered, - &services, binding, &attempt_processes, observability_producer.as_ref(), @@ -332,6 +271,7 @@ pub(super) async fn dispatch_work_application( deadline, command, ) + .await } WorkApplicationInvocationV1::Synthesize(command) => { let Ok(capability) = CapabilityId::new(*capability) else { @@ -344,7 +284,6 @@ pub(super) async fn dispatch_work_application( tracedecay_contracts::WorkProductBindingV1::new(capability, use_case.clone()); attempt_operations::synthesize( ®istered, - &services, binding, &attempt_processes, observability_producer.as_ref(), @@ -359,6 +298,7 @@ pub(super) async fn dispatch_work_application( deadline, command, ) + .await } WorkApplicationInvocationV1::AttemptStatus(request) => attempt_operations::attempt_status( ®istered, @@ -373,20 +313,23 @@ pub(super) async fn dispatch_work_application( deadline, request, ), - WorkApplicationInvocationV1::CancelAttempt(command) => attempt_operations::cancel_attempt( - ®istered, - &services, - &attempt_processes, - request_id, - &context, - canonical_request_id, - operation_key, - use_case, - input_digest, - observed_at, - deadline, - command, - ), + WorkApplicationInvocationV1::CancelAttempt(command) => { + attempt_operations::cancel_attempt( + ®istered, + &services, + &attempt_processes, + request_id, + &context, + canonical_request_id, + operation_key, + use_case, + input_digest, + observed_at, + deadline, + command, + ) + .await + } WorkApplicationInvocationV1::RetryAttempt(command) => { let Ok(capability) = CapabilityId::new(*capability) else { return DaemonInvocationResponse::problem( @@ -398,7 +341,6 @@ pub(super) async fn dispatch_work_application( tracedecay_contracts::WorkProductBindingV1::new(capability, use_case.clone()); attempt_operations::retry_attempt( ®istered, - &services, binding, &attempt_processes, observability_producer.as_ref(), @@ -413,6 +355,7 @@ pub(super) async fn dispatch_work_application( deadline, command, ) + .await } WorkApplicationInvocationV1::ListAttempts(request) => { let _span = tracing::trace_span!("daemon.service.work.list_attempts").entered(); @@ -731,6 +674,7 @@ pub(super) async fn dispatch_work_application( deadline, command, ) + .await } WorkApplicationInvocationV1::Views(request) => { let _span = tracing::trace_span!("daemon.service.work.views").entered(); @@ -834,50 +778,24 @@ pub(super) async fn dispatch_work_application( } } WorkApplicationInvocationV1::MutateGraph(request) => { - let _span = tracing::trace_span!("daemon.service.work.mutate_graph").entered(); - { - let Ok(capability) = CapabilityId::new(*capability) else { - return DaemonInvocationResponse::problem( - request_id, - DaemonInvocationProblem::Unavailable, - ); - }; - let binding = - tracedecay_contracts::WorkProductBindingV1::new(capability, use_case.clone()); - let product_services = - match tracedecay_application::work::RegisteredWorkProductServicesV1::attach( - ®istered.database, - binding.clone(), - ) { - Ok(services) => services, - Err(_) => { - return DaemonInvocationResponse::problem( - request_id, - DaemonInvocationProblem::Unavailable, - ); - } - }; - let mutated = preparation::current_work_product_revision_pins(®istered) - .and_then(|revisions| { - product_services - .mutations() - .mutate(&context, &binding, request, &revisions) - .map_err(work_product_problem) - }); - complete_work_effect( - ®istered, - request_id, - &context, - canonical_request_id, - operation_key, - use_case, - input_digest, - mutated, - observed_at, - deadline, - WorkApplicationOutcomeV1::MutateGraph, - ) - } + let mutated = tracing::Instrument::instrument( + preparation::mutate_product(®istered, &context, capability, &use_case, request), + tracing::trace_span!("daemon.service.work.mutate_graph"), + ) + .await; + complete_work_effect( + ®istered, + request_id, + &context, + canonical_request_id, + operation_key, + use_case, + input_digest, + mutated, + observed_at, + deadline, + WorkApplicationOutcomeV1::MutateGraph, + ) } WorkApplicationInvocationV1::PauseRun(command) => { let transition = tracing::Instrument::instrument( diff --git a/crates/tracedecay-daemon-service/src/invocation/work_attempt_exec.rs b/crates/tracedecay-daemon-service/src/invocation/work_attempt_exec.rs index 7412ecd358..a6273763f8 100644 --- a/crates/tracedecay-daemon-service/src/invocation/work_attempt_exec.rs +++ b/crates/tracedecay-daemon-service/src/invocation/work_attempt_exec.rs @@ -60,8 +60,8 @@ use tracedecay_application::observability::{ record_work_operation_resource, }; use tracedecay_contracts::{ - WorkAttemptEffectDispatchOutcomeV1, WorkAttemptEffectResolutionV1, WorkAttemptEffectServiceV1, - WorkAttemptEffectStoragePortV1, WorkAttemptEvidenceRecordV1, WorkAttemptProviderOutcomeV1, + ApplicationProblem, WorkAttemptEffectDispatchOutcomeV1, WorkAttemptEffectHolderV1, + WorkAttemptEffectResolutionV1, WorkAttemptEvidenceRecordV1, WorkAttemptProviderOutcomeV1, WorkProviderAvailabilityV1, WorkProviderFallbackRecordV1, WorkflowArtifactPayload, WorkflowArtifactStorePort, workflow_artifact_payload_digest, }; @@ -154,6 +154,157 @@ fn provider_stdout_artifacts( #[cfg(test)] mod tests; +/// The durable attempt and effect-holder steps the provider driver awaits +/// between process events. The registered Work services run each one as a +/// writer job, so a step never parks the task's thread on the writer. +pub(super) trait AttemptStepsV1: Sync { + fn mark_running( + &self, + context: &RequestContext, + identity: &WorkAttemptIdentityV1, + actual_route: WorkProviderRouteV1, + ) -> impl Future> + Send; + + fn mark_provider_unavailable( + &self, + context: &RequestContext, + identity: &WorkAttemptIdentityV1, + observed_at: UtcMicros, + ) -> impl Future> + Send; + + fn fail_recovery( + &self, + context: &RequestContext, + identity: &WorkAttemptIdentityV1, + evidence: &WorkAttemptEvidenceRecordV1, + ) -> impl Future> + Send; + + fn acknowledge_cancellation( + &self, + context: &RequestContext, + identity: &WorkAttemptIdentityV1, + acknowledged_at: UtcMicros, + ) -> impl Future> + Send; + + fn escalate_cancellation( + &self, + context: &RequestContext, + identity: &WorkAttemptIdentityV1, + escalated_at: UtcMicros, + ) -> impl Future> + Send; + + fn settle_with_artifacts( + &self, + context: &RequestContext, + identity: &WorkAttemptIdentityV1, + evidence: &WorkAttemptEvidenceRecordV1, + artifacts: Vec, + ) -> impl Future> + Send; + + fn record_dispatch( + &self, + context: &RequestContext, + attempt: &WorkAttemptV1, + dispatched_at: UtcMicros, + ) -> impl Future> + Send; + + fn settle_dispatch( + &self, + context: &RequestContext, + attempt: &WorkAttemptIdentityV1, + resolution: WorkAttemptEffectResolutionV1, + resolved_at: UtcMicros, + ) -> impl Future> + Send; +} + +impl AttemptStepsV1 for tracedecay_application::work::RegisteredWorkApplicationServicesV1 { + fn mark_running( + &self, + context: &RequestContext, + identity: &WorkAttemptIdentityV1, + actual_route: WorkProviderRouteV1, + ) -> impl Future> + Send { + self.mark_running(context.clone(), identity.clone(), actual_route) + } + + fn mark_provider_unavailable( + &self, + context: &RequestContext, + identity: &WorkAttemptIdentityV1, + observed_at: UtcMicros, + ) -> impl Future> + Send { + self.mark_provider_unavailable(context.clone(), identity.clone(), observed_at) + } + + fn fail_recovery( + &self, + context: &RequestContext, + identity: &WorkAttemptIdentityV1, + evidence: &WorkAttemptEvidenceRecordV1, + ) -> impl Future> + Send { + self.fail_recovery(context.clone(), identity.clone(), evidence.clone()) + } + + fn acknowledge_cancellation( + &self, + context: &RequestContext, + identity: &WorkAttemptIdentityV1, + acknowledged_at: UtcMicros, + ) -> impl Future> + Send { + self.acknowledge_cancellation(context.clone(), identity.clone(), acknowledged_at) + } + + fn escalate_cancellation( + &self, + context: &RequestContext, + identity: &WorkAttemptIdentityV1, + escalated_at: UtcMicros, + ) -> impl Future> + Send { + self.escalate_cancellation(context.clone(), identity.clone(), escalated_at) + } + + fn settle_with_artifacts( + &self, + context: &RequestContext, + identity: &WorkAttemptIdentityV1, + evidence: &WorkAttemptEvidenceRecordV1, + artifacts: Vec, + ) -> impl Future> + Send { + self.settle_with_artifacts( + context.clone(), + identity.clone(), + evidence.clone(), + artifacts, + ) + } + + fn record_dispatch( + &self, + context: &RequestContext, + attempt: &WorkAttemptV1, + dispatched_at: UtcMicros, + ) -> impl Future> + Send + { + self.record_effect_dispatch( + context.clone(), + attempt.identity().clone(), + attempt.execution().effect_state(), + dispatched_at, + attempt.execution().deadline(), + ) + } + + fn settle_dispatch( + &self, + context: &RequestContext, + attempt: &WorkAttemptIdentityV1, + resolution: WorkAttemptEffectResolutionV1, + resolved_at: UtcMicros, + ) -> impl Future> + Send { + self.settle_effect_dispatch(context.clone(), attempt.clone(), resolution, resolved_at) + } +} + /// How long an acknowledged cancellation may run before escalation to /// forced termination. const CANCELLATION_GRACE: std::time::Duration = std::time::Duration::from_secs(10); @@ -292,9 +443,11 @@ impl WorkAttemptProcessRegistryV1 { let task_key = key.clone(); let (start, started) = tokio::sync::oneshot::channel(); let future = task(Arc::clone(&cancellation)); - // The attempt calls synchronous Work storage ports (`mark_running`, - // the settle paths) that park their thread on the writer, so it runs - // on a blocking-pool thread rather than a native worker. + // The attempt's own Work steps are awaited writer jobs, but it still + // persists provider output through the synchronous workflow artifact + // store and reconciles the workflow fan-out afterwards, both of which + // park their thread on the writer, so it runs on a blocking-pool thread + // rather than a native worker. let Ok(task) = native_runtime::NativeTask::spawn_off_native_thread(async move { let admitted = tokio::select! { result = started => result.is_ok(), @@ -560,8 +713,6 @@ async fn run_attempt( return; } }; - let attempts = services.attempts(); - let attempt_effects = services.attempt_effects(); let workflow_artifacts = registered.database.workflow_storage().map_err(|error| { tracing::warn!( task = attempt.identity().task_id().as_str(), @@ -592,8 +743,7 @@ async fn run_attempt( Ok(selection) => match selection.provider.protocol { WorkProviderProtocol::CodexAppServerJsonRpc => { execute_app_server( - attempts, - attempt_effects, + &services, workflow_artifacts.as_ref().ok(), &context, &attempt, @@ -608,8 +758,7 @@ async fn run_attempt( } _ => { execute_provider_with_environment( - attempts, - attempt_effects, + &services, workflow_artifacts.as_ref().ok(), &context, &attempt, @@ -625,7 +774,7 @@ async fn run_attempt( }, Err(denial) => { settle_unstarted( - attempts, + &services, &context, &identity, &attempt, @@ -634,7 +783,8 @@ async fn run_attempt( }, denial.fallback, observability_producer.as_deref(), - ); + ) + .await; } } } @@ -795,32 +945,34 @@ fn availability_state(error: WorkExecutableBindingError) -> WorkProviderAvailabi /// Seals a terminal denial for an attempt whose provider never started: /// fence to `RecoveryRequired`, then fail recovery with the typed outcome. -fn settle_unstarted( - attempts: &tracedecay_contracts::WorkAttemptService, +async fn settle_unstarted( + steps: &impl AttemptStepsV1, context: &RequestContext, identity: &WorkAttemptIdentityV1, attempt: &WorkAttemptV1, outcome: WorkAttemptProviderOutcomeV1, provider_fallback: Option, observability_producer: Option<&BoundedObservabilityProducerV1>, -) where - S: tracedecay_contracts::WorkAttemptStoragePort, -{ +) { let observed_at = now_micros(); - if let Err(problem) = attempts.mark_provider_unavailable(context, identity, observed_at) { + if let Err(problem) = steps + .mark_provider_unavailable(context, identity, observed_at) + .await + { tracing::warn!( task = identity.task_id().as_str(), ?problem, "work attempt could not be fenced for provider unavailability" ); seal_cancellation_before_running( - attempts, + steps, context, attempt, None, provider_fallback, observability_producer, - ); + ) + .await; return; } let evidence = WorkAttemptEvidenceRecordV1 { @@ -834,7 +986,7 @@ fn settle_unstarted( provider_fallback, observed_at, }; - match attempts.fail_recovery(context, identity, &evidence) { + match steps.fail_recovery(context, identity, &evidence).await { Ok(settled) => { let _ = record_terminal_attempt_product_views(observability_producer, &settled); } @@ -852,20 +1004,19 @@ fn settle_unstarted( /// the race: the row is `CancellationRequested`, which cannot become /// `Running`, and no provider is left to exit. Seal it here as `Cancelled`. /// Any other refusal leaves the durable row as the authority. -fn seal_cancellation_before_running( - attempts: &tracedecay_contracts::WorkAttemptService, +async fn seal_cancellation_before_running( + steps: &impl AttemptStepsV1, context: &RequestContext, attempt: &WorkAttemptV1, actual_route: Option, provider_fallback: Option, observability_producer: Option<&BoundedObservabilityProducerV1>, -) where - S: tracedecay_contracts::WorkAttemptStoragePort, -{ +) { let identity = attempt.identity(); let observed_at = now_micros(); - if attempts + if steps .acknowledge_cancellation(context, identity, observed_at) + .await .is_err() { return; @@ -881,7 +1032,10 @@ fn seal_cancellation_before_running( provider_fallback, observed_at, }; - match attempts.settle_with_artifacts(context, identity, &evidence, Vec::new()) { + match steps + .settle_with_artifacts(context, identity, &evidence, Vec::new()) + .await + { Ok(settled) => { let _ = record_terminal_attempt_product_views(observability_producer, &settled); } @@ -901,21 +1055,12 @@ enum EffectDispatchAdmission { Unavailable, } -fn record_effect_dispatch( - effects: &WorkAttemptEffectServiceV1, +async fn record_effect_dispatch( + steps: &impl AttemptStepsV1, context: &RequestContext, attempt: &WorkAttemptV1, -) -> EffectDispatchAdmission -where - S: WorkAttemptEffectStoragePortV1, -{ - match effects.record_dispatch( - context, - attempt.identity().clone(), - attempt.execution().effect_state(), - now_micros(), - attempt.execution().deadline(), - ) { +) -> EffectDispatchAdmission { + match steps.record_dispatch(context, attempt, now_micros()).await { Ok(WorkAttemptEffectDispatchOutcomeV1::Recorded(_)) => EffectDispatchAdmission::Recorded, Ok(WorkAttemptEffectDispatchOutcomeV1::Replayed(_)) => { tracing::warn!( @@ -935,15 +1080,12 @@ where } } -fn settle_effect_dispatch( - effects: &WorkAttemptEffectServiceV1, +async fn settle_effect_dispatch( + steps: &impl AttemptStepsV1, context: &RequestContext, attempt: &WorkAttemptV1, provider_started: bool, -) -> bool -where - S: WorkAttemptEffectStoragePortV1, -{ +) -> bool { let resolution = if !provider_started || attempt.execution().effect_state() == tracedecay_domain::WorkEffectStateV1::Observational { @@ -951,7 +1093,10 @@ where } else { WorkAttemptEffectResolutionV1::Unknown }; - if let Err(problem) = effects.settle(context, attempt.identity(), resolution, now_micros()) { + if let Err(problem) = steps + .settle_dispatch(context, attempt.identity(), resolution, now_micros()) + .await + { tracing::warn!( task = attempt.identity().task_id().as_str(), ?problem, @@ -985,9 +1130,8 @@ fn admitted_provider_environment( level = "trace", skip_all )] -async fn execute_provider_with_environment( - attempts: &tracedecay_contracts::WorkAttemptService, - attempt_effects: &WorkAttemptEffectServiceV1, +async fn execute_provider_with_environment( + steps: &impl AttemptStepsV1, workflow_artifacts: Option<&tracedecay_rusqlite_runtime::workflow::WorkflowSqliteAuthority>, context: &RequestContext, attempt: &WorkAttemptV1, @@ -997,9 +1141,7 @@ async fn execute_provider_with_environment( observability_producer: Option<&BoundedObservabilityProducerV1>, topology_policy_digest: Option<&TopologyPolicyDigestV1>, timing: AttemptAdmissionTimingV1, -) where - S: tracedecay_contracts::WorkAttemptStoragePort + WorkAttemptEffectStoragePortV1, -{ +) { let identity = attempt.identity().clone(); let envelope = attempt.execution(); let resolved = &selection.provider; @@ -1017,12 +1159,12 @@ async fn execute_provider_with_environment( command.env(key, value); } - match record_effect_dispatch(attempt_effects, context, attempt) { + match record_effect_dispatch(steps, context, attempt).await { EffectDispatchAdmission::Recorded => {} EffectDispatchAdmission::Replayed => return, EffectDispatchAdmission::Unavailable => { settle_unstarted( - attempts, + steps, context, &identity, attempt, @@ -1031,7 +1173,8 @@ async fn execute_provider_with_environment( }, selection.fallback.clone(), observability_producer, - ); + ) + .await; return; } } @@ -1051,22 +1194,26 @@ async fn execute_provider_with_environment( ?error, "work attempt provider process could not be spawned" ); - if !settle_effect_dispatch(attempt_effects, context, attempt, false) { + if !settle_effect_dispatch(steps, context, attempt, false).await { return; } settle_unstarted( - attempts, + steps, context, &identity, attempt, WorkAttemptProviderOutcomeV1::LaunchFailed, selection.fallback.clone(), observability_producer, - ); + ) + .await; return; } }; - let running = match attempts.mark_running(context, &identity, selection.actual_route.clone()) { + let running = match steps + .mark_running(context, &identity, selection.actual_route.clone()) + .await + { Ok(running) => running, Err(problem) => { // The lease no longer admits this task (fenced by recovery or a @@ -1079,15 +1226,16 @@ async fn execute_provider_with_environment( ); process.signal(TerminationSignal::Kill); let _ = process.wait().await; - if settle_effect_dispatch(attempt_effects, context, attempt, true) { + if settle_effect_dispatch(steps, context, attempt, true).await { seal_cancellation_before_running( - attempts, + steps, context, attempt, Some(selection.actual_route.clone()), selection.fallback.clone(), observability_producer, - ); + ) + .await; } return; } @@ -1110,7 +1258,7 @@ async fn execute_provider_with_environment( let outcome = match ending { Ok(Some(Ok(status))) => exit_outcome(status), Ok(Some(Err(_))) => WorkAttemptProviderOutcomeV1::LaunchFailed, - Ok(None) => cancel_ladder(attempts, context, &identity, &mut process).await, + Ok(None) => cancel_ladder(steps, context, &identity, &mut process).await, // An unavailable timer fails closed as an elapsed wall budget. Err(_) => { let stalled_for = started.elapsed(); @@ -1151,10 +1299,13 @@ async fn execute_provider_with_environment( provider_fallback: selection.fallback.clone(), observed_at: now_micros(), }; - if !settle_effect_dispatch(attempt_effects, context, attempt, true) { + if !settle_effect_dispatch(steps, context, attempt, true).await { return; } - match attempts.settle_with_artifacts(context, &identity, &evidence, artifacts) { + match steps + .settle_with_artifacts(context, &identity, &evidence, artifacts) + .await + { Ok(settled) => { let _ = record_terminal_attempt_product_views(observability_producer, &settled); if let Some(observation) = @@ -1207,9 +1358,8 @@ struct AppServerSessionOutput { level = "trace", skip_all )] -async fn execute_app_server( - attempts: &tracedecay_contracts::WorkAttemptService, - attempt_effects: &WorkAttemptEffectServiceV1, +async fn execute_app_server( + steps: &impl AttemptStepsV1, workflow_artifacts: Option<&tracedecay_rusqlite_runtime::workflow::WorkflowSqliteAuthority>, context: &RequestContext, attempt: &WorkAttemptV1, @@ -1219,19 +1369,17 @@ async fn execute_app_server( observability_producer: Option<&BoundedObservabilityProducerV1>, topology_policy_digest: Option<&TopologyPolicyDigestV1>, timing: AttemptAdmissionTimingV1, -) where - S: tracedecay_contracts::WorkAttemptStoragePort + WorkAttemptEffectStoragePortV1, -{ +) { let identity = attempt.identity().clone(); let envelope = attempt.execution(); let snapshot = envelope.execution_snapshot(); - match record_effect_dispatch(attempt_effects, context, attempt) { + match record_effect_dispatch(steps, context, attempt).await { EffectDispatchAdmission::Recorded => {} EffectDispatchAdmission::Replayed => return, EffectDispatchAdmission::Unavailable => { settle_unstarted( - attempts, + steps, context, &identity, attempt, @@ -1240,7 +1388,8 @@ async fn execute_app_server( }, selection.fallback.clone(), observability_producer, - ); + ) + .await; return; } } @@ -1248,7 +1397,10 @@ async fn execute_app_server( // The launch and the protocol session are one indivisible blocking call // here, so the attempt is marked Running before it starts; a launch that // never happens surfaces as a typed failure on the settle path instead. - let running = match attempts.mark_running(context, &identity, selection.actual_route.clone()) { + let running = match steps + .mark_running(context, &identity, selection.actual_route.clone()) + .await + { Ok(running) => running, Err(problem) => { tracing::warn!( @@ -1256,15 +1408,16 @@ async fn execute_app_server( ?problem, "work attempt could not be marked running; app-server was not started" ); - if settle_effect_dispatch(attempt_effects, context, attempt, false) { + if settle_effect_dispatch(steps, context, attempt, false).await { seal_cancellation_before_running( - attempts, + steps, context, attempt, None, selection.fallback.clone(), observability_producer, - ); + ) + .await; } return; } @@ -1343,7 +1496,9 @@ async fn execute_app_server( // the whole process tree, so the ladder acknowledges and stops // there rather than pretending an interrupt was survived. if let Err(problem) = - attempts.acknowledge_cancellation(context, &identity, now_micros()) + steps + .acknowledge_cancellation(context, &identity, now_micros()) + .await { tracing::warn!( task = identity.task_id().as_str(), @@ -1433,14 +1588,19 @@ async fn execute_app_server( observed_at: now_micros(), }; if !settle_effect_dispatch( - attempt_effects, + steps, context, attempt, launch_receipt.started_at().is_some(), - ) { + ) + .await + { return; } - match attempts.settle_with_artifacts(context, &identity, &evidence, artifacts) { + match steps + .settle_with_artifacts(context, &identity, &evidence, artifacts) + .await + { Ok(settled) => { let _ = record_terminal_attempt_product_views(observability_producer, &settled); if let Some(observation) = launch_receipt.started_at().and_then(|started| { @@ -1513,16 +1673,16 @@ fn offer_no_progress_observation( level = "trace", skip_all )] -async fn cancel_ladder( - attempts: &tracedecay_contracts::WorkAttemptService, +async fn cancel_ladder( + steps: &impl AttemptStepsV1, context: &RequestContext, identity: &WorkAttemptIdentityV1, process: &mut ProviderProcess, -) -> WorkAttemptProviderOutcomeV1 -where - S: tracedecay_contracts::WorkAttemptStoragePort, -{ - if let Err(problem) = attempts.acknowledge_cancellation(context, identity, now_micros()) { +) -> WorkAttemptProviderOutcomeV1 { + if let Err(problem) = steps + .acknowledge_cancellation(context, identity, now_micros()) + .await + { tracing::warn!( task = identity.task_id().as_str(), ?problem, @@ -1534,7 +1694,10 @@ where .await .is_err() { - if let Err(problem) = attempts.escalate_cancellation(context, identity, now_micros()) { + if let Err(problem) = steps + .escalate_cancellation(context, identity, now_micros()) + .await + { tracing::warn!( task = identity.task_id().as_str(), ?problem, diff --git a/crates/tracedecay-daemon-service/src/invocation/work_attempt_exec/tests.rs b/crates/tracedecay-daemon-service/src/invocation/work_attempt_exec/tests.rs index a706409b38..4e8c3fea90 100644 --- a/crates/tracedecay-daemon-service/src/invocation/work_attempt_exec/tests.rs +++ b/crates/tracedecay-daemon-service/src/invocation/work_attempt_exec/tests.rs @@ -42,9 +42,9 @@ use tracedecay_contracts::{ DisclosureClass, ObservabilityHorizonV1, ObservabilityQueryPort, ObservabilityQueryV1, RequestId, ResolvedScope, WorkAttemptAdmissionKind, WorkAttemptCapacityV1, WorkAttemptCapacityVerdictV1, WorkAttemptEffectDispatchOutcomeV1, WorkAttemptEffectHolderV1, - WorkAttemptEffectResolutionV1, WorkAttemptEffectServiceV1, WorkAttemptEffectStorageErrorV1, - WorkAttemptEffectStoragePortV1, WorkAttemptInsertOutcome, WorkAttemptListPageV1, - WorkAttemptService, WorkAttemptStatusRequestV1, WorkAttemptStorageError, + WorkAttemptEffectReadPortV1, WorkAttemptEffectResolutionV1, WorkAttemptEffectServiceV1, + WorkAttemptEffectStorageErrorV1, WorkAttemptEffectStoragePortV1, WorkAttemptInsertOutcome, + WorkAttemptListPageV1, WorkAttemptService, WorkAttemptStatusRequestV1, WorkAttemptStorageError, WorkAttemptStoragePort, WorkAttemptStreamChannelV1, WorkAttemptStreamSummaryV1, }; use tracedecay_domain::{ @@ -455,7 +455,9 @@ impl WorkAttemptEffectStoragePortV1 for AttemptStore { rows.effects.insert(key, next.clone()); Ok(next) } +} +impl WorkAttemptEffectReadPortV1 for AttemptStore { fn load_effect_dispatch( &self, authority: &WorkAuthority, @@ -582,6 +584,120 @@ impl Fixture { } } +impl Fixture { + fn steps(&self) -> FixtureSteps<'_> { + FixtureSteps { + attempts: &self.attempts, + effects: &self.effects, + } + } +} + +/// The driver's steps answered synchronously by the in-memory services. +struct FixtureSteps<'fixture> { + attempts: &'fixture WorkAttemptService, + effects: &'fixture WorkAttemptEffectServiceV1, +} + +impl AttemptStepsV1 for FixtureSteps<'_> { + fn mark_running( + &self, + context: &RequestContext, + identity: &WorkAttemptIdentityV1, + actual_route: WorkProviderRouteV1, + ) -> impl Future> + Send { + std::future::ready(self.attempts.mark_running(context, identity, actual_route)) + } + + fn mark_provider_unavailable( + &self, + context: &RequestContext, + identity: &WorkAttemptIdentityV1, + observed_at: UtcMicros, + ) -> impl Future> + Send { + std::future::ready( + self.attempts + .mark_provider_unavailable(context, identity, observed_at), + ) + } + + fn fail_recovery( + &self, + context: &RequestContext, + identity: &WorkAttemptIdentityV1, + evidence: &WorkAttemptEvidenceRecordV1, + ) -> impl Future> + Send { + std::future::ready(self.attempts.fail_recovery(context, identity, evidence)) + } + + fn acknowledge_cancellation( + &self, + context: &RequestContext, + identity: &WorkAttemptIdentityV1, + acknowledged_at: UtcMicros, + ) -> impl Future> + Send { + std::future::ready(self.attempts.acknowledge_cancellation( + context, + identity, + acknowledged_at, + )) + } + + fn escalate_cancellation( + &self, + context: &RequestContext, + identity: &WorkAttemptIdentityV1, + escalated_at: UtcMicros, + ) -> impl Future> + Send { + std::future::ready( + self.attempts + .escalate_cancellation(context, identity, escalated_at), + ) + } + + fn settle_with_artifacts( + &self, + context: &RequestContext, + identity: &WorkAttemptIdentityV1, + evidence: &WorkAttemptEvidenceRecordV1, + artifacts: Vec, + ) -> impl Future> + Send { + std::future::ready( + self.attempts + .settle_with_artifacts(context, identity, evidence, artifacts), + ) + } + + fn record_dispatch( + &self, + context: &RequestContext, + attempt: &WorkAttemptV1, + dispatched_at: UtcMicros, + ) -> impl Future> + Send + { + std::future::ready(self.effects.record_dispatch( + context, + attempt.identity().clone(), + attempt.execution().effect_state(), + dispatched_at, + attempt.execution().deadline(), + )) + } + + fn settle_dispatch( + &self, + context: &RequestContext, + attempt: &WorkAttemptIdentityV1, + resolution: WorkAttemptEffectResolutionV1, + resolved_at: UtcMicros, + ) -> impl Future> + Send { + std::future::ready( + self.effects + .settle(context, attempt, resolution, resolved_at), + ) + } +} + /// Persists one canonically bound attempt in `Leased`, the state handed to the /// live provider path. Proposal selection is deliberately explicit here: this /// provider-runtime fixture does not invent routing evidence through a Work @@ -828,8 +944,7 @@ fn a_clean_provider_run_seals_succeeded_evidence_over_the_captured_stream() { let admitted_environment = admitted_provider_environment(fixture.attempt.execution().execution_snapshot()); execute_provider_with_environment( - &fixture.attempts, - &fixture.effects, + &fixture.steps(), Some(&workflow_artifacts), &fixture.context, &fixture.attempt, @@ -1013,8 +1128,7 @@ async fn initial_provider_child_uses_values_captured_for_that_spawn() { std::env::set_var(&ambient_secret, "ambient-secret"); } execute_provider_with_environment( - &fixture.attempts, - &fixture.effects, + &fixture.steps(), Some(&workflow_artifacts), &fixture.context, &fixture.attempt, @@ -1079,8 +1193,7 @@ fn stdout_past_the_admitted_cap_is_a_typed_overflow_not_a_silent_success() { let admitted_environment = admitted_provider_environment(fixture.attempt.execution().execution_snapshot()); execute_provider_with_environment( - &fixture.attempts, - &fixture.effects, + &fixture.steps(), Some(&workflow_artifacts), &fixture.context, &fixture.attempt, @@ -1261,10 +1374,10 @@ fn a_provider_that_ignores_interrupt_is_escalated_to_a_kill_on_the_record() { std::future::pending::<()>().await; }; + let steps = fixture.steps(); tokio::select! { () = execute_provider_with_environment( - &fixture.attempts, - &fixture.effects, + &steps, Some(&workflow_artifacts), &fixture.context, &fixture.attempt, @@ -1397,8 +1510,7 @@ async fn a_cancellation_requested_before_mark_running_seals_cancelled() { let route = requested_route(WorkProviderBackendV1::ClaudeCodeCli); execute_provider_with_environment( - &fixture.attempts, - &fixture.effects, + &fixture.steps(), Some(&workflow_artifacts), &fixture.context, &fixture.attempt, @@ -1502,8 +1614,7 @@ async fn a_wall_exhausted_provider_seals_timed_out_and_emits_the_no_progress_ter let admitted_environment = admitted_provider_environment(fixture.attempt.execution().execution_snapshot()); execute_provider_with_environment( - &fixture.attempts, - &fixture.effects, + &fixture.steps(), Some(&workflow_artifacts), &fixture.context, &fixture.attempt, @@ -1741,8 +1852,7 @@ async fn a_disqualified_app_server_falls_back_to_codex_cli_and_says_so_in_the_ev let admitted_environment = admitted_provider_environment(fixture.attempt.execution().execution_snapshot()); execute_provider_with_environment( - &fixture.attempts, - &fixture.effects, + &fixture.steps(), Some(&workflow_artifacts), &fixture.context, &fixture.attempt, @@ -1884,7 +1994,7 @@ async fn a_fallback_that_is_also_refused_keeps_both_denials_on_the_record() { // Sealed exactly as `run_attempt` seals it. settle_unstarted( - &fixture.attempts, + &fixture.steps(), &fixture.context, fixture.identity(), &fixture.attempt, @@ -1893,7 +2003,8 @@ async fn a_fallback_that_is_also_refused_keeps_both_denials_on_the_record() { }, denial.fallback.clone(), None, - ); + ) + .await; assert_eq!(fixture.state(), WorkAttemptStateV1::Failed); let evidence = fixture.sealed_evidence(); assert_eq!( @@ -1987,8 +2098,7 @@ async fn a_missing_provider_executable_seals_a_typed_denial_instead_of_panicking let admitted_environment = admitted_provider_environment(fixture.attempt.execution().execution_snapshot()); execute_provider_with_environment( - &fixture.attempts, - &fixture.effects, + &fixture.steps(), Some(&workflow_artifacts), &fixture.context, &fixture.attempt, @@ -2036,7 +2146,7 @@ async fn an_unavailable_provider_state_is_sealed_as_denial_evidence() { ); settle_unstarted( - &fixture.attempts, + &fixture.steps(), &fixture.context, fixture.identity(), &fixture.attempt, @@ -2045,7 +2155,8 @@ async fn an_unavailable_provider_state_is_sealed_as_denial_evidence() { }, None, None, - ); + ) + .await; assert_eq!(fixture.state(), WorkAttemptStateV1::Failed); assert_eq!( diff --git a/crates/tracedecay-rusqlite-runtime/src/work/effect_holder.rs b/crates/tracedecay-rusqlite-runtime/src/work/effect_holder.rs index de4f93828d..388fbce7cf 100644 --- a/crates/tracedecay-rusqlite-runtime/src/work/effect_holder.rs +++ b/crates/tracedecay-rusqlite-runtime/src/work/effect_holder.rs @@ -2,43 +2,65 @@ use serde::Deserialize; use tracedecay_contracts::{ - WorkAttemptEffectDispatchOutcomeV1, WorkAttemptEffectHolderV1, WorkAttemptEffectResolutionV1, - WorkAttemptEffectStorageErrorV1, WorkAttemptEffectStoragePortV1, + WorkAttemptEffectDispatchOutcomeV1, WorkAttemptEffectHolderV1, WorkAttemptEffectReadPortV1, + WorkAttemptEffectResolutionV1, WorkAttemptEffectStorageErrorV1, WorkAttemptEffectStoragePortV1, }; use tracedecay_domain::{UtcMicros, WorkAttemptIdentityV1, WorkAttemptV1, WorkAuthority}; -use crate::exact_sql::{ExactSqlTransaction, ExactSqlValue}; +use crate::exact_sql::ExactSqlValue; use crate::work::{ - RegisteredWorkQuery, WorkSqliteStorage, authority_params_owned, exact_sql_integer, - exact_sql_statement, exact_sql_text, registered_work_query, + RegisteredWorkQuery, RegisteredWorkWrite, WorkSqliteStorage, authority_params_owned, + exact_sql_integer, exact_sql_statement, exact_sql_text, registered_work_query, }; +use crate::work_run_control::WorkSqliteJobStorage; type StorageError = WorkAttemptEffectStorageErrorV1; -impl WorkAttemptEffectStoragePortV1 for WorkSqliteStorage { +impl WorkAttemptEffectReadPortV1 for WorkSqliteStorage { + fn load_effect_dispatch( + &self, + authority: &WorkAuthority, + attempt: &WorkAttemptIdentityV1, + ) -> Result, StorageError> { + let transaction = self + .handle() + .begin_deferred() + .map_err(|_| StorageError::Unavailable)?; + let holder = load_holder(&transaction, authority, attempt)?; + transaction + .commit() + .map_err(|_| StorageError::Unavailable)?; + Ok(holder) + } +} + +impl WorkAttemptEffectReadPortV1 for WorkSqliteJobStorage<'_> { + fn load_effect_dispatch( + &self, + authority: &WorkAuthority, + attempt: &WorkAttemptIdentityV1, + ) -> Result, StorageError> { + load_holder(self.transaction, authority, attempt) + } +} + +/// The writer job commits or rolls back the holder write with the job. +impl WorkAttemptEffectStoragePortV1 for WorkSqliteJobStorage<'_> { fn begin_effect_dispatch( &self, authority: &WorkAuthority, holder: &WorkAttemptEffectHolderV1, ) -> Result { holder.validate().map_err(|_| StorageError::Conflict)?; - let transaction = self - .handle() - .begin_immediate() - .map_err(|_| StorageError::Unavailable)?; - require_open_attempt(&transaction, authority, holder)?; - if let Some(existing) = load_holder(&transaction, authority, holder.attempt())? { - let _ = transaction.rollback(); + require_open_attempt(self.transaction, authority, holder)?; + if let Some(existing) = load_holder(self.transaction, authority, holder.attempt())? { return if same_dispatch(&existing, holder) { Ok(WorkAttemptEffectDispatchOutcomeV1::Replayed(existing)) } else { Err(StorageError::Conflict) }; } - insert_holder(&transaction, authority, holder)?; - transaction - .commit() - .map_err(|_| StorageError::Unavailable)?; + insert_holder(self.transaction, authority, holder)?; Ok(WorkAttemptEffectDispatchOutcomeV1::Recorded(holder.clone())) } @@ -49,22 +71,13 @@ impl WorkAttemptEffectStoragePortV1 for WorkSqliteStorage { resolution: WorkAttemptEffectResolutionV1, resolved_at: UtcMicros, ) -> Result { - let transaction = self - .handle() - .begin_immediate() - .map_err(|_| StorageError::Unavailable)?; - let existing = load_holder(&transaction, authority, attempt)? + let existing = load_holder(self.transaction, authority, attempt)? .ok_or(StorageError::NotFoundOrNotAuthorized)?; let next = match existing.resolution() { None => existing .with_resolution(resolution, resolved_at) .map_err(|_| StorageError::Conflict)?, - Some(current) if current == resolution => { - transaction - .commit() - .map_err(|_| StorageError::Unavailable)?; - return Ok(existing); - } + Some(current) if current == resolution => return Ok(existing), Some(WorkAttemptEffectResolutionV1::Unknown) if resolution == WorkAttemptEffectResolutionV1::NoEffect => { @@ -72,37 +85,15 @@ impl WorkAttemptEffectStoragePortV1 for WorkSqliteStorage { .with_resolution(resolution, resolved_at) .map_err(|_| StorageError::Conflict)? } - Some(_) => { - let _ = transaction.rollback(); - return Err(StorageError::Conflict); - } + Some(_) => return Err(StorageError::Conflict), }; - update_holder(&transaction, authority, &next)?; - transaction - .commit() - .map_err(|_| StorageError::Unavailable)?; + update_holder(self.transaction, authority, &next)?; Ok(next) } - - fn load_effect_dispatch( - &self, - authority: &WorkAuthority, - attempt: &WorkAttemptIdentityV1, - ) -> Result, StorageError> { - let transaction = self - .handle() - .begin_deferred() - .map_err(|_| StorageError::Unavailable)?; - let holder = load_holder(&transaction, authority, attempt)?; - transaction - .commit() - .map_err(|_| StorageError::Unavailable)?; - Ok(holder) - } } fn require_open_attempt( - transaction: &ExactSqlTransaction, + transaction: &impl RegisteredWorkWrite, authority: &WorkAuthority, holder: &WorkAttemptEffectHolderV1, ) -> Result<(), StorageError> { @@ -206,14 +197,14 @@ fn same_dispatch( } fn insert_holder( - transaction: &ExactSqlTransaction, + transaction: &impl RegisteredWorkWrite, authority: &WorkAuthority, holder: &WorkAttemptEffectHolderV1, ) -> Result<(), StorageError> { holder.validate().map_err(|_| StorageError::Conflict)?; let payload = serde_json::to_string(holder).map_err(|_| StorageError::Unavailable)?; transaction - .execute( + .work_execute( exact_sql_statement( "INSERT INTO work_attempt_effect_holders_v1 ( project_id, repository_id, worktree_id, actor_id, policy_digest, @@ -242,14 +233,14 @@ fn insert_holder( } fn update_holder( - transaction: &ExactSqlTransaction, + transaction: &impl RegisteredWorkWrite, authority: &WorkAuthority, holder: &WorkAttemptEffectHolderV1, ) -> Result<(), StorageError> { holder.validate().map_err(|_| StorageError::Conflict)?; let payload = serde_json::to_string(holder).map_err(|_| StorageError::Unavailable)?; let changed = transaction - .execute( + .work_execute( exact_sql_statement( "UPDATE work_attempt_effect_holders_v1 SET resolution = ?9, resolved_at = ?10, holder_payload = ?11 diff --git a/crates/tracedecay-rusqlite-runtime/src/work/retry.rs b/crates/tracedecay-rusqlite-runtime/src/work/retry.rs index 06399cd317..8d9699d9d5 100644 --- a/crates/tracedecay-rusqlite-runtime/src/work/retry.rs +++ b/crates/tracedecay-rusqlite-runtime/src/work/retry.rs @@ -11,9 +11,9 @@ use tracedecay_domain::{ canonical_sha256, }; -use crate::exact_sql::{ExactSqlTransaction, ExactSqlValue}; +use crate::exact_sql::ExactSqlValue; use crate::work::{ - WorkSqliteStorage, authority_params_owned, exact_sql_integer, exact_sql_statement, + RegisteredWorkWrite, authority_params_owned, exact_sql_integer, exact_sql_statement, exact_sql_text, registered_work_query, }; use crate::work_attempt::identity_params; @@ -25,21 +25,13 @@ struct StoredWorkAttemptV1 { synthesis: Option, } -impl WorkRetryStoragePortV1 for WorkSqliteStorage { +impl WorkRetryStoragePortV1 for crate::work_run_control::WorkSqliteJobStorage<'_> { fn retry_by_command( &self, authority: &WorkAuthority, command_id: &tracedecay_domain::WorkCommandId, ) -> Result, WorkAttemptStorageError> { - let transaction = self - .handle() - .begin_deferred() - .map_err(|_| WorkAttemptStorageError::Unavailable)?; - let outcome = replay(&transaction, authority, command_id.as_str())?; - transaction - .commit() - .map_err(|_| WorkAttemptStorageError::Unavailable)?; - Ok(outcome) + replay(self.transaction, authority, command_id.as_str()) } fn insert_retry_bounded( @@ -48,38 +40,26 @@ impl WorkRetryStoragePortV1 for WorkSqliteStorage { write: &WorkRetryWriteV1, concurrency: &TopologyConcurrencyPolicyV1, ) -> Result { - let transaction = self - .handle() - .begin_immediate() - .map_err(|_| WorkAttemptStorageError::Unavailable)?; - let outcome = - insert_retry_bounded_in_transaction(&transaction, authority, write, concurrency); - match outcome { - Ok(created @ WorkRetryAttemptOutcomeV1::Created { .. }) => { - transaction - .commit() - .map_err(|_| WorkAttemptStorageError::Unavailable)?; - Ok(created) - } - Ok(replayed @ WorkRetryAttemptOutcomeV1::Replayed { .. }) => { - transaction - .rollback() - .map_err(|_| WorkAttemptStorageError::Unavailable)?; - Ok(replayed) - } - Err(error) => { - transaction - .rollback() - .map_err(|_| WorkAttemptStorageError::Unavailable)?; - Err(error) - } - } + insert_retry_bounded_in_transaction(self.transaction, authority, write, concurrency) + } + + fn admit_retry( + &self, + admission: &tracedecay_contracts::WorkProductRetryAdmissionV1, + ) -> Result< + ( + tracedecay_contracts::WorkProductEventCommitV1, + WorkRetryAttemptOutcomeV1, + ), + tracedecay_contracts::WorkProductAttemptAdmissionErrorV1, + > { + crate::work_product::admit_retry(self.transaction, admission) } } /// Persist one retry and receipt without settling the caller-owned transaction. pub(crate) fn insert_retry_bounded_in_transaction( - transaction: &ExactSqlTransaction, + transaction: &impl RegisteredWorkWrite, authority: &WorkAuthority, write: &WorkRetryWriteV1, concurrency: &TopologyConcurrencyPolicyV1, @@ -132,7 +112,7 @@ fn validate_write(write: &WorkRetryWriteV1) -> Result<(), WorkAttemptStorageErro } fn replay( - transaction: &ExactSqlTransaction, + transaction: &impl RegisteredWorkWrite, authority: &WorkAuthority, command_id: &str, ) -> Result, WorkAttemptStorageError> { @@ -201,7 +181,7 @@ fn replay( } fn load_attempt( - transaction: &ExactSqlTransaction, + transaction: &impl RegisteredWorkWrite, authority: &WorkAuthority, identity: &WorkAttemptIdentityV1, ) -> Result, WorkAttemptStorageError> { @@ -263,7 +243,7 @@ const fn attempt_state(state: tracedecay_domain::WorkAttemptStateV1) -> &'static } fn require_original_retry_evidence( - transaction: &ExactSqlTransaction, + transaction: &impl RegisteredWorkWrite, authority: &WorkAuthority, write: &WorkRetryWriteV1, ) -> Result<(), WorkAttemptStorageError> { @@ -290,7 +270,7 @@ fn require_original_retry_evidence( } fn require_run_reservation( - transaction: &ExactSqlTransaction, + transaction: &impl RegisteredWorkWrite, authority: &WorkAuthority, identity: &WorkAttemptIdentityV1, ) -> Result<(), WorkAttemptStorageError> { @@ -320,7 +300,7 @@ fn require_run_reservation( } fn require_first_run_admission( - transaction: &ExactSqlTransaction, + transaction: &impl RegisteredWorkWrite, authority: &WorkAuthority, attempt: &WorkAttemptV1, ) -> Result<(), WorkAttemptStorageError> { @@ -357,7 +337,7 @@ fn require_first_run_admission( } fn insert_attempt( - transaction: &ExactSqlTransaction, + transaction: &impl RegisteredWorkWrite, authority: &WorkAuthority, attempt: &WorkAttemptV1, ) -> Result<(), WorkAttemptStorageError> { @@ -367,7 +347,7 @@ fn insert_attempt( }) .map_err(|_| WorkAttemptStorageError::Unavailable)?; transaction - .execute( + .work_execute( exact_sql_statement( "INSERT INTO work_attempts_v1 ( project_id, repository_id, worktree_id, actor_id, policy_digest, @@ -394,7 +374,7 @@ fn insert_attempt( } fn insert_receipt( - transaction: &ExactSqlTransaction, + transaction: &impl RegisteredWorkWrite, authority: &WorkAuthority, write: &WorkRetryWriteV1, ) -> Result<(), WorkAttemptStorageError> { @@ -405,7 +385,7 @@ fn insert_receipt( ) .map_err(|_| WorkAttemptStorageError::Unavailable)?; transaction - .execute( + .work_execute( exact_sql_statement( "INSERT INTO work_retry_receipts_v1 ( project_id, repository_id, worktree_id, actor_id, policy_digest, diff --git a/crates/tracedecay-rusqlite-runtime/src/work_attempt.rs b/crates/tracedecay-rusqlite-runtime/src/work_attempt.rs index 125886f2ef..4779414144 100644 --- a/crates/tracedecay-rusqlite-runtime/src/work_attempt.rs +++ b/crates/tracedecay-rusqlite-runtime/src/work_attempt.rs @@ -16,9 +16,10 @@ use tracedecay_domain::{ use crate::exact_sql::ExactSqlValue; use crate::work::{ - RegisteredWorkQuery, WorkSqliteStorage, authority_params_owned, exact_sql_integer, - exact_sql_statement, exact_sql_text, registered_work_query, + RegisteredWorkQuery, RegisteredWorkWrite, WorkSqliteStorage, authority_params_owned, + exact_sql_integer, exact_sql_statement, exact_sql_text, registered_work_query, }; +use crate::work_run_control::WorkSqliteJobStorage; mod rooted_evidence; @@ -29,6 +30,26 @@ struct StoredWorkAttemptV1 { synthesis: Option, } +impl StoredWorkAttemptV1 { + const fn admission_kind(&self) -> WorkAttemptAdmissionKind { + if self.synthesis.is_some() { + WorkAttemptAdmissionKind::Synthesis + } else { + WorkAttemptAdmissionKind::Ordinary + } + } +} + +fn load_stored( + source: &impl RegisteredWorkQuery, + authority: &WorkAuthority, + identity: &WorkAttemptIdentityV1, +) -> Result { + let payload = load_attempt_payload(source, authority, identity)? + .ok_or(WorkAttemptStorageError::NotFoundOrNotAuthorized)?; + decode_stored(&payload) +} + fn insert_attempt( storage: &WorkSqliteStorage, authority: &WorkAuthority, @@ -52,7 +73,7 @@ fn insert_attempt( /// Persist one ordinary attempt without settling the caller-owned transaction. pub(crate) fn insert_attempt_in_transaction( - transaction: &crate::exact_sql::ExactSqlTransaction, + transaction: &impl RegisteredWorkWrite, authority: &WorkAuthority, attempt: &WorkAttemptV1, concurrency: Option<&TopologyConcurrencyPolicyV1>, @@ -85,54 +106,199 @@ pub(crate) fn insert_attempt_in_transaction( } } -impl WorkAttemptStoragePort for WorkSqliteStorage { +/// The writer job's transaction commits or rolls back every statement here +/// with the job. +impl WorkAttemptStoragePort for WorkSqliteJobStorage<'_> { fn next_fence_epoch(&self, authority: &WorkAuthority) -> Result { - { - let _span = tracing::trace_span!("rusqlite.work_attempt.txn.fence_epoch").entered(); - { - let transaction = self - .handle() - .begin_immediate() - .map_err(|_| WorkAttemptStorageError::Unavailable)?; - { - let _span = - tracing::trace_span!("rusqlite.work_attempt.cas.fence_epoch").entered(); - { - transaction - .execute( - exact_sql_statement( - "INSERT INTO work_attempt_fences_v1 ( + next_fence_epoch_tx(self.transaction, authority) + } + + fn insert( + &self, + authority: &WorkAuthority, + attempt: &WorkAttemptV1, + ) -> Result { + insert_attempt_in_transaction(self.transaction, authority, attempt, None) + } + + fn insert_bounded( + &self, + authority: &WorkAuthority, + attempt: &WorkAttemptV1, + concurrency: &TopologyConcurrencyPolicyV1, + ) -> Result { + insert_attempt_in_transaction(self.transaction, authority, attempt, Some(concurrency)) + } + + fn admission_capacities( + &self, + authority: &WorkAuthority, + task_ids: &[TaskId], + concurrency: &TopologyConcurrencyPolicyV1, + ) -> Result, WorkAttemptStorageError> + { + crate::work::capacity::capacities(self.transaction, authority, task_ids, concurrency) + } + + fn load( + &self, + authority: &WorkAuthority, + identity: &WorkAttemptIdentityV1, + ) -> Result { + load_stored(self.transaction, authority, identity).map(|record| record.attempt) + } + + fn load_admission_kind( + &self, + authority: &WorkAuthority, + identity: &WorkAttemptIdentityV1, + ) -> Result { + load_stored(self.transaction, authority, identity).map(|record| record.admission_kind()) + } + + fn update( + &self, + authority: &WorkAuthority, + expected_fence: &tracedecay_domain::WorkLeaseFenceV1, + expected_state: WorkAttemptStateV1, + next: &WorkAttemptV1, + evidence: Option<&WorkAttemptEvidenceRecordV1>, + ) -> Result<(), WorkAttemptStorageError> { + update_tx( + self.transaction, + authority, + expected_fence, + expected_state, + next, + evidence, + ) + } + + fn open_attempts( + &self, + authority: &WorkAuthority, + ) -> Result, WorkAttemptStorageError> { + open_attempts_from(self.transaction, authority) + } + + fn list( + &self, + authority: &WorkAuthority, + start_after: Option<&WorkAttemptIdentityV1>, + limit: u32, + ) -> Result { + list_from(self.transaction, authority, start_after, limit) + } +} + +impl WorkSynthesisAdmissionStoragePort for WorkSqliteJobStorage<'_> { + fn insert_synthesis( + &self, + authority: &WorkAuthority, + record: &WorkSynthesisAdmissionRecordV1, + ) -> Result { + insert_synthesis_in_transaction(self.transaction, authority, record, None) + } + + fn insert_synthesis_bounded( + &self, + authority: &WorkAuthority, + record: &WorkSynthesisAdmissionRecordV1, + concurrency: &TopologyConcurrencyPolicyV1, + ) -> Result { + insert_synthesis_in_transaction(self.transaction, authority, record, Some(concurrency)) + } + + fn load_synthesis( + &self, + authority: &WorkAuthority, + identity: &WorkAttemptIdentityV1, + ) -> Result { + load_stored(self.transaction, authority, identity)? + .synthesis + .ok_or(WorkAttemptStorageError::AttemptConflict) + } + + fn admit_synthesis( + &self, + admission: &tracedecay_contracts::WorkProductSynthesisAdmissionV1, + ) -> Result< + ( + tracedecay_contracts::WorkProductEventCommitV1, + WorkSynthesisInsertOutcome, + ), + tracedecay_contracts::WorkProductAttemptAdmissionErrorV1, + > { + crate::work_product::admit_synthesis(self.transaction, admission) + } +} + +impl WorkSqliteJobStorage<'_> { + /// Loads one admitted synthesis source payload inside the job, where the + /// workflow artifact store's own transaction would wait on this writer. + pub fn load_workflow_artifact( + &self, + artifact: &tracedecay_domain::WorkArtifactRefV1, + ) -> Result< + tracedecay_contracts::WorkflowArtifactPayload, + tracedecay_contracts::WorkflowArtifactStoreError, + > { + let bytes = crate::workflow::run_journal::stored_payload_tx( + self.transaction, + artifact.digest().as_str(), + )? + .ok_or(tracedecay_contracts::WorkflowArtifactStoreError::Missing)?; + tracedecay_contracts::WorkflowArtifactPayload::new(artifact.clone(), bytes) + } +} + +fn next_fence_epoch_tx( + transaction: &impl RegisteredWorkWrite, + authority: &WorkAuthority, +) -> Result { + { + let _span = tracing::trace_span!("rusqlite.work_attempt.cas.fence_epoch").entered(); + transaction + .work_execute( + exact_sql_statement( + "INSERT INTO work_attempt_fences_v1 ( project_id, repository_id, worktree_id, actor_id, policy_digest, epoch ) VALUES (?1, ?2, ?3, ?4, ?5, 1) ON CONFLICT (project_id, repository_id, worktree_id, actor_id, policy_digest) DO UPDATE SET epoch = epoch + 1", - authority_params_owned(authority), - ) - .map_err(|_| WorkAttemptStorageError::Unavailable)?, - ) - .map_err(|_| WorkAttemptStorageError::Unavailable) - } - }?; - let rows = registered_work_query( - &transaction, - "SELECT epoch FROM work_attempt_fences_v1 - WHERE project_id = ?1 AND repository_id = ?2 AND worktree_id = ?3 - AND actor_id = ?4 AND policy_digest = ?5", authority_params_owned(authority), ) - .map_err(|_| WorkAttemptStorageError::Unavailable)?; - let epoch = rows - .rows - .first() - .and_then(|row| exact_sql_integer(&row.values, 0)) - .and_then(|value| u64::try_from(value).ok()) - .ok_or(WorkAttemptStorageError::Unavailable)?; - transaction - .commit() - .map_err(|_| WorkAttemptStorageError::Unavailable)?; - Ok(epoch) - } - } + .map_err(|_| WorkAttemptStorageError::Unavailable)?, + ) + .map_err(|_| WorkAttemptStorageError::Unavailable) + }?; + let rows = registered_work_query( + transaction, + "SELECT epoch FROM work_attempt_fences_v1 + WHERE project_id = ?1 AND repository_id = ?2 AND worktree_id = ?3 + AND actor_id = ?4 AND policy_digest = ?5", + authority_params_owned(authority), + ) + .map_err(|_| WorkAttemptStorageError::Unavailable)?; + rows.rows + .first() + .and_then(|row| exact_sql_integer(&row.values, 0)) + .and_then(|value| u64::try_from(value).ok()) + .ok_or(WorkAttemptStorageError::Unavailable) +} + +impl WorkAttemptStoragePort for WorkSqliteStorage { + fn next_fence_epoch(&self, authority: &WorkAuthority) -> Result { + let _span = tracing::trace_span!("rusqlite.work_attempt.txn.fence_epoch").entered(); + let transaction = self + .handle() + .begin_immediate() + .map_err(|_| WorkAttemptStorageError::Unavailable)?; + let epoch = next_fence_epoch_tx(&transaction, authority)?; + transaction + .commit() + .map_err(|_| WorkAttemptStorageError::Unavailable)?; + Ok(epoch) } fn insert( @@ -167,9 +333,7 @@ impl WorkAttemptStoragePort for WorkSqliteStorage { authority: &WorkAuthority, identity: &WorkAttemptIdentityV1, ) -> Result { - let payload = load_attempt_payload(self.handle(), authority, identity)? - .ok_or(WorkAttemptStorageError::NotFoundOrNotAuthorized)?; - Ok(decode_stored(&payload)?.attempt) + load_stored(self.handle(), authority, identity).map(|record| record.attempt) } fn load_admission_kind( @@ -177,14 +341,7 @@ impl WorkAttemptStoragePort for WorkSqliteStorage { authority: &WorkAuthority, identity: &WorkAttemptIdentityV1, ) -> Result { - let payload = load_attempt_payload(self.handle(), authority, identity)? - .ok_or(WorkAttemptStorageError::NotFoundOrNotAuthorized)?; - let record = decode_stored(&payload)?; - Ok(if record.synthesis.is_some() { - WorkAttemptAdmissionKind::Synthesis - } else { - WorkAttemptAdmissionKind::Ordinary - }) + load_stored(self.handle(), authority, identity).map(|record| record.admission_kind()) } fn update( @@ -195,105 +352,33 @@ impl WorkAttemptStoragePort for WorkSqliteStorage { next: &WorkAttemptV1, evidence: Option<&WorkAttemptEvidenceRecordV1>, ) -> Result<(), WorkAttemptStorageError> { - { - let _span = tracing::trace_span!("rusqlite.work_attempt.txn.update").entered(); - { - let evidence_payload = evidence - .map(serde_json::to_string) - .transpose() - .map_err(|_| WorkAttemptStorageError::Unavailable)?; - let transaction = self - .handle() - .begin_immediate() - .map_err(|_| WorkAttemptStorageError::Unavailable)?; - let existing = load_attempt_payload(&transaction, authority, next.identity())? - .ok_or(WorkAttemptStorageError::NotFoundOrNotAuthorized)?; - let mut record = decode_stored(&existing)?; - record.attempt = next.clone(); - let payload = serde_json::to_string(&record) - .map_err(|_| WorkAttemptStorageError::Unavailable)?; - let changed = { - let _span = tracing::trace_span!("rusqlite.work_attempt.cas.update").entered(); - { - transaction - .execute( - exact_sql_statement( - "UPDATE work_attempts_v1 SET - state = ?9, lease_id = ?10, fence_epoch = ?11, terminal = ?12, - attempt_payload = ?13, - evidence_payload = COALESCE(?14, evidence_payload) - WHERE project_id = ?1 AND repository_id = ?2 AND worktree_id = ?3 - AND actor_id = ?4 AND policy_digest = ?5 - AND task_id = ?6 AND run_id = ?7 AND attempt_id = ?8 - AND lease_id = ?15 AND fence_epoch = ?16 AND state = ?17", - authority_params_owned(authority) - .into_iter() - .chain(identity_params(next.identity())) - .chain([ - ExactSqlValue::Text(state_text(next.state())), - ExactSqlValue::Text( - next.lease().lease_id().as_str().to_owned(), - ), - ExactSqlValue::Integer( - i64::try_from(next.lease().epoch().get()).map_err( - |_| WorkAttemptStorageError::Unavailable, - )?, - ), - ExactSqlValue::Integer(i64::from(next.is_terminal())), - ExactSqlValue::Text(payload), - evidence_payload - .map_or(ExactSqlValue::Null, ExactSqlValue::Text), - ExactSqlValue::Text( - expected_fence.lease_id().as_str().to_owned(), - ), - ExactSqlValue::Integer( - i64::try_from(expected_fence.epoch().get()) - .map_err(|_| { - WorkAttemptStorageError::Unavailable - })?, - ), - ExactSqlValue::Text(state_text(expected_state)), - ]) - .collect(), - ) - .map_err(|_| WorkAttemptStorageError::Unavailable)?, - ) - .map_err(|_| WorkAttemptStorageError::Unavailable) - } - }?; - if changed.changed_rows != 1 { - let _ = transaction.rollback(); - return Err(WorkAttemptStorageError::FenceConflict); - } - crate::work_run_control::close_blocked_intervals_on_terminal_attempt( - &transaction, - authority, - next, - )?; - transaction - .commit() - .map_err(|_| WorkAttemptStorageError::Unavailable)?; - Ok(()) - } + let _span = tracing::trace_span!("rusqlite.work_attempt.txn.update").entered(); + let transaction = self + .handle() + .begin_immediate() + .map_err(|_| WorkAttemptStorageError::Unavailable)?; + if let Err(error) = update_tx( + &transaction, + authority, + expected_fence, + expected_state, + next, + evidence, + ) { + let _ = transaction.rollback(); + return Err(error); } + transaction + .commit() + .map_err(|_| WorkAttemptStorageError::Unavailable)?; + Ok(()) } fn open_attempts( &self, authority: &WorkAuthority, ) -> Result, WorkAttemptStorageError> { - let sql = format!( - "SELECT attempt.attempt_payload FROM work_attempts_v1 AS attempt - WHERE attempt.project_id = ?1 AND attempt.repository_id = ?2 - AND attempt.worktree_id = ?3 AND attempt.actor_id = ?4 - AND attempt.policy_digest = ?5 - AND {} - ORDER BY attempt.task_id, attempt.run_id, attempt.attempt_id", - crate::work::ACTIVE_ATTEMPT_PREDICATE - ); - let rows = registered_work_query(self.handle(), &sql, authority_params_owned(authority)) - .map_err(|_| WorkAttemptStorageError::Unavailable)?; - rows.rows.into_iter().map(attempt_from_row).collect() + open_attempts_from(self.handle(), authority) } fn has_open_attempts_in_exact_scope( @@ -328,94 +413,165 @@ impl WorkAttemptStoragePort for WorkSqliteStorage { start_after: Option<&WorkAttemptIdentityV1>, limit: u32, ) -> Result { - let authority_filter = "project_id = ?1 AND repository_id = ?2 AND worktree_id = ?3 - AND actor_id = ?4 AND policy_digest = ?5"; - let after_filter = if start_after.is_some() { - " AND (task_id, run_id, attempt_id) > (?6, ?7, ?8)" - } else { - "" - }; - let mut params = authority_params_owned(authority); - if let Some(start_after) = start_after { - params.extend(identity_params(start_after)); - } // One deferred transaction keeps the remaining count and the page on // the same consistent view of the attempt rows. - { - let _span = tracing::trace_span!("rusqlite.work_attempt.txn.list").entered(); - { - let transaction = self - .handle() - .begin_deferred() - .map_err(|_| WorkAttemptStorageError::Unavailable)?; - let counted = registered_work_query( - &transaction, - &format!( - "SELECT COUNT(*) FROM work_attempts_v1 WHERE {authority_filter}{after_filter}" - ), - params.clone(), - ) + let _span = tracing::trace_span!("rusqlite.work_attempt.txn.list").entered(); + let transaction = self + .handle() + .begin_deferred() .map_err(|_| WorkAttemptStorageError::Unavailable)?; - let remaining = counted - .rows - .first() - .and_then(|row| exact_sql_integer(&row.values, 0)) - .and_then(|value| u32::try_from(value).ok()) - .ok_or(WorkAttemptStorageError::Unavailable)?; - let limit_placeholder = params.len() + 1; - params.push(ExactSqlValue::Integer(i64::from(limit))); - let rows = registered_work_query( - &transaction, - &format!( - "SELECT attempt_payload FROM work_attempts_v1 - WHERE {authority_filter}{after_filter} - ORDER BY task_id, run_id, attempt_id - LIMIT ?{limit_placeholder}" - ), - params, + let page = list_from(&transaction, authority, start_after, limit)?; + transaction + .commit() + .map_err(|_| WorkAttemptStorageError::Unavailable)?; + Ok(page) + } +} + +/// Replaces the attempt row only while its lease fence and state are still +/// `expected_fence` and `expected_state`, closing its open blocked intervals +/// when the attempt turns terminal. +fn update_tx( + transaction: &impl RegisteredWorkWrite, + authority: &WorkAuthority, + expected_fence: &tracedecay_domain::WorkLeaseFenceV1, + expected_state: WorkAttemptStateV1, + next: &WorkAttemptV1, + evidence: Option<&WorkAttemptEvidenceRecordV1>, +) -> Result<(), WorkAttemptStorageError> { + let evidence_payload = evidence + .map(serde_json::to_string) + .transpose() + .map_err(|_| WorkAttemptStorageError::Unavailable)?; + let mut record = load_stored(transaction, authority, next.identity())?; + record.attempt = next.clone(); + let payload = + serde_json::to_string(&record).map_err(|_| WorkAttemptStorageError::Unavailable)?; + let changed = { + let _span = tracing::trace_span!("rusqlite.work_attempt.cas.update").entered(); + transaction + .work_execute( + exact_sql_statement( + "UPDATE work_attempts_v1 SET + state = ?9, lease_id = ?10, fence_epoch = ?11, terminal = ?12, + attempt_payload = ?13, + evidence_payload = COALESCE(?14, evidence_payload) + WHERE project_id = ?1 AND repository_id = ?2 AND worktree_id = ?3 + AND actor_id = ?4 AND policy_digest = ?5 + AND task_id = ?6 AND run_id = ?7 AND attempt_id = ?8 + AND lease_id = ?15 AND fence_epoch = ?16 AND state = ?17", + authority_params_owned(authority) + .into_iter() + .chain(identity_params(next.identity())) + .chain([ + ExactSqlValue::Text(state_text(next.state())), + ExactSqlValue::Text(next.lease().lease_id().as_str().to_owned()), + ExactSqlValue::Integer( + i64::try_from(next.lease().epoch().get()) + .map_err(|_| WorkAttemptStorageError::Unavailable)?, + ), + ExactSqlValue::Integer(i64::from(next.is_terminal())), + ExactSqlValue::Text(payload), + evidence_payload.map_or(ExactSqlValue::Null, ExactSqlValue::Text), + ExactSqlValue::Text(expected_fence.lease_id().as_str().to_owned()), + ExactSqlValue::Integer( + i64::try_from(expected_fence.epoch().get()) + .map_err(|_| WorkAttemptStorageError::Unavailable)?, + ), + ExactSqlValue::Text(state_text(expected_state)), + ]) + .collect(), ) - .map_err(|_| WorkAttemptStorageError::Unavailable)?; - transaction - .commit() - .map_err(|_| WorkAttemptStorageError::Unavailable)?; - let attempts = rows - .rows - .into_iter() - .map(attempt_from_row) - .collect::, _>>()?; - Ok(WorkAttemptListPageV1 { - attempts, - remaining, - }) - } - } + .map_err(|_| WorkAttemptStorageError::Unavailable)?, + ) + .map_err(|_| WorkAttemptStorageError::Unavailable) + }?; + if changed.changed_rows != 1 { + return Err(WorkAttemptStorageError::FenceConflict); } + crate::work_run_control::close_blocked_intervals_on_terminal_attempt( + transaction, + authority, + next, + ) } -fn insert_synthesis_record( - storage: &WorkSqliteStorage, +fn open_attempts_from( + source: &impl RegisteredWorkQuery, authority: &WorkAuthority, - record: &WorkSynthesisAdmissionRecordV1, - concurrency: Option<&TopologyConcurrencyPolicyV1>, -) -> Result { - { - let _span = tracing::trace_span!("rusqlite.work_attempt.txn.synthesis").entered(); - { - let transaction = storage - .handle() - .begin_immediate() - .map_err(|_| WorkAttemptStorageError::Unavailable)?; - let outcome = - insert_synthesis_in_transaction(&transaction, authority, record, concurrency); - let commit = matches!(outcome, Ok(WorkSynthesisInsertOutcome::Inserted)); - finish_immediate(transaction, outcome, commit) - } +) -> Result, WorkAttemptStorageError> { + let sql = format!( + "SELECT attempt.attempt_payload FROM work_attempts_v1 AS attempt + WHERE attempt.project_id = ?1 AND attempt.repository_id = ?2 + AND attempt.worktree_id = ?3 AND attempt.actor_id = ?4 + AND attempt.policy_digest = ?5 + AND {} + ORDER BY attempt.task_id, attempt.run_id, attempt.attempt_id", + crate::work::ACTIVE_ATTEMPT_PREDICATE + ); + let rows = registered_work_query(source, &sql, authority_params_owned(authority)) + .map_err(|_| WorkAttemptStorageError::Unavailable)?; + rows.rows.into_iter().map(attempt_from_row).collect() +} + +/// The page and its remaining count, both read through `source`; the caller +/// makes `source` one consistent view. +fn list_from( + source: &impl RegisteredWorkQuery, + authority: &WorkAuthority, + start_after: Option<&WorkAttemptIdentityV1>, + limit: u32, +) -> Result { + let authority_filter = "project_id = ?1 AND repository_id = ?2 AND worktree_id = ?3 + AND actor_id = ?4 AND policy_digest = ?5"; + let after_filter = if start_after.is_some() { + " AND (task_id, run_id, attempt_id) > (?6, ?7, ?8)" + } else { + "" + }; + let mut params = authority_params_owned(authority); + if let Some(start_after) = start_after { + params.extend(identity_params(start_after)); } + let counted = registered_work_query( + source, + &format!("SELECT COUNT(*) FROM work_attempts_v1 WHERE {authority_filter}{after_filter}"), + params.clone(), + ) + .map_err(|_| WorkAttemptStorageError::Unavailable)?; + let remaining = counted + .rows + .first() + .and_then(|row| exact_sql_integer(&row.values, 0)) + .and_then(|value| u32::try_from(value).ok()) + .ok_or(WorkAttemptStorageError::Unavailable)?; + let limit_placeholder = params.len() + 1; + params.push(ExactSqlValue::Integer(i64::from(limit))); + let rows = registered_work_query( + source, + &format!( + "SELECT attempt_payload FROM work_attempts_v1 + WHERE {authority_filter}{after_filter} + ORDER BY task_id, run_id, attempt_id + LIMIT ?{limit_placeholder}" + ), + params, + ) + .map_err(|_| WorkAttemptStorageError::Unavailable)?; + let attempts = rows + .rows + .into_iter() + .map(attempt_from_row) + .collect::, _>>()?; + Ok(WorkAttemptListPageV1 { + attempts, + remaining, + }) } /// Persist one synthesis attempt without settling the caller-owned transaction. pub(crate) fn insert_synthesis_in_transaction( - transaction: &crate::exact_sql::ExactSqlTransaction, + transaction: &impl RegisteredWorkWrite, authority: &WorkAuthority, record: &WorkSynthesisAdmissionRecordV1, concurrency: Option<&TopologyConcurrencyPolicyV1>, @@ -450,37 +606,6 @@ pub(crate) fn insert_synthesis_in_transaction( } } -impl WorkSynthesisAdmissionStoragePort for WorkSqliteStorage { - fn insert_synthesis( - &self, - authority: &WorkAuthority, - record: &WorkSynthesisAdmissionRecordV1, - ) -> Result { - insert_synthesis_record(self, authority, record, None) - } - - fn insert_synthesis_bounded( - &self, - authority: &WorkAuthority, - record: &WorkSynthesisAdmissionRecordV1, - concurrency: &TopologyConcurrencyPolicyV1, - ) -> Result { - insert_synthesis_record(self, authority, record, Some(concurrency)) - } - - fn load_synthesis( - &self, - authority: &WorkAuthority, - identity: &WorkAttemptIdentityV1, - ) -> Result { - let payload = load_attempt_payload(self.handle(), authority, identity)? - .ok_or(WorkAttemptStorageError::NotFoundOrNotAuthorized)?; - decode_stored(&payload)? - .synthesis - .ok_or(WorkAttemptStorageError::AttemptConflict) - } -} - impl WorkAttemptEvidenceReadPort for WorkSqliteStorage { fn evidence_page( &self, @@ -582,13 +707,13 @@ fn finish_immediate( } fn insert_attempt_row( - transaction: &crate::exact_sql::ExactSqlTransaction, + transaction: &impl RegisteredWorkWrite, authority: &WorkAuthority, attempt: &WorkAttemptV1, payload: String, ) -> Result<(), WorkAttemptStorageError> { transaction - .execute( + .work_execute( exact_sql_statement( "INSERT INTO work_attempts_v1 ( project_id, repository_id, worktree_id, actor_id, policy_digest, @@ -656,7 +781,7 @@ fn load_attempt_payload( /// attempt's immutable deadline and topology, so a caller cannot replace the /// run authority through a lexically earlier attempt ID. fn require_first_run_admission( - transaction: &crate::exact_sql::ExactSqlTransaction, + transaction: &impl RegisteredWorkWrite, authority: &WorkAuthority, attempt: &WorkAttemptV1, ) -> Result<(), WorkAttemptStorageError> { @@ -698,7 +823,7 @@ fn require_first_run_admission( /// attempt naming any other worktree root would write a tree the placement /// authority does not record for this run. pub(crate) fn require_placement_root( - transaction: &crate::exact_sql::ExactSqlTransaction, + transaction: &impl RegisteredWorkWrite, authority: &WorkAuthority, attempt: &WorkAttemptV1, ) -> Result<(), WorkAttemptStorageError> { @@ -727,7 +852,7 @@ pub(crate) fn require_placement_root( } fn require_run_reservation_admitted( - transaction: &crate::exact_sql::ExactSqlTransaction, + transaction: &impl RegisteredWorkWrite, authority: &WorkAuthority, identity: &WorkAttemptIdentityV1, ) -> Result<(), WorkAttemptStorageError> { diff --git a/crates/tracedecay-rusqlite-runtime/src/work_product.rs b/crates/tracedecay-rusqlite-runtime/src/work_product.rs index 3e05a071c2..994fb3a676 100644 --- a/crates/tracedecay-rusqlite-runtime/src/work_product.rs +++ b/crates/tracedecay-rusqlite-runtime/src/work_product.rs @@ -40,6 +40,7 @@ use crate::exact_sql::ExactSqlValue; use crate::work::{RegisteredWorkQuery, exact_sql_integer, exact_sql_text, registered_work_query}; mod attempt_admission; +pub(crate) use attempt_admission::{admit_retry, admit_synthesis}; mod authorization; mod events; mod evidence; diff --git a/crates/tracedecay-rusqlite-runtime/src/work_product/attempt_admission.rs b/crates/tracedecay-rusqlite-runtime/src/work_product/attempt_admission.rs index 72020dc0d3..0c950a4585 100644 --- a/crates/tracedecay-rusqlite-runtime/src/work_product/attempt_admission.rs +++ b/crates/tracedecay-rusqlite-runtime/src/work_product/attempt_admission.rs @@ -17,10 +17,50 @@ use tracedecay_domain::{WorkProductAuthorizedRelationScopeV1, WorkProductGraphV1 use super::{fold_graph, load_journal}; use crate::exact_sql::ExactSqlTransaction; -use crate::work::WorkSqliteStorage; +use crate::work::{RegisteredWorkWrite, WorkSqliteStorage}; +use crate::work_run_control::WorkSqliteJobStorage; type AdmissionError = WorkProductAttemptAdmissionErrorV1; +/// The writer job commits the product link and the attempt row together, or +/// rolls both back with any refusal the job returns. +impl WorkProductAttemptAdmissionPortV1 for WorkSqliteJobStorage<'_> { + fn admit_attempt( + &self, + admission: &WorkProductAttemptAdmissionV1, + ) -> Result { + admission.validate()?; + require_declared_authority(admission)?; + require_request_active(&admission.product_context)?; + admit_attempt_in_transaction(self.transaction, admission) + } +} + +/// Admits one retry inside a writer job, which commits or rolls back the +/// product link, the receipt, the attempt row and the workflow rebind +/// together. +pub(crate) fn admit_retry( + transaction: &impl RegisteredWorkWrite, + admission: &WorkProductRetryAdmissionV1, +) -> Result<(WorkProductEventCommitV1, WorkRetryAttemptOutcomeV1), AdmissionError> { + admission.validate()?; + require_declared_authority(&admission.admission)?; + require_request_active(&admission.admission.product_context)?; + admit_retry_in_transaction(transaction, admission) +} + +/// Admits one synthesis inside a writer job, which commits or rolls back the +/// product link, the attempt row and its synthesis record together. +pub(crate) fn admit_synthesis( + transaction: &impl RegisteredWorkWrite, + admission: &WorkProductSynthesisAdmissionV1, +) -> Result<(WorkProductEventCommitV1, WorkSynthesisInsertOutcome), AdmissionError> { + admission.validate()?; + require_declared_authority(&admission.admission)?; + require_request_active(&admission.admission.product_context)?; + admit_synthesis_in_transaction(transaction, admission) +} + impl WorkProductAttemptAdmissionPortV1 for WorkSqliteStorage { fn admit_attempt( &self, @@ -50,74 +90,10 @@ impl WorkProductAttemptAdmissionPortV1 for WorkSqliteStorage { Err(error) => rollback_after_failure(transaction, error), } } - - fn admit_retry( - &self, - admission: &WorkProductRetryAdmissionV1, - ) -> Result<(WorkProductEventCommitV1, WorkRetryAttemptOutcomeV1), AdmissionError> { - admission.validate()?; - require_declared_authority(&admission.admission)?; - require_request_active(&admission.admission.product_context)?; - let transaction = self - .handle() - .begin_immediate() - .map_err(|_| AdmissionError::Unavailable)?; - let outcome = admit_retry_in_transaction(&transaction, admission); - match outcome { - Ok((product, WorkRetryAttemptOutcomeV1::Created { receipt, attempt })) => { - transaction - .commit() - .map_err(|_| AdmissionError::DurabilityUncertain)?; - Ok(( - product, - WorkRetryAttemptOutcomeV1::Created { receipt, attempt }, - )) - } - Ok((product, WorkRetryAttemptOutcomeV1::Replayed { receipt, attempt })) => { - transaction - .rollback() - .map_err(|_| AdmissionError::DurabilityUncertain)?; - Ok(( - product, - WorkRetryAttemptOutcomeV1::Replayed { receipt, attempt }, - )) - } - Err(error) => rollback_after_failure(transaction, error), - } - } - - fn admit_synthesis( - &self, - admission: &WorkProductSynthesisAdmissionV1, - ) -> Result<(WorkProductEventCommitV1, WorkSynthesisInsertOutcome), AdmissionError> { - admission.validate()?; - require_declared_authority(&admission.admission)?; - require_request_active(&admission.admission.product_context)?; - let transaction = self - .handle() - .begin_immediate() - .map_err(|_| AdmissionError::Unavailable)?; - let outcome = admit_synthesis_in_transaction(&transaction, admission); - match outcome { - Ok((product, WorkSynthesisInsertOutcome::Inserted)) => { - transaction - .commit() - .map_err(|_| AdmissionError::DurabilityUncertain)?; - Ok((product, WorkSynthesisInsertOutcome::Inserted)) - } - Ok((product, WorkSynthesisInsertOutcome::Replayed(result))) => { - transaction - .rollback() - .map_err(|_| AdmissionError::DurabilityUncertain)?; - Ok((product, WorkSynthesisInsertOutcome::Replayed(result))) - } - Err(error) => rollback_after_failure(transaction, error), - } - } } fn admit_attempt_in_transaction( - transaction: &ExactSqlTransaction, + transaction: &impl RegisteredWorkWrite, admission: &WorkProductAttemptAdmissionV1, ) -> Result { let product = super::events::append_in_transaction( @@ -162,7 +138,7 @@ fn admit_attempt_in_transaction( } fn admit_retry_in_transaction( - transaction: &ExactSqlTransaction, + transaction: &impl RegisteredWorkWrite, admission: &WorkProductRetryAdmissionV1, ) -> Result<(WorkProductEventCommitV1, WorkRetryAttemptOutcomeV1), AdmissionError> { let product = super::events::append_in_transaction( @@ -221,7 +197,7 @@ fn map_workflow_error(error: WorkflowRunStorageError) -> AdmissionError { } fn admit_synthesis_in_transaction( - transaction: &ExactSqlTransaction, + transaction: &impl RegisteredWorkWrite, admission: &WorkProductSynthesisAdmissionV1, ) -> Result<(WorkProductEventCommitV1, WorkSynthesisInsertOutcome), AdmissionError> { let product = super::events::append_in_transaction( @@ -269,7 +245,7 @@ fn product_commit(outcome: &WorkProductEventCommitOutcomeV1) -> &WorkProductEven } fn graph_for_product( - transaction: &ExactSqlTransaction, + transaction: &impl RegisteredWorkWrite, context: &tracedecay_contracts::WorkProductPortContextV1, product: &WorkProductEventCommitV1, ) -> Result { diff --git a/crates/tracedecay-rusqlite-runtime/src/work_product/authorization.rs b/crates/tracedecay-rusqlite-runtime/src/work_product/authorization.rs index 2353d79b39..2bf695bf16 100644 --- a/crates/tracedecay-rusqlite-runtime/src/work_product/authorization.rs +++ b/crates/tracedecay-rusqlite-runtime/src/work_product/authorization.rs @@ -25,6 +25,7 @@ use tracedecay_contracts::{ use tracedecay_domain::UtcMicros; use crate::work::WorkSqliteStorage; +use crate::work_run_control::WorkSqliteJobStorage; impl WorkProductOwnerAuthorizationPortV1 for WorkSqliteStorage { fn authorize_scope( @@ -33,17 +34,35 @@ impl WorkProductOwnerAuthorizationPortV1 for WorkSqliteStorage { selection: &WorkProductSelectionScopeV1, _observed_at: UtcMicros, ) -> Result { - let shard = &self.handle().binding().shard_id; - if !selection_is_within_resolved_scope(context, selection) { - return Err(WorkProductOwnerAuthorizationErrorV1::NotAuthorized); - } - AuthorizedWorkProductScopeV1::new( - shard.brain_id.clone(), - shard.profile_id.clone(), - selection.clone(), - ) - .map_err(|_| WorkProductOwnerAuthorizationErrorV1::Unavailable) + authorize_for_owner(&self.handle().binding().shard_id, context, selection) + } +} + +impl WorkProductOwnerAuthorizationPortV1 for WorkSqliteJobStorage<'_> { + fn authorize_scope( + &self, + context: &RequestContext, + selection: &WorkProductSelectionScopeV1, + _observed_at: UtcMicros, + ) -> Result { + authorize_for_owner(self.owner, context, selection) + } +} + +fn authorize_for_owner( + shard: &tracedecay_store::StoreShardIdV1, + context: &RequestContext, + selection: &WorkProductSelectionScopeV1, +) -> Result { + if !selection_is_within_resolved_scope(context, selection) { + return Err(WorkProductOwnerAuthorizationErrorV1::NotAuthorized); } + AuthorizedWorkProductScopeV1::new( + shard.brain_id.clone(), + shard.profile_id.clone(), + selection.clone(), + ) + .map_err(|_| WorkProductOwnerAuthorizationErrorV1::Unavailable) } fn selection_is_within_resolved_scope( diff --git a/crates/tracedecay-rusqlite-runtime/src/work_product/events.rs b/crates/tracedecay-rusqlite-runtime/src/work_product/events.rs index 599af6fe17..38e2b32f0c 100644 --- a/crates/tracedecay-rusqlite-runtime/src/work_product/events.rs +++ b/crates/tracedecay-rusqlite-runtime/src/work_product/events.rs @@ -11,55 +11,49 @@ use tracedecay_domain::{ use super::{WORK_PRODUCT_EVENT_ID_DOMAIN, load_journal_tail, owner_params, selection_covers}; use crate::exact_sql::ExactSqlValue; -use crate::work::{WorkSqliteStorage, exact_sql_statement, exact_sql_text, registered_work_query}; +use crate::work::{ + RegisteredWorkQuery, RegisteredWorkWrite, WorkSqliteStorage, exact_sql_statement, + exact_sql_text, registered_work_query, +}; + +use crate::work_run_control::WorkSqliteJobStorage; type PortError = WorkProductEventPortErrorV1; -impl WorkProductEventPortV1 for WorkSqliteStorage { +/// The writer job commits the event and its verified version together, or +/// rolls both back with any refusal the job returns. +impl WorkProductEventPortV1 for WorkSqliteJobStorage<'_> { fn replay( &self, context: &WorkProductPortContextV1, command_id: &WorkCommandId, canonical_input_digest: &ManifestDigest, ) -> Result, PortError> { - let scope = context.authorized_scope(); - let rows = registered_work_query( - self.handle(), - "SELECT canonical_input_digest, event_payload FROM work_product_events_v1 - WHERE owner_brain_id = ?1 AND owner_profile_id = ?2 AND command_id = ?3", - owner_params(scope) - .into_iter() - .chain([ExactSqlValue::Text(command_id.as_str().to_owned())]) - .collect(), + replay_from( + self.transaction, + context, + command_id, + canonical_input_digest, ) - .map_err(|_| PortError::Unavailable)?; - let Some(row) = rows.rows.first() else { - return Ok(None); - }; - let stored_digest = exact_sql_text(&row.values, 0).ok_or(PortError::Unavailable)?; - if stored_digest != canonical_input_digest.as_str() { - // The same command id with different canonical input is a reused - // idempotency key, never a replay of this request. - return Err(PortError::IdempotencyConflict); - } - let event: WorkProductEventV1 = - serde_json::from_str(exact_sql_text(&row.values, 1).ok_or(PortError::Unavailable)?) - .map_err(|_| PortError::Unavailable)?; - // A replayed event must still be one this selection is authorized to - // see; otherwise its existence would leak through the idempotency - // channel. - if !selection_covers(scope.selection(), &event) { - return Err(PortError::NotFoundOrNotAuthorized); - } - let published = super::load_published_versions(self.handle(), scope) - .ok_or(PortError::Unavailable)? - .into_iter() - .find(|published| published.event_sequence == event.sequence()) - .ok_or(PortError::Unavailable)?; - let verified = super::verified_version(&published, &event).ok_or(PortError::Unavailable)?; - WorkProductEventCommitV1::new(event, verified) - .map(Some) - .map_err(|_| PortError::Unavailable) + } + + fn append_atomically( + &self, + context: &WorkProductPortContextV1, + draft: &WorkProductEventDraftV1, + ) -> Result { + append_in_transaction(self.transaction, context, draft) + } +} + +impl WorkProductEventPortV1 for WorkSqliteStorage { + fn replay( + &self, + context: &WorkProductPortContextV1, + command_id: &WorkCommandId, + canonical_input_digest: &ManifestDigest, + ) -> Result, PortError> { + replay_from(self.handle(), context, command_id, canonical_input_digest) } fn append_atomically( @@ -89,8 +83,53 @@ impl WorkProductEventPortV1 for WorkSqliteStorage { } } +fn replay_from( + source: &impl RegisteredWorkQuery, + context: &WorkProductPortContextV1, + command_id: &WorkCommandId, + canonical_input_digest: &ManifestDigest, +) -> Result, PortError> { + let scope = context.authorized_scope(); + let rows = registered_work_query( + source, + "SELECT canonical_input_digest, event_payload FROM work_product_events_v1 + WHERE owner_brain_id = ?1 AND owner_profile_id = ?2 AND command_id = ?3", + owner_params(scope) + .into_iter() + .chain([ExactSqlValue::Text(command_id.as_str().to_owned())]) + .collect(), + ) + .map_err(|_| PortError::Unavailable)?; + let Some(row) = rows.rows.first() else { + return Ok(None); + }; + let stored_digest = exact_sql_text(&row.values, 0).ok_or(PortError::Unavailable)?; + if stored_digest != canonical_input_digest.as_str() { + // The same command id with different canonical input is a reused + // idempotency key, never a replay of this request. + return Err(PortError::IdempotencyConflict); + } + let event: WorkProductEventV1 = + serde_json::from_str(exact_sql_text(&row.values, 1).ok_or(PortError::Unavailable)?) + .map_err(|_| PortError::Unavailable)?; + // A replayed event must still be one this selection is authorized to see; + // otherwise its existence would leak through the idempotency channel. + if !selection_covers(scope.selection(), &event) { + return Err(PortError::NotFoundOrNotAuthorized); + } + let published = super::load_published_versions(source, scope) + .ok_or(PortError::Unavailable)? + .into_iter() + .find(|published| published.event_sequence == event.sequence()) + .ok_or(PortError::Unavailable)?; + let verified = super::verified_version(&published, &event).ok_or(PortError::Unavailable)?; + WorkProductEventCommitV1::new(event, verified) + .map(Some) + .map_err(|_| PortError::Unavailable) +} + pub(super) fn append_in_transaction( - transaction: &crate::exact_sql::ExactSqlTransaction, + transaction: &impl RegisteredWorkWrite, context: &WorkProductPortContextV1, draft: &WorkProductEventDraftV1, ) -> Result { @@ -133,7 +172,7 @@ pub(super) fn append_in_transaction( } fn replay_in_transaction( - transaction: &crate::exact_sql::ExactSqlTransaction, + transaction: &impl RegisteredWorkWrite, context: &WorkProductPortContextV1, draft: &WorkProductEventDraftV1, ) -> Result, PortError> { @@ -202,7 +241,7 @@ fn mint_event( } fn insert_event( - transaction: &crate::exact_sql::ExactSqlTransaction, + transaction: &impl RegisteredWorkWrite, context: &WorkProductPortContextV1, event: &WorkProductEventV1, sequence: WorkProductEventSequenceV1, @@ -215,7 +254,7 @@ fn insert_event( None => ExactSqlValue::Null, }; transaction - .execute( + .work_execute( exact_sql_statement( "INSERT INTO work_product_events_v1 ( owner_brain_id, owner_profile_id, sequence, event_id, command_id, diff --git a/crates/tracedecay-rusqlite-runtime/src/work_product/publication.rs b/crates/tracedecay-rusqlite-runtime/src/work_product/publication.rs index 51129338b8..ebb36e5684 100644 --- a/crates/tracedecay-rusqlite-runtime/src/work_product/publication.rs +++ b/crates/tracedecay-rusqlite-runtime/src/work_product/publication.rs @@ -11,12 +11,14 @@ use tracedecay_domain::WorkProductEventV1; use super::{fold_graph, load_journal, owner_params, recovered_graph_digest, selection_covers}; use crate::exact_sql::ExactSqlValue; -use crate::work::{exact_sql_statement, exact_sql_text, registered_work_query}; +use crate::work::{ + RegisteredWorkWrite, exact_sql_statement, exact_sql_text, registered_work_query, +}; type PortError = WorkProductEventPortErrorV1; pub(super) fn publish_in_transaction( - transaction: &crate::exact_sql::ExactSqlTransaction, + transaction: &impl RegisteredWorkWrite, context: &WorkProductPortContextV1, event: &WorkProductEventV1, ) -> Result { @@ -75,7 +77,7 @@ pub(super) fn publish_in_transaction( } transaction - .execute( + .work_execute( exact_sql_statement( "INSERT INTO work_product_graph_versions_v1 ( owner_brain_id, owner_profile_id, graph_version, event_sequence, diff --git a/crates/tracedecay-rusqlite-runtime/src/work_run_control.rs b/crates/tracedecay-rusqlite-runtime/src/work_run_control.rs index 1eba82467b..263c12f77a 100644 --- a/crates/tracedecay-rusqlite-runtime/src/work_run_control.rs +++ b/crates/tracedecay-rusqlite-runtime/src/work_run_control.rs @@ -20,7 +20,7 @@ use tracedecay_domain::{ WorkRunControlV1, }; -use crate::exact_sql::{ExactSqlError, ExactSqlJobTransaction, ExactSqlTransaction, ExactSqlValue}; +use crate::exact_sql::{ExactSqlError, ExactSqlJobTransaction, ExactSqlValue}; use crate::work::{ ACTIVE_ATTEMPT_PREDICATE, RegisteredWorkQuery, RegisteredWorkWrite, WorkSqliteStorage, authority_params_owned, exact_sql_integer, exact_sql_statement, exact_sql_text, @@ -33,6 +33,11 @@ use crate::work::{ #[derive(Clone, Copy)] pub struct WorkSqliteJobStorage<'job> { pub(crate) transaction: &'job ExactSqlJobTransaction<'job>, + pub(crate) owner: &'job tracedecay_store::StoreShardIdV1, + /// The graph read made before the job. A job never replays the product + /// journal through the read port, so without one every graph read is + /// refused as unavailable. + pub(crate) graph: Option<&'job tracedecay_contracts::PrefetchedWorkGraphReadV1>, } impl WorkSqliteStorage { @@ -45,9 +50,95 @@ impl WorkSqliteStorage { R: Send + 'static, E: Send + 'static, { + self.run_job(None, job).await + } + + /// [`Self::transact`] for a job whose service reads the current product + /// graph: the job's storage answers that read from `graph`, read before + /// the job on a reader. + pub async fn transact_with_graph( + &self, + graph: tracedecay_contracts::PrefetchedWorkGraphReadV1, + job: F, + ) -> Result, ExactSqlError> + where + F: Fn(WorkSqliteJobStorage<'_>) -> Result + Send + Sync + 'static, + R: Send + 'static, + E: Send + 'static, + { + self.run_job(Some(graph), job).await + } + + async fn run_job( + &self, + graph: Option, + job: F, + ) -> Result, ExactSqlError> + where + F: Fn(WorkSqliteJobStorage<'_>) -> Result + Send + Sync + 'static, + R: Send + 'static, + E: Send + 'static, + { + let owner = self.handle().binding().shard_id.clone(); self.handle() - .transact(move |transaction| job(WorkSqliteJobStorage { transaction })) + .transact(move |transaction| { + job(WorkSqliteJobStorage { + transaction, + owner: &owner, + graph: graph.as_ref(), + }) + }) + .await + } + + /// Makes the current product graph read a writer job's service will ask + /// for, on the blocking pool, so the awaiting task parks no runtime thread + /// and the journal replay never holds the writer. A read that cannot run + /// prefetches nothing, and the job's service refuses its read. + pub async fn prefetch_graph( + &self, + read: impl FnOnce(&Self) -> tracedecay_contracts::PrefetchedWorkGraphReadV1 + Send + 'static, + ) -> tracedecay_contracts::PrefetchedWorkGraphReadV1 { + let storage = self.clone(); + crate::native_runtime::spawn_blocking(move || read(&storage)) .await + .unwrap_or_default() + } +} + +impl tracedecay_contracts::WorkRunReservationPort for WorkSqliteJobStorage<'_> { + fn load_run_control( + &self, + authority: &WorkAuthority, + task_id: &TaskId, + run_id: &RunId, + ) -> Result, WorkRunControlStorageError> { + load_run_control_from(self.transaction, authority, task_id, run_id) + } +} + +impl tracedecay_contracts::WorkRunReservationPort for WorkSqliteStorage { + fn load_run_control( + &self, + authority: &WorkAuthority, + task_id: &TaskId, + run_id: &RunId, + ) -> Result, WorkRunControlStorageError> { + load_run_control_from(self.handle(), authority, task_id, run_id) + } +} + +impl tracedecay_contracts::WorkGraphReadPortV1 for WorkSqliteJobStorage<'_> { + fn read_graph( + &self, + context: &tracedecay_contracts::WorkProductPortContextV1, + request: &tracedecay_contracts::WorkGraphReadRequestV1, + ) -> Result + { + self.graph.map_or( + Err(tracedecay_contracts::WorkGraphReadPortErrorV1::Unavailable), + |graph| graph.read_graph(context, request), + ) } } @@ -161,15 +252,6 @@ impl WorkRunControlStoragePort for WorkSqliteStorage { run_admission_from(self.handle(), authority, task_id, run_id) } - fn load_run_control( - &self, - authority: &WorkAuthority, - task_id: &TaskId, - run_id: &RunId, - ) -> Result, WorkRunControlStorageError> { - load_run_control_from(self.handle(), authority, task_id, run_id) - } - fn publish_run_control( &self, authority: &WorkAuthority, @@ -273,7 +355,7 @@ impl WorkRunControlStoragePort for WorkSqliteStorage { .entered(); { transaction - .execute( + .work_execute( exact_sql_statement( "UPDATE work_blocked_intervals_v1 SET observability_durable = 1 @@ -640,7 +722,7 @@ fn publish_run_control_tx( } fn load_blocked_interval_observation_cursor( - transaction: &ExactSqlTransaction, + transaction: &impl RegisteredWorkWrite, authority: &WorkAuthority, ) -> Result, WorkRunControlStorageError> { let rows = registered_work_query( @@ -661,7 +743,7 @@ fn load_blocked_interval_observation_cursor( } fn settled_blocked_interval_observation_page( - transaction: &ExactSqlTransaction, + transaction: &impl RegisteredWorkWrite, authority: &WorkAuthority, after: Option<&BlockedIntervalObservationCursor>, limit: u32, @@ -706,13 +788,13 @@ fn settled_blocked_interval_observation_page( } fn persist_blocked_interval_observation_cursor( - transaction: &ExactSqlTransaction, + transaction: &impl RegisteredWorkWrite, authority: &WorkAuthority, receipt: &WorkBlockedIntervalReceiptV1, ) -> Result<(), WorkRunControlStorageError> { let cursor = BlockedIntervalObservationCursor::from_receipt(receipt)?; transaction - .execute( + .work_execute( exact_sql_statement( "INSERT INTO work_blocked_interval_observation_cursors_v1 ( project_id, repository_id, worktree_id, actor_id, policy_digest, @@ -901,7 +983,7 @@ fn blocked_identity_params(receipt: &WorkBlockedIntervalReceiptV1) -> [ExactSqlV /// terminal CAS. The interval receipt cannot survive a terminal attempt with /// no end instant, and a crash can commit neither half independently. pub(crate) fn close_blocked_intervals_on_terminal_attempt( - transaction: &ExactSqlTransaction, + transaction: &impl RegisteredWorkWrite, authority: &WorkAuthority, next: &WorkAttemptV1, ) -> Result<(), WorkAttemptStorageError> { diff --git a/crates/tracedecay-rusqlite-runtime/src/workflow.rs b/crates/tracedecay-rusqlite-runtime/src/workflow.rs index 514f79503b..62c75e4efe 100644 --- a/crates/tracedecay-rusqlite-runtime/src/workflow.rs +++ b/crates/tracedecay-rusqlite-runtime/src/workflow.rs @@ -434,28 +434,30 @@ fn decode_json(payload: &str) -> Result { } pub(crate) fn query_tx( - transaction: &ExactSqlTransaction, + transaction: &impl crate::work::RegisteredWorkQuery, sql: &str, params: Vec, ) -> Result { - transaction.query(statement(sql, params)?) + transaction.work_query(statement(sql, params)?) } pub(crate) fn execute_tx( - transaction: &ExactSqlTransaction, + transaction: &impl crate::work::RegisteredWorkWrite, sql: &str, params: Vec, ) -> Result<(), ExactSqlError> { - transaction.execute(statement(sql, params)?).map(|_| ()) + transaction + .work_execute(statement(sql, params)?) + .map(|_| ()) } fn execute_tx_changed( - transaction: &ExactSqlTransaction, + transaction: &impl crate::work::RegisteredWorkWrite, sql: &str, params: Vec, ) -> Result { transaction - .execute(statement(sql, params)?) + .work_execute(statement(sql, params)?) .map(|result| result.changed_rows) } diff --git a/crates/tracedecay-rusqlite-runtime/src/workflow/run_journal.rs b/crates/tracedecay-rusqlite-runtime/src/workflow/run_journal.rs index bb1ab8162d..9e70262484 100644 --- a/crates/tracedecay-rusqlite-runtime/src/workflow/run_journal.rs +++ b/crates/tracedecay-rusqlite-runtime/src/workflow/run_journal.rs @@ -17,10 +17,7 @@ use tracedecay_domain::{ WorkflowRunProjection, canonical_sha256, }; -use super::{ - ExactSqlTransaction, ExactSqlValue, WorkflowSqliteAuthority, encode_json, execute_tx, query_tx, - sql_text, -}; +use super::{ExactSqlValue, WorkflowSqliteAuthority, encode_json, execute_tx, query_tx, sql_text}; use crate::work::{RegisteredWorkQuery, registered_work_query}; fn run_journal_unavailable(_: E) -> WorkflowRunStorageError { @@ -75,7 +72,7 @@ pub(crate) fn projection_from( } pub(crate) fn append_in_transaction( - transaction: &ExactSqlTransaction, + transaction: &impl crate::work::RegisteredWorkWrite, request: &WorkflowRunAppendRequest, ) -> Result { let payload = encode_json(&request.event).map_err(|_| WorkflowRunStorageError::Unavailable)?; @@ -292,8 +289,8 @@ fn artifact_store_unavailable(_: E) -> WorkflowArtifactStoreError { WorkflowArtifactStoreError::Unavailable } -fn stored_payload_tx( - transaction: &ExactSqlTransaction, +pub(crate) fn stored_payload_tx( + transaction: &impl crate::work::RegisteredWorkQuery, digest: &str, ) -> Result>, WorkflowArtifactStoreError> { let rows = query_tx( diff --git a/crates/tracedecay-rusqlite-runtime/tests/rusqlite_suite/work_attempt_storage.rs b/crates/tracedecay-rusqlite-runtime/tests/rusqlite_suite/work_attempt_storage.rs index b219144546..959e92e579 100644 --- a/crates/tracedecay-rusqlite-runtime/tests/rusqlite_suite/work_attempt_storage.rs +++ b/crates/tracedecay-rusqlite-runtime/tests/rusqlite_suite/work_attempt_storage.rs @@ -15,16 +15,16 @@ use std::{ use tracedecay_contracts::{ VerifiedWorkRetryFailureV1, WorkAttemptAdmissionKind, WorkAttemptCapacityScopeV1, WorkAttemptCapacityVerdictV1, WorkAttemptEffectDispatchOutcomeV1, WorkAttemptEffectHolderV1, - WorkAttemptEffectResolutionV1, WorkAttemptEffectStorageErrorV1, WorkAttemptEffectStoragePortV1, - WorkAttemptEvidenceReadPort, WorkAttemptEvidenceRecordV1, WorkAttemptInsertOutcome, - WorkAttemptProviderOutcomeV1, WorkAttemptReceiptReadPortV1, WorkAttemptStorageError, - WorkAttemptStoragePort, WorkOwnerObservationMarkOutcomeV1, WorkOwnerObservationStoragePortV1, - WorkRetryAttemptOutcomeV1, WorkRetryCauseV1, WorkRetryFailureSelectorV1, WorkRetryReceiptV1, - WorkRetrySourceV1, WorkRetryStoragePortV1, WorkRetryWriteV1, WorkRunControlStoragePort, - WorkRunControlTransitionPort, WorkSynthesisAdmissionRecordV1, - WorkSynthesisAdmissionStoragePort, WorkSynthesisAdmissionV1, WorkSynthesisEvidenceGroupV1, - WorkSynthesisInsertOutcome, WorkSynthesisSourceEnvelopeV1, WorkSynthesisSourceOutcomeV1, - WorkSynthesisSourceSetV1, WorkflowSynthesisDraft, + WorkAttemptEffectReadPortV1, WorkAttemptEffectResolutionV1, WorkAttemptEffectStorageErrorV1, + WorkAttemptEffectStoragePortV1, WorkAttemptEvidenceReadPort, WorkAttemptEvidenceRecordV1, + WorkAttemptInsertOutcome, WorkAttemptProviderOutcomeV1, WorkAttemptReceiptReadPortV1, + WorkAttemptStorageError, WorkAttemptStoragePort, WorkOwnerObservationMarkOutcomeV1, + WorkOwnerObservationStoragePortV1, WorkRetryAttemptOutcomeV1, WorkRetryCauseV1, + WorkRetryFailureSelectorV1, WorkRetryReceiptV1, WorkRetrySourceV1, WorkRetryStoragePortV1, + WorkRetryWriteV1, WorkRunControlStoragePort, WorkRunControlTransitionPort, + WorkSynthesisAdmissionRecordV1, WorkSynthesisAdmissionStoragePort, WorkSynthesisAdmissionV1, + WorkSynthesisEvidenceGroupV1, WorkSynthesisInsertOutcome, WorkSynthesisSourceEnvelopeV1, + WorkSynthesisSourceOutcomeV1, WorkSynthesisSourceSetV1, WorkflowSynthesisDraft, }; use tracedecay_domain::configuration::TopologyConcurrencyPolicyV1; use tracedecay_domain::{ @@ -43,6 +43,7 @@ use tracedecay_domain::{ }; use common::fixture_abs_root; +use tracedecay_rusqlite_runtime::work_run_control::WorkSqliteJobStorage; use tracedecay_rusqlite_runtime::workflow::install_workflow_schema; use work_registered_store::RegisteredWorkStore; @@ -468,10 +469,7 @@ fn retry_reservation_cannot_overbook_project_global_capacity() { let write = retry_write(&original); assert_eq!( - store - .storage() - .insert_retry_bounded(&retry_authority, &write, &concurrency(1, 1, 1)) - .unwrap_err(), + insert_retry(&store, &retry_authority, &write, &concurrency(1, 1, 1)).unwrap_err(), WorkAttemptStorageError::CapacityExceeded ); assert_eq!(store.count("work_attempts_v1"), 2); @@ -497,10 +495,13 @@ fn recovery_retry_replaces_the_original_across_active_attempt_census() { )); store.storage().insert(&authority, &original).unwrap(); - let outcome = store - .storage() - .insert_retry_bounded(&authority, &retry_write(&original), &concurrency(1, 1, 1)) - .unwrap(); + let outcome = insert_retry( + &store, + &authority, + &retry_write(&original), + &concurrency(1, 1, 1), + ) + .unwrap(); assert!(matches!(outcome, WorkRetryAttemptOutcomeV1::Created { .. })); let replacement = outcome.attempt().clone(); assert_eq!(store.count("work_attempts_v1"), 2); @@ -657,10 +658,7 @@ fn successful_retry_remains_pending_until_exact_durable_marker_cas() { )); store.storage().insert(&authority, &original).unwrap(); let write = retry_write(&original); - store - .storage() - .insert_retry_bounded(&authority, &write, &concurrency(1, 1, 1)) - .unwrap(); + insert_retry(&store, &authority, &write, &concurrency(1, 1, 1)).unwrap(); let pending = store .storage() @@ -810,10 +808,7 @@ fn effect_holder_is_exact_replayable_and_reconciles_unknown_across_restart() { ) .unwrap(); assert_eq!( - store - .storage() - .begin_effect_dispatch(&exact_authority, &first) - .unwrap(), + begin_dispatch(&store, &exact_authority, &first).unwrap(), WorkAttemptEffectDispatchOutcomeV1::Recorded(first.clone()) ); let replay = WorkAttemptEffectHolderV1::dispatched( @@ -824,10 +819,7 @@ fn effect_holder_is_exact_replayable_and_reconciles_unknown_across_restart() { ) .unwrap(); assert_eq!( - store - .storage() - .begin_effect_dispatch(&exact_authority, &replay) - .unwrap(), + begin_dispatch(&store, &exact_authority, &replay).unwrap(), WorkAttemptEffectDispatchOutcomeV1::Replayed(first), "an exact receipt replay never authorizes a second provider dispatch" ); @@ -839,36 +831,31 @@ fn effect_holder_is_exact_replayable_and_reconciles_unknown_across_restart() { ) .unwrap(); assert_eq!( - store - .storage() - .begin_effect_dispatch(&exact_authority, &conflicting_deadline) - .unwrap_err(), + begin_dispatch(&store, &exact_authority, &conflicting_deadline).unwrap_err(), WorkAttemptEffectStorageErrorV1::Conflict ); - let unknown = store - .storage() - .settle_effect_dispatch( - &exact_authority, - attempt.identity(), - WorkAttemptEffectResolutionV1::Unknown, - UtcMicros(50), - ) - .unwrap(); + let unknown = settle_dispatch( + &store, + &exact_authority, + attempt.identity(), + WorkAttemptEffectResolutionV1::Unknown, + UtcMicros(50), + ) + .unwrap(); assert_eq!( unknown.resolution(), Some(WorkAttemptEffectResolutionV1::Unknown) ); let restarted = store.restart("attempt-effect-holder"); - let no_effect = restarted - .storage() - .settle_effect_dispatch( - &exact_authority, - attempt.identity(), - WorkAttemptEffectResolutionV1::NoEffect, - UtcMicros(60), - ) - .unwrap(); + let no_effect = settle_dispatch( + &restarted, + &exact_authority, + attempt.identity(), + WorkAttemptEffectResolutionV1::NoEffect, + UtcMicros(60), + ) + .unwrap(); assert_eq!( no_effect.resolution(), Some(WorkAttemptEffectResolutionV1::NoEffect) @@ -1129,6 +1116,76 @@ fn recovery_required(attempt: &WorkAttemptV1) -> WorkAttemptV1 { .unwrap() } +/// Runs one storage call as a writer job, which commits only when it returns +/// `Ok`. +fn in_job( + store: &RegisteredWorkStore, + call: impl Fn(WorkSqliteJobStorage<'_>) -> Result + Send + Sync + 'static, +) -> Result { + tracedecay_rusqlite_runtime::native_runtime::block_on(store.storage().transact(call)) + .expect("native runtime") + .expect("the writer runs the job") +} + +fn begin_dispatch( + store: &RegisteredWorkStore, + authority: &WorkAuthority, + holder: &WorkAttemptEffectHolderV1, +) -> Result { + let (authority, holder) = (authority.clone(), holder.clone()); + in_job(store, move |storage| { + storage.begin_effect_dispatch(&authority, &holder) + }) +} + +fn settle_dispatch( + store: &RegisteredWorkStore, + authority: &WorkAuthority, + attempt: &WorkAttemptIdentityV1, + resolution: WorkAttemptEffectResolutionV1, + resolved_at: UtcMicros, +) -> Result { + let (authority, attempt) = (authority.clone(), attempt.clone()); + in_job(store, move |storage| { + storage.settle_effect_dispatch(&authority, &attempt, resolution, resolved_at) + }) +} + +fn insert_retry( + store: &RegisteredWorkStore, + authority: &WorkAuthority, + write: &WorkRetryWriteV1, + concurrency: &TopologyConcurrencyPolicyV1, +) -> Result { + let (authority, write, concurrency) = (authority.clone(), write.clone(), concurrency.clone()); + in_job(store, move |storage| { + storage.insert_retry_bounded(&authority, &write, &concurrency) + }) +} + +fn insert_synthesis( + store: &RegisteredWorkStore, + authority: &WorkAuthority, + record: &WorkSynthesisAdmissionRecordV1, + concurrency: &TopologyConcurrencyPolicyV1, +) -> Result { + let (authority, record, concurrency) = (authority.clone(), record.clone(), concurrency.clone()); + in_job(store, move |storage| { + storage.insert_synthesis_bounded(&authority, &record, &concurrency) + }) +} + +fn load_synthesis( + store: &RegisteredWorkStore, + authority: &WorkAuthority, + identity: &WorkAttemptIdentityV1, +) -> Result { + let (authority, identity) = (authority.clone(), identity.clone()); + in_job(store, move |storage| { + storage.load_synthesis(&authority, &identity) + }) +} + fn retry_write(original: &WorkAttemptV1) -> WorkRetryWriteV1 { let (failure, evidence_digest, observed_at, restart_reason) = match original.recovery() { WorkRecoveryStateV1::RecoveryRequired { @@ -1283,10 +1340,7 @@ fn insert_replays_identical_admissions_and_refuses_divergent_ones() { WorkAttemptAdmissionKind::Ordinary ); assert_eq!( - store - .storage() - .load_synthesis(&authority, first.identity()) - .unwrap_err(), + load_synthesis(&store, &authority, first.identity()).unwrap_err(), WorkAttemptStorageError::AttemptConflict ); // The same identity with different content is a conflict, not a refresh. @@ -1333,17 +1387,11 @@ fn synthesis_admission_replays_one_durable_result_and_refuses_changed_requests() let policy = concurrency(1, 1, 1); assert_eq!( - store - .storage() - .insert_synthesis_bounded(&authority, &record, &policy) - .unwrap(), + insert_synthesis(&store, &authority, &record, &policy).unwrap(), WorkSynthesisInsertOutcome::Inserted ); assert_eq!( - store - .storage() - .insert_synthesis_bounded(&authority, &record, &policy) - .unwrap(), + insert_synthesis(&store, &authority, &record, &policy).unwrap(), WorkSynthesisInsertOutcome::Replayed(Box::new(admission.clone())) ); assert_eq!(store.count("work_attempts_v1"), 1); @@ -1353,10 +1401,7 @@ fn synthesis_admission_replays_one_durable_result_and_refuses_changed_requests() result: admission, }; assert_eq!( - store - .storage() - .insert_synthesis_bounded(&authority, &changed, &policy) - .unwrap_err(), + insert_synthesis(&store, &authority, &changed, &policy).unwrap_err(), WorkAttemptStorageError::AttemptConflict ); assert_eq!(store.count("work_attempts_v1"), 1); @@ -1380,19 +1425,13 @@ fn synthesis_admission_replays_one_durable_result_and_refuses_changed_requests() ) .unwrap(); assert_eq!( - store - .storage() - .load_synthesis(&authority, admitted_attempt.identity()) - .unwrap(), + load_synthesis(&store, &authority, admitted_attempt.identity()).unwrap(), record ); let store = store.restart("attempt-synthesis-insert"); assert_eq!( - store - .storage() - .load_synthesis(&authority, admitted_attempt.identity()) - .unwrap(), + load_synthesis(&store, &authority, admitted_attempt.identity()).unwrap(), record ); assert_eq!( diff --git a/crates/tracedecay-rusqlite-runtime/tests/rusqlite_suite/work_run_control_storage.rs b/crates/tracedecay-rusqlite-runtime/tests/rusqlite_suite/work_run_control_storage.rs index 31655ae898..f747bb51f5 100644 --- a/crates/tracedecay-rusqlite-runtime/tests/rusqlite_suite/work_run_control_storage.rs +++ b/crates/tracedecay-rusqlite-runtime/tests/rusqlite_suite/work_run_control_storage.rs @@ -16,7 +16,7 @@ use std::collections::BTreeSet; use tracedecay_contracts::{ WorkAttemptStorageError, WorkAttemptStoragePort, WorkRunControlStorageError, - WorkRunControlStoragePort, WorkRunControlTransitionPort, + WorkRunControlStoragePort, WorkRunControlTransitionPort, WorkRunReservationPort, }; use tracedecay_domain::{ ActorId, AttemptId, CommitId, ConfigurationRevisionId, ConfigurationSnapshotId, ProjectId,