From 7efd8512a434264142b5592b07240e0e7e5e784f Mon Sep 17 00:00:00 2001 From: ScriptedAlchemy Date: Fri, 16 Oct 2026 18:41:32 +0000 Subject: [PATCH 1/4] refactor(work): admit attempts as one writer job StartAttempt, Synthesize, RetryAttempt and ResumeAttempts now run their contracts services inside one ExactSqlHandle::transact job over WorkSqliteJobStorage, and the awaiting task parks no native thread. StartAttempt's run reservation fence used to run in its own transaction before the admission transaction, so a pause committed in between surfaced as an identity conflict from the insert's fence; the fence, the replay check, the lease epoch, the product link and the attempt row now commit together, and a racing pause either lands first and is reported as the paused conflict or lands after the attempt. The current product graph the admissions read is replayed from the journal before the job, on the blocking pool, and handed to the job as a PrefetchedWorkGraphReadV1 that answers only that exact read; the link's graph-version compare-and-swap is its staleness fence. Synthesis hydrates its source payloads through the job storage, because the workflow artifact store's own transaction would wait on the writer running the job. The workflow rebind a retry carries is still read before the job and appended under its sequence check inside it. The attempt, effect-holder, owner-authorization and admission ports drop their Send + Sync bound so the job storage can implement them, sharing every SQL body with the handle storage through RegisteredWorkWrite. The reservation read moves to a WorkRunReservationPort. admit_retry and admit_synthesis move into the retry and synthesis storage ports; their handle implementations, and the handle retry and synthesis storage, had no other production caller and are deleted. Resume now fences every open attempt or none: a refusal partway through rolls back the attempts it had already fenced. --- .../src/work/registered.rs | 186 ++++- .../tests/application_suite/main.rs | 1 + .../application_suite/work_attempt_jobs.rs | 582 ++++++++++++++++ .../tests/application_suite/work_leaf_jobs.rs | 2 +- .../work_run_control_jobs.rs | 70 +- crates/tracedecay-contracts/src/lib.rs | 4 +- .../tracedecay-contracts/src/work_attempt.rs | 2 +- .../src/work_attempt/problem.rs | 6 + .../src/work_attempt/product_admission.rs | 39 +- .../product_synthesis_admission.rs | 10 +- .../src/work_attempt/synthesis_admission.rs | 10 + .../src/work_attempt_effect.rs | 2 +- .../tracedecay-contracts/src/work_evidence.rs | 4 +- .../src/work_product/attempt_admission.rs | 23 +- .../src/work_product/read.rs | 62 +- .../src/work_product/types.rs | 2 +- crates/tracedecay-contracts/src/work_retry.rs | 19 +- .../src/work_run_control.rs | 86 +-- .../src/work_synthesis.rs | 8 +- .../tests/contracts_suite/common/mod.rs | 153 ++--- .../work_run_control_service.rs | 17 +- .../src/invocation/work/attempt_operations.rs | 180 +++-- .../src/invocation/work/request_dispatch.rs | 7 +- .../src/work/effect_holder.rs | 162 +++-- .../src/work/retry.rs | 76 +-- .../src/work_attempt.rs | 641 +++++++++++------- .../src/work_product.rs | 1 + .../src/work_product/attempt_admission.rs | 114 ++-- .../src/work_product/authorization.rs | 39 +- .../src/work_product/events.rs | 13 +- .../src/work_product/publication.rs | 8 +- .../src/work_run_control.rs | 123 +++- .../src/workflow.rs | 14 +- .../src/workflow/run_journal.rs | 11 +- .../rusqlite_suite/work_attempt_storage.rs | 101 ++- .../work_run_control_storage.rs | 2 +- 36 files changed, 1929 insertions(+), 851 deletions(-) create mode 100644 crates/tracedecay-application/tests/application_suite/work_attempt_jobs.rs diff --git a/crates/tracedecay-application/src/work/registered.rs b/crates/tracedecay-application/src/work/registered.rs index b5ebb2ef3b..f8dc38aca1 100644 --- a/crates/tracedecay-application/src/work/registered.rs +++ b/crates/tracedecay-application/src/work/registered.rs @@ -98,13 +98,7 @@ pub struct RegisteredWorkProductServicesV1 { attempts: tracedecay_contracts::WorkProductAttemptServiceV1< tracedecay_rusqlite_runtime::work::WorkSqliteStorage, >, - synthesis: tracedecay_contracts::WorkProductSynthesisAttemptServiceV1< - tracedecay_rusqlite_runtime::work::WorkSqliteStorage, - >, - retry: tracedecay_contracts::WorkProductRetryServiceV1< - tracedecay_rusqlite_runtime::work::WorkSqliteStorage, - tracedecay_contracts::RuntimeWorkRetryEvidenceV1, - >, + storage: tracedecay_rusqlite_runtime::work::WorkSqliteStorage, } impl RegisteredWorkProductServicesV1 { @@ -137,13 +131,7 @@ impl RegisteredWorkProductServicesV1 { storage.clone(), ), attempts: tracedecay_contracts::WorkProductAttemptServiceV1::new(storage.clone()), - synthesis: tracedecay_contracts::WorkProductSynthesisAttemptServiceV1::new( - storage.clone(), - ), - retry: tracedecay_contracts::WorkProductRetryServiceV1::new( - storage, - tracedecay_contracts::RuntimeWorkRetryEvidenceV1, - ), + storage, }) } @@ -165,6 +153,137 @@ impl RegisteredWorkProductServicesV1 { }) } + /// Starts one attempt as a single writer transaction: the run's pause + /// fence, the replay check, the lease epoch, the product link and the + /// attempt row commit together, so a pause cannot land between the fence + /// and the reservation. The current product graph is read before the job, + /// on a reader; the link's graph-version compare-and-swap fences it. + pub async fn start_attempt( + &self, + context: tracedecay_contracts::RequestContext, + binding: tracedecay_contracts::WorkProductBindingV1, + revisions: tracedecay_contracts::WorkProductRevisionPinsV1, + topology: tracedecay_domain::WorkTopologyPolicyV1, + command: tracedecay_contracts::StartWorkAttemptCommand, + ) -> Result { + let graph = self + .storage + .prefetch_attempt_admission_graph(&context, command.occurred_at) + .await; + self.storage + .transact_with_graph(graph, move |storage| { + tracedecay_contracts::WorkRunControlService::new(storage).admit_reservation( + &context, + &command.task_id, + &command.run_id, + )?; + tracedecay_contracts::WorkProductAttemptServiceV1::new(storage) + .start_against_registered_topology( + &context, + &binding, + &revisions, + &topology, + command.clone(), + ) + }) + .await + .unwrap_or_else(|_| Err(attempt_unavailable())) + } + + /// Admits one synthesis attempt as a single writer transaction: the + /// replay check, the source reads, the run's pause fence, the source + /// payload hydration `prepare_instructions` does through the job storage + /// and the admission commit together. + #[allow(clippy::too_many_arguments)] + pub async fn synthesize( + &self, + context: tracedecay_contracts::RequestContext, + binding: tracedecay_contracts::WorkProductBindingV1, + revisions: tracedecay_contracts::WorkProductRevisionPinsV1, + topology: tracedecay_domain::WorkTopologyPolicyV1, + command: tracedecay_contracts::AdmitWorkSynthesisCommand, + prepare_instructions: F, + ) -> Result< + tracedecay_contracts::WorkSynthesisAttemptV1, + tracedecay_contracts::ApplicationProblem, + > + where + F: Fn( + &tracedecay_rusqlite_runtime::work_run_control::WorkSqliteJobStorage<'_>, + &tracedecay_contracts::StartWorkAttemptCommand, + &[tracedecay_domain::WorkAttemptV1], + ) -> Result + + Send + + Sync + + 'static, + { + let graph = self + .storage + .prefetch_attempt_admission_graph(&context, command.start.occurred_at) + .await; + self.storage + .transact_with_graph(graph, move |storage| { + tracedecay_contracts::admit_work_synthesis_against_registered_topology( + &tracedecay_contracts::WorkProductSynthesisAttemptServiceV1::new(storage), + &context, + &binding, + &revisions, + &topology, + command.clone(), + |start, sources| { + tracedecay_contracts::WorkRunControlService::new(storage) + .admit_reservation(&context, &start.task_id, &start.run_id)?; + prepare_instructions(&storage, start, sources) + }, + ) + }) + .await + .unwrap_or_else(|_| Err(attempt_unavailable())) + } + + /// Admits one retry as a single writer transaction: the replay check, the + /// effect-safety read, the lease epoch, the product link, the retry + /// receipt, the attempt row and the workflow child rebind commit together. + /// `workflow_rebind` is read before the job; its journal append is + /// sequence-checked inside it. + #[allow(clippy::too_many_arguments)] + pub async fn retry_attempt( + &self, + context: tracedecay_contracts::RequestContext, + binding: tracedecay_contracts::WorkProductBindingV1, + revisions: tracedecay_contracts::WorkProductRevisionPinsV1, + topology: tracedecay_domain::WorkTopologyPolicyV1, + command: tracedecay_contracts::RetryWorkAttemptCommandV1, + restarted_at: tracedecay_domain::UtcMicros, + workflow_rebind: Option, + ) -> Result< + tracedecay_contracts::WorkRetryAttemptOutcomeV1, + tracedecay_contracts::ApplicationProblem, + > { + let graph = self + .storage + .prefetch_attempt_admission_graph(&context, restarted_at) + .await; + self.storage + .transact_with_graph(graph, move |storage| { + tracedecay_contracts::WorkProductRetryServiceV1::new( + storage, + tracedecay_contracts::RuntimeWorkRetryEvidenceV1, + ) + .retry( + &context, + &binding, + &revisions, + &topology, + command.clone(), + restarted_at, + workflow_rebind.clone(), + ) + }) + .await + .unwrap_or_else(|_| Err(attempt_unavailable())) + } + pub const fn reads( &self, ) -> &tracedecay_contracts::WorkProductReadServiceV1< @@ -191,23 +310,6 @@ impl RegisteredWorkProductServicesV1 { > { &self.attempts } - - pub const fn synthesis( - &self, - ) -> &tracedecay_contracts::WorkProductSynthesisAttemptServiceV1< - tracedecay_rusqlite_runtime::work::WorkSqliteStorage, - > { - &self.synthesis - } - - pub const fn retry( - &self, - ) -> &tracedecay_contracts::WorkProductRetryServiceV1< - tracedecay_rusqlite_runtime::work::WorkSqliteStorage, - tracedecay_contracts::RuntimeWorkRetryEvidenceV1, - > { - &self.retry - } } pub struct RegisteredAuthorizedWorkProductServicesV1 { @@ -283,6 +385,24 @@ impl RegisteredWorkApplicationServicesV1 { .unwrap_or_else(|_| Err(run_control_unavailable())) } + /// Fences every open attempt after a restart as a single writer + /// transaction: the census and every fence commit together or not at all. + pub async fn resume_attempts( + &self, + context: tracedecay_contracts::RequestContext, + command: tracedecay_contracts::ResumeWorkAttemptsCommand, + ) -> Result< + tracedecay_contracts::WorkAttemptRecoveryReportV1, + tracedecay_contracts::ApplicationProblem, + > { + self.storage + .transact(move |storage| { + tracedecay_contracts::WorkAttemptService::new(storage).resume(&context, &command) + }) + .await + .unwrap_or_else(|_| Err(attempt_unavailable())) + } + /// Records one duplicate adjudication as a single writer transaction: the /// replay check, the revision compare and the append commit together. /// `current_evidence` is read before the job, outside the writer. @@ -566,6 +686,10 @@ fn run_control_unavailable() -> tracedecay_contracts::ApplicationProblem { tracedecay_contracts::WorkRunControlStorageError::Unavailable.into() } +fn attempt_unavailable() -> tracedecay_contracts::ApplicationProblem { + tracedecay_contracts::WorkAttemptStorageError::Unavailable.into() +} + fn attach_error(operation: &str, error: impl std::fmt::Display) -> TraceDecayError { TraceDecayError::Database { operation: operation.to_owned(), diff --git a/crates/tracedecay-application/tests/application_suite/main.rs b/crates/tracedecay-application/tests/application_suite/main.rs index 24b64fcf06..981a37b39b 100644 --- a/crates/tracedecay-application/tests/application_suite/main.rs +++ b/crates/tracedecay-application/tests/application_suite/main.rs @@ -17,6 +17,7 @@ mod observability_rollup_convergence; // root: the work_rollup bench mounts it, and the suite itself uses a subset. pub mod observability_runtime_contract; mod registered_scope_route; +mod work_attempt_jobs; mod work_leaf_jobs; mod work_run_control_jobs; mod work_service_composition; diff --git a/crates/tracedecay-application/tests/application_suite/work_attempt_jobs.rs b/crates/tracedecay-application/tests/application_suite/work_attempt_jobs.rs new file mode 100644 index 0000000000..5ddd98bf14 --- /dev/null +++ b/crates/tracedecay-application/tests/application_suite/work_attempt_jobs.rs @@ -0,0 +1,582 @@ +use std::collections::BTreeSet; +use std::time::Duration; + +use tracedecay_application::work::{ + RegisteredWorkApplicationServicesV1, RegisteredWorkProductServicesV1, +}; +use tracedecay_contracts::{ + AdmitWorkSynthesisCommand, ApplicationProblem, PauseWorkRunCommand, + PrepareWorkProductMutationRequestV1, RequestContext, ResumeWorkAttemptsCommand, + RetryWorkAttemptCommandV1, StartWorkAttemptCommand, WorkAttemptEvidenceRecordV1, + WorkAttemptProviderOutcomeV1, WorkAttemptStoragePort, WorkProductAuthorizedRelationScopeV1, + WorkProductBindingV1, WorkProductChangeDraftV1, WorkProductMutationReceiptV1, + WorkProductRevisionPinsV1, WorkProductSelectionScopeV1, WorkRetryAttemptOutcomeV1, + WorkRetryCauseV1, WorkRetryFailureSelectorV1, WorkRetrySourceV1, WorkSynthesisAttemptV1, + WorkflowArtifactPayload, WorkflowArtifactStorePort, workflow_artifact_payload_digest, +}; +use tracedecay_domain::test_fixtures::{digest, fixture_abs_root, id}; +use tracedecay_domain::{ + AttemptId, CatalogGenerationId, CommitId, ConfigurationRevisionId, InitiativeId, MilestoneId, + PolicyRevisionId, ProjectId, ProposalId, RepositoryId, RunId, UtcMicros, WorkArtifactId, + WorkArtifactRefV1, WorkAttemptIdentityV1, WorkAttemptStateV1, WorkCommandId, WorkEffectStateV1, + WorkHierarchyV1, WorkInitiativeV1, WorkItemInputV1, WorkItemV1, WorkMilestoneV1, WorkPlanId, + WorkPlanV1, WorkProposalDispositionV1, WorkProposalV1, WorkRouteDecisionV1, + WorkRunControlReasonV1, WorkScoreKindV1, WorkShapeAssessmentV1, WorkSizingV1, + WorkflowOperationRef, WorkflowOutputName, +}; +use tracedecay_global_db::RegisteredGlobalDb; +use tracedecay_global_db::tests::harness::RegisteredGlobalDbTestRuntime; +use tracedecay_rusqlite_runtime::native_runtime::{NativeTask, NativeTaskOutcome}; +use tracedecay_tool_catalog::{CapabilityId, UseCaseId}; + +use crate::work_leaf_jobs::on_native_worker; +use crate::work_run_control_jobs::{ + authority, context, context_for, execution_snapshot, route, task, +}; + +const PROJECT: &str = "project.run-control-jobs"; +const REPOSITORY: &str = "repository.run-control-jobs"; + +async fn project_runtime( + profile: &std::path::Path, + project: &std::path::Path, +) -> RegisteredGlobalDbTestRuntime { + tracedecay_global_db::register_registered_schema_installer(); + RegisteredGlobalDbTestRuntime::project(profile, project, id::(PROJECT)) + .await + .expect("registered project store opens") +} + +fn product_context() -> RequestContext { + context_for("start_attempt") +} + +fn binding() -> WorkProductBindingV1 { + WorkProductBindingV1::new( + CapabilityId::new("capability.work.start_attempt").unwrap(), + UseCaseId::new("use-case.work.start_attempt").unwrap(), + ) +} + +fn revisions() -> WorkProductRevisionPinsV1 { + WorkProductRevisionPinsV1 { + policy_revision_id: id::("policy-revision.attempt-jobs"), + configuration_revision_id: id::( + "configuration-revision.attempt-jobs", + ), + catalog_generation_id: id::("catalog-generation.attempt-jobs"), + } +} + +fn selection() -> WorkProductSelectionScopeV1 { + WorkProductSelectionScopeV1::relations(BTreeSet::from([ + WorkProductAuthorizedRelationScopeV1::Repository { + project_id: id::(PROJECT), + repository_id: id::(REPOSITORY), + }, + ])) + .unwrap() +} + +fn mutate( + product: &RegisteredWorkProductServicesV1, + change: WorkProductChangeDraftV1, + command_id: &str, + occurred_at: UtcMicros, +) -> WorkProductMutationReceiptV1 { + let context = product_context(); + let request = product + .mutations() + .prepare_mutation( + &context, + &binding(), + PrepareWorkProductMutationRequestV1 { + selection: selection(), + change, + causation_event_id: None, + evidence: Vec::new(), + }, + id::(command_id), + occurred_at, + revisions(), + ) + .expect("the mutation prepares"); + product + .mutations() + .mutate(&context, &binding(), request, &revisions()) + .expect("the mutation commits") +} + +/// Publishes `task()` with an accepted proposal and admitted execution, the +/// product graph every attempt admission is bound to. +fn admit_task(database: &RegisteredGlobalDb) { + let product = + RegisteredWorkProductServicesV1::attach(database, binding()).expect("product services"); + let created_at = UtcMicros(10); + let initiative_id = id::("initiative.attempt-jobs"); + let plan_id = id::("plan.attempt-jobs"); + let milestone_id = id::("milestone.attempt-jobs"); + let item = WorkItemV1::new(WorkItemInputV1 { + task_id: task(), + hierarchy: WorkHierarchyV1::new( + initiative_id.clone(), + plan_id.clone(), + milestone_id.clone(), + ), + title: "Run the admitted provider step".to_owned(), + dependencies: BTreeSet::new(), + informational_relations: BTreeSet::new(), + causal_candidates: BTreeSet::new(), + acceptance_criteria: Vec::new(), + effort: 1, + scheduled_at: None, + deadline: None, + created_at, + updated_at: created_at, + }) + .unwrap(); + let created = mutate( + &product, + WorkProductChangeDraftV1::CreateTask { + initiative: WorkInitiativeV1::new( + initiative_id.clone(), + "Attempt jobs".to_owned(), + created_at, + ) + .unwrap(), + plan: WorkPlanV1::new( + plan_id.clone(), + initiative_id, + "Attempt jobs plan".to_owned(), + created_at, + ) + .unwrap(), + milestone: WorkMilestoneV1::new( + milestone_id, + plan_id, + "Attempt jobs milestone".to_owned(), + created_at, + ) + .unwrap(), + item: Box::new(item), + }, + "command.attempt-jobs.create", + UtcMicros(10_000), + ); + let proposal = WorkProposalV1::new( + id::("proposal.attempt-jobs"), + task(), + created.verified_graph_version().graph_version(), + WorkShapeAssessmentV1::new(WorkScoreKindV1::Ordinal, 1, 1, 1, 1).unwrap(), + WorkSizingV1::new(WorkScoreKindV1::Ordinal, 1, 1, 1, "attempt jobs").unwrap(), + Vec::new(), + WorkRouteDecisionV1::selected(route(), Vec::new(), BTreeSet::new(), "none".to_owned()) + .unwrap(), + "Run the admitted provider step.".to_owned(), + digest('4'), + digest('5'), + ) + .unwrap(); + mutate( + &product, + WorkProductChangeDraftV1::DecideProposal { + proposal, + disposition: WorkProposalDispositionV1::Accepted, + }, + "command.attempt-jobs.accept", + UtcMicros(11_000), + ); + mutate( + &product, + WorkProductChangeDraftV1::AdmitExecution { task_id: task() }, + "command.attempt-jobs.admit", + UtcMicros(12_000), + ); +} + +fn start_command( + run_id: &str, + attempt_id: &str, + occurred_at: UtcMicros, +) -> StartWorkAttemptCommand { + StartWorkAttemptCommand { + task_id: task(), + run_id: id::(run_id), + attempt_id: id::(attempt_id), + operation: id::("operation.attempt-jobs.execute-provider"), + execution_snapshot: execution_snapshot(), + worktree_root: fixture_abs_root("/tmp/attempt-jobs"), + reference: None, + commit: id::("0123456789abcdef0123456789abcdef01234567"), + instructions: "Execute the admitted provider step.".to_owned(), + effect_state: WorkEffectStateV1::Observational, + occurred_at, + } +} + +async fn start( + product: &RegisteredWorkProductServicesV1, + command: StartWorkAttemptCommand, +) -> Result { + product + .start_attempt( + product_context(), + binding(), + revisions(), + execution_snapshot().topology().clone(), + command, + ) + .await +} + +fn identity(run_id: &str, attempt_id: &str) -> WorkAttemptIdentityV1 { + WorkAttemptIdentityV1::new(task(), id::(run_id), id::(attempt_id)).unwrap() +} + +#[tokio::test] +async fn attempt_admissions_awaited_on_a_native_worker_park_no_native_thread() { + let profile = tempfile::tempdir().expect("profile"); + let project = tempfile::tempdir().expect("project"); + let runtime = project_runtime(profile.path(), project.path()).await; + let database = runtime.project_database().expect("project database"); + admit_task(database); + let product = + RegisteredWorkProductServicesV1::attach(database, binding()).expect("product services"); + let work = RegisteredWorkApplicationServicesV1::attach(database).expect("Work services"); + let original = identity("run.attempt-jobs.native", "attempt.attempt-jobs.native"); + let retry_command = RetryWorkAttemptCommandV1 { + original_attempt: original.clone(), + new_attempt_id: id::("attempt.attempt-jobs.native.retry"), + failure: WorkRetryFailureSelectorV1 { + source: WorkRetrySourceV1::Runtime, + cause: WorkRetryCauseV1::RestartRecoveryRequired, + evidence_ref: "recovery-required".to_owned(), + }, + command_id: id::("command.attempt-jobs.native.retry"), + }; + let (outcome, writer_waits, reader_waits) = on_native_worker(async move { + let started = start( + &product, + start_command( + "run.attempt-jobs.native", + "attempt.attempt-jobs.native", + UtcMicros(20_000), + ), + ) + .await; + let replayed = start( + &product, + start_command( + "run.attempt-jobs.native", + "attempt.attempt-jobs.native", + UtcMicros(20_000), + ), + ) + .await; + let resumed = work + .resume_attempts( + product_context(), + ResumeWorkAttemptsCommand { + occurred_at: UtcMicros(30_000), + }, + ) + .await; + let retried = product + .retry_attempt( + product_context(), + binding(), + revisions(), + execution_snapshot().topology().clone(), + retry_command, + UtcMicros(40_000), + None, + ) + .await; + (started, replayed, resumed, retried) + }) + .await; + assert_eq!( + writer_waits, 0, + "the attempt admissions parked no native worker on the writer" + ); + assert_eq!( + reader_waits, 0, + "the attempt admissions parked no native worker on a reader" + ); + let (started, replayed, resumed, retried) = outcome; + let started = started.expect("the start commits"); + assert_eq!(started.state(), WorkAttemptStateV1::Leased); + assert_eq!( + replayed.expect("the same start replays"), + started, + "the replayed start returns the committed attempt" + ); + let resumed = resumed.expect("the resume commits"); + assert_eq!( + resumed + .recovery_required + .iter() + .map(|attempt| attempt.identity().clone()) + .collect::>(), + vec![original.clone()], + "the resume fenced the started attempt" + ); + let WorkRetryAttemptOutcomeV1::Created { attempt, .. } = retried.expect("the retry commits") + else { + panic!("the retry creates a replacement attempt"); + }; + let storage = database.work_storage().expect("Work storage"); + assert_eq!( + storage + .load(&authority(), &original) + .expect("the original reads") + .state(), + WorkAttemptStateV1::RecoveryRequired + ); + assert_eq!( + storage + .load(&authority(), attempt.identity()) + .expect("the replacement reads"), + attempt, + "the retry job committed its replacement attempt" + ); +} + +#[tokio::test] +async fn synthesis_awaited_on_a_native_worker_hydrates_sources_in_its_job() { + let profile = tempfile::tempdir().expect("profile"); + let project = tempfile::tempdir().expect("project"); + let runtime = project_runtime(profile.path(), project.path()).await; + let database = runtime.project_database().expect("project database"); + admit_task(database); + let product = + RegisteredWorkProductServicesV1::attach(database, binding()).expect("product services"); + let source = identity("run.attempt-jobs.source", "attempt.attempt-jobs.source"); + let source_product = + RegisteredWorkProductServicesV1::attach(database, binding()).expect("product services"); + let (started, _, _) = on_native_worker(async move { + start( + &source_product, + start_command( + "run.attempt-jobs.source", + "attempt.attempt-jobs.source", + UtcMicros(20_000), + ), + ) + .await + }) + .await; + started.expect("the source attempt starts"); + let bytes = b"source answer".to_vec(); + let artifact = WorkArtifactRefV1::new( + id::("artifact.provider.stdout"), + workflow_artifact_payload_digest(&bytes).unwrap(), + bytes.len() as u64, + ) + .unwrap(); + database + .workflow_storage() + .expect("workflow storage") + .persist(&WorkflowArtifactPayload::new(artifact.clone(), bytes).unwrap()) + .expect("the source payload persists"); + let attempts = RegisteredWorkApplicationServicesV1::attach(database).expect("Work services"); + attempts + .attempts() + .mark_running(&product_context(), &source, route()) + .expect("the source runs"); + attempts + .attempts() + .settle_with_artifacts( + &product_context(), + &source, + &WorkAttemptEvidenceRecordV1 { + identity: source.clone(), + requested_route: route(), + actual_route: Some(route()), + outcome: WorkAttemptProviderOutcomeV1::Exited { code: 0 }, + stdout: None, + stderr: None, + provider_session: None, + provider_fallback: None, + observed_at: UtcMicros(21_000), + }, + vec![artifact.clone()], + ) + .expect("the source succeeds"); + let command = AdmitWorkSynthesisCommand { + start: start_command( + "run.attempt-jobs.synthesis", + "attempt.attempt-jobs.synthesis", + UtcMicros(30_000), + ), + output_name: id::("output.attempt-jobs"), + sources: vec![source], + }; + let (outcome, writer_waits, reader_waits) = on_native_worker(async move { + product + .synthesize( + product_context(), + binding(), + revisions(), + execution_snapshot().topology().clone(), + command, + move |storage, start, sources| { + let payload = storage + .load_workflow_artifact(&sources[0].artifacts()[0]) + .map_err(|_| { + ApplicationProblem::from( + tracedecay_contracts::WorkAttemptStorageError::Unavailable, + ) + })?; + Ok(format!( + "{}\n{}", + start.instructions, + String::from_utf8_lossy(payload.bytes()) + )) + }, + ) + .await + }) + .await; + assert_eq!( + writer_waits, 0, + "synthesis parked no native worker on the writer" + ); + assert_eq!( + reader_waits, 0, + "synthesis parked no native worker on a reader" + ); + let WorkSynthesisAttemptV1::Admitted(admission) = outcome.expect("the synthesis commits") + else { + panic!("the cited source admits a synthesis"); + }; + assert_eq!( + admission.attempt.execution().instructions(), + "Execute the admitted provider step.\nsource answer", + "the job hydrated the source payload into the provider instructions" + ); + assert_eq!( + database + .work_storage() + .expect("Work storage") + .load(&authority(), admission.attempt.identity()) + .expect("the synthesis attempt reads"), + admission.attempt, + "the synthesis job committed its attempt" + ); +} + +/// A pause racing a start lands either before the start's job, which then +/// refuses the reservation as paused, or after it. It can no longer land +/// between the reservation fence and the attempt insert. +#[tokio::test] +async fn a_pause_racing_a_start_never_lands_between_its_fence_and_its_reservation() { + let profile = tempfile::tempdir().expect("profile"); + let project = tempfile::tempdir().expect("project"); + let runtime = project_runtime(profile.path(), project.path()).await; + let database = runtime.project_database().expect("project database"); + admit_task(database); + let run = "run.run-control-jobs"; + let storage = database.work_storage().expect("Work storage"); + let seed = crate::work_run_control_jobs::leased_attempt_of(id::(run), "seed"); + storage + .insert(&authority(), &seed) + .expect("the run holds a seed attempt"); + let attempts = RegisteredWorkApplicationServicesV1::attach(database).expect("Work services"); + attempts + .attempts() + .mark_running(&product_context(), seed.identity(), route()) + .expect("the seed runs"); + attempts + .attempts() + .settle( + &product_context(), + seed.identity(), + &WorkAttemptEvidenceRecordV1 { + identity: seed.identity().clone(), + requested_route: route(), + actual_route: Some(route()), + outcome: WorkAttemptProviderOutcomeV1::Exited { code: 1 }, + stdout: None, + stderr: None, + provider_session: None, + provider_fallback: None, + observed_at: UtcMicros(15_000), + }, + ) + .expect("the seed settles, so the run frees its capacity"); + let product = + RegisteredWorkProductServicesV1::attach(database, binding()).expect("product services"); + let work = RegisteredWorkApplicationServicesV1::attach(database).expect("Work services"); + // Hold the writer so the start's job and the pause's job queue behind it + // in the order they were submitted. + let blocker = NativeTask::spawn(async move { + storage + .transact(|_| { + std::thread::sleep(Duration::from_millis(400)); + Ok::<_, ()>(()) + }) + .await + .is_ok() + }) + .expect("the blocker spawns"); + tokio::time::sleep(Duration::from_millis(50)).await; + let starter = NativeTask::spawn(async move { + start( + &product, + start_command(run, "attempt.attempt-jobs.raced", UtcMicros(20_000)), + ) + .await + }) + .expect("the start spawns"); + tokio::time::sleep(Duration::from_millis(150)).await; + let pauser = NativeTask::spawn(async move { + work.pause_run( + context(), + PauseWorkRunCommand { + task_id: task(), + run_id: id::(run), + reason: WorkRunControlReasonV1::OperatorRequest, + expected_authority_version: None, + occurred_at: UtcMicros(25_000), + }, + ) + .await + .map(|transition| transition.control.state()) + }) + .expect("the pause spawns"); + let NativeTaskOutcome::Completed(blocked) = blocker.wait().await else { + panic!("the blocker completes"); + }; + assert!(blocked, "the blocker held the writer"); + let NativeTaskOutcome::Completed(started) = starter.wait().await else { + panic!("the start completes"); + }; + let NativeTaskOutcome::Completed(paused) = pauser.wait().await else { + panic!("the pause completes"); + }; + paused.expect("the pause commits"); + let raced = identity(run, "attempt.attempt-jobs.raced"); + let stored = database + .work_storage() + .expect("Work storage") + .load(&authority(), &raced); + match started { + Ok(attempt) => assert_eq!( + stored.expect("a committed start stored its attempt"), + attempt, + "the start committed before the pause" + ), + Err(problem) => { + assert_eq!( + problem + .diagnostic() + .map(|diagnostic| diagnostic.code.as_str()), + Some("application.work-run-control.paused"), + "a start the pause reached first is refused by the reservation fence, \ + never by a pause committed between the fence and the insert: {problem:?}" + ); + assert!(stored.is_err(), "a refused start stored no attempt"); + } + } +} diff --git a/crates/tracedecay-application/tests/application_suite/work_leaf_jobs.rs b/crates/tracedecay-application/tests/application_suite/work_leaf_jobs.rs index 4c931d8eba..074996fd65 100644 --- a/crates/tracedecay-application/tests/application_suite/work_leaf_jobs.rs +++ b/crates/tracedecay-application/tests/application_suite/work_leaf_jobs.rs @@ -41,7 +41,7 @@ async fn project_runtime( /// Runs `job` on a native runtime worker and returns its output with the /// writer and reader waits it parked that worker on. -async fn on_native_worker( +pub(crate) async fn on_native_worker( job: impl Future + Send + 'static, ) -> (T, u64, u64) { let writer_waits = native_thread_writer_waits(); diff --git a/crates/tracedecay-application/tests/application_suite/work_run_control_jobs.rs b/crates/tracedecay-application/tests/application_suite/work_run_control_jobs.rs index e5e284df64..a428bfe762 100644 --- a/crates/tracedecay-application/tests/application_suite/work_run_control_jobs.rs +++ b/crates/tracedecay-application/tests/application_suite/work_run_control_jobs.rs @@ -40,6 +40,12 @@ fn run() -> RunId { } pub(crate) fn context() -> RequestContext { + context_for("pause_run") +} + +/// A request context granted the `capability.work.{operation}` capability +/// and use case, under the same authority as [`context`]. +pub(crate) fn context_for(operation: &str) -> RequestContext { let scope = ResolvedScope::new( id::(PROJECT), id::(REPOSITORY), @@ -55,8 +61,8 @@ pub(crate) fn context() -> RequestContext { UtcMicros(1), UtcMicros(100_000), scope.clone(), - BTreeSet::from([CapabilityId::new("capability.work.pause_run").unwrap()]), - BTreeSet::from([UseCaseId::new("use-case.work.pause_run").unwrap()]), + BTreeSet::from([CapabilityId::new(format!("capability.work.{operation}")).unwrap()]), + BTreeSet::from([UseCaseId::new(format!("use-case.work.{operation}")).unwrap()]), DisclosureClass::Sensitive, ) .unwrap(); @@ -82,7 +88,7 @@ pub(crate) fn authority() -> WorkAuthority { .unwrap() } -fn route() -> WorkProviderRouteV1 { +pub(crate) fn route() -> WorkProviderRouteV1 { WorkProviderRouteV1::new( id::("provider.work.claude-code-cli"), id::("route.run-control-jobs.claude-code.v1"), @@ -111,37 +117,11 @@ pub(crate) fn leased_attempt_of(run_id: RunId, suffix: &str) -> WorkAttemptV1 { id::("proposal.run-control-jobs"), ) .unwrap(); - let snapshot = WorkExecutionSnapshot::new(WorkExecutionSnapshotInput { - configuration_revision_id: id::("configuration-revision.rcj.1"), - configuration_snapshot_id: id::("configuration-snapshot.rcj.1"), - effective_behavior_digest: digest('c'), - resolution_provenance_digest: digest('d'), - route: route(), - backend: WorkProviderBackendV1::ClaudeCodeCli, - protocol: WorkProviderProtocol::ClaudeStreamJson, - model: "claude-test".to_owned(), - executable: WorkExecutableReference::new( - "executable.claude.code-cli".to_owned(), - digest('e'), - ) - .unwrap(), - sandbox: WorkSandboxPolicy::Required, - approval: WorkApprovalPolicy::Never, - filesystem: WorkFilesystemPolicy::WorkspaceWrite, - egress: WorkEgressPolicy::Deny, - environment_allowlist: BTreeSet::new(), - credential_references: BTreeSet::new(), - limits: WorkExecutionLimits::new(128_000, 8_192, 16_384, 16_384, 65_536, 1).unwrap(), - deadline: UtcMicros(1_000_000), - fallback: WorkFallbackTopology::Disabled, - topology: tracedecay_domain::safe_work_topology_policy_v1(), - }) - .unwrap(); let envelope = WorkExecutionEnvelopeV1::new( identity.clone(), binding.clone(), id::("operation.run-control-jobs.execute-provider"), - snapshot, + execution_snapshot(), id::(PROJECT), id::(REPOSITORY), id::(WORKTREE), @@ -174,6 +154,36 @@ pub(crate) fn leased_attempt_of(run_id: RunId, suffix: &str) -> WorkAttemptV1 { .unwrap() } +/// The pinned execution snapshot every fixture attempt runs under. +pub(crate) fn execution_snapshot() -> WorkExecutionSnapshot { + WorkExecutionSnapshot::new(WorkExecutionSnapshotInput { + configuration_revision_id: id::("configuration-revision.rcj.1"), + configuration_snapshot_id: id::("configuration-snapshot.rcj.1"), + effective_behavior_digest: digest('c'), + resolution_provenance_digest: digest('d'), + route: route(), + backend: WorkProviderBackendV1::ClaudeCodeCli, + protocol: WorkProviderProtocol::ClaudeStreamJson, + model: "claude-test".to_owned(), + executable: WorkExecutableReference::new( + "executable.claude.code-cli".to_owned(), + digest('e'), + ) + .unwrap(), + sandbox: WorkSandboxPolicy::Required, + approval: WorkApprovalPolicy::Never, + filesystem: WorkFilesystemPolicy::WorkspaceWrite, + egress: WorkEgressPolicy::Deny, + environment_allowlist: BTreeSet::new(), + credential_references: BTreeSet::new(), + limits: WorkExecutionLimits::new(128_000, 8_192, 16_384, 16_384, 65_536, 1).unwrap(), + deadline: UtcMicros(1_000_000), + fallback: WorkFallbackTopology::Disabled, + topology: tracedecay_domain::safe_work_topology_policy_v1(), + }) + .unwrap() +} + #[tokio::test] async fn pause_and_resume_awaited_on_a_native_worker_park_no_native_thread() { tracedecay_global_db::register_registered_schema_installer(); diff --git a/crates/tracedecay-contracts/src/lib.rs b/crates/tracedecay-contracts/src/lib.rs index 55c7782800..ba7899ae4a 100644 --- a/crates/tracedecay-contracts/src/lib.rs +++ b/crates/tracedecay-contracts/src/lib.rs @@ -537,7 +537,7 @@ pub use work_product::{ AddWorkTaskRequestV1, AdmitWorkExecutionRequestV1, AdmittedWorkExecutionV1, AuthorizedWorkProductScopeV1, CreateWorkProductRequestV1, CreateWorkTaskRequestV1, DecideWorkProposalRequestV1, MAX_WORK_EVIDENCE_SELECTION_V1, - MAX_WORK_GRAPH_TEMPORAL_ENTRIES_V1, MAX_WORK_HISTORY_EVENTS_V1, + MAX_WORK_GRAPH_TEMPORAL_ENTRIES_V1, MAX_WORK_HISTORY_EVENTS_V1, PrefetchedWorkGraphReadV1, PrepareWorkProductMutationRequestV1, ReviewWorkProposalDispositionV1, ReviewWorkProposalRequestV1, SelectedWorkEvidenceV1, VerifiedWorkEvidenceExpansionV1, VerifiedWorkGraphVersionV1, WorkEvidenceExpandRequestV1, WorkEvidenceExpansionV1, @@ -569,7 +569,7 @@ pub use work_run_control::{ PauseWorkRunCommand, ResumeWorkRunCommand, WorkRunAdmissionV1, WorkRunControlFrontierV1, WorkRunControlReadingV1, WorkRunControlRequestV1, WorkRunControlService, WorkRunControlStorageError, WorkRunControlStoragePort, WorkRunControlTransitionPort, - WorkRunControlTransitionReceiptV1, WorkRunLiveAttemptV1, + WorkRunControlTransitionReceiptV1, WorkRunLiveAttemptV1, WorkRunReservationPort, }; pub use work_synthesis::{ AdmitWorkSynthesisCommand, WorkSynthesisAdmissionRecordV1, WorkSynthesisAdmissionV1, diff --git a/crates/tracedecay-contracts/src/work_attempt.rs b/crates/tracedecay-contracts/src/work_attempt.rs index ef23d19a7f..d3d2897eff 100644 --- a/crates/tracedecay-contracts/src/work_attempt.rs +++ b/crates/tracedecay-contracts/src/work_attempt.rs @@ -82,7 +82,7 @@ pub enum WorkAttemptInsertOutcome { /// Durable attempt persistence. Every transition is a compare-and-swap on the /// exact prior lease fence and state, so a fenced-out writer cannot advance a /// row it no longer owns. -pub trait WorkAttemptStoragePort: Send + Sync { +pub trait WorkAttemptStoragePort { /// Mints the next monotonic fence epoch for this authority scope. fn next_fence_epoch(&self, authority: &WorkAuthority) -> Result; diff --git a/crates/tracedecay-contracts/src/work_attempt/problem.rs b/crates/tracedecay-contracts/src/work_attempt/problem.rs index 9ba10a9031..e84d3b48c0 100644 --- a/crates/tracedecay-contracts/src/work_attempt/problem.rs +++ b/crates/tracedecay-contracts/src/work_attempt/problem.rs @@ -85,3 +85,9 @@ pub(super) fn denied_problem(code: &str, message: &str) -> ApplicationProblem { legal_actions: vec![LegalAction::Refresh], } } + +impl From for ApplicationProblem { + fn from(error: WorkAttemptStorageError) -> Self { + storage_problem(error) + } +} diff --git a/crates/tracedecay-contracts/src/work_attempt/product_admission.rs b/crates/tracedecay-contracts/src/work_attempt/product_admission.rs index 58a62debca..da8afbdb99 100644 --- a/crates/tracedecay-contracts/src/work_attempt/product_admission.rs +++ b/crates/tracedecay-contracts/src/work_attempt/product_admission.rs @@ -78,6 +78,37 @@ pub(crate) fn replayed_attempt_matches_command( Ok(attempt.identity() == identity && attempt.execution() == &expected) } +fn attempt_admission_selection( + context: &RequestContext, +) -> Result { + WorkProductSelectionScopeV1::relations(BTreeSet::from([ + WorkProductAuthorizedRelationScopeV1::Repository { + project_id: context.scope().project_id.clone(), + repository_id: context.scope().repository_id.clone(), + }, + ])) + .map_err(|_| invalid_start_problem()) +} + +impl crate::PrefetchedWorkGraphReadV1 { + /// The graph read an attempt admission (start, synthesis or retry) + /// observed at `observed_at` makes, read before its writer job. A context + /// with no valid selection prefetches nothing; the service refuses it. + pub fn for_attempt_admission( + storage: &S, + context: &RequestContext, + observed_at: UtcMicros, + ) -> Self + where + S: WorkGraphReadPortV1 + WorkProductOwnerAuthorizationPortV1, + { + attempt_admission_selection(context).map_or_else( + |_| Self::default(), + |selection| Self::current(storage, context, &selection, observed_at), + ) + } +} + /// Reads the current verified product graph under the exact relation scope /// resolved for this request. The caller cannot select a different profile or /// repository relation for attempt admission. @@ -91,13 +122,7 @@ where S: WorkGraphReadPortV1 + WorkProductOwnerAuthorizationPortV1, { admit_product_attempt_request(context, binding, observed_at)?; - let selection = WorkProductSelectionScopeV1::relations(BTreeSet::from([ - WorkProductAuthorizedRelationScopeV1::Repository { - project_id: context.scope().project_id.clone(), - repository_id: context.scope().repository_id.clone(), - }, - ])) - .map_err(|_| invalid_start_problem())?; + let selection = attempt_admission_selection(context)?; let authorized_scope = storage .authorize_scope(context, &selection, observed_at) .map_err(owner_problem)?; diff --git a/crates/tracedecay-contracts/src/work_attempt/product_synthesis_admission.rs b/crates/tracedecay-contracts/src/work_attempt/product_synthesis_admission.rs index ce24020bce..14a4784ea8 100644 --- a/crates/tracedecay-contracts/src/work_attempt/product_synthesis_admission.rs +++ b/crates/tracedecay-contracts/src/work_attempt/product_synthesis_admission.rs @@ -7,10 +7,9 @@ use tracedecay_domain::{ }; use crate::{ - ApplicationProblem, RequestContext, WorkGraphReadPortV1, WorkProductAttemptAdmissionPortV1, - WorkProductAttemptAdmissionV1, WorkProductBindingV1, WorkProductOwnerAuthorizationPortV1, - WorkProductRevisionPinsV1, WorkProductSynthesisAdmissionV1, WorkSynthesisAdmissionRecordV1, - WorkSynthesisAdmissionV1, + ApplicationProblem, RequestContext, WorkGraphReadPortV1, WorkProductAttemptAdmissionV1, + WorkProductBindingV1, WorkProductOwnerAuthorizationPortV1, WorkProductRevisionPinsV1, + WorkProductSynthesisAdmissionV1, WorkSynthesisAdmissionRecordV1, WorkSynthesisAdmissionV1, }; use super::{ @@ -39,8 +38,7 @@ impl WorkProductSynthesisAttemptServiceV1 where S: WorkSynthesisAdmissionStoragePort + WorkGraphReadPortV1 - + WorkProductOwnerAuthorizationPortV1 - + WorkProductAttemptAdmissionPortV1, + + WorkProductOwnerAuthorizationPortV1, { pub const fn new(storage: S) -> Self { Self { storage } diff --git a/crates/tracedecay-contracts/src/work_attempt/synthesis_admission.rs b/crates/tracedecay-contracts/src/work_attempt/synthesis_admission.rs index e8c3e5028c..4f122949c5 100644 --- a/crates/tracedecay-contracts/src/work_attempt/synthesis_admission.rs +++ b/crates/tracedecay-contracts/src/work_attempt/synthesis_admission.rs @@ -43,4 +43,14 @@ pub trait WorkSynthesisAdmissionStoragePort: WorkAttemptStoragePort { authority: &WorkAuthority, identity: &WorkAttemptIdentityV1, ) -> Result; + + /// Commits the accepted-attempt link, the attempt row and its synthesis + /// record as one transaction. + fn admit_synthesis( + &self, + admission: &crate::WorkProductSynthesisAdmissionV1, + ) -> Result< + (crate::WorkProductEventCommitV1, WorkSynthesisInsertOutcome), + crate::WorkProductAttemptAdmissionErrorV1, + >; } diff --git a/crates/tracedecay-contracts/src/work_attempt_effect.rs b/crates/tracedecay-contracts/src/work_attempt_effect.rs index 87120f17f0..819c0a89a9 100644 --- a/crates/tracedecay-contracts/src/work_attempt_effect.rs +++ b/crates/tracedecay-contracts/src/work_attempt_effect.rs @@ -157,7 +157,7 @@ pub enum WorkAttemptEffectStorageErrorV1 { /// Persistence owned by the exact Work-attempt authority. Implementations /// must make the immutable dispatch identity replay-safe and settlement CASed. -pub trait WorkAttemptEffectStoragePortV1: Send + Sync { +pub trait WorkAttemptEffectStoragePortV1 { fn begin_effect_dispatch( &self, authority: &WorkAuthority, diff --git a/crates/tracedecay-contracts/src/work_evidence.rs b/crates/tracedecay-contracts/src/work_evidence.rs index 060e8514b8..346071252e 100644 --- a/crates/tracedecay-contracts/src/work_evidence.rs +++ b/crates/tracedecay-contracts/src/work_evidence.rs @@ -519,7 +519,7 @@ pub struct WorkEvidenceRetrievalServiceV1 { impl WorkEvidenceRetrievalServiceV1 where R: WorkEvidenceRootReadPortV1, - A: WorkProductOwnerAuthorizationPortV1, + A: WorkProductOwnerAuthorizationPortV1 + Send + Sync, T: WorkAttemptReceiptReadPortV1, S: WorkTaskSessionPortV1, H: WorkAnchorHydrationPortV1, @@ -801,7 +801,7 @@ impl WorkTaskSessionReauthorizationPortV1 for WorkEvidenceRetrievalServiceV1 where R: WorkEvidenceRootReadPortV1, - A: WorkProductOwnerAuthorizationPortV1, + A: WorkProductOwnerAuthorizationPortV1 + Send + Sync, T: WorkAttemptReceiptReadPortV1, S: WorkTaskSessionPortV1, H: WorkAnchorHydrationPortV1, diff --git a/crates/tracedecay-contracts/src/work_product/attempt_admission.rs b/crates/tracedecay-contracts/src/work_product/attempt_admission.rs index 9c62cda2eb..505dfa4e0a 100644 --- a/crates/tracedecay-contracts/src/work_product/attempt_admission.rs +++ b/crates/tracedecay-contracts/src/work_product/attempt_admission.rs @@ -6,10 +6,7 @@ use tracedecay_domain::{ WorkProductEventPayloadV1, WorkflowRunEventKind, configuration::TopologyConcurrencyPolicyV1, }; -use crate::{ - WorkRetryAttemptOutcomeV1, WorkRetryWriteV1, WorkSynthesisAdmissionRecordV1, - WorkSynthesisInsertOutcome, WorkflowRunAppendRequest, -}; +use crate::{WorkRetryWriteV1, WorkSynthesisAdmissionRecordV1, WorkflowRunAppendRequest}; use super::{WorkProductEventCommitV1, WorkProductEventDraftV1, WorkProductPortContextV1}; @@ -183,25 +180,9 @@ fn selection_covers_authority( } /// Canonical one-transaction authority for product linkage and runtime rows. -pub trait WorkProductAttemptAdmissionPortV1: Send + Sync { +pub trait WorkProductAttemptAdmissionPortV1 { fn admit_attempt( &self, admission: &WorkProductAttemptAdmissionV1, ) -> Result; - - fn admit_retry( - &self, - admission: &WorkProductRetryAdmissionV1, - ) -> Result< - (WorkProductEventCommitV1, WorkRetryAttemptOutcomeV1), - WorkProductAttemptAdmissionErrorV1, - >; - - fn admit_synthesis( - &self, - admission: &WorkProductSynthesisAdmissionV1, - ) -> Result< - (WorkProductEventCommitV1, WorkSynthesisInsertOutcome), - WorkProductAttemptAdmissionErrorV1, - >; } diff --git a/crates/tracedecay-contracts/src/work_product/read.rs b/crates/tracedecay-contracts/src/work_product/read.rs index 97e4b01b0f..acd2e56626 100644 --- a/crates/tracedecay-contracts/src/work_product/read.rs +++ b/crates/tracedecay-contracts/src/work_product/read.rs @@ -657,7 +657,9 @@ impl From for WorkProductApplicationErrorV1 { } } -pub trait WorkGraphReadPortV1: Send + Sync { +/// It carries no `Send + Sync` bound so a storage scoped to one writer +/// transaction can implement it. +pub trait WorkGraphReadPortV1 { fn read_graph( &self, context: &WorkProductPortContextV1, @@ -678,6 +680,64 @@ where } } +/// One current graph read made on a reader before a write job, answered to +/// the job's service as an owned value, so the journal replay behind it never +/// holds the writer. The job's event compare-and-swap on the graph version is +/// its staleness fence. +/// +/// It answers only the exact scope and request it was read for; any other +/// read is refused as unavailable rather than answered from the wrong slice. +#[derive(Clone, Debug, Default)] +pub struct PrefetchedWorkGraphReadV1 { + read: Option<( + AuthorizedWorkProductScopeV1, + WorkGraphReadRequestV1, + Result, + )>, +} + +impl PrefetchedWorkGraphReadV1 { + /// Reads the current graph under `selection` as `context` observed it at + /// `observed_at`. A selection the owner refuses prefetches nothing: the + /// service refuses it again inside the job. + pub fn current( + storage: &S, + context: &RequestContext, + selection: &WorkProductSelectionScopeV1, + observed_at: UtcMicros, + ) -> Self + where + S: WorkGraphReadPortV1 + WorkProductOwnerAuthorizationPortV1, + { + let Ok(scope) = storage.authorize_scope(context, selection, observed_at) else { + return Self { read: None }; + }; + let port_context = WorkProductPortContextV1::from_request(context, scope, observed_at); + let request = WorkGraphReadRequestV1::current(selection.clone(), observed_at); + let read = storage.read_graph(&port_context, &request); + Self { + read: Some((port_context.authorized_scope().clone(), request, read)), + } + } +} + +impl WorkGraphReadPortV1 for PrefetchedWorkGraphReadV1 { + fn read_graph( + &self, + context: &WorkProductPortContextV1, + request: &WorkGraphReadRequestV1, + ) -> Result { + match &self.read { + Some((scope, prefetched, read)) + if scope == context.authorized_scope() && prefetched == request => + { + read.clone() + } + _ => Err(WorkGraphReadPortErrorV1::Unavailable), + } + } +} + pub struct WorkProductReadServiceV1 { graph: G, owner_authority: A, diff --git a/crates/tracedecay-contracts/src/work_product/types.rs b/crates/tracedecay-contracts/src/work_product/types.rs index 4b72b67043..d68165d145 100644 --- a/crates/tracedecay-contracts/src/work_product/types.rs +++ b/crates/tracedecay-contracts/src/work_product/types.rs @@ -118,7 +118,7 @@ pub enum WorkProductOwnerAuthorizationErrorV1 { /// Resolves the registered profile owner and authorizes every selected /// project/repository relation against the request context. -pub trait WorkProductOwnerAuthorizationPortV1: Send + Sync { +pub trait WorkProductOwnerAuthorizationPortV1 { fn authorize_scope( &self, context: &RequestContext, diff --git a/crates/tracedecay-contracts/src/work_retry.rs b/crates/tracedecay-contracts/src/work_retry.rs index f0042cfe16..bb0a0abce6 100644 --- a/crates/tracedecay-contracts/src/work_retry.rs +++ b/crates/tracedecay-contracts/src/work_retry.rs @@ -27,9 +27,9 @@ use crate::work_attempt_effect::{ }; use crate::{ ApplicationContractError, ApplicationProblem, RequestContext, RetryDirective, SafeDiagnostic, - WorkGraphReadPortV1, WorkProductAttemptAdmissionPortV1, WorkProductAttemptAdmissionV1, - WorkProductBindingV1, WorkProductOwnerAuthorizationPortV1, WorkProductRetryAdmissionV1, - WorkProductRevisionPinsV1, WorkflowFanOutAttemptBindingV1, WorkflowRunAppendRequest, + WorkGraphReadPortV1, WorkProductAttemptAdmissionV1, WorkProductBindingV1, + WorkProductOwnerAuthorizationPortV1, WorkProductRetryAdmissionV1, WorkProductRevisionPinsV1, + WorkflowFanOutAttemptBindingV1, WorkflowRunAppendRequest, }; const RETRY_INPUT_DIGEST_DOMAIN: &str = "tracedecay.application.work-retry-input.v1"; @@ -329,6 +329,16 @@ pub trait WorkRetryStoragePortV1: WorkAttemptStoragePort { write: &WorkRetryWriteV1, concurrency: &TopologyConcurrencyPolicyV1, ) -> Result; + + /// Commits the accepted-attempt link, the retry receipt, the attempt row + /// and any workflow child rebind as one transaction. + fn admit_retry( + &self, + admission: &WorkProductRetryAdmissionV1, + ) -> Result< + (crate::WorkProductEventCommitV1, WorkRetryAttemptOutcomeV1), + crate::WorkProductAttemptAdmissionErrorV1, + >; } /// Public retry admission over the verified Work product graph. The legacy @@ -352,8 +362,7 @@ where S: WorkRetryStoragePortV1 + WorkAttemptEffectStoragePortV1 + WorkGraphReadPortV1 - + WorkProductOwnerAuthorizationPortV1 - + WorkProductAttemptAdmissionPortV1, + + WorkProductOwnerAuthorizationPortV1, E: WorkRetryEvidencePortV1, { pub const fn new(storage: S, evidence: E) -> Self { diff --git a/crates/tracedecay-contracts/src/work_run_control.rs b/crates/tracedecay-contracts/src/work_run_control.rs index 17b357a44c..6bc15bfe0f 100644 --- a/crates/tracedecay-contracts/src/work_run_control.rs +++ b/crates/tracedecay-contracts/src/work_run_control.rs @@ -140,9 +140,23 @@ pub trait WorkRunControlTransitionPort { ) -> Result<(), WorkRunControlStorageError>; } +/// The control read a reservation fence makes. It carries no `Send + Sync` +/// bound so the reservation can be fenced inside the writer job that admits +/// the attempt. +pub trait WorkRunReservationPort { + /// The published control row for one run, or `None` when the run has never + /// been controlled. + fn load_run_control( + &self, + authority: &WorkAuthority, + task_id: &TaskId, + run_id: &RunId, + ) -> Result, WorkRunControlStorageError>; +} + /// The durable run-control rows and the attempt evidence they are derived /// from. -pub trait WorkRunControlStoragePort: Send + Sync { +pub trait WorkRunControlStoragePort: WorkRunReservationPort + Send + Sync { /// The admitted deadline and live attempt frontier for one run, or `None` /// when the run holds no durable attempt at all. Retry-superseded recovery /// records still count toward history, but are absent from the frontier. @@ -153,15 +167,6 @@ pub trait WorkRunControlStoragePort: Send + Sync { run_id: &RunId, ) -> Result, WorkRunControlStorageError>; - /// The published control row for one run, or `None` when the run has never - /// been controlled. - fn load_run_control( - &self, - authority: &WorkAuthority, - task_id: &TaskId, - run_id: &RunId, - ) -> Result, WorkRunControlStorageError>; - /// Publishes `next` under a compare-and-swap on the authority version the /// caller read. `expected` is `None` only for the first publication. fn publish_run_control( @@ -446,35 +451,8 @@ where impl WorkRunControlService where - S: WorkRunControlStoragePort, + S: WorkRunReservationPort, { - /// Reads the published control state for one run. - #[tracing::instrument(name = "application.work.run_control.read", level = "trace", skip_all)] - pub fn read( - &self, - context: &RequestContext, - request: &WorkRunControlRequestV1, - ) -> Result { - let authority = work_authority(context)?; - let admission = self.require_admission(&authority, &request.task_id, &request.run_id)?; - let control = self - .storage - .load_run_control(&authority, &request.task_id, &request.run_id) - .map_err(ApplicationProblem::from)?; - Ok(match control { - Some(control) => WorkRunControlReadingV1::Controlled { - control, - live_attempts: admission.live_attempts, - total_attempts: admission.total_attempts, - }, - None => WorkRunControlReadingV1::Uncontrolled { - deadline: admission.deadline, - live_attempts: admission.live_attempts, - total_attempts: admission.total_attempts, - }, - }) - } - /// Refuses a new attempt reservation while the run is paused. /// /// This is the fence Plan 32 requires: "pause and cancellation fence new @@ -504,6 +482,38 @@ where Some(_) | None => Ok(()), } } +} + +impl WorkRunControlService +where + S: WorkRunControlStoragePort, +{ + /// Reads the published control state for one run. + #[tracing::instrument(name = "application.work.run_control.read", level = "trace", skip_all)] + pub fn read( + &self, + context: &RequestContext, + request: &WorkRunControlRequestV1, + ) -> Result { + let authority = work_authority(context)?; + let admission = self.require_admission(&authority, &request.task_id, &request.run_id)?; + let control = self + .storage + .load_run_control(&authority, &request.task_id, &request.run_id) + .map_err(ApplicationProblem::from)?; + Ok(match control { + Some(control) => WorkRunControlReadingV1::Controlled { + control, + live_attempts: admission.live_attempts, + total_attempts: admission.total_attempts, + }, + None => WorkRunControlReadingV1::Uncontrolled { + deadline: admission.deadline, + live_attempts: admission.live_attempts, + total_attempts: admission.total_attempts, + }, + }) + } /// Reads the next bounded, cyclic recovery page of settled interval /// receipts. It skips only receipts whose exact owner fact was durably diff --git a/crates/tracedecay-contracts/src/work_synthesis.rs b/crates/tracedecay-contracts/src/work_synthesis.rs index b2ce1afc45..a6be265c74 100644 --- a/crates/tracedecay-contracts/src/work_synthesis.rs +++ b/crates/tracedecay-contracts/src/work_synthesis.rs @@ -33,9 +33,8 @@ use crate::work_attempt::{ }; use crate::workflow_synthesis::WorkflowSynthesisDraft; use crate::{ - ApplicationProblem, RequestContext, SafeDiagnostic, WorkGraphReadPortV1, - WorkProductAttemptAdmissionPortV1, WorkProductBindingV1, WorkProductOwnerAuthorizationPortV1, - WorkProductRevisionPinsV1, + ApplicationProblem, RequestContext, SafeDiagnostic, WorkGraphReadPortV1, WorkProductBindingV1, + WorkProductOwnerAuthorizationPortV1, WorkProductRevisionPinsV1, }; const WORK_SYNTHESIS_SOURCE_SET_DOMAIN: &str = @@ -208,8 +207,7 @@ pub fn admit_work_synthesis_against_registered_topology( where S: WorkSynthesisAdmissionStoragePort + WorkGraphReadPortV1 - + WorkProductOwnerAuthorizationPortV1 - + WorkProductAttemptAdmissionPortV1, + + WorkProductOwnerAuthorizationPortV1, F: FnOnce(&StartWorkAttemptCommand, &[WorkAttemptV1]) -> Result, { crate::require_registered_work_topology( diff --git a/crates/tracedecay-contracts/tests/contracts_suite/common/mod.rs b/crates/tracedecay-contracts/tests/contracts_suite/common/mod.rs index 172f08b826..9da9812aaa 100644 --- a/crates/tracedecay-contracts/tests/contracts_suite/common/mod.rs +++ b/crates/tracedecay-contracts/tests/contracts_suite/common/mod.rs @@ -604,89 +604,6 @@ impl WorkProductAttemptAdmissionPortV1 for WorkProductAttemptStore { attempt: admission.attempt.clone(), }) } - - fn admit_retry( - &self, - _admission: &tracedecay_contracts::WorkProductRetryAdmissionV1, - ) -> Result< - ( - WorkProductEventCommitV1, - tracedecay_contracts::WorkRetryAttemptOutcomeV1, - ), - WorkProductAttemptAdmissionErrorV1, - > { - Err(WorkProductAttemptAdmissionErrorV1::Unavailable) - } - - fn admit_synthesis( - &self, - admission: &tracedecay_contracts::WorkProductSynthesisAdmissionV1, - ) -> Result< - ( - WorkProductEventCommitV1, - tracedecay_contracts::WorkSynthesisInsertOutcome, - ), - WorkProductAttemptAdmissionErrorV1, - > { - admission.validate()?; - let attempt = &admission.admission.attempt; - let payload = serde_json::to_string(attempt) - .map_err(|_| WorkProductAttemptAdmissionErrorV1::Unavailable)?; - let key = attempt_key(&admission.admission.authority, attempt.identity()); - let mut rows = self - .inner - .lock() - .map_err(|_| WorkProductAttemptAdmissionErrorV1::Unavailable)?; - let product = rows - .events - .iter() - .find(|event| { - event.commit.event().command_id() == &admission.admission.product_draft.command_id - }) - .map(|event| event.commit.clone()); - let synthesis = rows.syntheses.get(&key).cloned(); - match (product, synthesis, rows.attempts.get(&key)) { - (Some(product), Some(existing), Some(existing_attempt)) - if existing.request_digest == admission.synthesis.request_digest - && existing.result == admission.synthesis.result - && existing_attempt == &payload => - { - return Ok(( - product, - WorkSynthesisInsertOutcome::Replayed(Box::new(existing.result)), - )); - } - (Some(_), Some(_), Some(_)) => { - return Err(WorkProductAttemptAdmissionErrorV1::IdentityConflict); - } - (None, None, None) => {} - _ => return Err(WorkProductAttemptAdmissionErrorV1::IdentityConflict), - } - let (next_graph, product) = product_commit_for(&rows, &admission.admission.product_draft)?; - attempt - .validate_graph_admission(&next_graph) - .map_err(|_| WorkProductAttemptAdmissionErrorV1::InvalidAdmission)?; - if matches!( - attempt_capacity( - &rows, - &admission.admission.authority, - attempt.identity().task_id(), - &admission.admission.concurrency, - ) - .map_err(|_| WorkProductAttemptAdmissionErrorV1::Unavailable)? - .verdict(), - WorkAttemptCapacityVerdictV1::Exhausted(_) - ) { - return Err(WorkProductAttemptAdmissionErrorV1::CapacityExceeded); - } - rows.graph = Some(next_graph); - rows.events.push(StoredProductEvent { - commit: product.clone(), - }); - rows.attempts.insert(key.clone(), payload); - rows.syntheses.insert(key, admission.synthesis.clone()); - Ok((product, WorkSynthesisInsertOutcome::Inserted)) - } } impl WorkAttemptStoragePort for WorkProductAttemptStore { @@ -913,4 +830,74 @@ impl WorkSynthesisAdmissionStoragePort for WorkProductAttemptStore { .cloned() .ok_or(WorkAttemptStorageError::AttemptConflict) } + + fn admit_synthesis( + &self, + admission: &tracedecay_contracts::WorkProductSynthesisAdmissionV1, + ) -> Result< + ( + WorkProductEventCommitV1, + tracedecay_contracts::WorkSynthesisInsertOutcome, + ), + WorkProductAttemptAdmissionErrorV1, + > { + admission.validate()?; + let attempt = &admission.admission.attempt; + let payload = serde_json::to_string(attempt) + .map_err(|_| WorkProductAttemptAdmissionErrorV1::Unavailable)?; + let key = attempt_key(&admission.admission.authority, attempt.identity()); + let mut rows = self + .inner + .lock() + .map_err(|_| WorkProductAttemptAdmissionErrorV1::Unavailable)?; + let product = rows + .events + .iter() + .find(|event| { + event.commit.event().command_id() == &admission.admission.product_draft.command_id + }) + .map(|event| event.commit.clone()); + let synthesis = rows.syntheses.get(&key).cloned(); + match (product, synthesis, rows.attempts.get(&key)) { + (Some(product), Some(existing), Some(existing_attempt)) + if existing.request_digest == admission.synthesis.request_digest + && existing.result == admission.synthesis.result + && existing_attempt == &payload => + { + return Ok(( + product, + WorkSynthesisInsertOutcome::Replayed(Box::new(existing.result)), + )); + } + (Some(_), Some(_), Some(_)) => { + return Err(WorkProductAttemptAdmissionErrorV1::IdentityConflict); + } + (None, None, None) => {} + _ => return Err(WorkProductAttemptAdmissionErrorV1::IdentityConflict), + } + let (next_graph, product) = product_commit_for(&rows, &admission.admission.product_draft)?; + attempt + .validate_graph_admission(&next_graph) + .map_err(|_| WorkProductAttemptAdmissionErrorV1::InvalidAdmission)?; + if matches!( + attempt_capacity( + &rows, + &admission.admission.authority, + attempt.identity().task_id(), + &admission.admission.concurrency, + ) + .map_err(|_| WorkProductAttemptAdmissionErrorV1::Unavailable)? + .verdict(), + WorkAttemptCapacityVerdictV1::Exhausted(_) + ) { + return Err(WorkProductAttemptAdmissionErrorV1::CapacityExceeded); + } + rows.graph = Some(next_graph); + rows.events.push(StoredProductEvent { + commit: product.clone(), + }); + rows.attempts.insert(key.clone(), payload); + rows.syntheses.insert(key, admission.synthesis.clone()); + Ok((product, WorkSynthesisInsertOutcome::Inserted)) + } } diff --git a/crates/tracedecay-contracts/tests/contracts_suite/work_run_control_service.rs b/crates/tracedecay-contracts/tests/contracts_suite/work_run_control_service.rs index 6dd7867053..ecc93bca97 100644 --- a/crates/tracedecay-contracts/tests/contracts_suite/work_run_control_service.rs +++ b/crates/tracedecay-contracts/tests/contracts_suite/work_run_control_service.rs @@ -22,6 +22,7 @@ use tracedecay_contracts::{ ResumeWorkRunCommand, WorkRunAdmissionV1, WorkRunControlFrontierV1, WorkRunControlReadingV1, WorkRunControlRequestV1, WorkRunControlService, WorkRunControlStorageError, WorkRunControlStoragePort, WorkRunControlTransitionPort, WorkRunLiveAttemptV1, + WorkRunReservationPort, }; use tracedecay_domain::{ ActorId, AttemptId, ProjectId, RepositoryId, RunId, TaskId, UtcMicros, WorkAuthority, @@ -215,29 +216,31 @@ impl WorkRunControlTransitionPort for TestStore { } } -impl WorkRunControlStoragePort for TestStore { - fn run_admission( +impl WorkRunReservationPort for TestStore { + fn load_run_control( &self, authority: &WorkAuthority, task_id: &TaskId, run_id: &RunId, - ) -> Result, WorkRunControlStorageError> { + ) -> Result, WorkRunControlStorageError> { Ok(self - .admissions + .controls .lock() .unwrap() .get(&(authority.clone(), task_id.clone(), run_id.clone())) .cloned()) } +} - fn load_run_control( +impl WorkRunControlStoragePort for TestStore { + fn run_admission( &self, authority: &WorkAuthority, task_id: &TaskId, run_id: &RunId, - ) -> Result, WorkRunControlStorageError> { + ) -> Result, WorkRunControlStorageError> { Ok(self - .controls + .admissions .lock() .unwrap() .get(&(authority.clone(), task_id.clone(), run_id.clone())) diff --git a/crates/tracedecay-daemon-service/src/invocation/work/attempt_operations.rs b/crates/tracedecay-daemon-service/src/invocation/work/attempt_operations.rs index 15672aa7d6..e6eb9acade 100644 --- a/crates/tracedecay-daemon-service/src/invocation/work/attempt_operations.rs +++ b/crates/tracedecay-daemon-service/src/invocation/work/attempt_operations.rs @@ -8,9 +8,9 @@ use tracedecay_contracts::{ AdmitWorkSynthesisCommand, ApplicationProblem, CancelWorkAttemptCommand, Deadline, RequestContext, RequestId, ResumeWorkAttemptsCommand, RetryWorkAttemptCommandV1, SafeDiagnostic, StartWorkAttemptCommand, WorkAttemptStatusRequestV1, WorkSynthesisAttemptV1, - WorkflowArtifactStorePort, }; use tracedecay_domain::{ManifestDigest, UtcMicros, WorkAttemptStateV1, WorkAttemptV1}; +use tracedecay_rusqlite_runtime::work_run_control::WorkSqliteJobStorage; use tracedecay_tool_catalog::UseCaseId; use tracedecay_application::work::{ @@ -31,16 +31,10 @@ fn consume_synthesis_bytes(remaining: &mut u64, bytes: u64) -> Result<(), Applic } fn synthesis_source_context( - registered: &RegisteredWorkRuntime, + artifacts: &WorkSqliteJobStorage<'_>, start: &StartWorkAttemptCommand, attempts: &[WorkAttemptV1], ) -> Result { - let artifacts = registered.database.workflow_storage().map_err(|_| { - ApplicationProblem::unavailable(SafeDiagnostic { - code: "application.work-synthesis.artifact-body-unavailable".to_owned(), - message: "The admitted synthesis source payload authority is unavailable.".to_owned(), - }) - })?; let mut remaining = start.execution_snapshot.limits().max_protocol_bytes(); consume_synthesis_bytes(&mut remaining, start.instructions.len() as u64)?; consume_synthesis_bytes(&mut remaining, 2)?; @@ -50,7 +44,7 @@ fn synthesis_source_context( let mut payloads = Vec::with_capacity(attempt.artifacts().len()); for artifact in attempt.artifacts() { consume_synthesis_bytes(&mut remaining, artifact.byte_length())?; - let payload = artifacts.load(artifact).map_err(|_| { + let payload = artifacts.load_workflow_artifact(artifact).map_err(|_| { ApplicationProblem::unavailable(SafeDiagnostic { code: "application.work-synthesis.artifact-body-unavailable".to_owned(), message: "An admitted synthesis source payload is absent or invalid." @@ -103,10 +97,8 @@ fn synthesis_source_context( } #[allow(clippy::too_many_arguments)] -#[tracing::instrument(name = "daemon.service.work.start_attempt", level = "trace", skip_all)] -pub(super) fn start_attempt( +pub(super) async fn start_attempt( registered: &RegisteredWorkRuntime, - services: &RegisteredWorkApplicationServicesV1, binding: tracedecay_contracts::WorkProductBindingV1, attempt_processes: &Arc, observability_producer: Option<&Arc>, @@ -121,30 +113,25 @@ pub(super) fn start_attempt( deadline: Deadline, command: StartWorkAttemptCommand, ) -> DaemonInvocationResponse { - let started = services - .run_control() - .admit_reservation(context, &command.task_id, &command.run_id) - .and_then(|()| { - RegisteredWorkProductServicesV1::attach(®istered.database, binding.clone()) - .map_err(|_| { - work_product_problem( - tracedecay_contracts::WorkProductApplicationErrorV1::GraphAuthorityUnavailable, - ) - }) - .and_then(|product| { - preparation::current_work_product_revision_pins(registered).and_then( - |revisions| { - product.attempts().start_against_registered_topology( - context, - &binding, - &revisions, - ®istered.work_topology_policy, - command, - ) - }, - ) - }) - }); + let started = match ( + attach_product(registered, &binding), + preparation::current_work_product_revision_pins(registered), + ) { + (Ok(product), Ok(revisions)) => { + tracing::Instrument::instrument( + product.start_attempt( + context.clone(), + binding, + revisions, + registered.work_topology_policy.clone(), + command, + ), + tracing::trace_span!("daemon.service.work.start_attempt"), + ) + .await + } + (Err(problem), _) | (_, Err(problem)) => Err(problem), + }; if let (Ok(attempt), Some(project_root)) = (&started, project_root) && attempt.state() == WorkAttemptStateV1::Leased { @@ -171,11 +158,20 @@ pub(super) fn start_attempt( ) } +fn attach_product( + registered: &RegisteredWorkRuntime, + binding: &tracedecay_contracts::WorkProductBindingV1, +) -> Result { + RegisteredWorkProductServicesV1::attach(®istered.database, binding.clone()).map_err(|_| { + work_product_problem( + tracedecay_contracts::WorkProductApplicationErrorV1::GraphAuthorityUnavailable, + ) + }) +} + #[allow(clippy::too_many_arguments)] -#[tracing::instrument(name = "daemon.service.work.synthesize", level = "trace", skip_all)] -pub(super) fn synthesize( +pub(super) async fn synthesize( registered: &RegisteredWorkRuntime, - services: &RegisteredWorkApplicationServicesV1, binding: tracedecay_contracts::WorkProductBindingV1, attempt_processes: &Arc, observability_producer: Option<&Arc>, @@ -190,31 +186,26 @@ pub(super) fn synthesize( deadline: Deadline, command: AdmitWorkSynthesisCommand, ) -> DaemonInvocationResponse { - let admitted = RegisteredWorkProductServicesV1::attach(®istered.database, binding.clone()) - .map_err(|_| { - work_product_problem( - tracedecay_contracts::WorkProductApplicationErrorV1::GraphAuthorityUnavailable, - ) - }) - .and_then(|product| { - let revisions = preparation::current_work_product_revision_pins(registered)?; - tracedecay_contracts::admit_work_synthesis_against_registered_topology( - product.synthesis(), - context, - &binding, - &revisions, - ®istered.work_topology_policy, - command, - |start, sources| { - services.run_control().admit_reservation( - context, - &start.task_id, - &start.run_id, - )?; - synthesis_source_context(registered, start, sources) - }, + let admitted = match ( + attach_product(registered, &binding), + preparation::current_work_product_revision_pins(registered), + ) { + (Ok(product), Ok(revisions)) => { + tracing::Instrument::instrument( + product.synthesize( + context.clone(), + binding, + revisions, + registered.work_topology_policy.clone(), + command, + synthesis_source_context, + ), + tracing::trace_span!("daemon.service.work.synthesize"), ) - }); + .await + } + (Err(problem), _) | (_, Err(problem)) => Err(problem), + }; if let (Ok(WorkSynthesisAttemptV1::Admitted(admission)), Some(project_root)) = (&admitted, project_root) && admission.attempt.state() == WorkAttemptStateV1::Leased @@ -308,10 +299,8 @@ pub(super) fn cancel_attempt( } #[allow(clippy::too_many_arguments)] -#[tracing::instrument(name = "daemon.service.work.retry_attempt", level = "trace", skip_all)] -pub(super) fn retry_attempt( +pub(super) async fn retry_attempt( registered: &RegisteredWorkRuntime, - _services: &RegisteredWorkApplicationServicesV1, binding: tracedecay_contracts::WorkProductBindingV1, attempt_processes: &Arc, observability_producer: Option<&Arc>, @@ -326,33 +315,35 @@ pub(super) fn retry_attempt( deadline: Deadline, command: RetryWorkAttemptCommandV1, ) -> DaemonInvocationResponse { - let retried = if project_root.is_none() { + let prepared = if project_root.is_none() { Err(ApplicationProblem::unavailable(SafeDiagnostic { code: "application.work-retry.project-root-unavailable".to_owned(), message: "The Work retry runtime owner is unavailable.".to_owned(), })) } else { - RegisteredWorkProductServicesV1::attach(®istered.database, binding.clone()) - .map_err(|_| { - work_product_problem( - tracedecay_contracts::WorkProductApplicationErrorV1::GraphAuthorityUnavailable, - ) - }) - .and_then(|product| { - preparation::current_work_product_revision_pins(registered).and_then(|revisions| { - let workflow_rebind = - workflow_retry_rebind(registered, &command.original_attempt)?; - product.retry().retry( - context, - &binding, - &revisions, - ®istered.work_topology_policy, - command, - observed_at, - workflow_rebind, - ) - }) - }) + attach_product(registered, &binding).and_then(|product| { + let revisions = preparation::current_work_product_revision_pins(registered)?; + let workflow_rebind = workflow_retry_rebind(registered, &command.original_attempt)?; + Ok((product, revisions, workflow_rebind)) + }) + }; + let retried = match prepared { + Ok((product, revisions, workflow_rebind)) => { + tracing::Instrument::instrument( + product.retry_attempt( + context.clone(), + binding, + revisions, + registered.work_topology_policy.clone(), + command, + observed_at, + workflow_rebind, + ), + tracing::trace_span!("daemon.service.work.retry_attempt"), + ) + .await + } + Err(problem) => Err(problem), }; if let Ok(outcome) = &retried { let _ = tracedecay_application::observability::record_work_retry_observation( @@ -432,12 +423,7 @@ fn workflow_retry_rebind( } #[allow(clippy::too_many_arguments)] -#[tracing::instrument( - name = "daemon.service.work.resume_attempts", - level = "trace", - skip_all -)] -pub(super) fn resume_attempts( +pub(super) async fn resume_attempts( registered: &RegisteredWorkRuntime, services: &RegisteredWorkApplicationServicesV1, attempt_processes: &Arc, @@ -463,7 +449,11 @@ pub(super) fn resume_attempts( "Work attempt recovery requires the current worktree to have no live provider holder.", )) } else { - services.attempts().resume(context, &command) + tracing::Instrument::instrument( + services.resume_attempts(context.clone(), command), + tracing::trace_span!("daemon.service.work.resume_attempts"), + ) + .await }; if report.is_ok() && let Some(project_root) = project_root diff --git a/crates/tracedecay-daemon-service/src/invocation/work/request_dispatch.rs b/crates/tracedecay-daemon-service/src/invocation/work/request_dispatch.rs index f41a54f89c..b1be54a470 100644 --- a/crates/tracedecay-daemon-service/src/invocation/work/request_dispatch.rs +++ b/crates/tracedecay-daemon-service/src/invocation/work/request_dispatch.rs @@ -317,7 +317,6 @@ pub(super) async fn dispatch_work_application( tracedecay_contracts::WorkProductBindingV1::new(capability, use_case.clone()); attempt_operations::start_attempt( ®istered, - &services, binding, &attempt_processes, observability_producer.as_ref(), @@ -332,6 +331,7 @@ pub(super) async fn dispatch_work_application( deadline, command, ) + .await } WorkApplicationInvocationV1::Synthesize(command) => { let Ok(capability) = CapabilityId::new(*capability) else { @@ -344,7 +344,6 @@ pub(super) async fn dispatch_work_application( tracedecay_contracts::WorkProductBindingV1::new(capability, use_case.clone()); attempt_operations::synthesize( ®istered, - &services, binding, &attempt_processes, observability_producer.as_ref(), @@ -359,6 +358,7 @@ pub(super) async fn dispatch_work_application( deadline, command, ) + .await } WorkApplicationInvocationV1::AttemptStatus(request) => attempt_operations::attempt_status( ®istered, @@ -398,7 +398,6 @@ pub(super) async fn dispatch_work_application( tracedecay_contracts::WorkProductBindingV1::new(capability, use_case.clone()); attempt_operations::retry_attempt( ®istered, - &services, binding, &attempt_processes, observability_producer.as_ref(), @@ -413,6 +412,7 @@ pub(super) async fn dispatch_work_application( deadline, command, ) + .await } WorkApplicationInvocationV1::ListAttempts(request) => { let _span = tracing::trace_span!("daemon.service.work.list_attempts").entered(); @@ -731,6 +731,7 @@ pub(super) async fn dispatch_work_application( deadline, command, ) + .await } WorkApplicationInvocationV1::Views(request) => { let _span = tracing::trace_span!("daemon.service.work.views").entered(); diff --git a/crates/tracedecay-rusqlite-runtime/src/work/effect_holder.rs b/crates/tracedecay-rusqlite-runtime/src/work/effect_holder.rs index de4f93828d..4d9770e95b 100644 --- a/crates/tracedecay-rusqlite-runtime/src/work/effect_holder.rs +++ b/crates/tracedecay-rusqlite-runtime/src/work/effect_holder.rs @@ -7,10 +7,10 @@ use tracedecay_contracts::{ }; use tracedecay_domain::{UtcMicros, WorkAttemptIdentityV1, WorkAttemptV1, WorkAuthority}; -use crate::exact_sql::{ExactSqlTransaction, ExactSqlValue}; +use crate::exact_sql::ExactSqlValue; use crate::work::{ - RegisteredWorkQuery, WorkSqliteStorage, authority_params_owned, exact_sql_integer, - exact_sql_statement, exact_sql_text, registered_work_query, + RegisteredWorkQuery, RegisteredWorkWrite, WorkSqliteStorage, authority_params_owned, + exact_sql_integer, exact_sql_statement, exact_sql_text, registered_work_query, }; type StorageError = WorkAttemptEffectStorageErrorV1; @@ -26,20 +26,8 @@ impl WorkAttemptEffectStoragePortV1 for WorkSqliteStorage { .handle() .begin_immediate() .map_err(|_| StorageError::Unavailable)?; - require_open_attempt(&transaction, authority, holder)?; - if let Some(existing) = load_holder(&transaction, authority, holder.attempt())? { - let _ = transaction.rollback(); - return if same_dispatch(&existing, holder) { - Ok(WorkAttemptEffectDispatchOutcomeV1::Replayed(existing)) - } else { - Err(StorageError::Conflict) - }; - } - insert_holder(&transaction, authority, holder)?; - transaction - .commit() - .map_err(|_| StorageError::Unavailable)?; - Ok(WorkAttemptEffectDispatchOutcomeV1::Recorded(holder.clone())) + let outcome = begin_effect_dispatch_tx(&transaction, authority, holder); + settle_immediate(transaction, outcome) } fn settle_effect_dispatch( @@ -53,35 +41,9 @@ impl WorkAttemptEffectStoragePortV1 for WorkSqliteStorage { .handle() .begin_immediate() .map_err(|_| StorageError::Unavailable)?; - let existing = load_holder(&transaction, authority, attempt)? - .ok_or(StorageError::NotFoundOrNotAuthorized)?; - let next = match existing.resolution() { - None => existing - .with_resolution(resolution, resolved_at) - .map_err(|_| StorageError::Conflict)?, - Some(current) if current == resolution => { - transaction - .commit() - .map_err(|_| StorageError::Unavailable)?; - return Ok(existing); - } - Some(WorkAttemptEffectResolutionV1::Unknown) - if resolution == WorkAttemptEffectResolutionV1::NoEffect => - { - existing - .with_resolution(resolution, resolved_at) - .map_err(|_| StorageError::Conflict)? - } - Some(_) => { - let _ = transaction.rollback(); - return Err(StorageError::Conflict); - } - }; - update_holder(&transaction, authority, &next)?; - transaction - .commit() - .map_err(|_| StorageError::Unavailable)?; - Ok(next) + let outcome = + settle_effect_dispatch_tx(&transaction, authority, attempt, resolution, resolved_at); + settle_immediate(transaction, outcome) } fn load_effect_dispatch( @@ -101,8 +63,106 @@ impl WorkAttemptEffectStoragePortV1 for WorkSqliteStorage { } } +/// The writer job commits or rolls back the holder write with the job. +impl WorkAttemptEffectStoragePortV1 for crate::work_run_control::WorkSqliteJobStorage<'_> { + fn begin_effect_dispatch( + &self, + authority: &WorkAuthority, + holder: &WorkAttemptEffectHolderV1, + ) -> Result { + holder.validate().map_err(|_| StorageError::Conflict)?; + begin_effect_dispatch_tx(self.transaction, authority, holder) + } + + fn settle_effect_dispatch( + &self, + authority: &WorkAuthority, + attempt: &WorkAttemptIdentityV1, + resolution: WorkAttemptEffectResolutionV1, + resolved_at: UtcMicros, + ) -> Result { + settle_effect_dispatch_tx( + self.transaction, + authority, + attempt, + resolution, + resolved_at, + ) + } + + fn load_effect_dispatch( + &self, + authority: &WorkAuthority, + attempt: &WorkAttemptIdentityV1, + ) -> Result, StorageError> { + load_holder(self.transaction, authority, attempt) + } +} + +fn settle_immediate( + transaction: crate::exact_sql::ExactSqlTransaction, + outcome: Result, +) -> Result { + match outcome { + Ok(value) => { + transaction + .commit() + .map_err(|_| StorageError::Unavailable)?; + Ok(value) + } + Err(error) => { + let _ = transaction.rollback(); + Err(error) + } + } +} + +fn begin_effect_dispatch_tx( + transaction: &impl RegisteredWorkWrite, + authority: &WorkAuthority, + holder: &WorkAttemptEffectHolderV1, +) -> Result { + require_open_attempt(transaction, authority, holder)?; + if let Some(existing) = load_holder(transaction, authority, holder.attempt())? { + return if same_dispatch(&existing, holder) { + Ok(WorkAttemptEffectDispatchOutcomeV1::Replayed(existing)) + } else { + Err(StorageError::Conflict) + }; + } + insert_holder(transaction, authority, holder)?; + Ok(WorkAttemptEffectDispatchOutcomeV1::Recorded(holder.clone())) +} + +fn settle_effect_dispatch_tx( + transaction: &impl RegisteredWorkWrite, + authority: &WorkAuthority, + attempt: &WorkAttemptIdentityV1, + resolution: WorkAttemptEffectResolutionV1, + resolved_at: UtcMicros, +) -> Result { + let existing = load_holder(transaction, authority, attempt)? + .ok_or(StorageError::NotFoundOrNotAuthorized)?; + let next = match existing.resolution() { + None => existing + .with_resolution(resolution, resolved_at) + .map_err(|_| StorageError::Conflict)?, + Some(current) if current == resolution => return Ok(existing), + Some(WorkAttemptEffectResolutionV1::Unknown) + if resolution == WorkAttemptEffectResolutionV1::NoEffect => + { + existing + .with_resolution(resolution, resolved_at) + .map_err(|_| StorageError::Conflict)? + } + Some(_) => return Err(StorageError::Conflict), + }; + update_holder(transaction, authority, &next)?; + Ok(next) +} + fn require_open_attempt( - transaction: &ExactSqlTransaction, + transaction: &impl RegisteredWorkWrite, authority: &WorkAuthority, holder: &WorkAttemptEffectHolderV1, ) -> Result<(), StorageError> { @@ -206,14 +266,14 @@ fn same_dispatch( } fn insert_holder( - transaction: &ExactSqlTransaction, + transaction: &impl RegisteredWorkWrite, authority: &WorkAuthority, holder: &WorkAttemptEffectHolderV1, ) -> Result<(), StorageError> { holder.validate().map_err(|_| StorageError::Conflict)?; let payload = serde_json::to_string(holder).map_err(|_| StorageError::Unavailable)?; transaction - .execute( + .work_execute( exact_sql_statement( "INSERT INTO work_attempt_effect_holders_v1 ( project_id, repository_id, worktree_id, actor_id, policy_digest, @@ -242,14 +302,14 @@ fn insert_holder( } fn update_holder( - transaction: &ExactSqlTransaction, + transaction: &impl RegisteredWorkWrite, authority: &WorkAuthority, holder: &WorkAttemptEffectHolderV1, ) -> Result<(), StorageError> { holder.validate().map_err(|_| StorageError::Conflict)?; let payload = serde_json::to_string(holder).map_err(|_| StorageError::Unavailable)?; let changed = transaction - .execute( + .work_execute( exact_sql_statement( "UPDATE work_attempt_effect_holders_v1 SET resolution = ?9, resolved_at = ?10, holder_payload = ?11 diff --git a/crates/tracedecay-rusqlite-runtime/src/work/retry.rs b/crates/tracedecay-rusqlite-runtime/src/work/retry.rs index 06399cd317..8d9699d9d5 100644 --- a/crates/tracedecay-rusqlite-runtime/src/work/retry.rs +++ b/crates/tracedecay-rusqlite-runtime/src/work/retry.rs @@ -11,9 +11,9 @@ use tracedecay_domain::{ canonical_sha256, }; -use crate::exact_sql::{ExactSqlTransaction, ExactSqlValue}; +use crate::exact_sql::ExactSqlValue; use crate::work::{ - WorkSqliteStorage, authority_params_owned, exact_sql_integer, exact_sql_statement, + RegisteredWorkWrite, authority_params_owned, exact_sql_integer, exact_sql_statement, exact_sql_text, registered_work_query, }; use crate::work_attempt::identity_params; @@ -25,21 +25,13 @@ struct StoredWorkAttemptV1 { synthesis: Option, } -impl WorkRetryStoragePortV1 for WorkSqliteStorage { +impl WorkRetryStoragePortV1 for crate::work_run_control::WorkSqliteJobStorage<'_> { fn retry_by_command( &self, authority: &WorkAuthority, command_id: &tracedecay_domain::WorkCommandId, ) -> Result, WorkAttemptStorageError> { - let transaction = self - .handle() - .begin_deferred() - .map_err(|_| WorkAttemptStorageError::Unavailable)?; - let outcome = replay(&transaction, authority, command_id.as_str())?; - transaction - .commit() - .map_err(|_| WorkAttemptStorageError::Unavailable)?; - Ok(outcome) + replay(self.transaction, authority, command_id.as_str()) } fn insert_retry_bounded( @@ -48,38 +40,26 @@ impl WorkRetryStoragePortV1 for WorkSqliteStorage { write: &WorkRetryWriteV1, concurrency: &TopologyConcurrencyPolicyV1, ) -> Result { - let transaction = self - .handle() - .begin_immediate() - .map_err(|_| WorkAttemptStorageError::Unavailable)?; - let outcome = - insert_retry_bounded_in_transaction(&transaction, authority, write, concurrency); - match outcome { - Ok(created @ WorkRetryAttemptOutcomeV1::Created { .. }) => { - transaction - .commit() - .map_err(|_| WorkAttemptStorageError::Unavailable)?; - Ok(created) - } - Ok(replayed @ WorkRetryAttemptOutcomeV1::Replayed { .. }) => { - transaction - .rollback() - .map_err(|_| WorkAttemptStorageError::Unavailable)?; - Ok(replayed) - } - Err(error) => { - transaction - .rollback() - .map_err(|_| WorkAttemptStorageError::Unavailable)?; - Err(error) - } - } + insert_retry_bounded_in_transaction(self.transaction, authority, write, concurrency) + } + + fn admit_retry( + &self, + admission: &tracedecay_contracts::WorkProductRetryAdmissionV1, + ) -> Result< + ( + tracedecay_contracts::WorkProductEventCommitV1, + WorkRetryAttemptOutcomeV1, + ), + tracedecay_contracts::WorkProductAttemptAdmissionErrorV1, + > { + crate::work_product::admit_retry(self.transaction, admission) } } /// Persist one retry and receipt without settling the caller-owned transaction. pub(crate) fn insert_retry_bounded_in_transaction( - transaction: &ExactSqlTransaction, + transaction: &impl RegisteredWorkWrite, authority: &WorkAuthority, write: &WorkRetryWriteV1, concurrency: &TopologyConcurrencyPolicyV1, @@ -132,7 +112,7 @@ fn validate_write(write: &WorkRetryWriteV1) -> Result<(), WorkAttemptStorageErro } fn replay( - transaction: &ExactSqlTransaction, + transaction: &impl RegisteredWorkWrite, authority: &WorkAuthority, command_id: &str, ) -> Result, WorkAttemptStorageError> { @@ -201,7 +181,7 @@ fn replay( } fn load_attempt( - transaction: &ExactSqlTransaction, + transaction: &impl RegisteredWorkWrite, authority: &WorkAuthority, identity: &WorkAttemptIdentityV1, ) -> Result, WorkAttemptStorageError> { @@ -263,7 +243,7 @@ const fn attempt_state(state: tracedecay_domain::WorkAttemptStateV1) -> &'static } fn require_original_retry_evidence( - transaction: &ExactSqlTransaction, + transaction: &impl RegisteredWorkWrite, authority: &WorkAuthority, write: &WorkRetryWriteV1, ) -> Result<(), WorkAttemptStorageError> { @@ -290,7 +270,7 @@ fn require_original_retry_evidence( } fn require_run_reservation( - transaction: &ExactSqlTransaction, + transaction: &impl RegisteredWorkWrite, authority: &WorkAuthority, identity: &WorkAttemptIdentityV1, ) -> Result<(), WorkAttemptStorageError> { @@ -320,7 +300,7 @@ fn require_run_reservation( } fn require_first_run_admission( - transaction: &ExactSqlTransaction, + transaction: &impl RegisteredWorkWrite, authority: &WorkAuthority, attempt: &WorkAttemptV1, ) -> Result<(), WorkAttemptStorageError> { @@ -357,7 +337,7 @@ fn require_first_run_admission( } fn insert_attempt( - transaction: &ExactSqlTransaction, + transaction: &impl RegisteredWorkWrite, authority: &WorkAuthority, attempt: &WorkAttemptV1, ) -> Result<(), WorkAttemptStorageError> { @@ -367,7 +347,7 @@ fn insert_attempt( }) .map_err(|_| WorkAttemptStorageError::Unavailable)?; transaction - .execute( + .work_execute( exact_sql_statement( "INSERT INTO work_attempts_v1 ( project_id, repository_id, worktree_id, actor_id, policy_digest, @@ -394,7 +374,7 @@ fn insert_attempt( } fn insert_receipt( - transaction: &ExactSqlTransaction, + transaction: &impl RegisteredWorkWrite, authority: &WorkAuthority, write: &WorkRetryWriteV1, ) -> Result<(), WorkAttemptStorageError> { @@ -405,7 +385,7 @@ fn insert_receipt( ) .map_err(|_| WorkAttemptStorageError::Unavailable)?; transaction - .execute( + .work_execute( exact_sql_statement( "INSERT INTO work_retry_receipts_v1 ( project_id, repository_id, worktree_id, actor_id, policy_digest, diff --git a/crates/tracedecay-rusqlite-runtime/src/work_attempt.rs b/crates/tracedecay-rusqlite-runtime/src/work_attempt.rs index 125886f2ef..4779414144 100644 --- a/crates/tracedecay-rusqlite-runtime/src/work_attempt.rs +++ b/crates/tracedecay-rusqlite-runtime/src/work_attempt.rs @@ -16,9 +16,10 @@ use tracedecay_domain::{ use crate::exact_sql::ExactSqlValue; use crate::work::{ - RegisteredWorkQuery, WorkSqliteStorage, authority_params_owned, exact_sql_integer, - exact_sql_statement, exact_sql_text, registered_work_query, + RegisteredWorkQuery, RegisteredWorkWrite, WorkSqliteStorage, authority_params_owned, + exact_sql_integer, exact_sql_statement, exact_sql_text, registered_work_query, }; +use crate::work_run_control::WorkSqliteJobStorage; mod rooted_evidence; @@ -29,6 +30,26 @@ struct StoredWorkAttemptV1 { synthesis: Option, } +impl StoredWorkAttemptV1 { + const fn admission_kind(&self) -> WorkAttemptAdmissionKind { + if self.synthesis.is_some() { + WorkAttemptAdmissionKind::Synthesis + } else { + WorkAttemptAdmissionKind::Ordinary + } + } +} + +fn load_stored( + source: &impl RegisteredWorkQuery, + authority: &WorkAuthority, + identity: &WorkAttemptIdentityV1, +) -> Result { + let payload = load_attempt_payload(source, authority, identity)? + .ok_or(WorkAttemptStorageError::NotFoundOrNotAuthorized)?; + decode_stored(&payload) +} + fn insert_attempt( storage: &WorkSqliteStorage, authority: &WorkAuthority, @@ -52,7 +73,7 @@ fn insert_attempt( /// Persist one ordinary attempt without settling the caller-owned transaction. pub(crate) fn insert_attempt_in_transaction( - transaction: &crate::exact_sql::ExactSqlTransaction, + transaction: &impl RegisteredWorkWrite, authority: &WorkAuthority, attempt: &WorkAttemptV1, concurrency: Option<&TopologyConcurrencyPolicyV1>, @@ -85,54 +106,199 @@ pub(crate) fn insert_attempt_in_transaction( } } -impl WorkAttemptStoragePort for WorkSqliteStorage { +/// The writer job's transaction commits or rolls back every statement here +/// with the job. +impl WorkAttemptStoragePort for WorkSqliteJobStorage<'_> { fn next_fence_epoch(&self, authority: &WorkAuthority) -> Result { - { - let _span = tracing::trace_span!("rusqlite.work_attempt.txn.fence_epoch").entered(); - { - let transaction = self - .handle() - .begin_immediate() - .map_err(|_| WorkAttemptStorageError::Unavailable)?; - { - let _span = - tracing::trace_span!("rusqlite.work_attempt.cas.fence_epoch").entered(); - { - transaction - .execute( - exact_sql_statement( - "INSERT INTO work_attempt_fences_v1 ( + next_fence_epoch_tx(self.transaction, authority) + } + + fn insert( + &self, + authority: &WorkAuthority, + attempt: &WorkAttemptV1, + ) -> Result { + insert_attempt_in_transaction(self.transaction, authority, attempt, None) + } + + fn insert_bounded( + &self, + authority: &WorkAuthority, + attempt: &WorkAttemptV1, + concurrency: &TopologyConcurrencyPolicyV1, + ) -> Result { + insert_attempt_in_transaction(self.transaction, authority, attempt, Some(concurrency)) + } + + fn admission_capacities( + &self, + authority: &WorkAuthority, + task_ids: &[TaskId], + concurrency: &TopologyConcurrencyPolicyV1, + ) -> Result, WorkAttemptStorageError> + { + crate::work::capacity::capacities(self.transaction, authority, task_ids, concurrency) + } + + fn load( + &self, + authority: &WorkAuthority, + identity: &WorkAttemptIdentityV1, + ) -> Result { + load_stored(self.transaction, authority, identity).map(|record| record.attempt) + } + + fn load_admission_kind( + &self, + authority: &WorkAuthority, + identity: &WorkAttemptIdentityV1, + ) -> Result { + load_stored(self.transaction, authority, identity).map(|record| record.admission_kind()) + } + + fn update( + &self, + authority: &WorkAuthority, + expected_fence: &tracedecay_domain::WorkLeaseFenceV1, + expected_state: WorkAttemptStateV1, + next: &WorkAttemptV1, + evidence: Option<&WorkAttemptEvidenceRecordV1>, + ) -> Result<(), WorkAttemptStorageError> { + update_tx( + self.transaction, + authority, + expected_fence, + expected_state, + next, + evidence, + ) + } + + fn open_attempts( + &self, + authority: &WorkAuthority, + ) -> Result, WorkAttemptStorageError> { + open_attempts_from(self.transaction, authority) + } + + fn list( + &self, + authority: &WorkAuthority, + start_after: Option<&WorkAttemptIdentityV1>, + limit: u32, + ) -> Result { + list_from(self.transaction, authority, start_after, limit) + } +} + +impl WorkSynthesisAdmissionStoragePort for WorkSqliteJobStorage<'_> { + fn insert_synthesis( + &self, + authority: &WorkAuthority, + record: &WorkSynthesisAdmissionRecordV1, + ) -> Result { + insert_synthesis_in_transaction(self.transaction, authority, record, None) + } + + fn insert_synthesis_bounded( + &self, + authority: &WorkAuthority, + record: &WorkSynthesisAdmissionRecordV1, + concurrency: &TopologyConcurrencyPolicyV1, + ) -> Result { + insert_synthesis_in_transaction(self.transaction, authority, record, Some(concurrency)) + } + + fn load_synthesis( + &self, + authority: &WorkAuthority, + identity: &WorkAttemptIdentityV1, + ) -> Result { + load_stored(self.transaction, authority, identity)? + .synthesis + .ok_or(WorkAttemptStorageError::AttemptConflict) + } + + fn admit_synthesis( + &self, + admission: &tracedecay_contracts::WorkProductSynthesisAdmissionV1, + ) -> Result< + ( + tracedecay_contracts::WorkProductEventCommitV1, + WorkSynthesisInsertOutcome, + ), + tracedecay_contracts::WorkProductAttemptAdmissionErrorV1, + > { + crate::work_product::admit_synthesis(self.transaction, admission) + } +} + +impl WorkSqliteJobStorage<'_> { + /// Loads one admitted synthesis source payload inside the job, where the + /// workflow artifact store's own transaction would wait on this writer. + pub fn load_workflow_artifact( + &self, + artifact: &tracedecay_domain::WorkArtifactRefV1, + ) -> Result< + tracedecay_contracts::WorkflowArtifactPayload, + tracedecay_contracts::WorkflowArtifactStoreError, + > { + let bytes = crate::workflow::run_journal::stored_payload_tx( + self.transaction, + artifact.digest().as_str(), + )? + .ok_or(tracedecay_contracts::WorkflowArtifactStoreError::Missing)?; + tracedecay_contracts::WorkflowArtifactPayload::new(artifact.clone(), bytes) + } +} + +fn next_fence_epoch_tx( + transaction: &impl RegisteredWorkWrite, + authority: &WorkAuthority, +) -> Result { + { + let _span = tracing::trace_span!("rusqlite.work_attempt.cas.fence_epoch").entered(); + transaction + .work_execute( + exact_sql_statement( + "INSERT INTO work_attempt_fences_v1 ( project_id, repository_id, worktree_id, actor_id, policy_digest, epoch ) VALUES (?1, ?2, ?3, ?4, ?5, 1) ON CONFLICT (project_id, repository_id, worktree_id, actor_id, policy_digest) DO UPDATE SET epoch = epoch + 1", - authority_params_owned(authority), - ) - .map_err(|_| WorkAttemptStorageError::Unavailable)?, - ) - .map_err(|_| WorkAttemptStorageError::Unavailable) - } - }?; - let rows = registered_work_query( - &transaction, - "SELECT epoch FROM work_attempt_fences_v1 - WHERE project_id = ?1 AND repository_id = ?2 AND worktree_id = ?3 - AND actor_id = ?4 AND policy_digest = ?5", authority_params_owned(authority), ) - .map_err(|_| WorkAttemptStorageError::Unavailable)?; - let epoch = rows - .rows - .first() - .and_then(|row| exact_sql_integer(&row.values, 0)) - .and_then(|value| u64::try_from(value).ok()) - .ok_or(WorkAttemptStorageError::Unavailable)?; - transaction - .commit() - .map_err(|_| WorkAttemptStorageError::Unavailable)?; - Ok(epoch) - } - } + .map_err(|_| WorkAttemptStorageError::Unavailable)?, + ) + .map_err(|_| WorkAttemptStorageError::Unavailable) + }?; + let rows = registered_work_query( + transaction, + "SELECT epoch FROM work_attempt_fences_v1 + WHERE project_id = ?1 AND repository_id = ?2 AND worktree_id = ?3 + AND actor_id = ?4 AND policy_digest = ?5", + authority_params_owned(authority), + ) + .map_err(|_| WorkAttemptStorageError::Unavailable)?; + rows.rows + .first() + .and_then(|row| exact_sql_integer(&row.values, 0)) + .and_then(|value| u64::try_from(value).ok()) + .ok_or(WorkAttemptStorageError::Unavailable) +} + +impl WorkAttemptStoragePort for WorkSqliteStorage { + fn next_fence_epoch(&self, authority: &WorkAuthority) -> Result { + let _span = tracing::trace_span!("rusqlite.work_attempt.txn.fence_epoch").entered(); + let transaction = self + .handle() + .begin_immediate() + .map_err(|_| WorkAttemptStorageError::Unavailable)?; + let epoch = next_fence_epoch_tx(&transaction, authority)?; + transaction + .commit() + .map_err(|_| WorkAttemptStorageError::Unavailable)?; + Ok(epoch) } fn insert( @@ -167,9 +333,7 @@ impl WorkAttemptStoragePort for WorkSqliteStorage { authority: &WorkAuthority, identity: &WorkAttemptIdentityV1, ) -> Result { - let payload = load_attempt_payload(self.handle(), authority, identity)? - .ok_or(WorkAttemptStorageError::NotFoundOrNotAuthorized)?; - Ok(decode_stored(&payload)?.attempt) + load_stored(self.handle(), authority, identity).map(|record| record.attempt) } fn load_admission_kind( @@ -177,14 +341,7 @@ impl WorkAttemptStoragePort for WorkSqliteStorage { authority: &WorkAuthority, identity: &WorkAttemptIdentityV1, ) -> Result { - let payload = load_attempt_payload(self.handle(), authority, identity)? - .ok_or(WorkAttemptStorageError::NotFoundOrNotAuthorized)?; - let record = decode_stored(&payload)?; - Ok(if record.synthesis.is_some() { - WorkAttemptAdmissionKind::Synthesis - } else { - WorkAttemptAdmissionKind::Ordinary - }) + load_stored(self.handle(), authority, identity).map(|record| record.admission_kind()) } fn update( @@ -195,105 +352,33 @@ impl WorkAttemptStoragePort for WorkSqliteStorage { next: &WorkAttemptV1, evidence: Option<&WorkAttemptEvidenceRecordV1>, ) -> Result<(), WorkAttemptStorageError> { - { - let _span = tracing::trace_span!("rusqlite.work_attempt.txn.update").entered(); - { - let evidence_payload = evidence - .map(serde_json::to_string) - .transpose() - .map_err(|_| WorkAttemptStorageError::Unavailable)?; - let transaction = self - .handle() - .begin_immediate() - .map_err(|_| WorkAttemptStorageError::Unavailable)?; - let existing = load_attempt_payload(&transaction, authority, next.identity())? - .ok_or(WorkAttemptStorageError::NotFoundOrNotAuthorized)?; - let mut record = decode_stored(&existing)?; - record.attempt = next.clone(); - let payload = serde_json::to_string(&record) - .map_err(|_| WorkAttemptStorageError::Unavailable)?; - let changed = { - let _span = tracing::trace_span!("rusqlite.work_attempt.cas.update").entered(); - { - transaction - .execute( - exact_sql_statement( - "UPDATE work_attempts_v1 SET - state = ?9, lease_id = ?10, fence_epoch = ?11, terminal = ?12, - attempt_payload = ?13, - evidence_payload = COALESCE(?14, evidence_payload) - WHERE project_id = ?1 AND repository_id = ?2 AND worktree_id = ?3 - AND actor_id = ?4 AND policy_digest = ?5 - AND task_id = ?6 AND run_id = ?7 AND attempt_id = ?8 - AND lease_id = ?15 AND fence_epoch = ?16 AND state = ?17", - authority_params_owned(authority) - .into_iter() - .chain(identity_params(next.identity())) - .chain([ - ExactSqlValue::Text(state_text(next.state())), - ExactSqlValue::Text( - next.lease().lease_id().as_str().to_owned(), - ), - ExactSqlValue::Integer( - i64::try_from(next.lease().epoch().get()).map_err( - |_| WorkAttemptStorageError::Unavailable, - )?, - ), - ExactSqlValue::Integer(i64::from(next.is_terminal())), - ExactSqlValue::Text(payload), - evidence_payload - .map_or(ExactSqlValue::Null, ExactSqlValue::Text), - ExactSqlValue::Text( - expected_fence.lease_id().as_str().to_owned(), - ), - ExactSqlValue::Integer( - i64::try_from(expected_fence.epoch().get()) - .map_err(|_| { - WorkAttemptStorageError::Unavailable - })?, - ), - ExactSqlValue::Text(state_text(expected_state)), - ]) - .collect(), - ) - .map_err(|_| WorkAttemptStorageError::Unavailable)?, - ) - .map_err(|_| WorkAttemptStorageError::Unavailable) - } - }?; - if changed.changed_rows != 1 { - let _ = transaction.rollback(); - return Err(WorkAttemptStorageError::FenceConflict); - } - crate::work_run_control::close_blocked_intervals_on_terminal_attempt( - &transaction, - authority, - next, - )?; - transaction - .commit() - .map_err(|_| WorkAttemptStorageError::Unavailable)?; - Ok(()) - } + let _span = tracing::trace_span!("rusqlite.work_attempt.txn.update").entered(); + let transaction = self + .handle() + .begin_immediate() + .map_err(|_| WorkAttemptStorageError::Unavailable)?; + if let Err(error) = update_tx( + &transaction, + authority, + expected_fence, + expected_state, + next, + evidence, + ) { + let _ = transaction.rollback(); + return Err(error); } + transaction + .commit() + .map_err(|_| WorkAttemptStorageError::Unavailable)?; + Ok(()) } fn open_attempts( &self, authority: &WorkAuthority, ) -> Result, WorkAttemptStorageError> { - let sql = format!( - "SELECT attempt.attempt_payload FROM work_attempts_v1 AS attempt - WHERE attempt.project_id = ?1 AND attempt.repository_id = ?2 - AND attempt.worktree_id = ?3 AND attempt.actor_id = ?4 - AND attempt.policy_digest = ?5 - AND {} - ORDER BY attempt.task_id, attempt.run_id, attempt.attempt_id", - crate::work::ACTIVE_ATTEMPT_PREDICATE - ); - let rows = registered_work_query(self.handle(), &sql, authority_params_owned(authority)) - .map_err(|_| WorkAttemptStorageError::Unavailable)?; - rows.rows.into_iter().map(attempt_from_row).collect() + open_attempts_from(self.handle(), authority) } fn has_open_attempts_in_exact_scope( @@ -328,94 +413,165 @@ impl WorkAttemptStoragePort for WorkSqliteStorage { start_after: Option<&WorkAttemptIdentityV1>, limit: u32, ) -> Result { - let authority_filter = "project_id = ?1 AND repository_id = ?2 AND worktree_id = ?3 - AND actor_id = ?4 AND policy_digest = ?5"; - let after_filter = if start_after.is_some() { - " AND (task_id, run_id, attempt_id) > (?6, ?7, ?8)" - } else { - "" - }; - let mut params = authority_params_owned(authority); - if let Some(start_after) = start_after { - params.extend(identity_params(start_after)); - } // One deferred transaction keeps the remaining count and the page on // the same consistent view of the attempt rows. - { - let _span = tracing::trace_span!("rusqlite.work_attempt.txn.list").entered(); - { - let transaction = self - .handle() - .begin_deferred() - .map_err(|_| WorkAttemptStorageError::Unavailable)?; - let counted = registered_work_query( - &transaction, - &format!( - "SELECT COUNT(*) FROM work_attempts_v1 WHERE {authority_filter}{after_filter}" - ), - params.clone(), - ) + let _span = tracing::trace_span!("rusqlite.work_attempt.txn.list").entered(); + let transaction = self + .handle() + .begin_deferred() .map_err(|_| WorkAttemptStorageError::Unavailable)?; - let remaining = counted - .rows - .first() - .and_then(|row| exact_sql_integer(&row.values, 0)) - .and_then(|value| u32::try_from(value).ok()) - .ok_or(WorkAttemptStorageError::Unavailable)?; - let limit_placeholder = params.len() + 1; - params.push(ExactSqlValue::Integer(i64::from(limit))); - let rows = registered_work_query( - &transaction, - &format!( - "SELECT attempt_payload FROM work_attempts_v1 - WHERE {authority_filter}{after_filter} - ORDER BY task_id, run_id, attempt_id - LIMIT ?{limit_placeholder}" - ), - params, + let page = list_from(&transaction, authority, start_after, limit)?; + transaction + .commit() + .map_err(|_| WorkAttemptStorageError::Unavailable)?; + Ok(page) + } +} + +/// Replaces the attempt row only while its lease fence and state are still +/// `expected_fence` and `expected_state`, closing its open blocked intervals +/// when the attempt turns terminal. +fn update_tx( + transaction: &impl RegisteredWorkWrite, + authority: &WorkAuthority, + expected_fence: &tracedecay_domain::WorkLeaseFenceV1, + expected_state: WorkAttemptStateV1, + next: &WorkAttemptV1, + evidence: Option<&WorkAttemptEvidenceRecordV1>, +) -> Result<(), WorkAttemptStorageError> { + let evidence_payload = evidence + .map(serde_json::to_string) + .transpose() + .map_err(|_| WorkAttemptStorageError::Unavailable)?; + let mut record = load_stored(transaction, authority, next.identity())?; + record.attempt = next.clone(); + let payload = + serde_json::to_string(&record).map_err(|_| WorkAttemptStorageError::Unavailable)?; + let changed = { + let _span = tracing::trace_span!("rusqlite.work_attempt.cas.update").entered(); + transaction + .work_execute( + exact_sql_statement( + "UPDATE work_attempts_v1 SET + state = ?9, lease_id = ?10, fence_epoch = ?11, terminal = ?12, + attempt_payload = ?13, + evidence_payload = COALESCE(?14, evidence_payload) + WHERE project_id = ?1 AND repository_id = ?2 AND worktree_id = ?3 + AND actor_id = ?4 AND policy_digest = ?5 + AND task_id = ?6 AND run_id = ?7 AND attempt_id = ?8 + AND lease_id = ?15 AND fence_epoch = ?16 AND state = ?17", + authority_params_owned(authority) + .into_iter() + .chain(identity_params(next.identity())) + .chain([ + ExactSqlValue::Text(state_text(next.state())), + ExactSqlValue::Text(next.lease().lease_id().as_str().to_owned()), + ExactSqlValue::Integer( + i64::try_from(next.lease().epoch().get()) + .map_err(|_| WorkAttemptStorageError::Unavailable)?, + ), + ExactSqlValue::Integer(i64::from(next.is_terminal())), + ExactSqlValue::Text(payload), + evidence_payload.map_or(ExactSqlValue::Null, ExactSqlValue::Text), + ExactSqlValue::Text(expected_fence.lease_id().as_str().to_owned()), + ExactSqlValue::Integer( + i64::try_from(expected_fence.epoch().get()) + .map_err(|_| WorkAttemptStorageError::Unavailable)?, + ), + ExactSqlValue::Text(state_text(expected_state)), + ]) + .collect(), ) - .map_err(|_| WorkAttemptStorageError::Unavailable)?; - transaction - .commit() - .map_err(|_| WorkAttemptStorageError::Unavailable)?; - let attempts = rows - .rows - .into_iter() - .map(attempt_from_row) - .collect::, _>>()?; - Ok(WorkAttemptListPageV1 { - attempts, - remaining, - }) - } - } + .map_err(|_| WorkAttemptStorageError::Unavailable)?, + ) + .map_err(|_| WorkAttemptStorageError::Unavailable) + }?; + if changed.changed_rows != 1 { + return Err(WorkAttemptStorageError::FenceConflict); } + crate::work_run_control::close_blocked_intervals_on_terminal_attempt( + transaction, + authority, + next, + ) } -fn insert_synthesis_record( - storage: &WorkSqliteStorage, +fn open_attempts_from( + source: &impl RegisteredWorkQuery, authority: &WorkAuthority, - record: &WorkSynthesisAdmissionRecordV1, - concurrency: Option<&TopologyConcurrencyPolicyV1>, -) -> Result { - { - let _span = tracing::trace_span!("rusqlite.work_attempt.txn.synthesis").entered(); - { - let transaction = storage - .handle() - .begin_immediate() - .map_err(|_| WorkAttemptStorageError::Unavailable)?; - let outcome = - insert_synthesis_in_transaction(&transaction, authority, record, concurrency); - let commit = matches!(outcome, Ok(WorkSynthesisInsertOutcome::Inserted)); - finish_immediate(transaction, outcome, commit) - } +) -> Result, WorkAttemptStorageError> { + let sql = format!( + "SELECT attempt.attempt_payload FROM work_attempts_v1 AS attempt + WHERE attempt.project_id = ?1 AND attempt.repository_id = ?2 + AND attempt.worktree_id = ?3 AND attempt.actor_id = ?4 + AND attempt.policy_digest = ?5 + AND {} + ORDER BY attempt.task_id, attempt.run_id, attempt.attempt_id", + crate::work::ACTIVE_ATTEMPT_PREDICATE + ); + let rows = registered_work_query(source, &sql, authority_params_owned(authority)) + .map_err(|_| WorkAttemptStorageError::Unavailable)?; + rows.rows.into_iter().map(attempt_from_row).collect() +} + +/// The page and its remaining count, both read through `source`; the caller +/// makes `source` one consistent view. +fn list_from( + source: &impl RegisteredWorkQuery, + authority: &WorkAuthority, + start_after: Option<&WorkAttemptIdentityV1>, + limit: u32, +) -> Result { + let authority_filter = "project_id = ?1 AND repository_id = ?2 AND worktree_id = ?3 + AND actor_id = ?4 AND policy_digest = ?5"; + let after_filter = if start_after.is_some() { + " AND (task_id, run_id, attempt_id) > (?6, ?7, ?8)" + } else { + "" + }; + let mut params = authority_params_owned(authority); + if let Some(start_after) = start_after { + params.extend(identity_params(start_after)); } + let counted = registered_work_query( + source, + &format!("SELECT COUNT(*) FROM work_attempts_v1 WHERE {authority_filter}{after_filter}"), + params.clone(), + ) + .map_err(|_| WorkAttemptStorageError::Unavailable)?; + let remaining = counted + .rows + .first() + .and_then(|row| exact_sql_integer(&row.values, 0)) + .and_then(|value| u32::try_from(value).ok()) + .ok_or(WorkAttemptStorageError::Unavailable)?; + let limit_placeholder = params.len() + 1; + params.push(ExactSqlValue::Integer(i64::from(limit))); + let rows = registered_work_query( + source, + &format!( + "SELECT attempt_payload FROM work_attempts_v1 + WHERE {authority_filter}{after_filter} + ORDER BY task_id, run_id, attempt_id + LIMIT ?{limit_placeholder}" + ), + params, + ) + .map_err(|_| WorkAttemptStorageError::Unavailable)?; + let attempts = rows + .rows + .into_iter() + .map(attempt_from_row) + .collect::, _>>()?; + Ok(WorkAttemptListPageV1 { + attempts, + remaining, + }) } /// Persist one synthesis attempt without settling the caller-owned transaction. pub(crate) fn insert_synthesis_in_transaction( - transaction: &crate::exact_sql::ExactSqlTransaction, + transaction: &impl RegisteredWorkWrite, authority: &WorkAuthority, record: &WorkSynthesisAdmissionRecordV1, concurrency: Option<&TopologyConcurrencyPolicyV1>, @@ -450,37 +606,6 @@ pub(crate) fn insert_synthesis_in_transaction( } } -impl WorkSynthesisAdmissionStoragePort for WorkSqliteStorage { - fn insert_synthesis( - &self, - authority: &WorkAuthority, - record: &WorkSynthesisAdmissionRecordV1, - ) -> Result { - insert_synthesis_record(self, authority, record, None) - } - - fn insert_synthesis_bounded( - &self, - authority: &WorkAuthority, - record: &WorkSynthesisAdmissionRecordV1, - concurrency: &TopologyConcurrencyPolicyV1, - ) -> Result { - insert_synthesis_record(self, authority, record, Some(concurrency)) - } - - fn load_synthesis( - &self, - authority: &WorkAuthority, - identity: &WorkAttemptIdentityV1, - ) -> Result { - let payload = load_attempt_payload(self.handle(), authority, identity)? - .ok_or(WorkAttemptStorageError::NotFoundOrNotAuthorized)?; - decode_stored(&payload)? - .synthesis - .ok_or(WorkAttemptStorageError::AttemptConflict) - } -} - impl WorkAttemptEvidenceReadPort for WorkSqliteStorage { fn evidence_page( &self, @@ -582,13 +707,13 @@ fn finish_immediate( } fn insert_attempt_row( - transaction: &crate::exact_sql::ExactSqlTransaction, + transaction: &impl RegisteredWorkWrite, authority: &WorkAuthority, attempt: &WorkAttemptV1, payload: String, ) -> Result<(), WorkAttemptStorageError> { transaction - .execute( + .work_execute( exact_sql_statement( "INSERT INTO work_attempts_v1 ( project_id, repository_id, worktree_id, actor_id, policy_digest, @@ -656,7 +781,7 @@ fn load_attempt_payload( /// attempt's immutable deadline and topology, so a caller cannot replace the /// run authority through a lexically earlier attempt ID. fn require_first_run_admission( - transaction: &crate::exact_sql::ExactSqlTransaction, + transaction: &impl RegisteredWorkWrite, authority: &WorkAuthority, attempt: &WorkAttemptV1, ) -> Result<(), WorkAttemptStorageError> { @@ -698,7 +823,7 @@ fn require_first_run_admission( /// attempt naming any other worktree root would write a tree the placement /// authority does not record for this run. pub(crate) fn require_placement_root( - transaction: &crate::exact_sql::ExactSqlTransaction, + transaction: &impl RegisteredWorkWrite, authority: &WorkAuthority, attempt: &WorkAttemptV1, ) -> Result<(), WorkAttemptStorageError> { @@ -727,7 +852,7 @@ pub(crate) fn require_placement_root( } fn require_run_reservation_admitted( - transaction: &crate::exact_sql::ExactSqlTransaction, + transaction: &impl RegisteredWorkWrite, authority: &WorkAuthority, identity: &WorkAttemptIdentityV1, ) -> Result<(), WorkAttemptStorageError> { diff --git a/crates/tracedecay-rusqlite-runtime/src/work_product.rs b/crates/tracedecay-rusqlite-runtime/src/work_product.rs index 3e05a071c2..994fb3a676 100644 --- a/crates/tracedecay-rusqlite-runtime/src/work_product.rs +++ b/crates/tracedecay-rusqlite-runtime/src/work_product.rs @@ -40,6 +40,7 @@ use crate::exact_sql::ExactSqlValue; use crate::work::{RegisteredWorkQuery, exact_sql_integer, exact_sql_text, registered_work_query}; mod attempt_admission; +pub(crate) use attempt_admission::{admit_retry, admit_synthesis}; mod authorization; mod events; mod evidence; diff --git a/crates/tracedecay-rusqlite-runtime/src/work_product/attempt_admission.rs b/crates/tracedecay-rusqlite-runtime/src/work_product/attempt_admission.rs index 72020dc0d3..0c950a4585 100644 --- a/crates/tracedecay-rusqlite-runtime/src/work_product/attempt_admission.rs +++ b/crates/tracedecay-rusqlite-runtime/src/work_product/attempt_admission.rs @@ -17,10 +17,50 @@ use tracedecay_domain::{WorkProductAuthorizedRelationScopeV1, WorkProductGraphV1 use super::{fold_graph, load_journal}; use crate::exact_sql::ExactSqlTransaction; -use crate::work::WorkSqliteStorage; +use crate::work::{RegisteredWorkWrite, WorkSqliteStorage}; +use crate::work_run_control::WorkSqliteJobStorage; type AdmissionError = WorkProductAttemptAdmissionErrorV1; +/// The writer job commits the product link and the attempt row together, or +/// rolls both back with any refusal the job returns. +impl WorkProductAttemptAdmissionPortV1 for WorkSqliteJobStorage<'_> { + fn admit_attempt( + &self, + admission: &WorkProductAttemptAdmissionV1, + ) -> Result { + admission.validate()?; + require_declared_authority(admission)?; + require_request_active(&admission.product_context)?; + admit_attempt_in_transaction(self.transaction, admission) + } +} + +/// Admits one retry inside a writer job, which commits or rolls back the +/// product link, the receipt, the attempt row and the workflow rebind +/// together. +pub(crate) fn admit_retry( + transaction: &impl RegisteredWorkWrite, + admission: &WorkProductRetryAdmissionV1, +) -> Result<(WorkProductEventCommitV1, WorkRetryAttemptOutcomeV1), AdmissionError> { + admission.validate()?; + require_declared_authority(&admission.admission)?; + require_request_active(&admission.admission.product_context)?; + admit_retry_in_transaction(transaction, admission) +} + +/// Admits one synthesis inside a writer job, which commits or rolls back the +/// product link, the attempt row and its synthesis record together. +pub(crate) fn admit_synthesis( + transaction: &impl RegisteredWorkWrite, + admission: &WorkProductSynthesisAdmissionV1, +) -> Result<(WorkProductEventCommitV1, WorkSynthesisInsertOutcome), AdmissionError> { + admission.validate()?; + require_declared_authority(&admission.admission)?; + require_request_active(&admission.admission.product_context)?; + admit_synthesis_in_transaction(transaction, admission) +} + impl WorkProductAttemptAdmissionPortV1 for WorkSqliteStorage { fn admit_attempt( &self, @@ -50,74 +90,10 @@ impl WorkProductAttemptAdmissionPortV1 for WorkSqliteStorage { Err(error) => rollback_after_failure(transaction, error), } } - - fn admit_retry( - &self, - admission: &WorkProductRetryAdmissionV1, - ) -> Result<(WorkProductEventCommitV1, WorkRetryAttemptOutcomeV1), AdmissionError> { - admission.validate()?; - require_declared_authority(&admission.admission)?; - require_request_active(&admission.admission.product_context)?; - let transaction = self - .handle() - .begin_immediate() - .map_err(|_| AdmissionError::Unavailable)?; - let outcome = admit_retry_in_transaction(&transaction, admission); - match outcome { - Ok((product, WorkRetryAttemptOutcomeV1::Created { receipt, attempt })) => { - transaction - .commit() - .map_err(|_| AdmissionError::DurabilityUncertain)?; - Ok(( - product, - WorkRetryAttemptOutcomeV1::Created { receipt, attempt }, - )) - } - Ok((product, WorkRetryAttemptOutcomeV1::Replayed { receipt, attempt })) => { - transaction - .rollback() - .map_err(|_| AdmissionError::DurabilityUncertain)?; - Ok(( - product, - WorkRetryAttemptOutcomeV1::Replayed { receipt, attempt }, - )) - } - Err(error) => rollback_after_failure(transaction, error), - } - } - - fn admit_synthesis( - &self, - admission: &WorkProductSynthesisAdmissionV1, - ) -> Result<(WorkProductEventCommitV1, WorkSynthesisInsertOutcome), AdmissionError> { - admission.validate()?; - require_declared_authority(&admission.admission)?; - require_request_active(&admission.admission.product_context)?; - let transaction = self - .handle() - .begin_immediate() - .map_err(|_| AdmissionError::Unavailable)?; - let outcome = admit_synthesis_in_transaction(&transaction, admission); - match outcome { - Ok((product, WorkSynthesisInsertOutcome::Inserted)) => { - transaction - .commit() - .map_err(|_| AdmissionError::DurabilityUncertain)?; - Ok((product, WorkSynthesisInsertOutcome::Inserted)) - } - Ok((product, WorkSynthesisInsertOutcome::Replayed(result))) => { - transaction - .rollback() - .map_err(|_| AdmissionError::DurabilityUncertain)?; - Ok((product, WorkSynthesisInsertOutcome::Replayed(result))) - } - Err(error) => rollback_after_failure(transaction, error), - } - } } fn admit_attempt_in_transaction( - transaction: &ExactSqlTransaction, + transaction: &impl RegisteredWorkWrite, admission: &WorkProductAttemptAdmissionV1, ) -> Result { let product = super::events::append_in_transaction( @@ -162,7 +138,7 @@ fn admit_attempt_in_transaction( } fn admit_retry_in_transaction( - transaction: &ExactSqlTransaction, + transaction: &impl RegisteredWorkWrite, admission: &WorkProductRetryAdmissionV1, ) -> Result<(WorkProductEventCommitV1, WorkRetryAttemptOutcomeV1), AdmissionError> { let product = super::events::append_in_transaction( @@ -221,7 +197,7 @@ fn map_workflow_error(error: WorkflowRunStorageError) -> AdmissionError { } fn admit_synthesis_in_transaction( - transaction: &ExactSqlTransaction, + transaction: &impl RegisteredWorkWrite, admission: &WorkProductSynthesisAdmissionV1, ) -> Result<(WorkProductEventCommitV1, WorkSynthesisInsertOutcome), AdmissionError> { let product = super::events::append_in_transaction( @@ -269,7 +245,7 @@ fn product_commit(outcome: &WorkProductEventCommitOutcomeV1) -> &WorkProductEven } fn graph_for_product( - transaction: &ExactSqlTransaction, + transaction: &impl RegisteredWorkWrite, context: &tracedecay_contracts::WorkProductPortContextV1, product: &WorkProductEventCommitV1, ) -> Result { diff --git a/crates/tracedecay-rusqlite-runtime/src/work_product/authorization.rs b/crates/tracedecay-rusqlite-runtime/src/work_product/authorization.rs index 2353d79b39..2bf695bf16 100644 --- a/crates/tracedecay-rusqlite-runtime/src/work_product/authorization.rs +++ b/crates/tracedecay-rusqlite-runtime/src/work_product/authorization.rs @@ -25,6 +25,7 @@ use tracedecay_contracts::{ use tracedecay_domain::UtcMicros; use crate::work::WorkSqliteStorage; +use crate::work_run_control::WorkSqliteJobStorage; impl WorkProductOwnerAuthorizationPortV1 for WorkSqliteStorage { fn authorize_scope( @@ -33,17 +34,35 @@ impl WorkProductOwnerAuthorizationPortV1 for WorkSqliteStorage { selection: &WorkProductSelectionScopeV1, _observed_at: UtcMicros, ) -> Result { - let shard = &self.handle().binding().shard_id; - if !selection_is_within_resolved_scope(context, selection) { - return Err(WorkProductOwnerAuthorizationErrorV1::NotAuthorized); - } - AuthorizedWorkProductScopeV1::new( - shard.brain_id.clone(), - shard.profile_id.clone(), - selection.clone(), - ) - .map_err(|_| WorkProductOwnerAuthorizationErrorV1::Unavailable) + authorize_for_owner(&self.handle().binding().shard_id, context, selection) + } +} + +impl WorkProductOwnerAuthorizationPortV1 for WorkSqliteJobStorage<'_> { + fn authorize_scope( + &self, + context: &RequestContext, + selection: &WorkProductSelectionScopeV1, + _observed_at: UtcMicros, + ) -> Result { + authorize_for_owner(self.owner, context, selection) + } +} + +fn authorize_for_owner( + shard: &tracedecay_store::StoreShardIdV1, + context: &RequestContext, + selection: &WorkProductSelectionScopeV1, +) -> Result { + if !selection_is_within_resolved_scope(context, selection) { + return Err(WorkProductOwnerAuthorizationErrorV1::NotAuthorized); } + AuthorizedWorkProductScopeV1::new( + shard.brain_id.clone(), + shard.profile_id.clone(), + selection.clone(), + ) + .map_err(|_| WorkProductOwnerAuthorizationErrorV1::Unavailable) } fn selection_is_within_resolved_scope( diff --git a/crates/tracedecay-rusqlite-runtime/src/work_product/events.rs b/crates/tracedecay-rusqlite-runtime/src/work_product/events.rs index 599af6fe17..18fb79da69 100644 --- a/crates/tracedecay-rusqlite-runtime/src/work_product/events.rs +++ b/crates/tracedecay-rusqlite-runtime/src/work_product/events.rs @@ -11,7 +11,10 @@ use tracedecay_domain::{ use super::{WORK_PRODUCT_EVENT_ID_DOMAIN, load_journal_tail, owner_params, selection_covers}; use crate::exact_sql::ExactSqlValue; -use crate::work::{WorkSqliteStorage, exact_sql_statement, exact_sql_text, registered_work_query}; +use crate::work::{ + RegisteredWorkWrite, WorkSqliteStorage, exact_sql_statement, exact_sql_text, + registered_work_query, +}; type PortError = WorkProductEventPortErrorV1; @@ -90,7 +93,7 @@ impl WorkProductEventPortV1 for WorkSqliteStorage { } pub(super) fn append_in_transaction( - transaction: &crate::exact_sql::ExactSqlTransaction, + transaction: &impl RegisteredWorkWrite, context: &WorkProductPortContextV1, draft: &WorkProductEventDraftV1, ) -> Result { @@ -133,7 +136,7 @@ pub(super) fn append_in_transaction( } fn replay_in_transaction( - transaction: &crate::exact_sql::ExactSqlTransaction, + transaction: &impl RegisteredWorkWrite, context: &WorkProductPortContextV1, draft: &WorkProductEventDraftV1, ) -> Result, PortError> { @@ -202,7 +205,7 @@ fn mint_event( } fn insert_event( - transaction: &crate::exact_sql::ExactSqlTransaction, + transaction: &impl RegisteredWorkWrite, context: &WorkProductPortContextV1, event: &WorkProductEventV1, sequence: WorkProductEventSequenceV1, @@ -215,7 +218,7 @@ fn insert_event( None => ExactSqlValue::Null, }; transaction - .execute( + .work_execute( exact_sql_statement( "INSERT INTO work_product_events_v1 ( owner_brain_id, owner_profile_id, sequence, event_id, command_id, diff --git a/crates/tracedecay-rusqlite-runtime/src/work_product/publication.rs b/crates/tracedecay-rusqlite-runtime/src/work_product/publication.rs index 51129338b8..ebb36e5684 100644 --- a/crates/tracedecay-rusqlite-runtime/src/work_product/publication.rs +++ b/crates/tracedecay-rusqlite-runtime/src/work_product/publication.rs @@ -11,12 +11,14 @@ use tracedecay_domain::WorkProductEventV1; use super::{fold_graph, load_journal, owner_params, recovered_graph_digest, selection_covers}; use crate::exact_sql::ExactSqlValue; -use crate::work::{exact_sql_statement, exact_sql_text, registered_work_query}; +use crate::work::{ + RegisteredWorkWrite, exact_sql_statement, exact_sql_text, registered_work_query, +}; type PortError = WorkProductEventPortErrorV1; pub(super) fn publish_in_transaction( - transaction: &crate::exact_sql::ExactSqlTransaction, + transaction: &impl RegisteredWorkWrite, context: &WorkProductPortContextV1, event: &WorkProductEventV1, ) -> Result { @@ -75,7 +77,7 @@ pub(super) fn publish_in_transaction( } transaction - .execute( + .work_execute( exact_sql_statement( "INSERT INTO work_product_graph_versions_v1 ( owner_brain_id, owner_profile_id, graph_version, event_sequence, diff --git a/crates/tracedecay-rusqlite-runtime/src/work_run_control.rs b/crates/tracedecay-rusqlite-runtime/src/work_run_control.rs index 1eba82467b..7dd44e1cd4 100644 --- a/crates/tracedecay-rusqlite-runtime/src/work_run_control.rs +++ b/crates/tracedecay-rusqlite-runtime/src/work_run_control.rs @@ -20,7 +20,7 @@ use tracedecay_domain::{ WorkRunControlV1, }; -use crate::exact_sql::{ExactSqlError, ExactSqlJobTransaction, ExactSqlTransaction, ExactSqlValue}; +use crate::exact_sql::{ExactSqlError, ExactSqlJobTransaction, ExactSqlValue}; use crate::work::{ ACTIVE_ATTEMPT_PREDICATE, RegisteredWorkQuery, RegisteredWorkWrite, WorkSqliteStorage, authority_params_owned, exact_sql_integer, exact_sql_statement, exact_sql_text, @@ -33,6 +33,11 @@ use crate::work::{ #[derive(Clone, Copy)] pub struct WorkSqliteJobStorage<'job> { pub(crate) transaction: &'job ExactSqlJobTransaction<'job>, + pub(crate) owner: &'job tracedecay_store::StoreShardIdV1, + /// The graph read made before the job. A job never replays the product + /// journal through the read port, so without one every graph read is + /// refused as unavailable. + pub(crate) graph: Option<&'job tracedecay_contracts::PrefetchedWorkGraphReadV1>, } impl WorkSqliteStorage { @@ -45,10 +50,103 @@ impl WorkSqliteStorage { R: Send + 'static, E: Send + 'static, { + self.run_job(None, job).await + } + + /// [`Self::transact`] for a job whose service reads the current product + /// graph: the job's storage answers that read from `graph`, read before + /// the job on a reader. + pub async fn transact_with_graph( + &self, + graph: tracedecay_contracts::PrefetchedWorkGraphReadV1, + job: F, + ) -> Result, ExactSqlError> + where + F: Fn(WorkSqliteJobStorage<'_>) -> Result + Send + Sync + 'static, + R: Send + 'static, + E: Send + 'static, + { + self.run_job(Some(graph), job).await + } + + async fn run_job( + &self, + graph: Option, + job: F, + ) -> Result, ExactSqlError> + where + F: Fn(WorkSqliteJobStorage<'_>) -> Result + Send + Sync + 'static, + R: Send + 'static, + E: Send + 'static, + { + let owner = self.handle().binding().shard_id.clone(); self.handle() - .transact(move |transaction| job(WorkSqliteJobStorage { transaction })) + .transact(move |transaction| { + job(WorkSqliteJobStorage { + transaction, + owner: &owner, + graph: graph.as_ref(), + }) + }) .await } + + /// Reads the current product graph an attempt admission observed at + /// `observed_at` makes, on the blocking pool, so the awaiting task parks + /// no runtime thread and the replay never holds the writer. + pub async fn prefetch_attempt_admission_graph( + &self, + context: &tracedecay_contracts::RequestContext, + observed_at: UtcMicros, + ) -> tracedecay_contracts::PrefetchedWorkGraphReadV1 { + let storage = self.clone(); + let context = context.clone(); + crate::native_runtime::spawn_blocking(move || { + tracedecay_contracts::PrefetchedWorkGraphReadV1::for_attempt_admission( + &storage, + &context, + observed_at, + ) + }) + .await + .unwrap_or_default() + } +} + +impl tracedecay_contracts::WorkRunReservationPort for WorkSqliteJobStorage<'_> { + fn load_run_control( + &self, + authority: &WorkAuthority, + task_id: &TaskId, + run_id: &RunId, + ) -> Result, WorkRunControlStorageError> { + load_run_control_from(self.transaction, authority, task_id, run_id) + } +} + +impl tracedecay_contracts::WorkRunReservationPort for WorkSqliteStorage { + fn load_run_control( + &self, + authority: &WorkAuthority, + task_id: &TaskId, + run_id: &RunId, + ) -> Result, WorkRunControlStorageError> { + load_run_control_from(self.handle(), authority, task_id, run_id) + } +} + +impl tracedecay_contracts::WorkGraphReadPortV1 for WorkSqliteJobStorage<'_> { + fn read_graph( + &self, + context: &tracedecay_contracts::WorkProductPortContextV1, + request: &tracedecay_contracts::WorkGraphReadRequestV1, + ) -> Result + { + self.graph.map_or( + Err(tracedecay_contracts::WorkGraphReadPortErrorV1::Unavailable), + |graph| graph.read_graph(context, request), + ) + } } impl WorkRunControlTransitionPort for WorkSqliteJobStorage<'_> { @@ -161,15 +259,6 @@ impl WorkRunControlStoragePort for WorkSqliteStorage { run_admission_from(self.handle(), authority, task_id, run_id) } - fn load_run_control( - &self, - authority: &WorkAuthority, - task_id: &TaskId, - run_id: &RunId, - ) -> Result, WorkRunControlStorageError> { - load_run_control_from(self.handle(), authority, task_id, run_id) - } - fn publish_run_control( &self, authority: &WorkAuthority, @@ -273,7 +362,7 @@ impl WorkRunControlStoragePort for WorkSqliteStorage { .entered(); { transaction - .execute( + .work_execute( exact_sql_statement( "UPDATE work_blocked_intervals_v1 SET observability_durable = 1 @@ -640,7 +729,7 @@ fn publish_run_control_tx( } fn load_blocked_interval_observation_cursor( - transaction: &ExactSqlTransaction, + transaction: &impl RegisteredWorkWrite, authority: &WorkAuthority, ) -> Result, WorkRunControlStorageError> { let rows = registered_work_query( @@ -661,7 +750,7 @@ fn load_blocked_interval_observation_cursor( } fn settled_blocked_interval_observation_page( - transaction: &ExactSqlTransaction, + transaction: &impl RegisteredWorkWrite, authority: &WorkAuthority, after: Option<&BlockedIntervalObservationCursor>, limit: u32, @@ -706,13 +795,13 @@ fn settled_blocked_interval_observation_page( } fn persist_blocked_interval_observation_cursor( - transaction: &ExactSqlTransaction, + transaction: &impl RegisteredWorkWrite, authority: &WorkAuthority, receipt: &WorkBlockedIntervalReceiptV1, ) -> Result<(), WorkRunControlStorageError> { let cursor = BlockedIntervalObservationCursor::from_receipt(receipt)?; transaction - .execute( + .work_execute( exact_sql_statement( "INSERT INTO work_blocked_interval_observation_cursors_v1 ( project_id, repository_id, worktree_id, actor_id, policy_digest, @@ -901,7 +990,7 @@ fn blocked_identity_params(receipt: &WorkBlockedIntervalReceiptV1) -> [ExactSqlV /// terminal CAS. The interval receipt cannot survive a terminal attempt with /// no end instant, and a crash can commit neither half independently. pub(crate) fn close_blocked_intervals_on_terminal_attempt( - transaction: &ExactSqlTransaction, + transaction: &impl RegisteredWorkWrite, authority: &WorkAuthority, next: &WorkAttemptV1, ) -> Result<(), WorkAttemptStorageError> { diff --git a/crates/tracedecay-rusqlite-runtime/src/workflow.rs b/crates/tracedecay-rusqlite-runtime/src/workflow.rs index 514f79503b..62c75e4efe 100644 --- a/crates/tracedecay-rusqlite-runtime/src/workflow.rs +++ b/crates/tracedecay-rusqlite-runtime/src/workflow.rs @@ -434,28 +434,30 @@ fn decode_json(payload: &str) -> Result { } pub(crate) fn query_tx( - transaction: &ExactSqlTransaction, + transaction: &impl crate::work::RegisteredWorkQuery, sql: &str, params: Vec, ) -> Result { - transaction.query(statement(sql, params)?) + transaction.work_query(statement(sql, params)?) } pub(crate) fn execute_tx( - transaction: &ExactSqlTransaction, + transaction: &impl crate::work::RegisteredWorkWrite, sql: &str, params: Vec, ) -> Result<(), ExactSqlError> { - transaction.execute(statement(sql, params)?).map(|_| ()) + transaction + .work_execute(statement(sql, params)?) + .map(|_| ()) } fn execute_tx_changed( - transaction: &ExactSqlTransaction, + transaction: &impl crate::work::RegisteredWorkWrite, sql: &str, params: Vec, ) -> Result { transaction - .execute(statement(sql, params)?) + .work_execute(statement(sql, params)?) .map(|result| result.changed_rows) } diff --git a/crates/tracedecay-rusqlite-runtime/src/workflow/run_journal.rs b/crates/tracedecay-rusqlite-runtime/src/workflow/run_journal.rs index bb1ab8162d..9e70262484 100644 --- a/crates/tracedecay-rusqlite-runtime/src/workflow/run_journal.rs +++ b/crates/tracedecay-rusqlite-runtime/src/workflow/run_journal.rs @@ -17,10 +17,7 @@ use tracedecay_domain::{ WorkflowRunProjection, canonical_sha256, }; -use super::{ - ExactSqlTransaction, ExactSqlValue, WorkflowSqliteAuthority, encode_json, execute_tx, query_tx, - sql_text, -}; +use super::{ExactSqlValue, WorkflowSqliteAuthority, encode_json, execute_tx, query_tx, sql_text}; use crate::work::{RegisteredWorkQuery, registered_work_query}; fn run_journal_unavailable(_: E) -> WorkflowRunStorageError { @@ -75,7 +72,7 @@ pub(crate) fn projection_from( } pub(crate) fn append_in_transaction( - transaction: &ExactSqlTransaction, + transaction: &impl crate::work::RegisteredWorkWrite, request: &WorkflowRunAppendRequest, ) -> Result { let payload = encode_json(&request.event).map_err(|_| WorkflowRunStorageError::Unavailable)?; @@ -292,8 +289,8 @@ fn artifact_store_unavailable(_: E) -> WorkflowArtifactStoreError { WorkflowArtifactStoreError::Unavailable } -fn stored_payload_tx( - transaction: &ExactSqlTransaction, +pub(crate) fn stored_payload_tx( + transaction: &impl crate::work::RegisteredWorkQuery, digest: &str, ) -> Result>, WorkflowArtifactStoreError> { let rows = query_tx( diff --git a/crates/tracedecay-rusqlite-runtime/tests/rusqlite_suite/work_attempt_storage.rs b/crates/tracedecay-rusqlite-runtime/tests/rusqlite_suite/work_attempt_storage.rs index b219144546..b69e7dc16f 100644 --- a/crates/tracedecay-rusqlite-runtime/tests/rusqlite_suite/work_attempt_storage.rs +++ b/crates/tracedecay-rusqlite-runtime/tests/rusqlite_suite/work_attempt_storage.rs @@ -43,6 +43,7 @@ use tracedecay_domain::{ }; use common::fixture_abs_root; +use tracedecay_rusqlite_runtime::work_run_control::WorkSqliteJobStorage; use tracedecay_rusqlite_runtime::workflow::install_workflow_schema; use work_registered_store::RegisteredWorkStore; @@ -468,10 +469,7 @@ fn retry_reservation_cannot_overbook_project_global_capacity() { let write = retry_write(&original); assert_eq!( - store - .storage() - .insert_retry_bounded(&retry_authority, &write, &concurrency(1, 1, 1)) - .unwrap_err(), + insert_retry(&store, &retry_authority, &write, &concurrency(1, 1, 1)).unwrap_err(), WorkAttemptStorageError::CapacityExceeded ); assert_eq!(store.count("work_attempts_v1"), 2); @@ -497,10 +495,13 @@ fn recovery_retry_replaces_the_original_across_active_attempt_census() { )); store.storage().insert(&authority, &original).unwrap(); - let outcome = store - .storage() - .insert_retry_bounded(&authority, &retry_write(&original), &concurrency(1, 1, 1)) - .unwrap(); + let outcome = insert_retry( + &store, + &authority, + &retry_write(&original), + &concurrency(1, 1, 1), + ) + .unwrap(); assert!(matches!(outcome, WorkRetryAttemptOutcomeV1::Created { .. })); let replacement = outcome.attempt().clone(); assert_eq!(store.count("work_attempts_v1"), 2); @@ -657,10 +658,7 @@ fn successful_retry_remains_pending_until_exact_durable_marker_cas() { )); store.storage().insert(&authority, &original).unwrap(); let write = retry_write(&original); - store - .storage() - .insert_retry_bounded(&authority, &write, &concurrency(1, 1, 1)) - .unwrap(); + insert_retry(&store, &authority, &write, &concurrency(1, 1, 1)).unwrap(); let pending = store .storage() @@ -1129,6 +1127,55 @@ fn recovery_required(attempt: &WorkAttemptV1) -> WorkAttemptV1 { .unwrap() } +/// Runs one storage call as a writer job, which commits only when it returns +/// `Ok`. +fn in_job( + store: &RegisteredWorkStore, + call: impl Fn(WorkSqliteJobStorage<'_>) -> Result + + Send + + Sync + + 'static, +) -> Result { + tracedecay_rusqlite_runtime::native_runtime::block_on(store.storage().transact(call)) + .expect("native runtime") + .expect("the writer runs the job") +} + +fn insert_retry( + store: &RegisteredWorkStore, + authority: &WorkAuthority, + write: &WorkRetryWriteV1, + concurrency: &TopologyConcurrencyPolicyV1, +) -> Result { + let (authority, write, concurrency) = (authority.clone(), write.clone(), concurrency.clone()); + in_job(store, move |storage| { + storage.insert_retry_bounded(&authority, &write, &concurrency) + }) +} + +fn insert_synthesis( + store: &RegisteredWorkStore, + authority: &WorkAuthority, + record: &WorkSynthesisAdmissionRecordV1, + concurrency: &TopologyConcurrencyPolicyV1, +) -> Result { + let (authority, record, concurrency) = (authority.clone(), record.clone(), concurrency.clone()); + in_job(store, move |storage| { + storage.insert_synthesis_bounded(&authority, &record, &concurrency) + }) +} + +fn load_synthesis( + store: &RegisteredWorkStore, + authority: &WorkAuthority, + identity: &WorkAttemptIdentityV1, +) -> Result { + let (authority, identity) = (authority.clone(), identity.clone()); + in_job(store, move |storage| { + storage.load_synthesis(&authority, &identity) + }) +} + fn retry_write(original: &WorkAttemptV1) -> WorkRetryWriteV1 { let (failure, evidence_digest, observed_at, restart_reason) = match original.recovery() { WorkRecoveryStateV1::RecoveryRequired { @@ -1283,10 +1330,7 @@ fn insert_replays_identical_admissions_and_refuses_divergent_ones() { WorkAttemptAdmissionKind::Ordinary ); assert_eq!( - store - .storage() - .load_synthesis(&authority, first.identity()) - .unwrap_err(), + load_synthesis(&store, &authority, first.identity()).unwrap_err(), WorkAttemptStorageError::AttemptConflict ); // The same identity with different content is a conflict, not a refresh. @@ -1333,17 +1377,11 @@ fn synthesis_admission_replays_one_durable_result_and_refuses_changed_requests() let policy = concurrency(1, 1, 1); assert_eq!( - store - .storage() - .insert_synthesis_bounded(&authority, &record, &policy) - .unwrap(), + insert_synthesis(&store, &authority, &record, &policy).unwrap(), WorkSynthesisInsertOutcome::Inserted ); assert_eq!( - store - .storage() - .insert_synthesis_bounded(&authority, &record, &policy) - .unwrap(), + insert_synthesis(&store, &authority, &record, &policy).unwrap(), WorkSynthesisInsertOutcome::Replayed(Box::new(admission.clone())) ); assert_eq!(store.count("work_attempts_v1"), 1); @@ -1353,10 +1391,7 @@ fn synthesis_admission_replays_one_durable_result_and_refuses_changed_requests() result: admission, }; assert_eq!( - store - .storage() - .insert_synthesis_bounded(&authority, &changed, &policy) - .unwrap_err(), + insert_synthesis(&store, &authority, &changed, &policy).unwrap_err(), WorkAttemptStorageError::AttemptConflict ); assert_eq!(store.count("work_attempts_v1"), 1); @@ -1380,19 +1415,13 @@ fn synthesis_admission_replays_one_durable_result_and_refuses_changed_requests() ) .unwrap(); assert_eq!( - store - .storage() - .load_synthesis(&authority, admitted_attempt.identity()) - .unwrap(), + load_synthesis(&store, &authority, admitted_attempt.identity()).unwrap(), record ); let store = store.restart("attempt-synthesis-insert"); assert_eq!( - store - .storage() - .load_synthesis(&authority, admitted_attempt.identity()) - .unwrap(), + load_synthesis(&store, &authority, admitted_attempt.identity()).unwrap(), record ); assert_eq!( diff --git a/crates/tracedecay-rusqlite-runtime/tests/rusqlite_suite/work_run_control_storage.rs b/crates/tracedecay-rusqlite-runtime/tests/rusqlite_suite/work_run_control_storage.rs index 31655ae898..f747bb51f5 100644 --- a/crates/tracedecay-rusqlite-runtime/tests/rusqlite_suite/work_run_control_storage.rs +++ b/crates/tracedecay-rusqlite-runtime/tests/rusqlite_suite/work_run_control_storage.rs @@ -16,7 +16,7 @@ use std::collections::BTreeSet; use tracedecay_contracts::{ WorkAttemptStorageError, WorkAttemptStoragePort, WorkRunControlStorageError, - WorkRunControlStoragePort, WorkRunControlTransitionPort, + WorkRunControlStoragePort, WorkRunControlTransitionPort, WorkRunReservationPort, }; use tracedecay_domain::{ ActorId, AttemptId, CommitId, ConfigurationRevisionId, ConfigurationSnapshotId, ProjectId, From 35c2097235659dfe48d6db561197b95828465764 Mon Sep 17 00:00:00 2001 From: ScriptedAlchemy Date: Fri, 16 Oct 2026 19:41:17 +0000 Subject: [PATCH 2/4] refactor(work): run provider driver steps as writer jobs The provider-attempt driver now awaits each durable step between process events as its own writer job: the effect dispatch record and settlement, mark_running, the provider-unavailable fence, the recovery failure, the cancellation acknowledgement and escalation, and the terminal settlement. Process waits stay on the attempt task. The driver reaches these steps through an AttemptStepsV1 trait; the registered Work services implement it with job-backed async methods, and the driver tests answer it from the in-memory services. The attempt task still runs on a blocking-pool thread. It persists provider output through the synchronous workflow artifact store and reconciles the workflow fan-out after the attempt, and both still park their thread on the writer. The effect-holder read moves to a WorkAttemptEffectReadPortV1 that retry safety, execution history and the leak evidence use. The handle storage implements only that read; its dispatch and settlement writes had no other production caller and are deleted, as is the sync attempt_effects service accessor. --- .../src/work/registered.rs | 164 +++++++- .../application_suite/work_attempt_jobs.rs | 193 ++++++++++ crates/tracedecay-contracts/src/lib.rs | 4 +- .../src/work_attempt_effect.rs | 32 +- .../src/work_execution_history.rs | 6 +- crates/tracedecay-contracts/src/work_retry.rs | 6 +- .../src/invocation/work_attempt_exec.rs | 349 +++++++++++++----- .../src/invocation/work_attempt_exec/tests.rs | 157 ++++++-- .../src/work/effect_holder.rs | 153 +++----- .../rusqlite_suite/work_attempt_storage.rs | 102 ++--- 10 files changed, 862 insertions(+), 304 deletions(-) diff --git a/crates/tracedecay-application/src/work/registered.rs b/crates/tracedecay-application/src/work/registered.rs index f8dc38aca1..4d4107e568 100644 --- a/crates/tracedecay-application/src/work/registered.rs +++ b/crates/tracedecay-application/src/work/registered.rs @@ -60,9 +60,6 @@ pub struct RegisteredWorkApplicationServicesV1 { attempts: tracedecay_contracts::WorkAttemptService< tracedecay_rusqlite_runtime::work::WorkSqliteStorage, >, - attempt_effects: tracedecay_contracts::WorkAttemptEffectServiceV1< - tracedecay_rusqlite_runtime::work::WorkSqliteStorage, - >, run_control: tracedecay_contracts::WorkRunControlService< tracedecay_rusqlite_runtime::work::WorkSqliteStorage, >, @@ -335,7 +332,6 @@ impl RegisteredWorkApplicationServicesV1 { let storage = db.work_storage()?; Ok(Self { attempts: tracedecay_contracts::WorkAttemptService::new(storage.clone()), - attempt_effects: tracedecay_contracts::WorkAttemptEffectServiceV1::new(storage.clone()), run_control: tracedecay_contracts::WorkRunControlService::new(storage.clone()), placement: tracedecay_contracts::WorkPlacementService::new(storage.clone()), artifact_hydration: tracedecay_contracts::WorkArtifactHydrationService::new( @@ -403,6 +399,158 @@ impl RegisteredWorkApplicationServicesV1 { .unwrap_or_else(|_| Err(attempt_unavailable())) } + /// Runs one provider-driver step of the attempt authority as a single + /// writer transaction: its load, its decision and its fenced update + /// commit together. + async fn attempt_step( + &self, + step: impl Fn( + &tracedecay_contracts::WorkAttemptService< + tracedecay_rusqlite_runtime::work_run_control::WorkSqliteJobStorage<'_>, + >, + ) -> Result + + Send + + Sync + + 'static, + ) -> Result + where + R: Send + 'static, + { + self.storage + .transact(move |storage| step(&tracedecay_contracts::WorkAttemptService::new(storage))) + .await + .unwrap_or_else(|_| Err(attempt_unavailable())) + } + + /// Marks the provider running under `actual_route`. + pub async fn mark_running( + &self, + context: tracedecay_contracts::RequestContext, + identity: tracedecay_domain::WorkAttemptIdentityV1, + actual_route: tracedecay_domain::WorkProviderRouteV1, + ) -> Result { + self.attempt_step(move |attempts| { + attempts.mark_running(&context, &identity, actual_route.clone()) + }) + .await + } + + /// Fences an attempt whose provider could not start for recovery. + pub async fn mark_provider_unavailable( + &self, + context: tracedecay_contracts::RequestContext, + identity: tracedecay_domain::WorkAttemptIdentityV1, + observed_at: tracedecay_domain::UtcMicros, + ) -> Result { + self.attempt_step(move |attempts| { + attempts.mark_provider_unavailable(&context, &identity, observed_at) + }) + .await + } + + /// Fails a recovery-required attempt with sealed denial evidence. + pub async fn fail_recovery( + &self, + context: tracedecay_contracts::RequestContext, + identity: tracedecay_domain::WorkAttemptIdentityV1, + evidence: tracedecay_contracts::WorkAttemptEvidenceRecordV1, + ) -> Result { + self.attempt_step(move |attempts| attempts.fail_recovery(&context, &identity, &evidence)) + .await + } + + /// Acknowledges a durable cancellation request. + pub async fn acknowledge_cancellation( + &self, + context: tracedecay_contracts::RequestContext, + identity: tracedecay_domain::WorkAttemptIdentityV1, + acknowledged_at: tracedecay_domain::UtcMicros, + ) -> Result { + self.attempt_step(move |attempts| { + attempts.acknowledge_cancellation(&context, &identity, acknowledged_at) + }) + .await + } + + /// Escalates an acknowledged cancellation to forced termination. + pub async fn escalate_cancellation( + &self, + context: tracedecay_contracts::RequestContext, + identity: tracedecay_domain::WorkAttemptIdentityV1, + escalated_at: tracedecay_domain::UtcMicros, + ) -> Result { + self.attempt_step(move |attempts| { + attempts.escalate_cancellation(&context, &identity, escalated_at) + }) + .await + } + + /// Seals an attempt with terminal evidence and its artifacts. + pub async fn settle_with_artifacts( + &self, + context: tracedecay_contracts::RequestContext, + identity: tracedecay_domain::WorkAttemptIdentityV1, + evidence: tracedecay_contracts::WorkAttemptEvidenceRecordV1, + artifacts: Vec, + ) -> Result { + self.attempt_step(move |attempts| { + attempts.settle_with_artifacts(&context, &identity, &evidence, artifacts.clone()) + }) + .await + } + + /// Records the immutable dispatch receipt before a provider launch, as a + /// single writer transaction. + pub async fn record_effect_dispatch( + &self, + context: tracedecay_contracts::RequestContext, + attempt: tracedecay_domain::WorkAttemptIdentityV1, + effect_state: tracedecay_domain::WorkEffectStateV1, + dispatched_at: tracedecay_domain::UtcMicros, + deadline: tracedecay_domain::UtcMicros, + ) -> Result< + tracedecay_contracts::WorkAttemptEffectDispatchOutcomeV1, + tracedecay_contracts::ApplicationProblem, + > { + self.storage + .transact(move |storage| { + tracedecay_contracts::WorkAttemptEffectServiceV1::new(storage).record_dispatch( + &context, + attempt.clone(), + effect_state, + dispatched_at, + deadline, + ) + }) + .await + .unwrap_or_else(|_| Err(attempt_unavailable())) + } + + /// Settles the dispatch receipt's effect resolution, as a single writer + /// transaction. + pub async fn settle_effect_dispatch( + &self, + context: tracedecay_contracts::RequestContext, + attempt: tracedecay_domain::WorkAttemptIdentityV1, + resolution: tracedecay_contracts::WorkAttemptEffectResolutionV1, + resolved_at: tracedecay_domain::UtcMicros, + ) -> Result< + tracedecay_contracts::WorkAttemptEffectHolderV1, + tracedecay_contracts::ApplicationProblem, + > { + self.storage + .transact(move |storage| { + tracedecay_contracts::WorkAttemptEffectServiceV1::new(storage).settle( + &context, + &attempt, + resolution, + resolved_at, + ) + }) + .await + .unwrap_or_else(|_| Err(attempt_unavailable())) + } + /// Records one duplicate adjudication as a single writer transaction: the /// replay check, the revision compare and the append commit together. /// `current_evidence` is read before the job, outside the writer. @@ -547,14 +695,6 @@ impl RegisteredWorkApplicationServicesV1 { &self.attempts } - pub const fn attempt_effects( - &self, - ) -> &tracedecay_contracts::WorkAttemptEffectServiceV1< - tracedecay_rusqlite_runtime::work::WorkSqliteStorage, - > { - &self.attempt_effects - } - /// The run-level pause/resume authority. /// /// It is a separate service from [`Self::attempts`] because the aggregate diff --git a/crates/tracedecay-application/tests/application_suite/work_attempt_jobs.rs b/crates/tracedecay-application/tests/application_suite/work_attempt_jobs.rs index 5ddd98bf14..d8ffa452cf 100644 --- a/crates/tracedecay-application/tests/application_suite/work_attempt_jobs.rs +++ b/crates/tracedecay-application/tests/application_suite/work_attempt_jobs.rs @@ -580,3 +580,196 @@ async fn a_pause_racing_a_start_never_lands_between_its_fence_and_its_reservatio } } } + +fn provider_evidence( + attempt: &WorkAttemptIdentityV1, + outcome: WorkAttemptProviderOutcomeV1, + observed_at: UtcMicros, +) -> WorkAttemptEvidenceRecordV1 { + WorkAttemptEvidenceRecordV1 { + identity: attempt.clone(), + requested_route: route(), + actual_route: Some(route()), + outcome, + stdout: None, + stderr: None, + provider_session: None, + provider_fallback: None, + observed_at, + } +} + +#[tokio::test] +async fn provider_driver_steps_awaited_on_a_native_worker_park_no_native_thread() { + let profile = tempfile::tempdir().expect("profile"); + let project = tempfile::tempdir().expect("project"); + let runtime = project_runtime(profile.path(), project.path()).await; + let database = runtime.project_database().expect("project database"); + let storage = database.work_storage().expect("Work storage"); + let run = id::("run.attempt-jobs.driver"); + let [succeeded, denied, cancelled] = ["succeeded", "denied", "cancelled"].map(|suffix| { + let attempt = crate::work_run_control_jobs::leased_attempt_of( + run.clone(), + &format!("attempt-jobs.driver.{suffix}"), + ); + storage + .insert(&authority(), &attempt) + .expect("the attempt is stored"); + attempt + }); + let (effect_state, deadline) = ( + succeeded.execution().effect_state(), + succeeded.execution().deadline(), + ); + let [succeeded, denied, cancelled] = + [succeeded, denied, cancelled].map(|attempt| attempt.identity().clone()); + let setup = RegisteredWorkApplicationServicesV1::attach(database).expect("Work services"); + setup + .attempts() + .mark_running(&product_context(), &cancelled, route()) + .expect("the cancelled attempt runs"); + setup + .attempts() + .request_cancellation( + &product_context(), + tracedecay_contracts::CancelWorkAttemptCommand { + task_id: task(), + run_id: run.clone(), + attempt_id: cancelled.attempt_id().clone(), + request_id: id("cancellation.attempt-jobs.driver"), + occurred_at: UtcMicros(20_000), + }, + ) + .expect("the cancellation is requested"); + let work = RegisteredWorkApplicationServicesV1::attach(database).expect("Work services"); + let identities = (succeeded.clone(), denied.clone(), cancelled.clone()); + let (outcome, writer_waits, reader_waits) = on_native_worker(async move { + let (succeeded, denied, cancelled) = identities; + let context = product_context(); + let dispatched = work + .record_effect_dispatch( + context.clone(), + succeeded.clone(), + effect_state, + UtcMicros(21_000), + deadline, + ) + .await + .map(|_| ()); + let running = work + .mark_running(context.clone(), succeeded.clone(), route()) + .await + .map(|attempt| attempt.state()); + let effect = work + .settle_effect_dispatch( + context.clone(), + succeeded.clone(), + tracedecay_contracts::WorkAttemptEffectResolutionV1::NoEffect, + UtcMicros(22_000), + ) + .await + .map(|holder| holder.resolution()); + let settled = work + .settle_with_artifacts( + context.clone(), + succeeded.clone(), + provider_evidence( + &succeeded, + WorkAttemptProviderOutcomeV1::Exited { code: 0 }, + UtcMicros(23_000), + ), + Vec::new(), + ) + .await + .map(|attempt| attempt.state()); + let fenced = work + .mark_provider_unavailable(context.clone(), denied.clone(), UtcMicros(24_000)) + .await + .map(|attempt| attempt.state()); + let failed = work + .fail_recovery( + context.clone(), + denied.clone(), + WorkAttemptEvidenceRecordV1 { + actual_route: None, + ..provider_evidence( + &denied, + WorkAttemptProviderOutcomeV1::LaunchFailed, + UtcMicros(25_000), + ) + }, + ) + .await + .map(|attempt| attempt.state()); + let acknowledged = work + .acknowledge_cancellation(context.clone(), cancelled.clone(), UtcMicros(26_000)) + .await + .map(|attempt| attempt.state()); + let escalated = work + .escalate_cancellation(context.clone(), cancelled.clone(), UtcMicros(27_000)) + .await + .map(|attempt| attempt.state()); + let sealed = work + .settle_with_artifacts( + context, + cancelled.clone(), + provider_evidence( + &cancelled, + WorkAttemptProviderOutcomeV1::Cancelled, + UtcMicros(28_000), + ), + Vec::new(), + ) + .await + .map(|attempt| attempt.state()); + vec![ + dispatched.map(|()| None), + running.map(Some), + effect.map(|_| None), + settled.map(Some), + fenced.map(Some), + failed.map(Some), + acknowledged.map(Some), + escalated.map(Some), + sealed.map(Some), + ] + }) + .await; + assert_eq!( + writer_waits, 0, + "the provider driver's steps parked no native worker on the writer" + ); + assert_eq!( + reader_waits, 0, + "the provider driver's steps parked no native worker on a reader" + ); + assert_eq!( + outcome, + vec![ + Ok(None), + Ok(Some(WorkAttemptStateV1::Running)), + Ok(None), + Ok(Some(WorkAttemptStateV1::Succeeded)), + Ok(Some(WorkAttemptStateV1::RecoveryRequired)), + Ok(Some(WorkAttemptStateV1::Failed)), + Ok(Some(WorkAttemptStateV1::CancellationAcknowledged)), + Ok(Some(WorkAttemptStateV1::CancellationEscalated)), + Ok(Some(WorkAttemptStateV1::Cancelled)), + ], + "every driver step committed its transition" + ); + for (identity, state) in [ + (&succeeded, WorkAttemptStateV1::Succeeded), + (&denied, WorkAttemptStateV1::Failed), + (&cancelled, WorkAttemptStateV1::Cancelled), + ] { + assert_eq!( + storage + .load(&authority(), identity) + .expect("the attempt reads") + .state(), + state, + "the driver's jobs committed the terminal state" + ); + } +} diff --git a/crates/tracedecay-contracts/src/lib.rs b/crates/tracedecay-contracts/src/lib.rs index ba7899ae4a..ea8c5c64be 100644 --- a/crates/tracedecay-contracts/src/lib.rs +++ b/crates/tracedecay-contracts/src/lib.rs @@ -458,8 +458,8 @@ pub use work_attempt::{ }; pub use work_attempt_effect::{ WorkAttemptEffectDispatchOutcomeV1, WorkAttemptEffectHolderErrorV1, WorkAttemptEffectHolderV1, - WorkAttemptEffectResolutionV1, WorkAttemptEffectServiceV1, WorkAttemptEffectStorageErrorV1, - WorkAttemptEffectStoragePortV1, + WorkAttemptEffectReadPortV1, WorkAttemptEffectResolutionV1, WorkAttemptEffectServiceV1, + WorkAttemptEffectStorageErrorV1, WorkAttemptEffectStoragePortV1, }; pub use work_catalog::{ WORK_APPLICATION_OPERATION_IDS_V1, work_executable_binding, work_executable_binding_registry, diff --git a/crates/tracedecay-contracts/src/work_attempt_effect.rs b/crates/tracedecay-contracts/src/work_attempt_effect.rs index 819c0a89a9..f819e172d1 100644 --- a/crates/tracedecay-contracts/src/work_attempt_effect.rs +++ b/crates/tracedecay-contracts/src/work_attempt_effect.rs @@ -155,9 +155,18 @@ pub enum WorkAttemptEffectStorageErrorV1 { Unavailable, } +/// The effect-holder read: retry safety and execution history consult it. +pub trait WorkAttemptEffectReadPortV1 { + fn load_effect_dispatch( + &self, + authority: &WorkAuthority, + attempt: &WorkAttemptIdentityV1, + ) -> Result, WorkAttemptEffectStorageErrorV1>; +} + /// Persistence owned by the exact Work-attempt authority. Implementations /// must make the immutable dispatch identity replay-safe and settlement CASed. -pub trait WorkAttemptEffectStoragePortV1 { +pub trait WorkAttemptEffectStoragePortV1: WorkAttemptEffectReadPortV1 { fn begin_effect_dispatch( &self, authority: &WorkAuthority, @@ -171,12 +180,6 @@ pub trait WorkAttemptEffectStoragePortV1 { resolution: WorkAttemptEffectResolutionV1, resolved_at: UtcMicros, ) -> Result; - - fn load_effect_dispatch( - &self, - authority: &WorkAuthority, - attempt: &WorkAttemptIdentityV1, - ) -> Result, WorkAttemptEffectStorageErrorV1>; } /// Application boundary used by the daemon provider runtime and leak source. @@ -184,14 +187,16 @@ pub struct WorkAttemptEffectServiceV1 { storage: S, } -impl WorkAttemptEffectServiceV1 -where - S: WorkAttemptEffectStoragePortV1, -{ +impl WorkAttemptEffectServiceV1 { pub const fn new(storage: S) -> Self { Self { storage } } +} +impl WorkAttemptEffectServiceV1 +where + S: WorkAttemptEffectStoragePortV1, +{ #[tracing::instrument( name = "application.work.attempt_effect.record_dispatch", level = "trace", @@ -231,7 +236,12 @@ where .settle_effect_dispatch(&authority, attempt, resolution, resolved_at) .map_err(effect_problem) } +} +impl WorkAttemptEffectServiceV1 +where + S: WorkAttemptEffectReadPortV1, +{ #[tracing::instrument( name = "application.work.attempt_effect.load", level = "trace", diff --git a/crates/tracedecay-contracts/src/work_execution_history.rs b/crates/tracedecay-contracts/src/work_execution_history.rs index e43e185a92..3dafa747fc 100644 --- a/crates/tracedecay-contracts/src/work_execution_history.rs +++ b/crates/tracedecay-contracts/src/work_execution_history.rs @@ -14,8 +14,8 @@ use tracedecay_domain::{ use crate::work::work_authority; use crate::{ - ApplicationProblem, RequestContext, SafeDiagnostic, WorkAttemptEffectStorageErrorV1, - WorkAttemptEffectStoragePortV1, WorkAttemptListCoverageV1, WorkAttemptListV1, + ApplicationProblem, RequestContext, SafeDiagnostic, WorkAttemptEffectReadPortV1, + WorkAttemptEffectStorageErrorV1, WorkAttemptListCoverageV1, WorkAttemptListV1, }; #[derive(Clone, Debug, Deserialize, Serialize, JsonSchema, PartialEq, Eq)] @@ -85,7 +85,7 @@ pub fn project_work_execution_history( attempts: WorkAttemptListV1, ) -> Result where - S: WorkAttemptEffectStoragePortV1, + S: WorkAttemptEffectReadPortV1, { let authority = work_authority(context)?; let WorkAttemptListV1::Listed { diff --git a/crates/tracedecay-contracts/src/work_retry.rs b/crates/tracedecay-contracts/src/work_retry.rs index bb0a0abce6..c4996db110 100644 --- a/crates/tracedecay-contracts/src/work_retry.rs +++ b/crates/tracedecay-contracts/src/work_retry.rs @@ -23,7 +23,7 @@ use crate::work_attempt::{ product_attempt_projection_binding, }; use crate::work_attempt_effect::{ - WorkAttemptEffectResolutionV1, WorkAttemptEffectStorageErrorV1, WorkAttemptEffectStoragePortV1, + WorkAttemptEffectReadPortV1, WorkAttemptEffectResolutionV1, WorkAttemptEffectStorageErrorV1, }; use crate::{ ApplicationContractError, ApplicationProblem, RequestContext, RetryDirective, SafeDiagnostic, @@ -360,7 +360,7 @@ pub struct WorkflowFanOutRetryRebindV1 { impl WorkProductRetryServiceV1 where S: WorkRetryStoragePortV1 - + WorkAttemptEffectStoragePortV1 + + WorkAttemptEffectReadPortV1 + WorkGraphReadPortV1 + WorkProductOwnerAuthorizationPortV1, E: WorkRetryEvidencePortV1, @@ -619,7 +619,7 @@ fn require_retry_effect_safe( original: &WorkAttemptV1, ) -> Result<(), ApplicationProblem> where - S: WorkAttemptEffectStoragePortV1, + S: WorkAttemptEffectReadPortV1, { if original.execution().effect_state() != WorkEffectStateV1::CompoundNonRepeatable { return Ok(()); diff --git a/crates/tracedecay-daemon-service/src/invocation/work_attempt_exec.rs b/crates/tracedecay-daemon-service/src/invocation/work_attempt_exec.rs index 7412ecd358..a6273763f8 100644 --- a/crates/tracedecay-daemon-service/src/invocation/work_attempt_exec.rs +++ b/crates/tracedecay-daemon-service/src/invocation/work_attempt_exec.rs @@ -60,8 +60,8 @@ use tracedecay_application::observability::{ record_work_operation_resource, }; use tracedecay_contracts::{ - WorkAttemptEffectDispatchOutcomeV1, WorkAttemptEffectResolutionV1, WorkAttemptEffectServiceV1, - WorkAttemptEffectStoragePortV1, WorkAttemptEvidenceRecordV1, WorkAttemptProviderOutcomeV1, + ApplicationProblem, WorkAttemptEffectDispatchOutcomeV1, WorkAttemptEffectHolderV1, + WorkAttemptEffectResolutionV1, WorkAttemptEvidenceRecordV1, WorkAttemptProviderOutcomeV1, WorkProviderAvailabilityV1, WorkProviderFallbackRecordV1, WorkflowArtifactPayload, WorkflowArtifactStorePort, workflow_artifact_payload_digest, }; @@ -154,6 +154,157 @@ fn provider_stdout_artifacts( #[cfg(test)] mod tests; +/// The durable attempt and effect-holder steps the provider driver awaits +/// between process events. The registered Work services run each one as a +/// writer job, so a step never parks the task's thread on the writer. +pub(super) trait AttemptStepsV1: Sync { + fn mark_running( + &self, + context: &RequestContext, + identity: &WorkAttemptIdentityV1, + actual_route: WorkProviderRouteV1, + ) -> impl Future> + Send; + + fn mark_provider_unavailable( + &self, + context: &RequestContext, + identity: &WorkAttemptIdentityV1, + observed_at: UtcMicros, + ) -> impl Future> + Send; + + fn fail_recovery( + &self, + context: &RequestContext, + identity: &WorkAttemptIdentityV1, + evidence: &WorkAttemptEvidenceRecordV1, + ) -> impl Future> + Send; + + fn acknowledge_cancellation( + &self, + context: &RequestContext, + identity: &WorkAttemptIdentityV1, + acknowledged_at: UtcMicros, + ) -> impl Future> + Send; + + fn escalate_cancellation( + &self, + context: &RequestContext, + identity: &WorkAttemptIdentityV1, + escalated_at: UtcMicros, + ) -> impl Future> + Send; + + fn settle_with_artifacts( + &self, + context: &RequestContext, + identity: &WorkAttemptIdentityV1, + evidence: &WorkAttemptEvidenceRecordV1, + artifacts: Vec, + ) -> impl Future> + Send; + + fn record_dispatch( + &self, + context: &RequestContext, + attempt: &WorkAttemptV1, + dispatched_at: UtcMicros, + ) -> impl Future> + Send; + + fn settle_dispatch( + &self, + context: &RequestContext, + attempt: &WorkAttemptIdentityV1, + resolution: WorkAttemptEffectResolutionV1, + resolved_at: UtcMicros, + ) -> impl Future> + Send; +} + +impl AttemptStepsV1 for tracedecay_application::work::RegisteredWorkApplicationServicesV1 { + fn mark_running( + &self, + context: &RequestContext, + identity: &WorkAttemptIdentityV1, + actual_route: WorkProviderRouteV1, + ) -> impl Future> + Send { + self.mark_running(context.clone(), identity.clone(), actual_route) + } + + fn mark_provider_unavailable( + &self, + context: &RequestContext, + identity: &WorkAttemptIdentityV1, + observed_at: UtcMicros, + ) -> impl Future> + Send { + self.mark_provider_unavailable(context.clone(), identity.clone(), observed_at) + } + + fn fail_recovery( + &self, + context: &RequestContext, + identity: &WorkAttemptIdentityV1, + evidence: &WorkAttemptEvidenceRecordV1, + ) -> impl Future> + Send { + self.fail_recovery(context.clone(), identity.clone(), evidence.clone()) + } + + fn acknowledge_cancellation( + &self, + context: &RequestContext, + identity: &WorkAttemptIdentityV1, + acknowledged_at: UtcMicros, + ) -> impl Future> + Send { + self.acknowledge_cancellation(context.clone(), identity.clone(), acknowledged_at) + } + + fn escalate_cancellation( + &self, + context: &RequestContext, + identity: &WorkAttemptIdentityV1, + escalated_at: UtcMicros, + ) -> impl Future> + Send { + self.escalate_cancellation(context.clone(), identity.clone(), escalated_at) + } + + fn settle_with_artifacts( + &self, + context: &RequestContext, + identity: &WorkAttemptIdentityV1, + evidence: &WorkAttemptEvidenceRecordV1, + artifacts: Vec, + ) -> impl Future> + Send { + self.settle_with_artifacts( + context.clone(), + identity.clone(), + evidence.clone(), + artifacts, + ) + } + + fn record_dispatch( + &self, + context: &RequestContext, + attempt: &WorkAttemptV1, + dispatched_at: UtcMicros, + ) -> impl Future> + Send + { + self.record_effect_dispatch( + context.clone(), + attempt.identity().clone(), + attempt.execution().effect_state(), + dispatched_at, + attempt.execution().deadline(), + ) + } + + fn settle_dispatch( + &self, + context: &RequestContext, + attempt: &WorkAttemptIdentityV1, + resolution: WorkAttemptEffectResolutionV1, + resolved_at: UtcMicros, + ) -> impl Future> + Send { + self.settle_effect_dispatch(context.clone(), attempt.clone(), resolution, resolved_at) + } +} + /// How long an acknowledged cancellation may run before escalation to /// forced termination. const CANCELLATION_GRACE: std::time::Duration = std::time::Duration::from_secs(10); @@ -292,9 +443,11 @@ impl WorkAttemptProcessRegistryV1 { let task_key = key.clone(); let (start, started) = tokio::sync::oneshot::channel(); let future = task(Arc::clone(&cancellation)); - // The attempt calls synchronous Work storage ports (`mark_running`, - // the settle paths) that park their thread on the writer, so it runs - // on a blocking-pool thread rather than a native worker. + // The attempt's own Work steps are awaited writer jobs, but it still + // persists provider output through the synchronous workflow artifact + // store and reconciles the workflow fan-out afterwards, both of which + // park their thread on the writer, so it runs on a blocking-pool thread + // rather than a native worker. let Ok(task) = native_runtime::NativeTask::spawn_off_native_thread(async move { let admitted = tokio::select! { result = started => result.is_ok(), @@ -560,8 +713,6 @@ async fn run_attempt( return; } }; - let attempts = services.attempts(); - let attempt_effects = services.attempt_effects(); let workflow_artifacts = registered.database.workflow_storage().map_err(|error| { tracing::warn!( task = attempt.identity().task_id().as_str(), @@ -592,8 +743,7 @@ async fn run_attempt( Ok(selection) => match selection.provider.protocol { WorkProviderProtocol::CodexAppServerJsonRpc => { execute_app_server( - attempts, - attempt_effects, + &services, workflow_artifacts.as_ref().ok(), &context, &attempt, @@ -608,8 +758,7 @@ async fn run_attempt( } _ => { execute_provider_with_environment( - attempts, - attempt_effects, + &services, workflow_artifacts.as_ref().ok(), &context, &attempt, @@ -625,7 +774,7 @@ async fn run_attempt( }, Err(denial) => { settle_unstarted( - attempts, + &services, &context, &identity, &attempt, @@ -634,7 +783,8 @@ async fn run_attempt( }, denial.fallback, observability_producer.as_deref(), - ); + ) + .await; } } } @@ -795,32 +945,34 @@ fn availability_state(error: WorkExecutableBindingError) -> WorkProviderAvailabi /// Seals a terminal denial for an attempt whose provider never started: /// fence to `RecoveryRequired`, then fail recovery with the typed outcome. -fn settle_unstarted( - attempts: &tracedecay_contracts::WorkAttemptService, +async fn settle_unstarted( + steps: &impl AttemptStepsV1, context: &RequestContext, identity: &WorkAttemptIdentityV1, attempt: &WorkAttemptV1, outcome: WorkAttemptProviderOutcomeV1, provider_fallback: Option, observability_producer: Option<&BoundedObservabilityProducerV1>, -) where - S: tracedecay_contracts::WorkAttemptStoragePort, -{ +) { let observed_at = now_micros(); - if let Err(problem) = attempts.mark_provider_unavailable(context, identity, observed_at) { + if let Err(problem) = steps + .mark_provider_unavailable(context, identity, observed_at) + .await + { tracing::warn!( task = identity.task_id().as_str(), ?problem, "work attempt could not be fenced for provider unavailability" ); seal_cancellation_before_running( - attempts, + steps, context, attempt, None, provider_fallback, observability_producer, - ); + ) + .await; return; } let evidence = WorkAttemptEvidenceRecordV1 { @@ -834,7 +986,7 @@ fn settle_unstarted( provider_fallback, observed_at, }; - match attempts.fail_recovery(context, identity, &evidence) { + match steps.fail_recovery(context, identity, &evidence).await { Ok(settled) => { let _ = record_terminal_attempt_product_views(observability_producer, &settled); } @@ -852,20 +1004,19 @@ fn settle_unstarted( /// the race: the row is `CancellationRequested`, which cannot become /// `Running`, and no provider is left to exit. Seal it here as `Cancelled`. /// Any other refusal leaves the durable row as the authority. -fn seal_cancellation_before_running( - attempts: &tracedecay_contracts::WorkAttemptService, +async fn seal_cancellation_before_running( + steps: &impl AttemptStepsV1, context: &RequestContext, attempt: &WorkAttemptV1, actual_route: Option, provider_fallback: Option, observability_producer: Option<&BoundedObservabilityProducerV1>, -) where - S: tracedecay_contracts::WorkAttemptStoragePort, -{ +) { let identity = attempt.identity(); let observed_at = now_micros(); - if attempts + if steps .acknowledge_cancellation(context, identity, observed_at) + .await .is_err() { return; @@ -881,7 +1032,10 @@ fn seal_cancellation_before_running( provider_fallback, observed_at, }; - match attempts.settle_with_artifacts(context, identity, &evidence, Vec::new()) { + match steps + .settle_with_artifacts(context, identity, &evidence, Vec::new()) + .await + { Ok(settled) => { let _ = record_terminal_attempt_product_views(observability_producer, &settled); } @@ -901,21 +1055,12 @@ enum EffectDispatchAdmission { Unavailable, } -fn record_effect_dispatch( - effects: &WorkAttemptEffectServiceV1, +async fn record_effect_dispatch( + steps: &impl AttemptStepsV1, context: &RequestContext, attempt: &WorkAttemptV1, -) -> EffectDispatchAdmission -where - S: WorkAttemptEffectStoragePortV1, -{ - match effects.record_dispatch( - context, - attempt.identity().clone(), - attempt.execution().effect_state(), - now_micros(), - attempt.execution().deadline(), - ) { +) -> EffectDispatchAdmission { + match steps.record_dispatch(context, attempt, now_micros()).await { Ok(WorkAttemptEffectDispatchOutcomeV1::Recorded(_)) => EffectDispatchAdmission::Recorded, Ok(WorkAttemptEffectDispatchOutcomeV1::Replayed(_)) => { tracing::warn!( @@ -935,15 +1080,12 @@ where } } -fn settle_effect_dispatch( - effects: &WorkAttemptEffectServiceV1, +async fn settle_effect_dispatch( + steps: &impl AttemptStepsV1, context: &RequestContext, attempt: &WorkAttemptV1, provider_started: bool, -) -> bool -where - S: WorkAttemptEffectStoragePortV1, -{ +) -> bool { let resolution = if !provider_started || attempt.execution().effect_state() == tracedecay_domain::WorkEffectStateV1::Observational { @@ -951,7 +1093,10 @@ where } else { WorkAttemptEffectResolutionV1::Unknown }; - if let Err(problem) = effects.settle(context, attempt.identity(), resolution, now_micros()) { + if let Err(problem) = steps + .settle_dispatch(context, attempt.identity(), resolution, now_micros()) + .await + { tracing::warn!( task = attempt.identity().task_id().as_str(), ?problem, @@ -985,9 +1130,8 @@ fn admitted_provider_environment( level = "trace", skip_all )] -async fn execute_provider_with_environment( - attempts: &tracedecay_contracts::WorkAttemptService, - attempt_effects: &WorkAttemptEffectServiceV1, +async fn execute_provider_with_environment( + steps: &impl AttemptStepsV1, workflow_artifacts: Option<&tracedecay_rusqlite_runtime::workflow::WorkflowSqliteAuthority>, context: &RequestContext, attempt: &WorkAttemptV1, @@ -997,9 +1141,7 @@ async fn execute_provider_with_environment( observability_producer: Option<&BoundedObservabilityProducerV1>, topology_policy_digest: Option<&TopologyPolicyDigestV1>, timing: AttemptAdmissionTimingV1, -) where - S: tracedecay_contracts::WorkAttemptStoragePort + WorkAttemptEffectStoragePortV1, -{ +) { let identity = attempt.identity().clone(); let envelope = attempt.execution(); let resolved = &selection.provider; @@ -1017,12 +1159,12 @@ async fn execute_provider_with_environment( command.env(key, value); } - match record_effect_dispatch(attempt_effects, context, attempt) { + match record_effect_dispatch(steps, context, attempt).await { EffectDispatchAdmission::Recorded => {} EffectDispatchAdmission::Replayed => return, EffectDispatchAdmission::Unavailable => { settle_unstarted( - attempts, + steps, context, &identity, attempt, @@ -1031,7 +1173,8 @@ async fn execute_provider_with_environment( }, selection.fallback.clone(), observability_producer, - ); + ) + .await; return; } } @@ -1051,22 +1194,26 @@ async fn execute_provider_with_environment( ?error, "work attempt provider process could not be spawned" ); - if !settle_effect_dispatch(attempt_effects, context, attempt, false) { + if !settle_effect_dispatch(steps, context, attempt, false).await { return; } settle_unstarted( - attempts, + steps, context, &identity, attempt, WorkAttemptProviderOutcomeV1::LaunchFailed, selection.fallback.clone(), observability_producer, - ); + ) + .await; return; } }; - let running = match attempts.mark_running(context, &identity, selection.actual_route.clone()) { + let running = match steps + .mark_running(context, &identity, selection.actual_route.clone()) + .await + { Ok(running) => running, Err(problem) => { // The lease no longer admits this task (fenced by recovery or a @@ -1079,15 +1226,16 @@ async fn execute_provider_with_environment( ); process.signal(TerminationSignal::Kill); let _ = process.wait().await; - if settle_effect_dispatch(attempt_effects, context, attempt, true) { + if settle_effect_dispatch(steps, context, attempt, true).await { seal_cancellation_before_running( - attempts, + steps, context, attempt, Some(selection.actual_route.clone()), selection.fallback.clone(), observability_producer, - ); + ) + .await; } return; } @@ -1110,7 +1258,7 @@ async fn execute_provider_with_environment( let outcome = match ending { Ok(Some(Ok(status))) => exit_outcome(status), Ok(Some(Err(_))) => WorkAttemptProviderOutcomeV1::LaunchFailed, - Ok(None) => cancel_ladder(attempts, context, &identity, &mut process).await, + Ok(None) => cancel_ladder(steps, context, &identity, &mut process).await, // An unavailable timer fails closed as an elapsed wall budget. Err(_) => { let stalled_for = started.elapsed(); @@ -1151,10 +1299,13 @@ async fn execute_provider_with_environment( provider_fallback: selection.fallback.clone(), observed_at: now_micros(), }; - if !settle_effect_dispatch(attempt_effects, context, attempt, true) { + if !settle_effect_dispatch(steps, context, attempt, true).await { return; } - match attempts.settle_with_artifacts(context, &identity, &evidence, artifacts) { + match steps + .settle_with_artifacts(context, &identity, &evidence, artifacts) + .await + { Ok(settled) => { let _ = record_terminal_attempt_product_views(observability_producer, &settled); if let Some(observation) = @@ -1207,9 +1358,8 @@ struct AppServerSessionOutput { level = "trace", skip_all )] -async fn execute_app_server( - attempts: &tracedecay_contracts::WorkAttemptService, - attempt_effects: &WorkAttemptEffectServiceV1, +async fn execute_app_server( + steps: &impl AttemptStepsV1, workflow_artifacts: Option<&tracedecay_rusqlite_runtime::workflow::WorkflowSqliteAuthority>, context: &RequestContext, attempt: &WorkAttemptV1, @@ -1219,19 +1369,17 @@ async fn execute_app_server( observability_producer: Option<&BoundedObservabilityProducerV1>, topology_policy_digest: Option<&TopologyPolicyDigestV1>, timing: AttemptAdmissionTimingV1, -) where - S: tracedecay_contracts::WorkAttemptStoragePort + WorkAttemptEffectStoragePortV1, -{ +) { let identity = attempt.identity().clone(); let envelope = attempt.execution(); let snapshot = envelope.execution_snapshot(); - match record_effect_dispatch(attempt_effects, context, attempt) { + match record_effect_dispatch(steps, context, attempt).await { EffectDispatchAdmission::Recorded => {} EffectDispatchAdmission::Replayed => return, EffectDispatchAdmission::Unavailable => { settle_unstarted( - attempts, + steps, context, &identity, attempt, @@ -1240,7 +1388,8 @@ async fn execute_app_server( }, selection.fallback.clone(), observability_producer, - ); + ) + .await; return; } } @@ -1248,7 +1397,10 @@ async fn execute_app_server( // The launch and the protocol session are one indivisible blocking call // here, so the attempt is marked Running before it starts; a launch that // never happens surfaces as a typed failure on the settle path instead. - let running = match attempts.mark_running(context, &identity, selection.actual_route.clone()) { + let running = match steps + .mark_running(context, &identity, selection.actual_route.clone()) + .await + { Ok(running) => running, Err(problem) => { tracing::warn!( @@ -1256,15 +1408,16 @@ async fn execute_app_server( ?problem, "work attempt could not be marked running; app-server was not started" ); - if settle_effect_dispatch(attempt_effects, context, attempt, false) { + if settle_effect_dispatch(steps, context, attempt, false).await { seal_cancellation_before_running( - attempts, + steps, context, attempt, None, selection.fallback.clone(), observability_producer, - ); + ) + .await; } return; } @@ -1343,7 +1496,9 @@ async fn execute_app_server( // the whole process tree, so the ladder acknowledges and stops // there rather than pretending an interrupt was survived. if let Err(problem) = - attempts.acknowledge_cancellation(context, &identity, now_micros()) + steps + .acknowledge_cancellation(context, &identity, now_micros()) + .await { tracing::warn!( task = identity.task_id().as_str(), @@ -1433,14 +1588,19 @@ async fn execute_app_server( observed_at: now_micros(), }; if !settle_effect_dispatch( - attempt_effects, + steps, context, attempt, launch_receipt.started_at().is_some(), - ) { + ) + .await + { return; } - match attempts.settle_with_artifacts(context, &identity, &evidence, artifacts) { + match steps + .settle_with_artifacts(context, &identity, &evidence, artifacts) + .await + { Ok(settled) => { let _ = record_terminal_attempt_product_views(observability_producer, &settled); if let Some(observation) = launch_receipt.started_at().and_then(|started| { @@ -1513,16 +1673,16 @@ fn offer_no_progress_observation( level = "trace", skip_all )] -async fn cancel_ladder( - attempts: &tracedecay_contracts::WorkAttemptService, +async fn cancel_ladder( + steps: &impl AttemptStepsV1, context: &RequestContext, identity: &WorkAttemptIdentityV1, process: &mut ProviderProcess, -) -> WorkAttemptProviderOutcomeV1 -where - S: tracedecay_contracts::WorkAttemptStoragePort, -{ - if let Err(problem) = attempts.acknowledge_cancellation(context, identity, now_micros()) { +) -> WorkAttemptProviderOutcomeV1 { + if let Err(problem) = steps + .acknowledge_cancellation(context, identity, now_micros()) + .await + { tracing::warn!( task = identity.task_id().as_str(), ?problem, @@ -1534,7 +1694,10 @@ where .await .is_err() { - if let Err(problem) = attempts.escalate_cancellation(context, identity, now_micros()) { + if let Err(problem) = steps + .escalate_cancellation(context, identity, now_micros()) + .await + { tracing::warn!( task = identity.task_id().as_str(), ?problem, diff --git a/crates/tracedecay-daemon-service/src/invocation/work_attempt_exec/tests.rs b/crates/tracedecay-daemon-service/src/invocation/work_attempt_exec/tests.rs index a706409b38..4e8c3fea90 100644 --- a/crates/tracedecay-daemon-service/src/invocation/work_attempt_exec/tests.rs +++ b/crates/tracedecay-daemon-service/src/invocation/work_attempt_exec/tests.rs @@ -42,9 +42,9 @@ use tracedecay_contracts::{ DisclosureClass, ObservabilityHorizonV1, ObservabilityQueryPort, ObservabilityQueryV1, RequestId, ResolvedScope, WorkAttemptAdmissionKind, WorkAttemptCapacityV1, WorkAttemptCapacityVerdictV1, WorkAttemptEffectDispatchOutcomeV1, WorkAttemptEffectHolderV1, - WorkAttemptEffectResolutionV1, WorkAttemptEffectServiceV1, WorkAttemptEffectStorageErrorV1, - WorkAttemptEffectStoragePortV1, WorkAttemptInsertOutcome, WorkAttemptListPageV1, - WorkAttemptService, WorkAttemptStatusRequestV1, WorkAttemptStorageError, + WorkAttemptEffectReadPortV1, WorkAttemptEffectResolutionV1, WorkAttemptEffectServiceV1, + WorkAttemptEffectStorageErrorV1, WorkAttemptEffectStoragePortV1, WorkAttemptInsertOutcome, + WorkAttemptListPageV1, WorkAttemptService, WorkAttemptStatusRequestV1, WorkAttemptStorageError, WorkAttemptStoragePort, WorkAttemptStreamChannelV1, WorkAttemptStreamSummaryV1, }; use tracedecay_domain::{ @@ -455,7 +455,9 @@ impl WorkAttemptEffectStoragePortV1 for AttemptStore { rows.effects.insert(key, next.clone()); Ok(next) } +} +impl WorkAttemptEffectReadPortV1 for AttemptStore { fn load_effect_dispatch( &self, authority: &WorkAuthority, @@ -582,6 +584,120 @@ impl Fixture { } } +impl Fixture { + fn steps(&self) -> FixtureSteps<'_> { + FixtureSteps { + attempts: &self.attempts, + effects: &self.effects, + } + } +} + +/// The driver's steps answered synchronously by the in-memory services. +struct FixtureSteps<'fixture> { + attempts: &'fixture WorkAttemptService, + effects: &'fixture WorkAttemptEffectServiceV1, +} + +impl AttemptStepsV1 for FixtureSteps<'_> { + fn mark_running( + &self, + context: &RequestContext, + identity: &WorkAttemptIdentityV1, + actual_route: WorkProviderRouteV1, + ) -> impl Future> + Send { + std::future::ready(self.attempts.mark_running(context, identity, actual_route)) + } + + fn mark_provider_unavailable( + &self, + context: &RequestContext, + identity: &WorkAttemptIdentityV1, + observed_at: UtcMicros, + ) -> impl Future> + Send { + std::future::ready( + self.attempts + .mark_provider_unavailable(context, identity, observed_at), + ) + } + + fn fail_recovery( + &self, + context: &RequestContext, + identity: &WorkAttemptIdentityV1, + evidence: &WorkAttemptEvidenceRecordV1, + ) -> impl Future> + Send { + std::future::ready(self.attempts.fail_recovery(context, identity, evidence)) + } + + fn acknowledge_cancellation( + &self, + context: &RequestContext, + identity: &WorkAttemptIdentityV1, + acknowledged_at: UtcMicros, + ) -> impl Future> + Send { + std::future::ready(self.attempts.acknowledge_cancellation( + context, + identity, + acknowledged_at, + )) + } + + fn escalate_cancellation( + &self, + context: &RequestContext, + identity: &WorkAttemptIdentityV1, + escalated_at: UtcMicros, + ) -> impl Future> + Send { + std::future::ready( + self.attempts + .escalate_cancellation(context, identity, escalated_at), + ) + } + + fn settle_with_artifacts( + &self, + context: &RequestContext, + identity: &WorkAttemptIdentityV1, + evidence: &WorkAttemptEvidenceRecordV1, + artifacts: Vec, + ) -> impl Future> + Send { + std::future::ready( + self.attempts + .settle_with_artifacts(context, identity, evidence, artifacts), + ) + } + + fn record_dispatch( + &self, + context: &RequestContext, + attempt: &WorkAttemptV1, + dispatched_at: UtcMicros, + ) -> impl Future> + Send + { + std::future::ready(self.effects.record_dispatch( + context, + attempt.identity().clone(), + attempt.execution().effect_state(), + dispatched_at, + attempt.execution().deadline(), + )) + } + + fn settle_dispatch( + &self, + context: &RequestContext, + attempt: &WorkAttemptIdentityV1, + resolution: WorkAttemptEffectResolutionV1, + resolved_at: UtcMicros, + ) -> impl Future> + Send { + std::future::ready( + self.effects + .settle(context, attempt, resolution, resolved_at), + ) + } +} + /// Persists one canonically bound attempt in `Leased`, the state handed to the /// live provider path. Proposal selection is deliberately explicit here: this /// provider-runtime fixture does not invent routing evidence through a Work @@ -828,8 +944,7 @@ fn a_clean_provider_run_seals_succeeded_evidence_over_the_captured_stream() { let admitted_environment = admitted_provider_environment(fixture.attempt.execution().execution_snapshot()); execute_provider_with_environment( - &fixture.attempts, - &fixture.effects, + &fixture.steps(), Some(&workflow_artifacts), &fixture.context, &fixture.attempt, @@ -1013,8 +1128,7 @@ async fn initial_provider_child_uses_values_captured_for_that_spawn() { std::env::set_var(&ambient_secret, "ambient-secret"); } execute_provider_with_environment( - &fixture.attempts, - &fixture.effects, + &fixture.steps(), Some(&workflow_artifacts), &fixture.context, &fixture.attempt, @@ -1079,8 +1193,7 @@ fn stdout_past_the_admitted_cap_is_a_typed_overflow_not_a_silent_success() { let admitted_environment = admitted_provider_environment(fixture.attempt.execution().execution_snapshot()); execute_provider_with_environment( - &fixture.attempts, - &fixture.effects, + &fixture.steps(), Some(&workflow_artifacts), &fixture.context, &fixture.attempt, @@ -1261,10 +1374,10 @@ fn a_provider_that_ignores_interrupt_is_escalated_to_a_kill_on_the_record() { std::future::pending::<()>().await; }; + let steps = fixture.steps(); tokio::select! { () = execute_provider_with_environment( - &fixture.attempts, - &fixture.effects, + &steps, Some(&workflow_artifacts), &fixture.context, &fixture.attempt, @@ -1397,8 +1510,7 @@ async fn a_cancellation_requested_before_mark_running_seals_cancelled() { let route = requested_route(WorkProviderBackendV1::ClaudeCodeCli); execute_provider_with_environment( - &fixture.attempts, - &fixture.effects, + &fixture.steps(), Some(&workflow_artifacts), &fixture.context, &fixture.attempt, @@ -1502,8 +1614,7 @@ async fn a_wall_exhausted_provider_seals_timed_out_and_emits_the_no_progress_ter let admitted_environment = admitted_provider_environment(fixture.attempt.execution().execution_snapshot()); execute_provider_with_environment( - &fixture.attempts, - &fixture.effects, + &fixture.steps(), Some(&workflow_artifacts), &fixture.context, &fixture.attempt, @@ -1741,8 +1852,7 @@ async fn a_disqualified_app_server_falls_back_to_codex_cli_and_says_so_in_the_ev let admitted_environment = admitted_provider_environment(fixture.attempt.execution().execution_snapshot()); execute_provider_with_environment( - &fixture.attempts, - &fixture.effects, + &fixture.steps(), Some(&workflow_artifacts), &fixture.context, &fixture.attempt, @@ -1884,7 +1994,7 @@ async fn a_fallback_that_is_also_refused_keeps_both_denials_on_the_record() { // Sealed exactly as `run_attempt` seals it. settle_unstarted( - &fixture.attempts, + &fixture.steps(), &fixture.context, fixture.identity(), &fixture.attempt, @@ -1893,7 +2003,8 @@ async fn a_fallback_that_is_also_refused_keeps_both_denials_on_the_record() { }, denial.fallback.clone(), None, - ); + ) + .await; assert_eq!(fixture.state(), WorkAttemptStateV1::Failed); let evidence = fixture.sealed_evidence(); assert_eq!( @@ -1987,8 +2098,7 @@ async fn a_missing_provider_executable_seals_a_typed_denial_instead_of_panicking let admitted_environment = admitted_provider_environment(fixture.attempt.execution().execution_snapshot()); execute_provider_with_environment( - &fixture.attempts, - &fixture.effects, + &fixture.steps(), Some(&workflow_artifacts), &fixture.context, &fixture.attempt, @@ -2036,7 +2146,7 @@ async fn an_unavailable_provider_state_is_sealed_as_denial_evidence() { ); settle_unstarted( - &fixture.attempts, + &fixture.steps(), &fixture.context, fixture.identity(), &fixture.attempt, @@ -2045,7 +2155,8 @@ async fn an_unavailable_provider_state_is_sealed_as_denial_evidence() { }, None, None, - ); + ) + .await; assert_eq!(fixture.state(), WorkAttemptStateV1::Failed); assert_eq!( diff --git a/crates/tracedecay-rusqlite-runtime/src/work/effect_holder.rs b/crates/tracedecay-rusqlite-runtime/src/work/effect_holder.rs index 4d9770e95b..388fbce7cf 100644 --- a/crates/tracedecay-rusqlite-runtime/src/work/effect_holder.rs +++ b/crates/tracedecay-rusqlite-runtime/src/work/effect_holder.rs @@ -2,8 +2,8 @@ use serde::Deserialize; use tracedecay_contracts::{ - WorkAttemptEffectDispatchOutcomeV1, WorkAttemptEffectHolderV1, WorkAttemptEffectResolutionV1, - WorkAttemptEffectStorageErrorV1, WorkAttemptEffectStoragePortV1, + WorkAttemptEffectDispatchOutcomeV1, WorkAttemptEffectHolderV1, WorkAttemptEffectReadPortV1, + WorkAttemptEffectResolutionV1, WorkAttemptEffectStorageErrorV1, WorkAttemptEffectStoragePortV1, }; use tracedecay_domain::{UtcMicros, WorkAttemptIdentityV1, WorkAttemptV1, WorkAuthority}; @@ -12,40 +12,11 @@ use crate::work::{ RegisteredWorkQuery, RegisteredWorkWrite, WorkSqliteStorage, authority_params_owned, exact_sql_integer, exact_sql_statement, exact_sql_text, registered_work_query, }; +use crate::work_run_control::WorkSqliteJobStorage; type StorageError = WorkAttemptEffectStorageErrorV1; -impl WorkAttemptEffectStoragePortV1 for WorkSqliteStorage { - fn begin_effect_dispatch( - &self, - authority: &WorkAuthority, - holder: &WorkAttemptEffectHolderV1, - ) -> Result { - holder.validate().map_err(|_| StorageError::Conflict)?; - let transaction = self - .handle() - .begin_immediate() - .map_err(|_| StorageError::Unavailable)?; - let outcome = begin_effect_dispatch_tx(&transaction, authority, holder); - settle_immediate(transaction, outcome) - } - - fn settle_effect_dispatch( - &self, - authority: &WorkAuthority, - attempt: &WorkAttemptIdentityV1, - resolution: WorkAttemptEffectResolutionV1, - resolved_at: UtcMicros, - ) -> Result { - let transaction = self - .handle() - .begin_immediate() - .map_err(|_| StorageError::Unavailable)?; - let outcome = - settle_effect_dispatch_tx(&transaction, authority, attempt, resolution, resolved_at); - settle_immediate(transaction, outcome) - } - +impl WorkAttemptEffectReadPortV1 for WorkSqliteStorage { fn load_effect_dispatch( &self, authority: &WorkAuthority, @@ -63,15 +34,34 @@ impl WorkAttemptEffectStoragePortV1 for WorkSqliteStorage { } } +impl WorkAttemptEffectReadPortV1 for WorkSqliteJobStorage<'_> { + fn load_effect_dispatch( + &self, + authority: &WorkAuthority, + attempt: &WorkAttemptIdentityV1, + ) -> Result, StorageError> { + load_holder(self.transaction, authority, attempt) + } +} + /// The writer job commits or rolls back the holder write with the job. -impl WorkAttemptEffectStoragePortV1 for crate::work_run_control::WorkSqliteJobStorage<'_> { +impl WorkAttemptEffectStoragePortV1 for WorkSqliteJobStorage<'_> { fn begin_effect_dispatch( &self, authority: &WorkAuthority, holder: &WorkAttemptEffectHolderV1, ) -> Result { holder.validate().map_err(|_| StorageError::Conflict)?; - begin_effect_dispatch_tx(self.transaction, authority, holder) + require_open_attempt(self.transaction, authority, holder)?; + if let Some(existing) = load_holder(self.transaction, authority, holder.attempt())? { + return if same_dispatch(&existing, holder) { + Ok(WorkAttemptEffectDispatchOutcomeV1::Replayed(existing)) + } else { + Err(StorageError::Conflict) + }; + } + insert_holder(self.transaction, authority, holder)?; + Ok(WorkAttemptEffectDispatchOutcomeV1::Recorded(holder.clone())) } fn settle_effect_dispatch( @@ -81,84 +71,25 @@ impl WorkAttemptEffectStoragePortV1 for crate::work_run_control::WorkSqliteJobSt resolution: WorkAttemptEffectResolutionV1, resolved_at: UtcMicros, ) -> Result { - settle_effect_dispatch_tx( - self.transaction, - authority, - attempt, - resolution, - resolved_at, - ) - } - - fn load_effect_dispatch( - &self, - authority: &WorkAuthority, - attempt: &WorkAttemptIdentityV1, - ) -> Result, StorageError> { - load_holder(self.transaction, authority, attempt) - } -} - -fn settle_immediate( - transaction: crate::exact_sql::ExactSqlTransaction, - outcome: Result, -) -> Result { - match outcome { - Ok(value) => { - transaction - .commit() - .map_err(|_| StorageError::Unavailable)?; - Ok(value) - } - Err(error) => { - let _ = transaction.rollback(); - Err(error) - } - } -} - -fn begin_effect_dispatch_tx( - transaction: &impl RegisteredWorkWrite, - authority: &WorkAuthority, - holder: &WorkAttemptEffectHolderV1, -) -> Result { - require_open_attempt(transaction, authority, holder)?; - if let Some(existing) = load_holder(transaction, authority, holder.attempt())? { - return if same_dispatch(&existing, holder) { - Ok(WorkAttemptEffectDispatchOutcomeV1::Replayed(existing)) - } else { - Err(StorageError::Conflict) + let existing = load_holder(self.transaction, authority, attempt)? + .ok_or(StorageError::NotFoundOrNotAuthorized)?; + let next = match existing.resolution() { + None => existing + .with_resolution(resolution, resolved_at) + .map_err(|_| StorageError::Conflict)?, + Some(current) if current == resolution => return Ok(existing), + Some(WorkAttemptEffectResolutionV1::Unknown) + if resolution == WorkAttemptEffectResolutionV1::NoEffect => + { + existing + .with_resolution(resolution, resolved_at) + .map_err(|_| StorageError::Conflict)? + } + Some(_) => return Err(StorageError::Conflict), }; + update_holder(self.transaction, authority, &next)?; + Ok(next) } - insert_holder(transaction, authority, holder)?; - Ok(WorkAttemptEffectDispatchOutcomeV1::Recorded(holder.clone())) -} - -fn settle_effect_dispatch_tx( - transaction: &impl RegisteredWorkWrite, - authority: &WorkAuthority, - attempt: &WorkAttemptIdentityV1, - resolution: WorkAttemptEffectResolutionV1, - resolved_at: UtcMicros, -) -> Result { - let existing = load_holder(transaction, authority, attempt)? - .ok_or(StorageError::NotFoundOrNotAuthorized)?; - let next = match existing.resolution() { - None => existing - .with_resolution(resolution, resolved_at) - .map_err(|_| StorageError::Conflict)?, - Some(current) if current == resolution => return Ok(existing), - Some(WorkAttemptEffectResolutionV1::Unknown) - if resolution == WorkAttemptEffectResolutionV1::NoEffect => - { - existing - .with_resolution(resolution, resolved_at) - .map_err(|_| StorageError::Conflict)? - } - Some(_) => return Err(StorageError::Conflict), - }; - update_holder(transaction, authority, &next)?; - Ok(next) } fn require_open_attempt( diff --git a/crates/tracedecay-rusqlite-runtime/tests/rusqlite_suite/work_attempt_storage.rs b/crates/tracedecay-rusqlite-runtime/tests/rusqlite_suite/work_attempt_storage.rs index b69e7dc16f..959e92e579 100644 --- a/crates/tracedecay-rusqlite-runtime/tests/rusqlite_suite/work_attempt_storage.rs +++ b/crates/tracedecay-rusqlite-runtime/tests/rusqlite_suite/work_attempt_storage.rs @@ -15,16 +15,16 @@ use std::{ use tracedecay_contracts::{ VerifiedWorkRetryFailureV1, WorkAttemptAdmissionKind, WorkAttemptCapacityScopeV1, WorkAttemptCapacityVerdictV1, WorkAttemptEffectDispatchOutcomeV1, WorkAttemptEffectHolderV1, - WorkAttemptEffectResolutionV1, WorkAttemptEffectStorageErrorV1, WorkAttemptEffectStoragePortV1, - WorkAttemptEvidenceReadPort, WorkAttemptEvidenceRecordV1, WorkAttemptInsertOutcome, - WorkAttemptProviderOutcomeV1, WorkAttemptReceiptReadPortV1, WorkAttemptStorageError, - WorkAttemptStoragePort, WorkOwnerObservationMarkOutcomeV1, WorkOwnerObservationStoragePortV1, - WorkRetryAttemptOutcomeV1, WorkRetryCauseV1, WorkRetryFailureSelectorV1, WorkRetryReceiptV1, - WorkRetrySourceV1, WorkRetryStoragePortV1, WorkRetryWriteV1, WorkRunControlStoragePort, - WorkRunControlTransitionPort, WorkSynthesisAdmissionRecordV1, - WorkSynthesisAdmissionStoragePort, WorkSynthesisAdmissionV1, WorkSynthesisEvidenceGroupV1, - WorkSynthesisInsertOutcome, WorkSynthesisSourceEnvelopeV1, WorkSynthesisSourceOutcomeV1, - WorkSynthesisSourceSetV1, WorkflowSynthesisDraft, + WorkAttemptEffectReadPortV1, WorkAttemptEffectResolutionV1, WorkAttemptEffectStorageErrorV1, + WorkAttemptEffectStoragePortV1, WorkAttemptEvidenceReadPort, WorkAttemptEvidenceRecordV1, + WorkAttemptInsertOutcome, WorkAttemptProviderOutcomeV1, WorkAttemptReceiptReadPortV1, + WorkAttemptStorageError, WorkAttemptStoragePort, WorkOwnerObservationMarkOutcomeV1, + WorkOwnerObservationStoragePortV1, WorkRetryAttemptOutcomeV1, WorkRetryCauseV1, + WorkRetryFailureSelectorV1, WorkRetryReceiptV1, WorkRetrySourceV1, WorkRetryStoragePortV1, + WorkRetryWriteV1, WorkRunControlStoragePort, WorkRunControlTransitionPort, + WorkSynthesisAdmissionRecordV1, WorkSynthesisAdmissionStoragePort, WorkSynthesisAdmissionV1, + WorkSynthesisEvidenceGroupV1, WorkSynthesisInsertOutcome, WorkSynthesisSourceEnvelopeV1, + WorkSynthesisSourceOutcomeV1, WorkSynthesisSourceSetV1, WorkflowSynthesisDraft, }; use tracedecay_domain::configuration::TopologyConcurrencyPolicyV1; use tracedecay_domain::{ @@ -808,10 +808,7 @@ fn effect_holder_is_exact_replayable_and_reconciles_unknown_across_restart() { ) .unwrap(); assert_eq!( - store - .storage() - .begin_effect_dispatch(&exact_authority, &first) - .unwrap(), + begin_dispatch(&store, &exact_authority, &first).unwrap(), WorkAttemptEffectDispatchOutcomeV1::Recorded(first.clone()) ); let replay = WorkAttemptEffectHolderV1::dispatched( @@ -822,10 +819,7 @@ fn effect_holder_is_exact_replayable_and_reconciles_unknown_across_restart() { ) .unwrap(); assert_eq!( - store - .storage() - .begin_effect_dispatch(&exact_authority, &replay) - .unwrap(), + begin_dispatch(&store, &exact_authority, &replay).unwrap(), WorkAttemptEffectDispatchOutcomeV1::Replayed(first), "an exact receipt replay never authorizes a second provider dispatch" ); @@ -837,36 +831,31 @@ fn effect_holder_is_exact_replayable_and_reconciles_unknown_across_restart() { ) .unwrap(); assert_eq!( - store - .storage() - .begin_effect_dispatch(&exact_authority, &conflicting_deadline) - .unwrap_err(), + begin_dispatch(&store, &exact_authority, &conflicting_deadline).unwrap_err(), WorkAttemptEffectStorageErrorV1::Conflict ); - let unknown = store - .storage() - .settle_effect_dispatch( - &exact_authority, - attempt.identity(), - WorkAttemptEffectResolutionV1::Unknown, - UtcMicros(50), - ) - .unwrap(); + let unknown = settle_dispatch( + &store, + &exact_authority, + attempt.identity(), + WorkAttemptEffectResolutionV1::Unknown, + UtcMicros(50), + ) + .unwrap(); assert_eq!( unknown.resolution(), Some(WorkAttemptEffectResolutionV1::Unknown) ); let restarted = store.restart("attempt-effect-holder"); - let no_effect = restarted - .storage() - .settle_effect_dispatch( - &exact_authority, - attempt.identity(), - WorkAttemptEffectResolutionV1::NoEffect, - UtcMicros(60), - ) - .unwrap(); + let no_effect = settle_dispatch( + &restarted, + &exact_authority, + attempt.identity(), + WorkAttemptEffectResolutionV1::NoEffect, + UtcMicros(60), + ) + .unwrap(); assert_eq!( no_effect.resolution(), Some(WorkAttemptEffectResolutionV1::NoEffect) @@ -1129,18 +1118,39 @@ fn recovery_required(attempt: &WorkAttemptV1) -> WorkAttemptV1 { /// Runs one storage call as a writer job, which commits only when it returns /// `Ok`. -fn in_job( +fn in_job( store: &RegisteredWorkStore, - call: impl Fn(WorkSqliteJobStorage<'_>) -> Result - + Send - + Sync - + 'static, -) -> Result { + call: impl Fn(WorkSqliteJobStorage<'_>) -> Result + Send + Sync + 'static, +) -> Result { tracedecay_rusqlite_runtime::native_runtime::block_on(store.storage().transact(call)) .expect("native runtime") .expect("the writer runs the job") } +fn begin_dispatch( + store: &RegisteredWorkStore, + authority: &WorkAuthority, + holder: &WorkAttemptEffectHolderV1, +) -> Result { + let (authority, holder) = (authority.clone(), holder.clone()); + in_job(store, move |storage| { + storage.begin_effect_dispatch(&authority, &holder) + }) +} + +fn settle_dispatch( + store: &RegisteredWorkStore, + authority: &WorkAuthority, + attempt: &WorkAttemptIdentityV1, + resolution: WorkAttemptEffectResolutionV1, + resolved_at: UtcMicros, +) -> Result { + let (authority, attempt) = (authority.clone(), attempt.clone()); + in_job(store, move |storage| { + storage.settle_effect_dispatch(&authority, &attempt, resolution, resolved_at) + }) +} + fn insert_retry( store: &RegisteredWorkStore, authority: &WorkAuthority, From 9d0df7a466c3f10edec703a218a25e5694965d54 Mon Sep 17 00:00:00 2001 From: ScriptedAlchemy Date: Fri, 16 Oct 2026 19:50:58 +0000 Subject: [PATCH 3/4] refactor(work): record attempt cancellations as writer jobs CancelAttempt now runs the contracts cancellation request inside one ExactSqlHandle::transact job over WorkSqliteJobStorage: the load, the replay or conflict check and the fenced update commit together, and the awaiting task parks no native thread. The in-process signal to a live provider task still follows the committed request. The handle-based request_cancellation keeps one production caller, the workflow fan-out fail-fast path, which is still synchronous. --- .../src/work/registered.rs | 12 ++++ .../application_suite/work_attempt_jobs.rs | 68 +++++++++++++++++++ .../src/invocation/work/attempt_operations.rs | 9 ++- .../src/invocation/work/request_dispatch.rs | 31 +++++---- 4 files changed, 103 insertions(+), 17 deletions(-) diff --git a/crates/tracedecay-application/src/work/registered.rs b/crates/tracedecay-application/src/work/registered.rs index 4d4107e568..10b7cfc1a6 100644 --- a/crates/tracedecay-application/src/work/registered.rs +++ b/crates/tracedecay-application/src/work/registered.rs @@ -422,6 +422,18 @@ impl RegisteredWorkApplicationServicesV1 { .unwrap_or_else(|_| Err(attempt_unavailable())) } + /// Records a cancellation request against an open attempt as a single + /// writer transaction: the load, the conflict check and the fenced update + /// commit together. + pub async fn cancel_attempt( + &self, + context: tracedecay_contracts::RequestContext, + command: tracedecay_contracts::CancelWorkAttemptCommand, + ) -> Result { + self.attempt_step(move |attempts| attempts.request_cancellation(&context, command.clone())) + .await + } + /// Marks the provider running under `actual_route`. pub async fn mark_running( &self, diff --git a/crates/tracedecay-application/tests/application_suite/work_attempt_jobs.rs b/crates/tracedecay-application/tests/application_suite/work_attempt_jobs.rs index d8ffa452cf..2cf18439f7 100644 --- a/crates/tracedecay-application/tests/application_suite/work_attempt_jobs.rs +++ b/crates/tracedecay-application/tests/application_suite/work_attempt_jobs.rs @@ -773,3 +773,71 @@ async fn provider_driver_steps_awaited_on_a_native_worker_park_no_native_thread( ); } } + +#[tokio::test] +async fn cancellation_awaited_on_a_native_worker_parks_no_native_thread() { + let profile = tempfile::tempdir().expect("profile"); + let project = tempfile::tempdir().expect("project"); + let runtime = project_runtime(profile.path(), project.path()).await; + let database = runtime.project_database().expect("project database"); + let storage = database.work_storage().expect("Work storage"); + let attempt = crate::work_run_control_jobs::leased_attempt_of( + id::("run.attempt-jobs.cancel"), + "attempt-jobs.cancel", + ); + storage + .insert(&authority(), &attempt) + .expect("the attempt is stored"); + let identity = attempt.identity().clone(); + let command = |request_id: &str| tracedecay_contracts::CancelWorkAttemptCommand { + task_id: task(), + run_id: identity.run_id().clone(), + attempt_id: identity.attempt_id().clone(), + request_id: id(request_id), + occurred_at: UtcMicros(20_000), + }; + let (first, replay, other) = ( + command("cancellation.attempt-jobs.1"), + command("cancellation.attempt-jobs.1"), + command("cancellation.attempt-jobs.2"), + ); + let work = RegisteredWorkApplicationServicesV1::attach(database).expect("Work services"); + let (outcome, writer_waits, reader_waits) = on_native_worker(async move { + ( + work.cancel_attempt(product_context(), first).await, + work.cancel_attempt(product_context(), replay).await, + work.cancel_attempt(product_context(), other).await, + ) + }) + .await; + assert_eq!( + writer_waits, 0, + "the cancellation parked no native worker on the writer" + ); + assert_eq!( + reader_waits, 0, + "the cancellation parked no native worker on a reader" + ); + let (requested, replayed, conflicting) = outcome; + let requested = requested.expect("the cancellation commits"); + assert_eq!(requested.state(), WorkAttemptStateV1::CancellationRequested); + assert_eq!( + replayed.expect("the same request replays"), + requested, + "the same request replays the committed attempt" + ); + assert_eq!( + conflicting + .expect_err("a different request conflicts") + .diagnostic() + .map(|diagnostic| diagnostic.code.clone()), + Some("application.work-attempt.cancellation-conflict".to_owned()) + ); + assert_eq!( + storage + .load(&authority(), &identity) + .expect("the attempt reads"), + requested, + "the job committed the cancellation request" + ); +} diff --git a/crates/tracedecay-daemon-service/src/invocation/work/attempt_operations.rs b/crates/tracedecay-daemon-service/src/invocation/work/attempt_operations.rs index e6eb9acade..bf2b9591dd 100644 --- a/crates/tracedecay-daemon-service/src/invocation/work/attempt_operations.rs +++ b/crates/tracedecay-daemon-service/src/invocation/work/attempt_operations.rs @@ -264,8 +264,7 @@ pub(super) fn attempt_status( } #[allow(clippy::too_many_arguments)] -#[tracing::instrument(name = "daemon.service.work.cancel_attempt", level = "trace", skip_all)] -pub(super) fn cancel_attempt( +pub(super) async fn cancel_attempt( registered: &RegisteredWorkRuntime, services: &RegisteredWorkApplicationServicesV1, attempt_processes: &Arc, @@ -279,7 +278,11 @@ pub(super) fn cancel_attempt( deadline: Deadline, command: CancelWorkAttemptCommand, ) -> DaemonInvocationResponse { - let cancelled = services.attempts().request_cancellation(context, command); + let cancelled = tracing::Instrument::instrument( + services.cancel_attempt(context.clone(), command), + tracing::trace_span!("daemon.service.work.cancel_attempt"), + ) + .await; if let Ok(attempt) = &cancelled { attempt_processes.signal_cancellation(&context.scope().worktree_id, attempt.identity()); } diff --git a/crates/tracedecay-daemon-service/src/invocation/work/request_dispatch.rs b/crates/tracedecay-daemon-service/src/invocation/work/request_dispatch.rs index b1be54a470..71fbf7a952 100644 --- a/crates/tracedecay-daemon-service/src/invocation/work/request_dispatch.rs +++ b/crates/tracedecay-daemon-service/src/invocation/work/request_dispatch.rs @@ -373,20 +373,23 @@ pub(super) async fn dispatch_work_application( deadline, request, ), - WorkApplicationInvocationV1::CancelAttempt(command) => attempt_operations::cancel_attempt( - ®istered, - &services, - &attempt_processes, - request_id, - &context, - canonical_request_id, - operation_key, - use_case, - input_digest, - observed_at, - deadline, - command, - ), + WorkApplicationInvocationV1::CancelAttempt(command) => { + attempt_operations::cancel_attempt( + ®istered, + &services, + &attempt_processes, + request_id, + &context, + canonical_request_id, + operation_key, + use_case, + input_digest, + observed_at, + deadline, + command, + ) + .await + } WorkApplicationInvocationV1::RetryAttempt(command) => { let Ok(capability) = CapabilityId::new(*capability) else { return DaemonInvocationResponse::problem( From 41700e61ab4fbc0ca3717794a9b438fb12776a58 Mon Sep 17 00:00:00 2001 From: ScriptedAlchemy Date: Fri, 16 Oct 2026 20:27:56 +0000 Subject: [PATCH 4/4] refactor(work): commit product mutations as writer jobs Create, ReviewProposal, AcceptProposal, AdmitExecution and MutateGraph now run the contracts product mutation service inside one ExactSqlHandle::transact job over WorkSqliteJobStorage: the revision check, the command replay check and the event append with its verified version commit together, and the awaiting task parks no native thread. They all route through one RegisteredWorkProductServicesV1::mutate. The current graph a mutation checks its expected version against is replayed from the journal before the job, on the blocking pool, and handed to the job as a PrefetchedWorkGraphReadV1; the append's graph-version compare-and-swap remains the staleness fence. AdmitExecution still prepares its execution snapshot from a reader before the job. WorkProductEventPortV1 drops its Send + Sync bound so the job storage can implement it, sharing the replay and append bodies with the handle storage. The handle append keeps production callers in the workflow fan-out, which is still synchronous. --- .../src/work/registered.rs | 67 +++- .../application_suite/work_attempt_jobs.rs | 97 +++-- .../src/work_product/mutation.rs | 20 ++ .../src/work_product/mutation/contracts.rs | 20 +- .../src/invocation/work/preparation.rs | 66 +++- .../src/invocation/work/request_dispatch.rs | 336 +++++++----------- .../src/work_product/events.rs | 116 +++--- .../src/work_run_control.rs | 25 +- 8 files changed, 444 insertions(+), 303 deletions(-) diff --git a/crates/tracedecay-application/src/work/registered.rs b/crates/tracedecay-application/src/work/registered.rs index 10b7cfc1a6..b2e89ee0b0 100644 --- a/crates/tracedecay-application/src/work/registered.rs +++ b/crates/tracedecay-application/src/work/registered.rs @@ -163,9 +163,17 @@ impl RegisteredWorkProductServicesV1 { topology: tracedecay_domain::WorkTopologyPolicyV1, command: tracedecay_contracts::StartWorkAttemptCommand, ) -> Result { + let prefetch_context = context.clone(); + let observed_at = command.occurred_at; let graph = self .storage - .prefetch_attempt_admission_graph(&context, command.occurred_at) + .prefetch_graph(move |storage| { + tracedecay_contracts::PrefetchedWorkGraphReadV1::for_attempt_admission( + storage, + &prefetch_context, + observed_at, + ) + }) .await; self.storage .transact_with_graph(graph, move |storage| { @@ -187,6 +195,43 @@ impl RegisteredWorkProductServicesV1 { .unwrap_or_else(|_| Err(attempt_unavailable())) } + /// Commits one product graph mutation as a single writer transaction: the + /// revision check, the command replay check and the event append with its + /// verified version commit together. The current graph the mutation + /// checks its expected version against is read before the job, on a + /// reader; the append's graph-version compare-and-swap fences it. + pub async fn mutate( + &self, + context: tracedecay_contracts::RequestContext, + binding: tracedecay_contracts::WorkProductBindingV1, + request: tracedecay_contracts::WorkProductMutationRequestV1, + revisions: tracedecay_contracts::WorkProductRevisionPinsV1, + ) -> Result< + tracedecay_contracts::WorkProductMutationReceiptV1, + tracedecay_contracts::WorkProductApplicationErrorV1, + > { + let (prefetch_context, prefetch_request) = (context.clone(), request.clone()); + let graph = self + .storage + .prefetch_graph(move |storage| { + tracedecay_contracts::PrefetchedWorkGraphReadV1::for_mutation( + storage, + &prefetch_context, + &prefetch_request, + ) + }) + .await; + self.storage + .transact_with_graph(graph, move |storage| { + tracedecay_contracts::WorkProductMutationServiceV1::new(storage, storage, storage) + .mutate(&context, &binding, request.clone(), &revisions) + }) + .await + .unwrap_or(Err( + tracedecay_contracts::WorkProductApplicationErrorV1::EventAuthorityUnavailable, + )) + } + /// Admits one synthesis attempt as a single writer transaction: the /// replay check, the source reads, the run's pause fence, the source /// payload hydration `prepare_instructions` does through the job storage @@ -214,9 +259,17 @@ impl RegisteredWorkProductServicesV1 { + Sync + 'static, { + let prefetch_context = context.clone(); + let observed_at = command.start.occurred_at; let graph = self .storage - .prefetch_attempt_admission_graph(&context, command.start.occurred_at) + .prefetch_graph(move |storage| { + tracedecay_contracts::PrefetchedWorkGraphReadV1::for_attempt_admission( + storage, + &prefetch_context, + observed_at, + ) + }) .await; self.storage .transact_with_graph(graph, move |storage| { @@ -257,9 +310,17 @@ impl RegisteredWorkProductServicesV1 { tracedecay_contracts::WorkRetryAttemptOutcomeV1, tracedecay_contracts::ApplicationProblem, > { + let prefetch_context = context.clone(); + let observed_at = restarted_at; let graph = self .storage - .prefetch_attempt_admission_graph(&context, restarted_at) + .prefetch_graph(move |storage| { + tracedecay_contracts::PrefetchedWorkGraphReadV1::for_attempt_admission( + storage, + &prefetch_context, + observed_at, + ) + }) .await; self.storage .transact_with_graph(graph, move |storage| { diff --git a/crates/tracedecay-application/tests/application_suite/work_attempt_jobs.rs b/crates/tracedecay-application/tests/application_suite/work_attempt_jobs.rs index 2cf18439f7..f39e5d33e1 100644 --- a/crates/tracedecay-application/tests/application_suite/work_attempt_jobs.rs +++ b/crates/tracedecay-application/tests/application_suite/work_attempt_jobs.rs @@ -78,17 +78,22 @@ fn selection() -> WorkProductSelectionScopeV1 { .unwrap() } -fn mutate( - product: &RegisteredWorkProductServicesV1, +/// Prepares one mutation from the current head, commits it as a writer job +/// on a native worker, and adds the native waits the commit parked to +/// `waits`. +async fn mutate( + database: &RegisteredGlobalDb, change: WorkProductChangeDraftV1, command_id: &str, occurred_at: UtcMicros, + waits: &mut (u64, u64), ) -> WorkProductMutationReceiptV1 { - let context = product_context(); + let product = + RegisteredWorkProductServicesV1::attach(database, binding()).expect("product services"); let request = product .mutations() .prepare_mutation( - &context, + &product_context(), &binding(), PrepareWorkProductMutationRequestV1 { selection: selection(), @@ -101,17 +106,22 @@ fn mutate( revisions(), ) .expect("the mutation prepares"); - product - .mutations() - .mutate(&context, &binding(), request, &revisions()) - .expect("the mutation commits") + let (receipt, writer_waits, reader_waits) = on_native_worker(async move { + product + .mutate(product_context(), binding(), request, revisions()) + .await + }) + .await; + waits.0 += writer_waits; + waits.1 += reader_waits; + receipt.expect("the mutation commits") } /// Publishes `task()` with an accepted proposal and admitted execution, the -/// product graph every attempt admission is bound to. -fn admit_task(database: &RegisteredGlobalDb) { - let product = - RegisteredWorkProductServicesV1::attach(database, binding()).expect("product services"); +/// product graph every attempt admission is bound to, and returns the native +/// writer and reader waits its mutation jobs parked. +async fn admit_task(database: &RegisteredGlobalDb) -> (u64, u64) { + let mut waits = (0, 0); let created_at = UtcMicros(10); let initiative_id = id::("initiative.attempt-jobs"); let plan_id = id::("plan.attempt-jobs"); @@ -136,7 +146,7 @@ fn admit_task(database: &RegisteredGlobalDb) { }) .unwrap(); let created = mutate( - &product, + database, WorkProductChangeDraftV1::CreateTask { initiative: WorkInitiativeV1::new( initiative_id.clone(), @@ -162,7 +172,9 @@ fn admit_task(database: &RegisteredGlobalDb) { }, "command.attempt-jobs.create", UtcMicros(10_000), - ); + &mut waits, + ) + .await; let proposal = WorkProposalV1::new( id::("proposal.attempt-jobs"), task(), @@ -178,20 +190,25 @@ fn admit_task(database: &RegisteredGlobalDb) { ) .unwrap(); mutate( - &product, + database, WorkProductChangeDraftV1::DecideProposal { proposal, disposition: WorkProposalDispositionV1::Accepted, }, "command.attempt-jobs.accept", UtcMicros(11_000), - ); + &mut waits, + ) + .await; mutate( - &product, + database, WorkProductChangeDraftV1::AdmitExecution { task_id: task() }, "command.attempt-jobs.admit", UtcMicros(12_000), - ); + &mut waits, + ) + .await; + waits } fn start_command( @@ -239,7 +256,7 @@ async fn attempt_admissions_awaited_on_a_native_worker_park_no_native_thread() { let project = tempfile::tempdir().expect("project"); let runtime = project_runtime(profile.path(), project.path()).await; let database = runtime.project_database().expect("project database"); - admit_task(database); + admit_task(database).await; let product = RegisteredWorkProductServicesV1::attach(database, binding()).expect("product services"); let work = RegisteredWorkApplicationServicesV1::attach(database).expect("Work services"); @@ -348,7 +365,7 @@ async fn synthesis_awaited_on_a_native_worker_hydrates_sources_in_its_job() { let project = tempfile::tempdir().expect("project"); let runtime = project_runtime(profile.path(), project.path()).await; let database = runtime.project_database().expect("project database"); - admit_task(database); + admit_task(database).await; let product = RegisteredWorkProductServicesV1::attach(database, binding()).expect("product services"); let source = identity("run.attempt-jobs.source", "attempt.attempt-jobs.source"); @@ -475,7 +492,7 @@ async fn a_pause_racing_a_start_never_lands_between_its_fence_and_its_reservatio let project = tempfile::tempdir().expect("project"); let runtime = project_runtime(profile.path(), project.path()).await; let database = runtime.project_database().expect("project database"); - admit_task(database); + admit_task(database).await; let run = "run.run-control-jobs"; let storage = database.work_storage().expect("Work storage"); let seed = crate::work_run_control_jobs::leased_attempt_of(id::(run), "seed"); @@ -841,3 +858,41 @@ async fn cancellation_awaited_on_a_native_worker_parks_no_native_thread() { "the job committed the cancellation request" ); } + +#[tokio::test] +async fn product_mutations_awaited_on_a_native_worker_park_no_native_thread() { + let profile = tempfile::tempdir().expect("profile"); + let project = tempfile::tempdir().expect("project"); + let runtime = project_runtime(profile.path(), project.path()).await; + let database = runtime.project_database().expect("project database"); + let (writer_waits, reader_waits) = admit_task(database).await; + assert_eq!( + writer_waits, 0, + "the create, proposal and admission jobs parked no native worker on the writer" + ); + assert_eq!( + reader_waits, 0, + "the create, proposal and admission jobs parked no native worker on a reader" + ); + let product = + RegisteredWorkProductServicesV1::attach(database, binding()).expect("product services"); + let read = product + .reads() + .read_graph( + &product_context(), + tracedecay_contracts::WorkGraphReadRequestV1::current(selection(), UtcMicros(13_000)), + ) + .expect("the graph reads") + .into_current_snapshot() + .expect("the graph has a head"); + let item = read.graph().item(&task()).expect("the task is published"); + assert!( + item.is_execution_admitted(), + "the admission job committed the execution admission" + ); + assert_eq!( + item.accepted_proposal(), + Some(&id::("proposal.attempt-jobs")), + "the proposal job committed the acceptance" + ); +} diff --git a/crates/tracedecay-contracts/src/work_product/mutation.rs b/crates/tracedecay-contracts/src/work_product/mutation.rs index de274b335f..292a7bb444 100644 --- a/crates/tracedecay-contracts/src/work_product/mutation.rs +++ b/crates/tracedecay-contracts/src/work_product/mutation.rs @@ -22,6 +22,26 @@ pub use contracts::*; const WORK_PRODUCT_MUTATION_DIGEST_DOMAIN: &str = "tracedecay.application.work-product-mutation.final-v2"; +impl super::PrefetchedWorkGraphReadV1 { + /// The current graph read a submitted mutation checks its expected version + /// against, read before its writer job. + pub fn for_mutation( + storage: &S, + context: &RequestContext, + request: &WorkProductMutationRequestV1, + ) -> Self + where + S: WorkGraphReadPortV1 + WorkProductOwnerAuthorizationPortV1, + { + Self::current( + storage, + context, + request.selection(), + request.mutation_identity().occurred_at, + ) + } +} + pub struct WorkProductMutationServiceV1 { graph: G, owner_authority: A, diff --git a/crates/tracedecay-contracts/src/work_product/mutation/contracts.rs b/crates/tracedecay-contracts/src/work_product/mutation/contracts.rs index 8010566849..0427c926da 100644 --- a/crates/tracedecay-contracts/src/work_product/mutation/contracts.rs +++ b/crates/tracedecay-contracts/src/work_product/mutation/contracts.rs @@ -144,7 +144,10 @@ impl WorkProductEventCommitV1 { /// /// A successful call commits both records in one transaction. There is no /// intermediate appended-but-unpublished state for a restart to reconcile. -pub trait WorkProductEventPortV1: Send + Sync { +/// +/// It carries no `Send + Sync` bound so a storage scoped to one writer +/// transaction can implement it. +pub trait WorkProductEventPortV1 { fn replay( &self, context: &WorkProductPortContextV1, @@ -424,4 +427,19 @@ impl WorkProductMutationRequestV1 { Self::RecordHandoff(request) => &request.mutation, } } + + pub const fn selection(&self) -> &WorkProductSelectionScopeV1 { + match self { + Self::Create(request) => &request.selection, + Self::AddTask(request) => &request.selection, + Self::CreateTask(request) => &request.selection, + Self::DecideProposal(request) => &request.selection, + Self::DecideRelationReplan(request) => &request.selection, + Self::ApplyRelationReplan(request) => &request.selection, + Self::AcceptTask(request) => &request.selection, + Self::AdmitExecution(request) => &request.selection, + Self::LinkAcceptedAttempt(request) => &request.selection, + Self::RecordHandoff(request) => &request.selection, + } + } } diff --git a/crates/tracedecay-daemon-service/src/invocation/work/preparation.rs b/crates/tracedecay-daemon-service/src/invocation/work/preparation.rs index f3f1784d0f..b8465cb7c3 100644 --- a/crates/tracedecay-daemon-service/src/invocation/work/preparation.rs +++ b/crates/tracedecay-daemon-service/src/invocation/work/preparation.rs @@ -318,13 +318,65 @@ pub(super) fn repository_selection( .map_err(|_| work_product_authority_unavailable()) } -pub(super) fn decide_product_proposal( +/// Commits one product graph mutation as a writer job. +pub(super) async fn mutate_product( registered: &RegisteredWorkRuntime, context: &RequestContext, capability: &str, use_case: &UseCaseId, - request: tracedecay_contracts::DecideWorkProposalRequestV1, + request: tracedecay_contracts::WorkProductMutationRequestV1, ) -> Result { + let (services, binding) = product_services(registered, capability, use_case)?; + let revisions = current_work_product_revision_pins(registered)?; + services + .mutate(context.clone(), binding, request, revisions) + .await + .map_err(work_product_problem) +} + +/// Admits execution of one task: the execution snapshot is prepared from a +/// reader before the admission event commits as a writer job. +pub(super) async fn admit_execution( + registered: &RegisteredWorkRuntime, + context: &RequestContext, + capability: &str, + use_case: &UseCaseId, + command: tracedecay_contracts::AdmitWorkExecutionRequestV1, +) -> Result { + let (services, binding) = product_services(registered, capability, use_case)?; + let revisions = current_work_product_revision_pins(registered)?; + if command.mutation.revisions != revisions { + return Err(work_product_problem( + tracedecay_contracts::WorkProductApplicationErrorV1::RevisionConflict, + )); + } + let execution_snapshot = prepare_execution_snapshot(registered, context, &binding, &command)?; + let mutation = services + .mutate( + context.clone(), + binding, + tracedecay_contracts::WorkProductMutationRequestV1::AdmitExecution(command), + revisions, + ) + .await + .map_err(work_product_problem)?; + Ok(tracedecay_contracts::AdmittedWorkExecutionV1 { + mutation, + execution_snapshot, + }) +} + +fn product_services( + registered: &RegisteredWorkRuntime, + capability: &str, + use_case: &UseCaseId, +) -> Result< + ( + tracedecay_application::work::RegisteredWorkProductServicesV1, + tracedecay_contracts::WorkProductBindingV1, + ), + ApplicationProblem, +> { let capability = CapabilityId::new(capability).map_err(|_| work_product_authority_unavailable())?; let binding = tracedecay_contracts::WorkProductBindingV1::new(capability, use_case.clone()); @@ -333,15 +385,7 @@ pub(super) fn decide_product_proposal( binding.clone(), ) .map_err(|_| work_product_authority_unavailable())?; - if request.mutation.revisions != current_work_product_revision_pins(registered)? { - return Err(work_product_problem( - tracedecay_contracts::WorkProductApplicationErrorV1::RevisionConflict, - )); - } - services - .mutations() - .decide_proposal(context, &binding, request) - .map_err(work_product_problem) + Ok((services, binding)) } fn work_product_authority_unavailable() -> ApplicationProblem { diff --git a/crates/tracedecay-daemon-service/src/invocation/work/request_dispatch.rs b/crates/tracedecay-daemon-service/src/invocation/work/request_dispatch.rs index 71fbf7a952..fdf1cb4bb5 100644 --- a/crates/tracedecay-daemon-service/src/invocation/work/request_dispatch.rs +++ b/crates/tracedecay-daemon-service/src/invocation/work/request_dispatch.rs @@ -118,193 +118,133 @@ pub(super) async fn dispatch_work_application( ) } WorkApplicationInvocationV1::Create(command) => { - let _span = tracing::trace_span!("daemon.service.work.create").entered(); - { - let Ok(capability_id) = CapabilityId::new(*capability) else { - return DaemonInvocationResponse::problem( - request_id, - DaemonInvocationProblem::Unavailable, - ); - }; - let binding = tracedecay_contracts::WorkProductBindingV1::new( - capability_id, - use_case.clone(), - ); - let created = tracedecay_application::work::RegisteredWorkProductServicesV1::attach( - ®istered.database, - binding.clone(), - ) - .map_err(|_| { - tracedecay_contracts::WorkProductApplicationErrorV1::GraphAuthorityUnavailable - }) - .and_then(|product| { - preparation::current_work_product_revision_pins(®istered) - .map_err(|_| { - tracedecay_contracts::WorkProductApplicationErrorV1::GraphAuthorityUnavailable - }) - .and_then(|revisions| { - if command.mutation.revisions != revisions { - return Err( - tracedecay_contracts::WorkProductApplicationErrorV1::RevisionConflict, - ); - } - product - .mutations() - .create_task(&context, &binding, command) - }) - }) - .map_err(work_product_problem); - complete_work_effect( + let created = tracing::Instrument::instrument( + preparation::mutate_product( ®istered, - request_id, &context, - canonical_request_id, - operation_key, - use_case, - input_digest, - created, - observed_at, - deadline, - WorkApplicationOutcomeV1::Create, - ) - } + capability, + &use_case, + tracedecay_contracts::WorkProductMutationRequestV1::CreateTask(Box::new( + command, + )), + ), + tracing::trace_span!("daemon.service.work.create"), + ) + .await; + complete_work_effect( + ®istered, + request_id, + &context, + canonical_request_id, + operation_key, + use_case, + input_digest, + created, + observed_at, + deadline, + WorkApplicationOutcomeV1::Create, + ) } WorkApplicationInvocationV1::ReviewProposal(request) => { - let _span = tracing::trace_span!("daemon.service.work.review_proposal").entered(); - { - let proposal_ref = request.proposal.proposal_id().as_str().to_owned(); - let command_ref = request.mutation.command_id.as_str().to_owned(); - let disposition = request.disposition; - let occurred_at = request.mutation.occurred_at; - let result = preparation::decide_product_proposal( + let proposal_ref = request.proposal.proposal_id().as_str().to_owned(); + let command_ref = request.mutation.command_id.as_str().to_owned(); + let disposition = request.disposition; + let occurred_at = request.mutation.occurred_at; + let result = tracing::Instrument::instrument( + preparation::mutate_product( ®istered, &context, capability, &use_case, - request.into(), + tracedecay_contracts::WorkProductMutationRequestV1::DecideProposal( + request.into(), + ), + ), + tracing::trace_span!("daemon.service.work.review_proposal"), + ) + .await; + if result.is_ok() { + let _ = tracedecay_application::observability::record_reliance_decision( + observability_producer.as_deref(), + &proposal_ref, + &command_ref, + Some(disposition), + occurred_at, ); - if result.is_ok() { - let _ = tracedecay_application::observability::record_reliance_decision( - observability_producer.as_deref(), - &proposal_ref, - &command_ref, - Some(disposition), - occurred_at, - ); - } - complete_work_effect( - ®istered, - request_id, - &context, - canonical_request_id, - operation_key, - use_case, - input_digest, - result, - observed_at, - deadline, - WorkApplicationOutcomeV1::ReviewProposal, - ) } + complete_work_effect( + ®istered, + request_id, + &context, + canonical_request_id, + operation_key, + use_case, + input_digest, + result, + observed_at, + deadline, + WorkApplicationOutcomeV1::ReviewProposal, + ) } WorkApplicationInvocationV1::AcceptProposal(command) => { - let _span = tracing::trace_span!("daemon.service.work.accept_proposal").entered(); - { - let proposal_ref = command.proposal.proposal_id().as_str().to_owned(); - let command_ref = command.mutation.command_id.as_str().to_owned(); - let occurred_at = command.mutation.occurred_at; - let result = preparation::decide_product_proposal( + let proposal_ref = command.proposal.proposal_id().as_str().to_owned(); + let command_ref = command.mutation.command_id.as_str().to_owned(); + let occurred_at = command.mutation.occurred_at; + let result = tracing::Instrument::instrument( + preparation::mutate_product( ®istered, &context, capability, &use_case, - command.into(), + tracedecay_contracts::WorkProductMutationRequestV1::DecideProposal( + command.into(), + ), + ), + tracing::trace_span!("daemon.service.work.accept_proposal"), + ) + .await; + if result.is_ok() { + let _ = tracedecay_application::observability::record_reliance_decision( + observability_producer.as_deref(), + &proposal_ref, + &command_ref, + None, + occurred_at, ); - if result.is_ok() { - let _ = tracedecay_application::observability::record_reliance_decision( - observability_producer.as_deref(), - &proposal_ref, - &command_ref, - None, - occurred_at, - ); - } - complete_work_effect( - ®istered, - request_id, - &context, - canonical_request_id, - operation_key, - use_case, - input_digest, - result, - observed_at, - deadline, - WorkApplicationOutcomeV1::AcceptProposal, - ) } + complete_work_effect( + ®istered, + request_id, + &context, + canonical_request_id, + operation_key, + use_case, + input_digest, + result, + observed_at, + deadline, + WorkApplicationOutcomeV1::AcceptProposal, + ) } WorkApplicationInvocationV1::AdmitExecution(command) => { - let _span = tracing::trace_span!("daemon.service.work.admit_execution").entered(); - { - let result = CapabilityId::new(*capability) - .map_err(|_| { - work_product_problem( - tracedecay_contracts::WorkProductApplicationErrorV1::GraphAuthorityUnavailable, - ) - }) - .and_then(|capability| { - let binding = tracedecay_contracts::WorkProductBindingV1::new( - capability, - use_case.clone(), - ); - tracedecay_application::work::RegisteredWorkProductServicesV1::attach( - ®istered.database, - binding.clone(), - ) - .map_err(|_| { - work_product_problem( - tracedecay_contracts::WorkProductApplicationErrorV1::GraphAuthorityUnavailable, - ) - }) - .and_then(|product| { - let revisions = - preparation::current_work_product_revision_pins(®istered)?; - if command.mutation.revisions != revisions { - return Err(work_product_problem( - tracedecay_contracts::WorkProductApplicationErrorV1::RevisionConflict, - )); - } - let execution_snapshot = preparation::prepare_execution_snapshot( - ®istered, - &context, - &binding, - &command, - )?; - let mutation = product - .mutations() - .admit_execution(&context, &binding, command) - .map_err(work_product_problem)?; - Ok(tracedecay_contracts::AdmittedWorkExecutionV1 { - mutation, - execution_snapshot, - }) - }) - }); - complete_work_effect( - ®istered, - request_id, - &context, - canonical_request_id, - operation_key, - use_case, - input_digest, - result, - observed_at, - deadline, - WorkApplicationOutcomeV1::AdmitExecution, - ) - } + let result = tracing::Instrument::instrument( + preparation::admit_execution(®istered, &context, capability, &use_case, command), + tracing::trace_span!("daemon.service.work.admit_execution"), + ) + .await; + complete_work_effect( + ®istered, + request_id, + &context, + canonical_request_id, + operation_key, + use_case, + input_digest, + result, + observed_at, + deadline, + WorkApplicationOutcomeV1::AdmitExecution, + ) } WorkApplicationInvocationV1::StartAttempt(command) => { let Ok(capability) = CapabilityId::new(*capability) else { @@ -838,50 +778,24 @@ pub(super) async fn dispatch_work_application( } } WorkApplicationInvocationV1::MutateGraph(request) => { - let _span = tracing::trace_span!("daemon.service.work.mutate_graph").entered(); - { - let Ok(capability) = CapabilityId::new(*capability) else { - return DaemonInvocationResponse::problem( - request_id, - DaemonInvocationProblem::Unavailable, - ); - }; - let binding = - tracedecay_contracts::WorkProductBindingV1::new(capability, use_case.clone()); - let product_services = - match tracedecay_application::work::RegisteredWorkProductServicesV1::attach( - ®istered.database, - binding.clone(), - ) { - Ok(services) => services, - Err(_) => { - return DaemonInvocationResponse::problem( - request_id, - DaemonInvocationProblem::Unavailable, - ); - } - }; - let mutated = preparation::current_work_product_revision_pins(®istered) - .and_then(|revisions| { - product_services - .mutations() - .mutate(&context, &binding, request, &revisions) - .map_err(work_product_problem) - }); - complete_work_effect( - ®istered, - request_id, - &context, - canonical_request_id, - operation_key, - use_case, - input_digest, - mutated, - observed_at, - deadline, - WorkApplicationOutcomeV1::MutateGraph, - ) - } + let mutated = tracing::Instrument::instrument( + preparation::mutate_product(®istered, &context, capability, &use_case, request), + tracing::trace_span!("daemon.service.work.mutate_graph"), + ) + .await; + complete_work_effect( + ®istered, + request_id, + &context, + canonical_request_id, + operation_key, + use_case, + input_digest, + mutated, + observed_at, + deadline, + WorkApplicationOutcomeV1::MutateGraph, + ) } WorkApplicationInvocationV1::PauseRun(command) => { let transition = tracing::Instrument::instrument( diff --git a/crates/tracedecay-rusqlite-runtime/src/work_product/events.rs b/crates/tracedecay-rusqlite-runtime/src/work_product/events.rs index 18fb79da69..38e2b32f0c 100644 --- a/crates/tracedecay-rusqlite-runtime/src/work_product/events.rs +++ b/crates/tracedecay-rusqlite-runtime/src/work_product/events.rs @@ -12,57 +12,48 @@ use tracedecay_domain::{ use super::{WORK_PRODUCT_EVENT_ID_DOMAIN, load_journal_tail, owner_params, selection_covers}; use crate::exact_sql::ExactSqlValue; use crate::work::{ - RegisteredWorkWrite, WorkSqliteStorage, exact_sql_statement, exact_sql_text, - registered_work_query, + RegisteredWorkQuery, RegisteredWorkWrite, WorkSqliteStorage, exact_sql_statement, + exact_sql_text, registered_work_query, }; +use crate::work_run_control::WorkSqliteJobStorage; + type PortError = WorkProductEventPortErrorV1; -impl WorkProductEventPortV1 for WorkSqliteStorage { +/// The writer job commits the event and its verified version together, or +/// rolls both back with any refusal the job returns. +impl WorkProductEventPortV1 for WorkSqliteJobStorage<'_> { fn replay( &self, context: &WorkProductPortContextV1, command_id: &WorkCommandId, canonical_input_digest: &ManifestDigest, ) -> Result, PortError> { - let scope = context.authorized_scope(); - let rows = registered_work_query( - self.handle(), - "SELECT canonical_input_digest, event_payload FROM work_product_events_v1 - WHERE owner_brain_id = ?1 AND owner_profile_id = ?2 AND command_id = ?3", - owner_params(scope) - .into_iter() - .chain([ExactSqlValue::Text(command_id.as_str().to_owned())]) - .collect(), + replay_from( + self.transaction, + context, + command_id, + canonical_input_digest, ) - .map_err(|_| PortError::Unavailable)?; - let Some(row) = rows.rows.first() else { - return Ok(None); - }; - let stored_digest = exact_sql_text(&row.values, 0).ok_or(PortError::Unavailable)?; - if stored_digest != canonical_input_digest.as_str() { - // The same command id with different canonical input is a reused - // idempotency key, never a replay of this request. - return Err(PortError::IdempotencyConflict); - } - let event: WorkProductEventV1 = - serde_json::from_str(exact_sql_text(&row.values, 1).ok_or(PortError::Unavailable)?) - .map_err(|_| PortError::Unavailable)?; - // A replayed event must still be one this selection is authorized to - // see; otherwise its existence would leak through the idempotency - // channel. - if !selection_covers(scope.selection(), &event) { - return Err(PortError::NotFoundOrNotAuthorized); - } - let published = super::load_published_versions(self.handle(), scope) - .ok_or(PortError::Unavailable)? - .into_iter() - .find(|published| published.event_sequence == event.sequence()) - .ok_or(PortError::Unavailable)?; - let verified = super::verified_version(&published, &event).ok_or(PortError::Unavailable)?; - WorkProductEventCommitV1::new(event, verified) - .map(Some) - .map_err(|_| PortError::Unavailable) + } + + fn append_atomically( + &self, + context: &WorkProductPortContextV1, + draft: &WorkProductEventDraftV1, + ) -> Result { + append_in_transaction(self.transaction, context, draft) + } +} + +impl WorkProductEventPortV1 for WorkSqliteStorage { + fn replay( + &self, + context: &WorkProductPortContextV1, + command_id: &WorkCommandId, + canonical_input_digest: &ManifestDigest, + ) -> Result, PortError> { + replay_from(self.handle(), context, command_id, canonical_input_digest) } fn append_atomically( @@ -92,6 +83,51 @@ impl WorkProductEventPortV1 for WorkSqliteStorage { } } +fn replay_from( + source: &impl RegisteredWorkQuery, + context: &WorkProductPortContextV1, + command_id: &WorkCommandId, + canonical_input_digest: &ManifestDigest, +) -> Result, PortError> { + let scope = context.authorized_scope(); + let rows = registered_work_query( + source, + "SELECT canonical_input_digest, event_payload FROM work_product_events_v1 + WHERE owner_brain_id = ?1 AND owner_profile_id = ?2 AND command_id = ?3", + owner_params(scope) + .into_iter() + .chain([ExactSqlValue::Text(command_id.as_str().to_owned())]) + .collect(), + ) + .map_err(|_| PortError::Unavailable)?; + let Some(row) = rows.rows.first() else { + return Ok(None); + }; + let stored_digest = exact_sql_text(&row.values, 0).ok_or(PortError::Unavailable)?; + if stored_digest != canonical_input_digest.as_str() { + // The same command id with different canonical input is a reused + // idempotency key, never a replay of this request. + return Err(PortError::IdempotencyConflict); + } + let event: WorkProductEventV1 = + serde_json::from_str(exact_sql_text(&row.values, 1).ok_or(PortError::Unavailable)?) + .map_err(|_| PortError::Unavailable)?; + // A replayed event must still be one this selection is authorized to see; + // otherwise its existence would leak through the idempotency channel. + if !selection_covers(scope.selection(), &event) { + return Err(PortError::NotFoundOrNotAuthorized); + } + let published = super::load_published_versions(source, scope) + .ok_or(PortError::Unavailable)? + .into_iter() + .find(|published| published.event_sequence == event.sequence()) + .ok_or(PortError::Unavailable)?; + let verified = super::verified_version(&published, &event).ok_or(PortError::Unavailable)?; + WorkProductEventCommitV1::new(event, verified) + .map(Some) + .map_err(|_| PortError::Unavailable) +} + pub(super) fn append_in_transaction( transaction: &impl RegisteredWorkWrite, context: &WorkProductPortContextV1, diff --git a/crates/tracedecay-rusqlite-runtime/src/work_run_control.rs b/crates/tracedecay-rusqlite-runtime/src/work_run_control.rs index 7dd44e1cd4..263c12f77a 100644 --- a/crates/tracedecay-rusqlite-runtime/src/work_run_control.rs +++ b/crates/tracedecay-rusqlite-runtime/src/work_run_control.rs @@ -91,25 +91,18 @@ impl WorkSqliteStorage { .await } - /// Reads the current product graph an attempt admission observed at - /// `observed_at` makes, on the blocking pool, so the awaiting task parks - /// no runtime thread and the replay never holds the writer. - pub async fn prefetch_attempt_admission_graph( + /// Makes the current product graph read a writer job's service will ask + /// for, on the blocking pool, so the awaiting task parks no runtime thread + /// and the journal replay never holds the writer. A read that cannot run + /// prefetches nothing, and the job's service refuses its read. + pub async fn prefetch_graph( &self, - context: &tracedecay_contracts::RequestContext, - observed_at: UtcMicros, + read: impl FnOnce(&Self) -> tracedecay_contracts::PrefetchedWorkGraphReadV1 + Send + 'static, ) -> tracedecay_contracts::PrefetchedWorkGraphReadV1 { let storage = self.clone(); - let context = context.clone(); - crate::native_runtime::spawn_blocking(move || { - tracedecay_contracts::PrefetchedWorkGraphReadV1::for_attempt_admission( - &storage, - &context, - observed_at, - ) - }) - .await - .unwrap_or_default() + crate::native_runtime::spawn_blocking(move || read(&storage)) + .await + .unwrap_or_default() } }