Repository navigation
refactor(work): run adjudications and placements as writer jobs #4142
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Changes from all commits
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change | ||||||||||||
|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
|
|
@@ -283,6 +283,142 @@ impl RegisteredWorkApplicationServicesV1 { | |||||||||||||
| .unwrap_or_else(|_| Err(run_control_unavailable())) | ||||||||||||||
| } | ||||||||||||||
|
|
||||||||||||||
| /// Records one duplicate adjudication as a single writer transaction: the | ||||||||||||||
| /// replay check, the revision compare and the append commit together. | ||||||||||||||
| /// `current_evidence` is read before the job, outside the writer. | ||||||||||||||
| pub async fn adjudicate_duplicate( | ||||||||||||||
| &self, | ||||||||||||||
| context: tracedecay_contracts::RequestContext, | ||||||||||||||
| command: tracedecay_domain::WorkDuplicateAdjudicationCommandV1, | ||||||||||||||
| current_evidence: tracedecay_domain::WorkDuplicateAdjudicationEvidenceV1, | ||||||||||||||
| ) -> Result< | ||||||||||||||
| tracedecay_contracts::WorkDuplicateAdjudicationAppendOutcomeV1, | ||||||||||||||
| tracedecay_contracts::ApplicationProblem, | ||||||||||||||
| > { | ||||||||||||||
| self.storage | ||||||||||||||
| .transact(move |storage| { | ||||||||||||||
| tracedecay_contracts::WorkDuplicateAdjudicationServiceV1::new(storage).adjudicate( | ||||||||||||||
| &context, | ||||||||||||||
| command.clone(), | ||||||||||||||
| current_evidence.clone(), | ||||||||||||||
| ) | ||||||||||||||
| }) | ||||||||||||||
| .await | ||||||||||||||
| .unwrap_or_else(|_| { | ||||||||||||||
| Err( | ||||||||||||||
| tracedecay_contracts::WorkDuplicateAdjudicationStorageErrorV1::Unavailable | ||||||||||||||
| .into(), | ||||||||||||||
| ) | ||||||||||||||
| }) | ||||||||||||||
| } | ||||||||||||||
|
|
||||||||||||||
| /// Records one leak adjudication as a single writer transaction: the | ||||||||||||||
| /// command replay check, the verdict over `evidence` and the | ||||||||||||||
| /// revision-compared append commit together. `evidence` is collected | ||||||||||||||
| /// before the job, outside the writer. | ||||||||||||||
| pub async fn adjudicate_leak<E>( | ||||||||||||||
| &self, | ||||||||||||||
| context: tracedecay_contracts::RequestContext, | ||||||||||||||
| command: tracedecay_contracts::AdjudicateWorkLeakCommandV1, | ||||||||||||||
| evidence: E, | ||||||||||||||
| scan_started_at: tracedecay_domain::UtcMicros, | ||||||||||||||
| scan_deadline: tracedecay_domain::UtcMicros, | ||||||||||||||
| ) -> Result< | ||||||||||||||
| tracedecay_contracts::WorkLeakAdjudicationOutcomeV1, | ||||||||||||||
| tracedecay_contracts::ApplicationProblem, | ||||||||||||||
| > | ||||||||||||||
| where | ||||||||||||||
| E: tracedecay_contracts::WorkLeakEvidencePortV1 + Clone + 'static, | ||||||||||||||
| { | ||||||||||||||
| self.storage | ||||||||||||||
| .transact(move |storage| { | ||||||||||||||
| tracedecay_contracts::WorkLeakAdjudicationServiceV1::new(storage, evidence.clone()) | ||||||||||||||
| .adjudicate(&context, command.clone(), scan_started_at, scan_deadline) | ||||||||||||||
| }) | ||||||||||||||
| .await | ||||||||||||||
| .unwrap_or_else(|_| { | ||||||||||||||
| Err(tracedecay_contracts::WorkLeakAdjudicationStorageErrorV1::Unavailable.into()) | ||||||||||||||
| }) | ||||||||||||||
| } | ||||||||||||||
|
|
||||||||||||||
| /// Admits one placement as a single writer transaction: the replay check, | ||||||||||||||
| /// the exclusivity read and the publication commit together. `observe` | ||||||||||||||
| /// reads the target (Git) before the job, never on the writer. | ||||||||||||||
| pub async fn admit_placement( | ||||||||||||||
| &self, | ||||||||||||||
| context: tracedecay_contracts::RequestContext, | ||||||||||||||
| command: tracedecay_contracts::AdmitWorkPlacementCommand, | ||||||||||||||
| observe: impl FnOnce( | ||||||||||||||
| &tracedecay_domain::WorkPlacementTargetV1, | ||||||||||||||
| ) -> Result< | ||||||||||||||
| tracedecay_domain::WorkPlacementObservationV1, | ||||||||||||||
| tracedecay_contracts::ApplicationProblem, | ||||||||||||||
| >, | ||||||||||||||
| ) -> Result<tracedecay_domain::WorkPlacementV1, tracedecay_contracts::ApplicationProblem> { | ||||||||||||||
| let observation = observe(&command.target)?; | ||||||||||||||
| self.storage | ||||||||||||||
| .transact(move |storage| { | ||||||||||||||
| tracedecay_contracts::WorkPlacementService::new(storage).admit_placement( | ||||||||||||||
| &context, | ||||||||||||||
| command.clone(), | ||||||||||||||
| |_| Ok(observation), | ||||||||||||||
| ) | ||||||||||||||
| }) | ||||||||||||||
| .await | ||||||||||||||
| .unwrap_or_else(|_| Err(placement_unavailable())) | ||||||||||||||
| } | ||||||||||||||
|
|
||||||||||||||
| /// Releases one placement. The placement is read on a reader and its | ||||||||||||||
| /// target observed before the job; the job then publishes only while the | ||||||||||||||
| /// placement still has the observed target and the expected authority | ||||||||||||||
| /// version, so the Git observation never runs on the writer. | ||||||||||||||
| pub async fn release_placement( | ||||||||||||||
| &self, | ||||||||||||||
| context: tracedecay_contracts::RequestContext, | ||||||||||||||
| command: tracedecay_contracts::ReleaseWorkPlacementCommand, | ||||||||||||||
| observe: impl FnOnce( | ||||||||||||||
| &tracedecay_domain::WorkPlacementTargetV1, | ||||||||||||||
| ) -> Result< | ||||||||||||||
| tracedecay_domain::WorkPlacementObservationV1, | ||||||||||||||
| tracedecay_contracts::ApplicationProblem, | ||||||||||||||
| >, | ||||||||||||||
| ) -> Result<tracedecay_domain::WorkPlacementV1, tracedecay_contracts::ApplicationProblem> { | ||||||||||||||
| let authority = tracedecay_contracts::work_authority(&context)?; | ||||||||||||||
| let identity = tracedecay_domain::WorkPlacementIdentityV1::new( | ||||||||||||||
| command.task_id.clone(), | ||||||||||||||
| command.run_id.clone(), | ||||||||||||||
| ); | ||||||||||||||
| let current = self | ||||||||||||||
| .storage | ||||||||||||||
| .load_placement_async(&authority, &identity) | ||||||||||||||
| .await? | ||||||||||||||
| .ok_or(tracedecay_contracts::WorkPlacementStorageError::NotFoundOrNotAuthorized)?; | ||||||||||||||
| if current.authority_version() != command.expected_authority_version { | ||||||||||||||
| return Err(tracedecay_contracts::WorkPlacementStorageError::AuthorityConflict.into()); | ||||||||||||||
|
Comment on lines
+396
to
+397
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. 🟡 Stale placement masks admission refusal For a cancelled or expired release with a stale version,
Suggested change
Was this helpful? React with 👍 or 👎 to provide feedback. |
||||||||||||||
| } | ||||||||||||||
| let observation = observe(current.target())?; | ||||||||||||||
| let observed_target = current.target().clone(); | ||||||||||||||
| self.storage | ||||||||||||||
| .transact(move |storage| { | ||||||||||||||
| tracedecay_contracts::WorkPlacementService::new(storage).release( | ||||||||||||||
| &context, | ||||||||||||||
| command.clone(), | ||||||||||||||
| |target| { | ||||||||||||||
| if target == &observed_target { | ||||||||||||||
| Ok(observation) | ||||||||||||||
| } else { | ||||||||||||||
| Err( | ||||||||||||||
| tracedecay_contracts::WorkPlacementStorageError::AuthorityConflict | ||||||||||||||
| .into(), | ||||||||||||||
| ) | ||||||||||||||
| } | ||||||||||||||
| }, | ||||||||||||||
| ) | ||||||||||||||
| }) | ||||||||||||||
| .await | ||||||||||||||
| .unwrap_or_else(|_| Err(placement_unavailable())) | ||||||||||||||
| } | ||||||||||||||
|
|
||||||||||||||
| pub fn attempts( | ||||||||||||||
| &self, | ||||||||||||||
| ) -> &tracedecay_contracts::WorkAttemptService< | ||||||||||||||
|
|
@@ -422,6 +558,10 @@ pub fn authorized_work_intelligence_service( | |||||||||||||
| )) | ||||||||||||||
| } | ||||||||||||||
|
|
||||||||||||||
| fn placement_unavailable() -> tracedecay_contracts::ApplicationProblem { | ||||||||||||||
| tracedecay_contracts::WorkPlacementStorageError::Unavailable.into() | ||||||||||||||
| } | ||||||||||||||
|
|
||||||||||||||
| fn run_control_unavailable() -> tracedecay_contracts::ApplicationProblem { | ||||||||||||||
| tracedecay_contracts::WorkRunControlStorageError::Unavailable.into() | ||||||||||||||
| } | ||||||||||||||
|
|
||||||||||||||
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
🔴 Queued leak scan records stale verdict
When Work state changes while the writer is queued,
adjudicate_leakclassifies an older snapshot as current.DaemonWorkLeakEvidenceV1::inspecttimestamps that snapshot only after the job starts, so it can publish a false leak verdict.Learn more
The daemon gathers the attempt, process-owner status, placement, effect and delivery census before enqueuing this job in read_leak_evidence. The writer can process other work before this job begins. inspect then assigns a new completion timestamp to the previously gathered snapshot and derives a verdict from it. State transitions during the queue wait are invisible, but the receipt attributes the verdict to the later scan completion.
Example: A running attempt has no live owner when the daemon reads it. The owner resumes before the queued job starts. The job still records
AttemptWithoutLiveOwneras though the scan had just found an ownerless attempt.Recommended fix: Determine the scan result and its completion timestamp before enqueueing, or revalidate the versioned evidence at the writer boundary before committing. Keep the replay and revision comparison in the writer job; do not timestamp frozen evidence at job execution time.
Was this helpful? React with 👍 or 👎 to provide feedback.