diff --git a/CHANGELOG.md b/CHANGELOG.md index 6110fb9c..876a40f4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,19 @@ ## Unreleased +## 2.0.8 - 2026-09-08 + +- Concurrent embedded child completions persist their outcomes before checking + the parallel barrier, preventing a lost parent wake-up under MySQL repeatable + reads. Existing repair paths recover stranded parents from durable child + terminal history without rerunning completed children. +- A child that handles an activity failure and completes no longer propagates + historical failure metadata as its result. Genuine terminal failures retain + their own failure identity across embedded and service-mode completion paths. +- Parent-notification errors roll back the workflow task for repair instead of + appending a workflow failure after completion. This does not rewrite existing + histories that already contain contradictory terminal events. + ## 2.0.7 - 2026-09-08 - Activity metadata projections no longer download or decode external arguments diff --git a/composer.json b/composer.json index 9bcdf143..c010f049 100644 --- a/composer.json +++ b/composer.json @@ -79,7 +79,7 @@ "dev-main": "2.0.x-dev" }, "durable-workflow": { - "product-train": "2.0.7", + "product-train": "2.0.8", "laravel-embedded-upgrade-contract": "resources/laravel-embedded-upgrade-contract.json", "laravel-dependency-security-policy": "resources/laravel-dependency-security-policy.json" }, diff --git a/src/V2/Support/ChildRunHistory.php b/src/V2/Support/ChildRunHistory.php index 4a7e8c20..91393ecd 100644 --- a/src/V2/Support/ChildRunHistory.php +++ b/src/V2/Support/ChildRunHistory.php @@ -557,6 +557,20 @@ public static function resolvedStatus( }; } + public static function terminalFailureForRun(WorkflowRun $childRun): ?WorkflowFailure + { + if (self::resolvedStatus(null, $childRun) === RunStatus::Completed) { + return null; + } + + $failureId = self::terminalEventForRun($childRun)?->payload['failure_id'] ?? null; + + return $childRun->failures->first( + static fn (WorkflowFailure $failure): bool => ! $failure->handled + && ($failureId === null || $failure->id === $failureId) + ); + } + public static function outputForResolution( WorkflowHistoryEvent $resolutionEvent, ?WorkflowRun $childRun = null, diff --git a/src/V2/Support/DefaultWorkflowTaskBridge.php b/src/V2/Support/DefaultWorkflowTaskBridge.php index 416c8606..1867715c 100644 --- a/src/V2/Support/DefaultWorkflowTaskBridge.php +++ b/src/V2/Support/DefaultWorkflowTaskBridge.php @@ -4672,7 +4672,7 @@ private function recordChildResolution( ) ->sortByDesc('sequence') ->first(); - $failure = $childRun->failures->first(); + $failure = ChildRunHistory::terminalFailureForRun($childRun); $parallelMetadataPath = ChildRunHistory::parallelGroupPathForSequence($run, $sequence); $parallelMetadata = ParallelChildGroup::payloadForPath($parallelMetadataPath); $childOutput = $childTerminalEvent?->event_type === HistoryEventType::WorkflowCompleted diff --git a/src/V2/Support/TaskRepairCandidates.php b/src/V2/Support/TaskRepairCandidates.php index fb0037f5..6f130495 100644 --- a/src/V2/Support/TaskRepairCandidates.php +++ b/src/V2/Support/TaskRepairCandidates.php @@ -643,7 +643,32 @@ private static function missingTaskRunQuery( ?string $queue = null, ) { $query = WorkflowRunSummary::query() - ->where('liveness_state', 'repair_needed') + ->where(static function ($candidate): void { + $candidate->where('liveness_state', 'repair_needed') + ->orWhere(static function ($waiting): void { + $waiting->where('liveness_state', 'waiting_for_child') + ->whereDoesntHave('run.tasks', static function ($task): void { + $task->where('task_type', 'workflow') + ->whereIn('status', [TaskStatus::Ready->value, TaskStatus::Leased->value]); + }) + ->whereHas('run.childLinks', static function ($link): void { + $link->where('link_type', 'child_workflow'); + }) + ->whereDoesntHave('run.childLinks', static function ($link): void { + $link->where('link_type', 'child_workflow') + ->where(static function ($unclosed): void { + $unclosed->whereDoesntHave('childRun') + ->orWhereHas('childRun', static function ($child): void { + $child->whereIn('status', [ + RunStatus::Pending->value, + RunStatus::Running->value, + RunStatus::Waiting->value, + ]); + }); + }); + }); + }); + }) ->whereNull('next_task_id') ->whereIn('status', [RunStatus::Pending->value, RunStatus::Running->value, RunStatus::Waiting->value]); diff --git a/src/V2/Support/WorkflowExecutor.php b/src/V2/Support/WorkflowExecutor.php index 86874ad7..9e023b6b 100644 --- a/src/V2/Support/WorkflowExecutor.php +++ b/src/V2/Support/WorkflowExecutor.php @@ -2007,6 +2007,54 @@ public function timeoutIfDeadlineExpired(WorkflowRun $run, WorkflowTask $task): return true; } + /** + * The caller holds the parent run lock; child terminal history, not row status, authorizes recovery. + */ + public function recoverClosedChildResolutions(WorkflowRun $parentRun): void + { + if ($parentRun->status->isTerminal() || $parentRun->tasks() + ->where('task_type', TaskType::Workflow->value) + ->whereIn('status', [TaskStatus::Ready->value, TaskStatus::Leased->value]) + ->lockForUpdate() + ->exists()) { + return; + } + + $parentRun->setRelation('historyEvents', $parentRun->historyEvents()->lockForUpdate()->get()); + foreach (ChildRunHistory::knownSequences($parentRun) as $sequence) { + if (ChildRunHistory::resolutionEventForSequence($parentRun, $sequence) !== null) { + continue; + } + try { + WorkflowStepHistory::assertCompatible($parentRun, $sequence, WorkflowStepHistory::CHILD_WORKFLOW); + WorkflowStepHistory::assertTypedHistoryRecorded( + $parentRun, + $sequence, + WorkflowStepHistory::CHILD_WORKFLOW + ); + } catch (HistoryEventShapeMismatchException) { + continue; + } + + $childRun = ChildRunHistory::childRunForSequence($parentRun, $sequence); + if (! $childRun instanceof WorkflowRun || ! $childRun->historyEvents->contains( + static fn (WorkflowHistoryEvent $event): bool => in_array($event->event_type, [ + HistoryEventType::WorkflowCompleted, + HistoryEventType::WorkflowFailed, + HistoryEventType::WorkflowCancelled, + HistoryEventType::WorkflowTerminated, + ], true) + )) { + continue; + } + if ($this->startChildRetryIfAvailable($parentRun, $sequence, $childRun) === null) { + $this->recordChildResolution($parentRun, null, $sequence, $childRun); + } + $parentRun->unsetRelation('childLinks'); + $parentRun->setRelation('historyEvents', $parentRun->historyEvents()->lockForUpdate()->get()); + } + } + private function scheduleActivity( WorkflowRun $run, WorkflowTask $task, @@ -2920,7 +2968,7 @@ private function recordChildResolution( ) ->sortByDesc('sequence') ->first(); - $failure = $childRun->failures->first(); + $failure = ChildRunHistory::terminalFailureForRun($childRun); $parallelMetadataPath = ChildRunHistory::parallelGroupPathForSequence($run, $sequence); $parallelMetadata = ParallelChildGroup::payloadForPath($parallelMetadataPath); $childOutput = $childTerminalEvent?->event_type === HistoryEventType::WorkflowCompleted @@ -3718,6 +3766,12 @@ private function failRun( string $sourceKind, string $sourceId, ): void { + if ($run->status === RunStatus::Completed && $run->historyEvents() + ->where('event_type', HistoryEventType::WorkflowCompleted->value)->exists()) { + // Completion side effects failed, not workflow code. Roll back this task attempt. + throw $throwable; + } + if ($throwable instanceof UnresolvedWorkflowFailureException) { $this->blockReplayUntilFailureCanBeRestored($run, $task, $throwable); @@ -4353,21 +4407,18 @@ private function dispatchParentResumeTasks(WorkflowRun $childRun): void continue; } - $parallelMetadataPath = ChildRunHistory::parallelGroupPathForSequence( - $parentRun, - $parentReference['parent_sequence'], - ); - $childStatus = ChildRunHistory::resolvedStatus(null, $childRun); - - if ( - $parallelMetadataPath !== [] - && $childStatus instanceof RunStatus - && ! ParallelChildGroup::shouldWakeParentOnChildClosure( + try { + WorkflowStepHistory::assertCompatible( $parentRun, - $parallelMetadataPath, - $childStatus - ) - ) { + $parentReference['parent_sequence'], + WorkflowStepHistory::CHILD_WORKFLOW, + ); + WorkflowStepHistory::assertTypedHistoryRecorded( + $parentRun, + $parentReference['parent_sequence'], + WorkflowStepHistory::CHILD_WORKFLOW, + ); + } catch (HistoryEventShapeMismatchException) { $this->projectRun( $parentRun->fresh([ 'instance', @@ -4385,12 +4436,29 @@ private function dispatchParentResumeTasks(WorkflowRun $childRun): void continue; } + // Every closer persists its outcome before checking the shared barrier. + // The next parent-lock holder must not depend on a stale child snapshot. + $resolutionEvent = $this->recordChildResolution( + $parentRun, + null, + $parentReference['parent_sequence'], + $childRun, + ); + $parentTaskPayload = WorkflowTaskPayload::forChildResolution($resolutionEvent); + $parallelMetadataPath = ChildRunHistory::parallelGroupPathForSequence( + $parentRun, + $parentReference['parent_sequence'], + ); + $childStatus = ChildRunHistory::resolvedStatus($resolutionEvent, $childRun); + if ( $parallelMetadataPath !== [] - && ! $this->recordClosedParallelChildResolutions( + && $childStatus instanceof RunStatus + && ! ParallelChildGroup::shouldWakeParentOnChildClosure( $parentRun, $parallelMetadataPath, - $parentReference['parent_sequence'], + $childStatus, + lockHistoryForUpdate: true, ) ) { $this->projectRun( @@ -4410,18 +4478,14 @@ private function dispatchParentResumeTasks(WorkflowRun $childRun): void continue; } - try { - WorkflowStepHistory::assertCompatible( - $parentRun, - $parentReference['parent_sequence'], - WorkflowStepHistory::CHILD_WORKFLOW, - ); - WorkflowStepHistory::assertTypedHistoryRecorded( + if ( + $parallelMetadataPath !== [] + && ! $this->recordClosedParallelChildResolutions( $parentRun, + $parallelMetadataPath, $parentReference['parent_sequence'], - WorkflowStepHistory::CHILD_WORKFLOW, - ); - } catch (HistoryEventShapeMismatchException) { + ) + ) { $this->projectRun( $parentRun->fresh([ 'instance', @@ -4438,14 +4502,6 @@ private function dispatchParentResumeTasks(WorkflowRun $childRun): void continue; } - - $resolutionEvent = $this->recordChildResolution( - $parentRun, - null, - $parentReference['parent_sequence'], - $childRun, - ); - $parentTaskPayload = WorkflowTaskPayload::forChildResolution($resolutionEvent); } $hasOpenWorkflowTask = WorkflowTask::query() diff --git a/src/V2/TaskWatchdog.php b/src/V2/TaskWatchdog.php index 0937efae..7f4460b2 100644 --- a/src/V2/TaskWatchdog.php +++ b/src/V2/TaskWatchdog.php @@ -25,6 +25,7 @@ use Workflow\V2\Support\TaskRepairCandidates; use Workflow\V2\Support\TaskRepairPolicy; use Workflow\V2\Support\WorkerCompatibilityFleet; +use Workflow\V2\Support\WorkflowExecutor; /** * Periodic repair loop that scans for stuck workflow tasks, expired activity @@ -286,6 +287,7 @@ private static function recoverMissingTask(string $runId): array ->lockForUpdate() ->findOrFail($runId); + app(WorkflowExecutor::class)->recoverClosedChildResolutions($run); $summary = self::historyProjectionRole()->projectRun($run); if ($summary->liveness_state !== 'repair_needed' || $summary->next_task_id !== null) { diff --git a/src/V2/WorkflowStub.php b/src/V2/WorkflowStub.php index 434dc493..b1f2518c 100644 --- a/src/V2/WorkflowStub.php +++ b/src/V2/WorkflowStub.php @@ -1598,6 +1598,7 @@ public function attemptRepair(): CommandResult ->get() ); + app(\Workflow\V2\Support\WorkflowExecutor::class)->recoverClosedChildResolutions($run); $summary = self::projectRun($run); if (in_array($summary->liveness_state, ['repair_needed', 'workflow_replay_blocked'], true)) { @@ -4523,7 +4524,8 @@ private function createParentResumeTasks(WorkflowRun $childRun): array && ! ParallelChildGroup::shouldWakeParentOnChildClosure( $parentRun, $parallelMetadataPath, - $childStatus + $childStatus, + lockHistoryForUpdate: true, ) ) { self::projectRun($parentRun, self::PROJECTION_RUN_RELATIONS_WITH_CHILDREN); @@ -4604,7 +4606,7 @@ private function recordParentChildResolution( ) ->sortByDesc('sequence') ->first(); - $failure = $childRun->failures->first(); + $failure = ChildRunHistory::terminalFailureForRun($childRun); $parallelMetadataPath = ChildRunHistory::parallelGroupPathForSequence($parentRun, $sequence); $parallelMetadata = ParallelChildGroup::payloadForPath($parallelMetadataPath); $childOutput = $childTerminalEvent?->event_type === HistoryEventType::WorkflowCompleted diff --git a/tests/Feature/V2/V2ChildTerminalOutcomeTest.php b/tests/Feature/V2/V2ChildTerminalOutcomeTest.php new file mode 100644 index 00000000..769f5fda --- /dev/null +++ b/tests/Feature/V2/V2ChildTerminalOutcomeTest.php @@ -0,0 +1,177 @@ +completeRecoveredChild(serviceMode: false); + } + + public function testServiceModeCompletedChildDoesNotPropagateItsHandledActivityFailure(): void + { + $this->completeRecoveredChild(serviceMode: true); + } + + public function testParentNotificationFailureRetriesTheTaskWithoutChangingTheChildOutcome(): void + { + $this->completeRecoveredChild(serviceMode: false, interruptNotification: true); + } + + public function testGenuineChildFailureUsesTheTerminalFailureNotAnEarlierHandledFailure(): void + { + [$parent, $childId] = $this->prepareRecoveredChild(); + $handledFailure = WorkflowFailure::query()->where('workflow_run_id', $childId)->sole(); + $task = $this->readyTask($childId); + $bridge = $this->app->make(WorkflowTaskBridge::class); + $this->assertTrue($bridge->claimStatus($task->id, 'terminal-failure-worker')['claimed']); + $completion = $bridge->complete($task->id, [[ + 'type' => 'fail_workflow', + 'message' => 'Terminal failure after recovery', + 'exception_class' => RuntimeException::class, + ]]); + $this->assertTrue($completion['completed']); + $terminalEvent = WorkflowHistoryEvent::query()->where('workflow_run_id', $childId) + ->where('event_type', HistoryEventType::WorkflowFailed->value)->sole(); + $resolution = WorkflowHistoryEvent::query()->where('workflow_run_id', $parent->runId()) + ->where('event_type', HistoryEventType::ChildRunFailed->value)->sole(); + $this->assertNotSame($handledFailure->id, $resolution->payload['failure_id']); + $this->assertSame($terminalEvent->payload['failure_id'], $resolution->payload['failure_id']); + $this->assertSame('Terminal failure after recovery', $resolution->payload['message']); + $this->runReadyTask($parent->runId()); + $this->assertTrue($parent->refresh()->failed()); + } + + private function completeRecoveredChild(bool $serviceMode, bool $interruptNotification = false): void + { + [$parent, $childId] = $this->prepareRecoveredChild(); + + if ($interruptNotification) { + $eventName = 'eloquent.creating: ' . WorkflowHistoryEvent::class; + Event::listen($eventName, static function (WorkflowHistoryEvent $event): void { + if ($event->event_type === HistoryEventType::ChildRunCompleted) { + throw new RuntimeException('Parent notification unavailable'); + } + }); + try { + $this->runReadyTask($childId); + $this->fail('The task infrastructure error must remain visible.'); + } catch (RuntimeException $exception) { + $this->assertSame('Parent notification unavailable', $exception->getMessage()); + } finally { + Event::forget($eventName); + } + $this->assertSame(RunStatus::Waiting, WorkflowRun::query()->findOrFail($childId)->status); + $this->assertSame(0, WorkflowHistoryEvent::query()->where('workflow_run_id', $childId) + ->whereIn( + 'event_type', + [HistoryEventType::WorkflowCompleted->value, HistoryEventType::WorkflowFailed->value] + ) + ->count()); + $this->assertSame('repair_dispatched', WorkflowStub::loadRun($childId)->attemptRepair()->outcome()); + } + + if ($serviceMode) { + $task = $this->readyTask($childId); + $bridge = $this->app->make(WorkflowTaskBridge::class); + $this->assertTrue($bridge->claimStatus($task->id, 'handled-failure-worker')['claimed']); + $completion = $bridge->complete($task->id, [[ + 'type' => 'complete_workflow', + 'result' => Avro::serialize('Hello, Recovered!'), + 'payload_codec' => 'avro', + ]]); + $this->assertTrue($completion['completed']); + } else { + $this->runReadyTask($childId); + } + + $this->assertSame(RunStatus::Completed, WorkflowRun::query()->findOrFail($childId)->status); + $this->assertSame([HistoryEventType::WorkflowCompleted], WorkflowHistoryEvent::query() + ->where('workflow_run_id', $childId) + ->whereIn( + 'event_type', + [HistoryEventType::WorkflowCompleted->value, HistoryEventType::WorkflowFailed->value] + ) + ->orderBy('sequence') + ->pluck('event_type') + ->all()); + $resolution = WorkflowHistoryEvent::query()->where('workflow_run_id', $parent->runId()) + ->where('event_type', HistoryEventType::ChildRunCompleted->value)->sole(); + foreach ([ + 'failure_id', + 'failure_category', + 'exception', + 'exception_type', + 'exception_class', + 'message', + 'code', + ] as $key) { + $this->assertArrayNotHasKey($key, $resolution->payload); + } + $this->runReadyTask($parent->runId()); + $this->assertTrue($parent->refresh()->completed()); + $this->assertSame('Hello, Recovered!', $parent->output()); + } + + /** + * @return array{WorkflowStub, string} + */ + private function prepareRecoveredChild(): array + { + self::stopWorkers(); + Queue::fake(); + $parent = WorkflowStub::make(TestHandledFailureParentWorkflow::class); + $parent->start(); + $this->runReadyTask($parent->runId()); + $link = WorkflowLink::query()->where('parent_workflow_run_id', $parent->runId())->sole(); + $childId = $link->child_workflow_run_id; + + $this->runReadyTask($childId); + $this->runReadyTask($childId); + $this->runReadyTask($childId); + $this->runReadyTask($childId); + $this->assertSame(1, WorkflowFailure::query()->where('workflow_run_id', $childId) + ->where('handled', true) + ->count()); + + return [$parent, $childId]; + } + + private function readyTask(string $runId): WorkflowTask + { + return WorkflowTask::query()->where('workflow_run_id', $runId) + ->where('status', TaskStatus::Ready->value)->sole(); + } + + private function runReadyTask(string $runId): void + { + $task = $this->readyTask($runId); + $job = $task->task_type === TaskType::Activity + ? new RunActivityTask($task->id) + : new RunWorkflowTask($task->id); + $this->app->call([$job, 'handle']); + } +} diff --git a/tests/Feature/V2/V2EmbeddedParallelChildConcurrencyTest.php b/tests/Feature/V2/V2EmbeddedParallelChildConcurrencyTest.php new file mode 100644 index 00000000..73d47b18 --- /dev/null +++ b/tests/Feature/V2/V2EmbeddedParallelChildConcurrencyTest.php @@ -0,0 +1,242 @@ +start(4); + $workflows[] = $workflow; + } + foreach ($workflows as $workflow) { + $this->waitForWorkflow($workflow, static fn (WorkflowStub $run): bool => ! $run->refresh()->running()); + $this->assertTrue($workflow->refresh()->completed()); + $this->assertSame( + ['Hello, 0!', 'Hello, 1!', 'Hello, 2!', 'Hello, 3!'], + array_column($workflow->output(), 'greeting') + ); + $childIds = WorkflowLink::query()->where('parent_workflow_run_id', $workflow->runId()) + ->pluck('child_workflow_run_id') + ->all(); + $this->assertCount(4, $childIds); + $this->assertSame(4, ActivityExecution::query()->whereIn('workflow_run_id', $childIds) + ->where('attempt_count', 1) + ->where('status', 'completed') + ->count()); + $this->assertSame(4, WorkflowHistoryEvent::query()->where('workflow_run_id', $workflow->runId()) + ->where('event_type', HistoryEventType::ChildRunCompleted->value)->count()); + } + } + + public function testIncompleteChildGroupIsNotARepairCandidateAndDoesNotWake(): void + { + [$workflow] = $this->strandedParent(closedChildren: 1); + $parentId = $workflow->runId(); + $this->assertSame([], TaskRepairCandidates::runIds(runIds: [$parentId])); + $this->assertSame('repair_not_needed', WorkflowStub::loadRun($parentId)->attemptRepair()->outcome()); + $this->assertSame(0, $this->openTasks($parentId)->count()); + $this->assertSame('waiting_for_child', $workflow->refresh()->summary()->liveness_state); + } + + public function testMutableClosedRowsWithoutTerminalHistoryCannotAuthorizeRecovery(): void + { + [$workflow, $childIds] = $this->strandedParent(); + WorkflowHistoryEvent::query()->where('workflow_run_id', $childIds[0]) + ->where('event_type', HistoryEventType::WorkflowCompleted->value)->delete(); + $parentId = $workflow->runId(); + $this->assertSame(0, TaskWatchdog::runPass(runIds: [$parentId])['repaired_missing_tasks']); + $this->assertSame(0, $this->openTasks($parentId)->count()); + $this->assertSame(1, WorkflowHistoryEvent::query()->where('workflow_run_id', $parentId) + ->where('event_type', HistoryEventType::ChildRunCompleted->value)->count()); + } + + public function testExplicitRepairRecoversStrandedParentWithoutDuplicateEventsOrTasks(): void + { + [$workflow, $childIds] = $this->strandedParent(); + $parentId = $workflow->runId(); + $this->assertSame('repair_dispatched', WorkflowStub::loadRun($parentId)->attemptRepair()->outcome()); + $this->assertSame('repair_not_needed', WorkflowStub::loadRun($parentId)->attemptRepair()->outcome()); + $this->assertSame(1, $this->openTasks($parentId)->count()); + $this->assertSame(2, WorkflowHistoryEvent::query()->where('workflow_run_id', $parentId) + ->where('event_type', HistoryEventType::ChildRunCompleted->value)->count()); + $this->runWorkflowTask($parentId); + $this->assertTrue($workflow->refresh()->completed()); + $this->assertSame($childIds, array_column($workflow->output()['children'], 'run_id')); + } + + public function testRepairPassRecoversAlreadyClosedChildrenWithoutParentResolutions(): void + { + [$workflow, $childIds] = $this->strandedParent(); + $parentId = $workflow->runId(); + DB::purge(); + DB::reconnect(); + $this->assertSame([$parentId], TaskRepairCandidates::runIds(runIds: [$parentId])); + $report = TaskWatchdog::runPass(runIds: [$parentId]); + $this->assertSame(1, $report['repaired_missing_tasks']); + $this->assertSame([], $report['missing_run_failures']); + $this->assertSame(1, $this->openTasks($parentId)->count()); + $this->assertSame(2, WorkflowHistoryEvent::query()->where('workflow_run_id', $parentId) + ->where('event_type', HistoryEventType::ChildRunCompleted->value)->count()); + $this->assertSame(0, TaskWatchdog::runPass(runIds: [$parentId])['repaired_missing_tasks']); + $this->assertSame(1, $this->openTasks($parentId)->count()); + $this->runWorkflowTask($parentId); + $this->assertTrue($workflow->refresh()->completed()); + $this->assertSame($childIds, array_column($workflow->output()['children'], 'run_id')); + } + + public function testOlderCompletionSnapshotStillObservesCommittedSiblingAndWakesParent(): void + { + if (DB::connection()->getDriverName() !== 'mysql' || ! function_exists('pcntl_fork')) { + $this->markTestSkipped('MySQL repeatable-read and process control are required.'); + } + + self::stopWorkers(); + Queue::fake(); + $workflow = WorkflowStub::make(TestParallelChildWorkflow::class, 'embedded-parallel-snapshot'); + $workflow->start(0, 0); + $parentId = $workflow->runId(); + $this->runWorkflowTask($parentId); + $childIds = WorkflowLink::query()->where('parent_workflow_run_id', $parentId) + ->orderBy('sequence') + ->pluck('child_workflow_run_id') + ->all(); + $this->assertCount(2, $childIds); + + $sockets = stream_socket_pair(STREAM_PF_UNIX, STREAM_SOCK_STREAM, STREAM_IPPROTO_IP); + $this->assertIsArray($sockets); + DB::purge(); + $pid = pcntl_fork(); + $this->assertNotSame(-1, $pid); + + if ($pid === 0) { + fclose($sockets[0]); + try { + DB::reconnect(); + DB::statement('SET SESSION TRANSACTION ISOLATION LEVEL REPEATABLE READ'); + DB::transaction(function () use ($childIds, $sockets): void { + // Both child rows are visible before the sibling commits. + WorkflowRun::query()->whereIn('id', $childIds)->get(); + fwrite($sockets[1], "snapshot\n"); + if (fgets($sockets[1]) !== "complete\n") { + throw new \RuntimeException('Missing completion release.'); + } + $this->runWorkflowTask($childIds[1]); + }); + fwrite($sockets[1], "completed\n"); + fclose($sockets[1]); + exit(0); + } catch (\Throwable $error) { + fwrite($sockets[1], $error::class . ': ' . $error->getMessage() . "\n"); + fclose($sockets[1]); + exit(1); + } + } + + fclose($sockets[1]); + stream_set_timeout($sockets[0], 20); + try { + $this->assertSame("snapshot\n", fgets($sockets[0])); + DB::reconnect(); + $this->runWorkflowTask($childIds[0]); + $this->assertSame(0, $this->openTasks($parentId)->count()); + fwrite($sockets[0], "complete\n"); + $this->assertSame("completed\n", fgets($sockets[0])); + pcntl_waitpid($pid, $status); + $this->assertSame(0, pcntl_wexitstatus($status)); + $pid = 0; + } finally { + fclose($sockets[0]); + if ($pid > 0) { + posix_kill($pid, SIGKILL); + pcntl_waitpid($pid, $status); + } + DB::purge(); + DB::reconnect(); + } + + $this->assertSame(2, WorkflowRun::query()->whereIn('id', $childIds) + ->where('status', RunStatus::Completed->value)->count()); + $this->assertSame(1, $this->openTasks($parentId)->count()); + $this->assertSame(2, WorkflowHistoryEvent::query()->where('workflow_run_id', $parentId) + ->where('event_type', HistoryEventType::ChildRunCompleted->value)->count()); + $this->runWorkflowTask($parentId); + $this->assertTrue($workflow->refresh()->completed()); + $this->assertSame($childIds, array_column($workflow->output()['children'], 'run_id')); + } + + /** + * @return array{WorkflowStub, list} + */ + private function strandedParent(int $closedChildren = 2): array + { + self::stopWorkers(); + Queue::fake(); + $workflow = WorkflowStub::make(TestParallelChildWorkflow::class, 'stranded-parallel-parent'); + $workflow->start(0, 0); + $parentId = $workflow->runId(); + $this->runWorkflowTask($parentId); + $childIds = WorkflowLink::query()->where('parent_workflow_run_id', $parentId) + ->orderBy('sequence') + ->pluck('child_workflow_run_id') + ->all(); + foreach (array_slice($childIds, 0, $closedChildren) as $childId) { + $this->runWorkflowTask($childId); + } + + // Reconstruct the 2.0.7 lost-wake state, preserving each child's durable result. + $this->openTasks($parentId) + ->delete(); + WorkflowHistoryEvent::query()->where('workflow_run_id', $parentId) + ->where('event_type', HistoryEventType::ChildRunCompleted->value)->delete(); + $run = WorkflowRun::query()->findOrFail($parentId); + $run->forceFill([ + 'last_history_sequence' => $run->historyEvents() + ->max('sequence'), + ])->save(); + RunSummaryProjector::project($run->fresh()); + $this->assertSame('waiting_for_child', WorkflowRunSummary::query()->findOrFail($parentId)->liveness_state); + + return [$workflow, $childIds]; + } + + private function openTasks(string $runId): \Illuminate\Database\Eloquent\Builder + { + return WorkflowTask::query()->where('workflow_run_id', $runId) + ->where('task_type', TaskType::Workflow->value) + ->whereIn('status', [TaskStatus::Ready->value, TaskStatus::Leased->value]); + } + + private function runWorkflowTask(string $runId): void + { + $task = $this->openTasks($runId) + ->sole(); + $this->app->call([new RunWorkflowTask($task->id), 'handle']); + } +} diff --git a/tests/Feature/V2/V2EmbeddedReplayRegressionCorpusTest.php b/tests/Feature/V2/V2EmbeddedReplayRegressionCorpusTest.php index 34651d42..14574fd9 100644 --- a/tests/Feature/V2/V2EmbeddedReplayRegressionCorpusTest.php +++ b/tests/Feature/V2/V2EmbeddedReplayRegressionCorpusTest.php @@ -29,6 +29,7 @@ use Workflow\V2\Models\WorkflowSearchAttribute; use Workflow\V2\Models\WorkflowTask; use Workflow\V2\Support\ConditionWaits; +use Workflow\V2\Support\DefaultHistoryProjectionRole; use Workflow\V2\Support\EmbeddedV2HistoryImport; use Workflow\V2\Support\HistoryExport; use Workflow\V2\Support\QueryStateReplayer; @@ -36,6 +37,7 @@ use Workflow\V2\Support\WorkflowFiberRunner; use Workflow\V2\Support\WorkflowReplayer; use Workflow\V2\Support\WorkflowStep; +use Workflow\V2\TaskWatchdog; use Workflow\V2\Workflow; use Workflow\V2\WorkflowStub; @@ -68,7 +70,10 @@ public function testFixturesExecuteThroughDeclaredReplayConsumers(): void $this->assertLegacyParallelChildRawHistoryGap($fixture); } - if (($fixture['id'] ?? null) === 'parallel-child-group-durable-command-sequence') { + if (in_array($fixture['id'] ?? null, [ + 'parallel-child-group-durable-command-sequence', + 'parallel-child-group-stranded-parent-recovery', + ], true)) { $this->assertStandaloneParallelChildBarrierFixture($fixture); } @@ -339,6 +344,27 @@ private function assertStandaloneParallelChildBarrierFixture(array $fixture): vo ->where('event_type', HistoryEventType::ChildRunCompleted->value) ->count()); + if ($fixture['id'] === 'parallel-child-group-stranded-parent-recovery') { + $this->app->instance(HistoryProjectionRole::class, new DefaultHistoryProjectionRole()); + // Keep closed child histories, but reconstruct the parent's pre-fix lost-wake state. + WorkflowTask::query()->where('workflow_run_id', $parentRun->id) + ->whereIn('status', [TaskStatus::Ready->value, TaskStatus::Leased->value])->delete(); + WorkflowHistoryEvent::query()->where('workflow_run_id', $parentRun->id) + ->where('event_type', HistoryEventType::ChildRunCompleted->value)->delete(); + $parentRun->refresh() + ->forceFill([ + 'last_history_sequence' => $parentRun->historyEvents() + ->max('sequence'), + ])->save(); + $projection = $this->app->make(HistoryProjectionRole::class); + $this->assertSame('waiting_for_child', $projection->projectRun($parentRun->fresh())->liveness_state); + DB::purge(); + DB::reconnect(); + $this->assertSame(1, TaskWatchdog::runPass(runIds: [$parentRun->id])['repaired_missing_tasks']); + $this->assertSame(0, TaskWatchdog::runPass(runIds: [$parentRun->id])['repaired_missing_tasks']); + $this->assertSame(1, $openParentTaskCount()); + } + /** @var WorkflowTask $replacementTask */ $replacementTask = WorkflowTask::query() ->where('workflow_run_id', $parentRun->id) @@ -361,14 +387,7 @@ private function assertStandaloneParallelChildBarrierFixture(array $fixture): vo $this->assertTrue($coldReplay->completed); $this->assertSame(['complete_workflow'], array_column($coldReplay->commands, 'type')); - $this->assertSame([ - [ - 'child' => 'first', - ], - [ - 'child' => 'second', - ], - ], $coldReplay->result['children'] ?? null); + $this->assertSame($fixture['expected']['result']['children'], $coldReplay->result['children'] ?? null); $this->assertSame($stub->workflowId(), $coldReplay->result['workflow_id'] ?? null); $this->assertSame($parentRun->id, $coldReplay->result['run_id'] ?? null); diff --git a/tests/Feature/V2/V2WorkflowTest.php b/tests/Feature/V2/V2WorkflowTest.php index bf079119..a7a401e2 100644 --- a/tests/Feature/V2/V2WorkflowTest.php +++ b/tests/Feature/V2/V2WorkflowTest.php @@ -3469,7 +3469,7 @@ public function testParallelChildAllWaitsForLastSuccessfulChildBeforeResumingPar $this->runReadyTaskForRun($firstChildRunId, TaskType::Workflow); - $this->assertSame(0, WorkflowHistoryEvent::query() + $this->assertSame(1, WorkflowHistoryEvent::query() ->where('workflow_run_id', $parentRunId) ->where('event_type', 'ChildRunCompleted') ->count()); diff --git a/tests/Fixtures/V2/ReplayRegression/parallel-child-group-stranded-parent-recovery.json b/tests/Fixtures/V2/ReplayRegression/parallel-child-group-stranded-parent-recovery.json new file mode 100644 index 00000000..68c24b5f --- /dev/null +++ b/tests/Fixtures/V2/ReplayRegression/parallel-child-group-stranded-parent-recovery.json @@ -0,0 +1,143 @@ +{ + "$schema": "https://raw.githubusercontent.com/durable-workflow/.github/main/regression-corpus/evidence-schema.json", + "fixture_schema": "durable-workflow.replay-regression/v1", + "id": "parallel-child-group-stranded-parent-recovery", + "protocol_version": "1.0", + "bindings": [ + "php" + ], + "workflow": { + "type": "Tests\\Fixtures\\V2\\TestParallelChildReplayWorkflow", + "arguments": [ + "recovered-first", + "recovered-second" + ], + "payload_codec": "avro" + }, + "history": [ + { + "sequence": 1, + "event_type": "WorkflowStarted", + "payload": {}, + "recorded_at": "2026-09-08T12:00:01+00:00" + }, + { + "sequence": 2, + "event_type": "ChildWorkflowScheduled", + "payload": { + "sequence": 1, + "child_workflow_type": "Tests\\Fixtures\\V2\\TestChildGreetingWorkflow", + "parallel_group_id": "parallel-children:1:2", + "parallel_group_kind": "child", + "parallel_group_base_sequence": 1, + "parallel_group_size": 2, + "parallel_group_index": 0, + "parallel_group_path": [ + { + "parallel_group_id": "parallel-children:1:2", + "parallel_group_kind": "child", + "parallel_group_base_sequence": 1, + "parallel_group_size": 2, + "parallel_group_index": 0 + } + ] + }, + "recorded_at": "2026-09-08T12:00:02+00:00" + }, + { + "sequence": 3, + "event_type": "ChildWorkflowScheduled", + "payload": { + "sequence": 2, + "child_workflow_type": "Tests\\Fixtures\\V2\\TestChildGreetingWorkflow", + "parallel_group_id": "parallel-children:1:2", + "parallel_group_kind": "child", + "parallel_group_base_sequence": 1, + "parallel_group_size": 2, + "parallel_group_index": 1, + "parallel_group_path": [ + { + "parallel_group_id": "parallel-children:1:2", + "parallel_group_kind": "child", + "parallel_group_base_sequence": 1, + "parallel_group_size": 2, + "parallel_group_index": 1 + } + ] + }, + "recorded_at": "2026-09-08T12:00:03+00:00" + }, + { + "sequence": 4, + "event_type": "ChildRunCompleted", + "payload": { + "sequence": 2, + "child_workflow_type": "Tests\\Fixtures\\V2\\TestChildGreetingWorkflow", + "parallel_group_id": "parallel-children:1:2", + "parallel_group_kind": "child", + "parallel_group_base_sequence": 1, + "parallel_group_size": 2, + "parallel_group_index": 1, + "parallel_group_path": [ + { + "parallel_group_id": "parallel-children:1:2", + "parallel_group_kind": "child", + "parallel_group_base_sequence": 1, + "parallel_group_size": 2, + "parallel_group_index": 1 + } + ], + "child_status": "completed", + "payload_codec": "avro", + "output": "wwHioz3/VYAiNw4CCmNoaWxkCiByZWNvdmVyZWQtc2Vjb25kAA==" + }, + "recorded_at": "2026-09-08T12:00:04+00:00" + }, + { + "sequence": 5, + "event_type": "ChildRunCompleted", + "payload": { + "sequence": 1, + "child_workflow_type": "Tests\\Fixtures\\V2\\TestChildGreetingWorkflow", + "parallel_group_id": "parallel-children:1:2", + "parallel_group_kind": "child", + "parallel_group_base_sequence": 1, + "parallel_group_size": 2, + "parallel_group_index": 0, + "parallel_group_path": [ + { + "parallel_group_id": "parallel-children:1:2", + "parallel_group_kind": "child", + "parallel_group_base_sequence": 1, + "parallel_group_size": 2, + "parallel_group_index": 0 + } + ], + "child_status": "completed", + "payload_codec": "avro", + "output": "wwHioz3/VYAiNw4CCmNoaWxkCh5yZWNvdmVyZWQtZmlyc3QA" + }, + "recorded_at": "2026-09-08T12:00:05+00:00" + } + ], + "expected": { + "completed": true, + "result": { + "children": [ + { + "child": "recovered-first" + }, + { + "child": "recovered-second" + } + ], + "workflow_id": "regression-corpus-parallel-child-group-stranded-parent-recovery", + "run_id": "regression-corpus-run-parallel-child-group-stranded-parent-recovery" + }, + "commands": [ + { + "type": "complete_workflow" + } + ] + } +} diff --git a/tests/Fixtures/V2/TestHandledFailureParentWorkflow.php b/tests/Fixtures/V2/TestHandledFailureParentWorkflow.php new file mode 100644 index 00000000..602ce9e7 --- /dev/null +++ b/tests/Fixtures/V2/TestHandledFailureParentWorkflow.php @@ -0,0 +1,16 @@ + child(TestChildGreetingWorkflow::class, (string) $index); + } + + return all($calls); + } +}