Handle temporary worker database unavailability without losing poll identity

app/Auth/AuthException.php

@@ -3,20 +3,22 @@
 namespace App\Auth;
 
 use RuntimeException;
+use Throwable;
 
 final class AuthException extends RuntimeException
 {
     private function __construct(
         private readonly int $status,
         private readonly string $reason,
         string $message,
+        ?Throwable $previous = null,
     ) {
-        parent::__construct($message);
+        parent::__construct($message, 0, $previous);
     }
 
-    public static function configuration(string $message): self
+    public static function configuration(string $message, ?Throwable $previous = null): self
     {
-        return new self(500, 'server_error', $message);
+        return new self(500, 'server_error', $message, $previous);
     }
 
     public static function unauthenticated(string $message): self

app/Auth/ConfiguredAuthProvider.php

@@ -111,8 +111,8 @@ private function authenticateToken(Request $request): Principal
         if ($runtimeCredentialsEnabled) {
             try {
                 $credential = RuntimeCredential::activeForToken($provided);
-            } catch (\Throwable) {
-                throw AuthException::configuration('Database runtime credential authentication is enabled but unavailable.');
+            } catch (\Throwable $exception) {
+                throw AuthException::configuration('Database runtime credential authentication is enabled but unavailable.', $exception);
             }
 
             if ($credential instanceof RuntimeCredential) {

app/Http/Middleware/Authenticate.php

@@ -5,6 +5,7 @@
 use App\Auth\AuthException;
 use App\Auth\Principal;
 use App\Contracts\AuthProvider;
+use App\Support\BackendUnavailable;
 use App\Support\ControlPlaneProtocol;
 use App\Support\RuntimeExternalPayloadAudit;
 use App\Support\WorkerProtocol;
@@ -32,6 +33,11 @@ public function handle(Request $request, Closure $next): Response
         try {
             $principal = $this->authProvider->authenticate($request);
         } catch (AuthException $exception) {
+            if (BackendUnavailable::is($exception)
+                && ($response = BackendUnavailable::workerResponse($request)) !== null) {
+                return $response;
+            }
+
             return self::error($request, $exception->status(), $exception->reason(), $exception->getMessage());
         }
 

app/Http/Middleware/RequireWorkflowBootstrapReady.php

@@ -2,6 +2,7 @@
 
 namespace App\Http\Middleware;
 
+use App\Support\BackendUnavailable;
 use App\Support\ControlPlaneProtocol;
 use App\Support\ServerReadiness;
 use App\Support\WorkerProtocol;
@@ -19,6 +20,11 @@ public function __construct(
     public function handle(Request $request, Closure $next): Response
     {
         $status = $this->readiness->bootstrapStatus();
+        if (($status['reason'] ?? null) === 'backend_unavailable'
+            && ($response = BackendUnavailable::workerResponse($request)) !== null) {
+            return $response;
+        }
+
         $blockedBy = is_array($status['blocked_by'] ?? null)
             ? array_values(array_filter($status['blocked_by'], static fn (mixed $value): bool => is_string($value) && $value !== ''))
             : [];

app/Support/BackendUnavailable.php

@@ -0,0 +1,85 @@
+<?php
+
+namespace App\Support;
+
+use Illuminate\Http\JsonResponse;
+use Illuminate\Http\Request;
+use PDOException;
+use Throwable;
+
+final class BackendUnavailable
+{
+    public static function is(Throwable $exception): bool
+    {
+        for ($current = $exception; $current !== null; $current = $current->getPrevious()) {
+            if (! $current instanceof PDOException || ! is_array($current->errorInfo)) {
+                continue;
+            }
+
+            [$state, $code] = $current->errorInfo + [null, null];
+
+            // Use driver fields, never SQL text or customer-controlled bindings.
+            if (($state === 'HY000' && in_array((int) $code, [2002, 2003, 2006, 2013, 2055], true))
+                || in_array($state, ['08000', '08001', '08003', '08006', '08007', '08S01', '57P01', '57P02', '57P03'], true)) {
+                return true;
+            }
+        }
+
+        return false;
+    }
+
+    public static function workerResponse(Request $request): ?JsonResponse
+    {
+        if (! WorkerProtocol::isWorkerPlaneRequest($request)
+            || ! WorkerProtocol::requestUsesCompatibleProtocolVersion($request)) {
+            return null;
+        }
+
+        $operation = match (true) {
+            $request->is('api/worker/workflow-tasks/poll') => 'poll_workflow_task',
+            $request->is('api/worker/activity-tasks/poll') => 'poll_activity_task',
+            $request->is('api/worker/query-tasks/poll') => 'poll_query_task',
+            $request->is('api/worker/update-validation-tasks/poll') => 'poll_update_validation_task',
+            $request->is('api/worker/register') => 'register_worker',
+            $request->is('api/worker/heartbeat') => 'heartbeat_worker',
+            default => null,
+        };
+        if ($operation === null) {
+            return null;
+        }
+
+        $workerId = self::identity($request->input('worker_id'));
+        $taskQueue = self::identity($request->input('task_queue'));
+        $pollId = self::identity($request->input('poll_request_id'));
+        $isPoll = str_starts_with($operation, 'poll_');
+        $payload = [
+            'reason' => 'backend_unavailable',
+            'message' => 'The database is temporarily unavailable. Retry the same request with backoff; its outcome may be unknown.',
+            'operation' => $operation,
+            'outcome' => 'unknown',
+            'worker_id' => $workerId,
+            'task_queue' => $taskQueue,
+            'retryable' => $workerId !== null
+                && ($operation === 'heartbeat_worker' || $taskQueue !== null)
+                && (! $isPoll || $pollId !== null),
+            'retry_after_seconds' => 1,
+        ];
+        if ($isPoll) {
+            // A lost connection can follow COMMIT. Reusing this ID lets native
+            // lease bindings reconcile it; task=null is not proof of no lease.
+            $payload += [
+                'task' => null,
+                'poll_status' => 'backend_unavailable',
+                'poll_request_id' => $pollId,
+                'retry_same_poll_request_id' => true,
+            ];
+        }
+
+        return WorkerProtocol::json($payload, 503)->header('Retry-After', '1');
+    }
+
+    private static function identity(mixed $value): ?string
+    {
+        return is_string($value) && $value !== '' && strlen($value) <= 255 ? $value : null;
+    }
+}

app/Support/ServerReadiness.php

@@ -92,6 +92,7 @@ private function databaseCheck(): array
             return [
                 'status' => 'unavailable',
                 'message' => $exception->getMessage(),
+                'reason' => BackendUnavailable::is($exception) ? 'backend_unavailable' : null,
             ];
         }
     }
@@ -109,6 +110,7 @@ private function migrationCheck(): array
             return [
                 'status' => 'unavailable',
                 'message' => $exception->getMessage(),
+                'reason' => BackendUnavailable::is($exception) ? 'backend_unavailable' : null,
             ];
         }
 
@@ -243,6 +245,7 @@ private function queueCheck(): array
             return $check + [
                 'status' => 'unavailable',
                 'message' => $exception->getMessage(),
+                'reason' => BackendUnavailable::is($exception) ? 'backend_unavailable' : null,
             ];
         }
     }
@@ -555,17 +558,20 @@ private function workflowCheck(array $checks): array
     private function bootstrapCheck(array $checks): array
     {
         $blockedBy = [];
+        $reason = null;
 
         foreach (['database', 'migrations', 'queue'] as $key) {
             if (! self::statusAllowsReady($checks[$key]['status'] ?? null)) {
                 $blockedBy[] = $key;
+                $reason ??= $checks[$key]['reason'] ?? null;
             }
         }
 
         if ($blockedBy !== []) {
             return [
                 'status' => 'blocked',
                 'blocked_by' => $blockedBy,
+                'reason' => $reason,
                 'remediation' => 'Restore database connectivity and run server-bootstrap to migrate workflow and configured queue storage before serving workflow v2 traffic.',
             ];
         }
@@ -717,7 +723,7 @@ private function normalizeWorkflowCheck(array $check): array
             'checks' => is_array($check['checks'] ?? null) ? array_values($check['checks']) : [],
         ];
 
-        foreach (['blocked_by', 'message', 'remediation'] as $key) {
+        foreach (['blocked_by', 'message', 'remediation', 'reason'] as $key) {
             if (array_key_exists($key, $check)) {
                 $normalized[$key] = $check[$key];
             }

bootstrap/app.php

@@ -4,6 +4,7 @@
 use App\Http\Middleware\EnforcePayloadLimits;
 use App\Http\Middleware\RemoveServerHeader;
 use App\Support\BackendLockPressure;
+use App\Support\BackendUnavailable;
 use App\Support\ControlPlaneFailureDiagnostics;
 use App\Support\ControlPlaneOperation;
 use App\Support\ControlPlaneProtocol;
@@ -40,6 +41,12 @@
         ]);
     })
     ->withExceptions(function (Exceptions $exceptions) {
+        $exceptions->render(function (Throwable $exception, Request $request) {
+            return BackendUnavailable::is($exception)
+                ? BackendUnavailable::workerResponse($request)
+                : null;
+        });
+
         $exceptions->render(function (NamespaceDurableStateException $exception, Request $request) {
             $payload = array_filter([
                 'schema' => 'durable-workflow.v2.namespace-durable-state-error.v1',

composer.json

@@ -48,7 +48,7 @@
     },
     "extra": {
         "durable-workflow": {
-            "product-train": "2.2.0"
+            "product-train": "2.2.1"
         },
         "laravel": {
             "dont-discover": []

docker-compose.dedicated-matching.yml

@@ -32,13 +32,13 @@ name: durable-workflow-server
 # daemon reports `shape: dedicated`.
 
 # Generated by scripts/ci/sync-source-release.mjs. Do not edit the fallback.
-x-server-image: &server-image ${DW_SERVER_IMAGE:-durableworkflow/server:${DW_SERVER_TAG:-2.2.0}}
+x-server-image: &server-image ${DW_SERVER_IMAGE:-durableworkflow/server:${DW_SERVER_TAG:-2.2.1}}
 
 x-server-environment: &server-environment
   APP_NAME: "Durable Workflow Server"
   APP_ENV: ${APP_ENV:-local}
   DW_SERVER_KEY: ${DW_SERVER_KEY:-}
-  APP_VERSION: ${APP_VERSION:-${DW_SERVER_TAG:-2.2.0}}
+  APP_VERSION: ${APP_VERSION:-${DW_SERVER_TAG:-2.2.1}}
   APP_DEBUG: ${APP_DEBUG:-false}
   DB_CONNECTION: mysql
   DB_HOST: mysql

docker-compose.memo-rolling.yml

@@ -49,14 +49,14 @@ services:
     command: ["server-bootstrap"]
     environment:
       <<: *runtime-environment
-      APP_VERSION: ${APP_VERSION:-2.2.0}
+      APP_VERSION: ${APP_VERSION:-2.2.1}
 
   successor:
     image: ${DW_MEMO_SUCCESSOR_IMAGE:-durable-workflow/server-memo-rolling:local}
     ports: !override []
     environment:
       <<: *runtime-environment
-      APP_VERSION: ${APP_VERSION:-2.2.0}
+      APP_VERSION: ${APP_VERSION:-2.2.1}
       DW_SERVER_ID: memo-successor
       DW_SERVER_TOPOLOGY_SHAPE: standalone_server
       DW_SERVER_PROCESS_CLASS: server_http_node

docker-compose.published.yml

@@ -1,13 +1,13 @@
 name: durable-workflow-server
 
 # Generated by scripts/ci/sync-source-release.mjs. Do not edit the fallback.
-x-server-image: &server-image ${DW_SERVER_IMAGE:-durableworkflow/server:${DW_SERVER_TAG:-2.2.0}}
+x-server-image: &server-image ${DW_SERVER_IMAGE:-durableworkflow/server:${DW_SERVER_TAG:-2.2.1}}
 
 x-server-environment: &server-environment
   APP_NAME: "Durable Workflow Server"
   APP_ENV: ${APP_ENV:-local}
   DW_SERVER_KEY: ${DW_SERVER_KEY:-}
-  APP_VERSION: ${APP_VERSION:-${DW_SERVER_TAG:-2.2.0}}
+  APP_VERSION: ${APP_VERSION:-${DW_SERVER_TAG:-2.2.1}}
   APP_DEBUG: ${APP_DEBUG:-false}
   LOG_CHANNEL: ${LOG_CHANNEL:-stderr}
   LOG_LEVEL: ${LOG_LEVEL:-info}

docker-compose.small-cluster.yml

@@ -12,7 +12,7 @@ x-server-build: &server-build
 x-server-environment: &server-environment
   APP_NAME: "Durable Workflow Server"
   APP_ENV: testing
-  APP_VERSION: ${APP_VERSION:-2.2.0}
+  APP_VERSION: ${APP_VERSION:-2.2.1}
   APP_DEBUG: "false"
   DW_SERVER_KEY: ${DW_SERVER_KEY:-base64:5Zt4nUhlCm3DD0nLXZJQdHiwPfb56yGo9gNV/g3jYbY=}
   DB_CONNECTION: ${DW_SMALL_CLUSTER_DB:-mysql}

docker-compose.yml

@@ -15,7 +15,7 @@ services:
       DW_SERVER_KEY: "${DW_SERVER_KEY:-}"
       DW_SERVER_TOPOLOGY_SHAPE: standalone_server
       DW_SERVER_PROCESS_CLASS: server_http_node
-      APP_VERSION: "${APP_VERSION:-2.2.0}"
+      APP_VERSION: "${APP_VERSION:-2.2.1}"
       APP_DEBUG: "false"
       DB_CONNECTION: mysql
       DB_HOST: mysql
@@ -62,7 +62,7 @@ services:
       APP_NAME: "Durable Workflow Server"
       APP_ENV: local
       DW_SERVER_KEY: "${DW_SERVER_KEY:-}"
-      APP_VERSION: "${APP_VERSION:-2.2.0}"
+      APP_VERSION: "${APP_VERSION:-2.2.1}"
       APP_DEBUG: "false"
       DB_CONNECTION: mysql
       DB_HOST: mysql
@@ -123,7 +123,7 @@ services:
       DW_SERVER_KEY: "${DW_SERVER_KEY:-}"
       DW_SERVER_TOPOLOGY_SHAPE: standalone_server
       DW_SERVER_PROCESS_CLASS: worker_node
-      APP_VERSION: "${APP_VERSION:-2.2.0}"
+      APP_VERSION: "${APP_VERSION:-2.2.1}"
       DB_CONNECTION: mysql
       DB_HOST: mysql
       DB_PORT: 3306
@@ -176,7 +176,7 @@ services:
       DW_SERVER_KEY: "${DW_SERVER_KEY:-}"
       DW_SERVER_TOPOLOGY_SHAPE: standalone_server
       DW_SERVER_PROCESS_CLASS: scheduler_node
-      APP_VERSION: "${APP_VERSION:-2.2.0}"
+      APP_VERSION: "${APP_VERSION:-2.2.1}"
       DB_CONNECTION: mysql
       DB_HOST: mysql
       DB_PORT: 3306

k8s/README.md

@@ -13,23 +13,23 @@ The checked-in manifests are synchronized with the repository's stable source
 release and pin its Docker Hub tag:
 
 ```text
-durableworkflow/server:2.2.0
+durableworkflow/server:2.2.1
 ```
 
 Before production use, patch every workload image to the exact published tag or
 digest you intend to run:
 
 ```bash
 kubectl set image -n durable-workflow deploy/durable-workflow-server \
-  server=durableworkflow/server:2.2.0
+  server=durableworkflow/server:2.2.1
 kubectl set image -n durable-workflow deploy/durable-workflow-worker \
-  worker=durableworkflow/server:2.2.0
+  worker=durableworkflow/server:2.2.1
 kubectl set image -n durable-workflow cronjob/durable-workflow-scheduler \
-  scheduler=durableworkflow/server:2.2.0
+  scheduler=durableworkflow/server:2.2.1
 ```
 
 GitHub Container Registry publishes the same release line at
-`ghcr.io/durable-workflow/server:2.2.0`. Digest pinning is preferred for strict
+`ghcr.io/durable-workflow/server:2.2.1`. Digest pinning is preferred for strict
 change control.
 
 The manifests expect you to provide: