diff --git a/app/Auth/AuthException.php b/app/Auth/AuthException.php index 49772976..8d38d5ae 100644 --- a/app/Auth/AuthException.php +++ b/app/Auth/AuthException.php @@ -3,6 +3,7 @@ namespace App\Auth; use RuntimeException; +use Throwable; final class AuthException extends RuntimeException { @@ -10,13 +11,14 @@ private function __construct( private readonly int $status, private readonly string $reason, string $message, + ?Throwable $previous = null, ) { - parent::__construct($message); + parent::__construct($message, 0, $previous); } - public static function configuration(string $message): self + public static function configuration(string $message, ?Throwable $previous = null): self { - return new self(500, 'server_error', $message); + return new self(500, 'server_error', $message, $previous); } public static function unauthenticated(string $message): self diff --git a/app/Auth/ConfiguredAuthProvider.php b/app/Auth/ConfiguredAuthProvider.php index dbafea60..a638d041 100644 --- a/app/Auth/ConfiguredAuthProvider.php +++ b/app/Auth/ConfiguredAuthProvider.php @@ -111,8 +111,8 @@ private function authenticateToken(Request $request): Principal if ($runtimeCredentialsEnabled) { try { $credential = RuntimeCredential::activeForToken($provided); - } catch (\Throwable) { - throw AuthException::configuration('Database runtime credential authentication is enabled but unavailable.'); + } catch (\Throwable $exception) { + throw AuthException::configuration('Database runtime credential authentication is enabled but unavailable.', $exception); } if ($credential instanceof RuntimeCredential) { diff --git a/app/Http/Middleware/Authenticate.php b/app/Http/Middleware/Authenticate.php index 4063864e..3a856172 100644 --- a/app/Http/Middleware/Authenticate.php +++ b/app/Http/Middleware/Authenticate.php @@ -5,6 +5,7 @@ use App\Auth\AuthException; use App\Auth\Principal; use App\Contracts\AuthProvider; +use App\Support\BackendUnavailable; use App\Support\ControlPlaneProtocol; use App\Support\RuntimeExternalPayloadAudit; use App\Support\WorkerProtocol; @@ -32,6 +33,11 @@ public function handle(Request $request, Closure $next): Response try { $principal = $this->authProvider->authenticate($request); } catch (AuthException $exception) { + if (BackendUnavailable::is($exception) + && ($response = BackendUnavailable::workerResponse($request)) !== null) { + return $response; + } + return self::error($request, $exception->status(), $exception->reason(), $exception->getMessage()); } diff --git a/app/Http/Middleware/RequireWorkflowBootstrapReady.php b/app/Http/Middleware/RequireWorkflowBootstrapReady.php index fee3e372..fd9f528a 100644 --- a/app/Http/Middleware/RequireWorkflowBootstrapReady.php +++ b/app/Http/Middleware/RequireWorkflowBootstrapReady.php @@ -2,6 +2,7 @@ namespace App\Http\Middleware; +use App\Support\BackendUnavailable; use App\Support\ControlPlaneProtocol; use App\Support\ServerReadiness; use App\Support\WorkerProtocol; @@ -19,6 +20,11 @@ public function __construct( public function handle(Request $request, Closure $next): Response { $status = $this->readiness->bootstrapStatus(); + if (($status['reason'] ?? null) === 'backend_unavailable' + && ($response = BackendUnavailable::workerResponse($request)) !== null) { + return $response; + } + $blockedBy = is_array($status['blocked_by'] ?? null) ? array_values(array_filter($status['blocked_by'], static fn (mixed $value): bool => is_string($value) && $value !== '')) : []; diff --git a/app/Support/BackendUnavailable.php b/app/Support/BackendUnavailable.php new file mode 100644 index 00000000..2302b702 --- /dev/null +++ b/app/Support/BackendUnavailable.php @@ -0,0 +1,85 @@ +getPrevious()) { + if (! $current instanceof PDOException || ! is_array($current->errorInfo)) { + continue; + } + + [$state, $code] = $current->errorInfo + [null, null]; + + // Use driver fields, never SQL text or customer-controlled bindings. + if (($state === 'HY000' && in_array((int) $code, [2002, 2003, 2006, 2013, 2055], true)) + || in_array($state, ['08000', '08001', '08003', '08006', '08007', '08S01', '57P01', '57P02', '57P03'], true)) { + return true; + } + } + + return false; + } + + public static function workerResponse(Request $request): ?JsonResponse + { + if (! WorkerProtocol::isWorkerPlaneRequest($request) + || ! WorkerProtocol::requestUsesCompatibleProtocolVersion($request)) { + return null; + } + + $operation = match (true) { + $request->is('api/worker/workflow-tasks/poll') => 'poll_workflow_task', + $request->is('api/worker/activity-tasks/poll') => 'poll_activity_task', + $request->is('api/worker/query-tasks/poll') => 'poll_query_task', + $request->is('api/worker/update-validation-tasks/poll') => 'poll_update_validation_task', + $request->is('api/worker/register') => 'register_worker', + $request->is('api/worker/heartbeat') => 'heartbeat_worker', + default => null, + }; + if ($operation === null) { + return null; + } + + $workerId = self::identity($request->input('worker_id')); + $taskQueue = self::identity($request->input('task_queue')); + $pollId = self::identity($request->input('poll_request_id')); + $isPoll = str_starts_with($operation, 'poll_'); + $payload = [ + 'reason' => 'backend_unavailable', + 'message' => 'The database is temporarily unavailable. Retry the same request with backoff; its outcome may be unknown.', + 'operation' => $operation, + 'outcome' => 'unknown', + 'worker_id' => $workerId, + 'task_queue' => $taskQueue, + 'retryable' => $workerId !== null + && ($operation === 'heartbeat_worker' || $taskQueue !== null) + && (! $isPoll || $pollId !== null), + 'retry_after_seconds' => 1, + ]; + if ($isPoll) { + // A lost connection can follow COMMIT. Reusing this ID lets native + // lease bindings reconcile it; task=null is not proof of no lease. + $payload += [ + 'task' => null, + 'poll_status' => 'backend_unavailable', + 'poll_request_id' => $pollId, + 'retry_same_poll_request_id' => true, + ]; + } + + return WorkerProtocol::json($payload, 503)->header('Retry-After', '1'); + } + + private static function identity(mixed $value): ?string + { + return is_string($value) && $value !== '' && strlen($value) <= 255 ? $value : null; + } +} diff --git a/app/Support/ServerReadiness.php b/app/Support/ServerReadiness.php index d9501d65..bf9abebd 100644 --- a/app/Support/ServerReadiness.php +++ b/app/Support/ServerReadiness.php @@ -92,6 +92,7 @@ private function databaseCheck(): array return [ 'status' => 'unavailable', 'message' => $exception->getMessage(), + 'reason' => BackendUnavailable::is($exception) ? 'backend_unavailable' : null, ]; } } @@ -109,6 +110,7 @@ private function migrationCheck(): array return [ 'status' => 'unavailable', 'message' => $exception->getMessage(), + 'reason' => BackendUnavailable::is($exception) ? 'backend_unavailable' : null, ]; } @@ -243,6 +245,7 @@ private function queueCheck(): array return $check + [ 'status' => 'unavailable', 'message' => $exception->getMessage(), + 'reason' => BackendUnavailable::is($exception) ? 'backend_unavailable' : null, ]; } } @@ -555,10 +558,12 @@ private function workflowCheck(array $checks): array private function bootstrapCheck(array $checks): array { $blockedBy = []; + $reason = null; foreach (['database', 'migrations', 'queue'] as $key) { if (! self::statusAllowsReady($checks[$key]['status'] ?? null)) { $blockedBy[] = $key; + $reason ??= $checks[$key]['reason'] ?? null; } } @@ -566,6 +571,7 @@ private function bootstrapCheck(array $checks): array return [ 'status' => 'blocked', 'blocked_by' => $blockedBy, + 'reason' => $reason, 'remediation' => 'Restore database connectivity and run server-bootstrap to migrate workflow and configured queue storage before serving workflow v2 traffic.', ]; } @@ -717,7 +723,7 @@ private function normalizeWorkflowCheck(array $check): array 'checks' => is_array($check['checks'] ?? null) ? array_values($check['checks']) : [], ]; - foreach (['blocked_by', 'message', 'remediation'] as $key) { + foreach (['blocked_by', 'message', 'remediation', 'reason'] as $key) { if (array_key_exists($key, $check)) { $normalized[$key] = $check[$key]; } diff --git a/bootstrap/app.php b/bootstrap/app.php index 10904bcb..d6f735bd 100644 --- a/bootstrap/app.php +++ b/bootstrap/app.php @@ -4,6 +4,7 @@ use App\Http\Middleware\EnforcePayloadLimits; use App\Http\Middleware\RemoveServerHeader; use App\Support\BackendLockPressure; +use App\Support\BackendUnavailable; use App\Support\ControlPlaneFailureDiagnostics; use App\Support\ControlPlaneOperation; use App\Support\ControlPlaneProtocol; @@ -40,6 +41,12 @@ ]); }) ->withExceptions(function (Exceptions $exceptions) { + $exceptions->render(function (Throwable $exception, Request $request) { + return BackendUnavailable::is($exception) + ? BackendUnavailable::workerResponse($request) + : null; + }); + $exceptions->render(function (NamespaceDurableStateException $exception, Request $request) { $payload = array_filter([ 'schema' => 'durable-workflow.v2.namespace-durable-state-error.v1', diff --git a/composer.json b/composer.json index f70de7f5..31febed9 100644 --- a/composer.json +++ b/composer.json @@ -48,7 +48,7 @@ }, "extra": { "durable-workflow": { - "product-train": "2.2.0" + "product-train": "2.2.1" }, "laravel": { "dont-discover": [] diff --git a/composer.lock b/composer.lock index fefc9bb2..fcf08cad 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "3ae0ae62f86d838b1bb51b98c0595c25", + "content-hash": "74de81dfe7587091bf9b40d221ffb410", "packages": [ { "name": "apache/avro", diff --git a/docker-compose.dedicated-matching.yml b/docker-compose.dedicated-matching.yml index f6ca4283..93264db2 100644 --- a/docker-compose.dedicated-matching.yml +++ b/docker-compose.dedicated-matching.yml @@ -32,13 +32,13 @@ name: durable-workflow-server # daemon reports `shape: dedicated`. # Generated by scripts/ci/sync-source-release.mjs. Do not edit the fallback. -x-server-image: &server-image ${DW_SERVER_IMAGE:-durableworkflow/server:${DW_SERVER_TAG:-2.2.0}} +x-server-image: &server-image ${DW_SERVER_IMAGE:-durableworkflow/server:${DW_SERVER_TAG:-2.2.1}} x-server-environment: &server-environment APP_NAME: "Durable Workflow Server" APP_ENV: ${APP_ENV:-local} DW_SERVER_KEY: ${DW_SERVER_KEY:-} - APP_VERSION: ${APP_VERSION:-${DW_SERVER_TAG:-2.2.0}} + APP_VERSION: ${APP_VERSION:-${DW_SERVER_TAG:-2.2.1}} APP_DEBUG: ${APP_DEBUG:-false} DB_CONNECTION: mysql DB_HOST: mysql diff --git a/docker-compose.memo-rolling.yml b/docker-compose.memo-rolling.yml index 8da222b7..29a2b6ed 100644 --- a/docker-compose.memo-rolling.yml +++ b/docker-compose.memo-rolling.yml @@ -49,14 +49,14 @@ services: command: ["server-bootstrap"] environment: <<: *runtime-environment - APP_VERSION: ${APP_VERSION:-2.2.0} + APP_VERSION: ${APP_VERSION:-2.2.1} successor: image: ${DW_MEMO_SUCCESSOR_IMAGE:-durable-workflow/server-memo-rolling:local} ports: !override [] environment: <<: *runtime-environment - APP_VERSION: ${APP_VERSION:-2.2.0} + APP_VERSION: ${APP_VERSION:-2.2.1} DW_SERVER_ID: memo-successor DW_SERVER_TOPOLOGY_SHAPE: standalone_server DW_SERVER_PROCESS_CLASS: server_http_node diff --git a/docker-compose.published.yml b/docker-compose.published.yml index c920149b..7a5f924a 100644 --- a/docker-compose.published.yml +++ b/docker-compose.published.yml @@ -1,13 +1,13 @@ name: durable-workflow-server # Generated by scripts/ci/sync-source-release.mjs. Do not edit the fallback. -x-server-image: &server-image ${DW_SERVER_IMAGE:-durableworkflow/server:${DW_SERVER_TAG:-2.2.0}} +x-server-image: &server-image ${DW_SERVER_IMAGE:-durableworkflow/server:${DW_SERVER_TAG:-2.2.1}} x-server-environment: &server-environment APP_NAME: "Durable Workflow Server" APP_ENV: ${APP_ENV:-local} DW_SERVER_KEY: ${DW_SERVER_KEY:-} - APP_VERSION: ${APP_VERSION:-${DW_SERVER_TAG:-2.2.0}} + APP_VERSION: ${APP_VERSION:-${DW_SERVER_TAG:-2.2.1}} APP_DEBUG: ${APP_DEBUG:-false} LOG_CHANNEL: ${LOG_CHANNEL:-stderr} LOG_LEVEL: ${LOG_LEVEL:-info} diff --git a/docker-compose.small-cluster.yml b/docker-compose.small-cluster.yml index 52b496c0..b9265a9a 100644 --- a/docker-compose.small-cluster.yml +++ b/docker-compose.small-cluster.yml @@ -12,7 +12,7 @@ x-server-build: &server-build x-server-environment: &server-environment APP_NAME: "Durable Workflow Server" APP_ENV: testing - APP_VERSION: ${APP_VERSION:-2.2.0} + APP_VERSION: ${APP_VERSION:-2.2.1} APP_DEBUG: "false" DW_SERVER_KEY: ${DW_SERVER_KEY:-base64:5Zt4nUhlCm3DD0nLXZJQdHiwPfb56yGo9gNV/g3jYbY=} DB_CONNECTION: ${DW_SMALL_CLUSTER_DB:-mysql} diff --git a/docker-compose.yml b/docker-compose.yml index f281e0a5..180a187c 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -15,7 +15,7 @@ services: DW_SERVER_KEY: "${DW_SERVER_KEY:-}" DW_SERVER_TOPOLOGY_SHAPE: standalone_server DW_SERVER_PROCESS_CLASS: server_http_node - APP_VERSION: "${APP_VERSION:-2.2.0}" + APP_VERSION: "${APP_VERSION:-2.2.1}" APP_DEBUG: "false" DB_CONNECTION: mysql DB_HOST: mysql @@ -62,7 +62,7 @@ services: APP_NAME: "Durable Workflow Server" APP_ENV: local DW_SERVER_KEY: "${DW_SERVER_KEY:-}" - APP_VERSION: "${APP_VERSION:-2.2.0}" + APP_VERSION: "${APP_VERSION:-2.2.1}" APP_DEBUG: "false" DB_CONNECTION: mysql DB_HOST: mysql @@ -123,7 +123,7 @@ services: DW_SERVER_KEY: "${DW_SERVER_KEY:-}" DW_SERVER_TOPOLOGY_SHAPE: standalone_server DW_SERVER_PROCESS_CLASS: worker_node - APP_VERSION: "${APP_VERSION:-2.2.0}" + APP_VERSION: "${APP_VERSION:-2.2.1}" DB_CONNECTION: mysql DB_HOST: mysql DB_PORT: 3306 @@ -176,7 +176,7 @@ services: DW_SERVER_KEY: "${DW_SERVER_KEY:-}" DW_SERVER_TOPOLOGY_SHAPE: standalone_server DW_SERVER_PROCESS_CLASS: scheduler_node - APP_VERSION: "${APP_VERSION:-2.2.0}" + APP_VERSION: "${APP_VERSION:-2.2.1}" DB_CONNECTION: mysql DB_HOST: mysql DB_PORT: 3306 diff --git a/k8s/README.md b/k8s/README.md index 6f0ef60c..70c12d19 100644 --- a/k8s/README.md +++ b/k8s/README.md @@ -13,7 +13,7 @@ The checked-in manifests are synchronized with the repository's stable source release and pin its Docker Hub tag: ```text -durableworkflow/server:2.2.0 +durableworkflow/server:2.2.1 ``` Before production use, patch every workload image to the exact published tag or @@ -21,15 +21,15 @@ digest you intend to run: ```bash kubectl set image -n durable-workflow deploy/durable-workflow-server \ - server=durableworkflow/server:2.2.0 + server=durableworkflow/server:2.2.1 kubectl set image -n durable-workflow deploy/durable-workflow-worker \ - worker=durableworkflow/server:2.2.0 + worker=durableworkflow/server:2.2.1 kubectl set image -n durable-workflow cronjob/durable-workflow-scheduler \ - scheduler=durableworkflow/server:2.2.0 + scheduler=durableworkflow/server:2.2.1 ``` GitHub Container Registry publishes the same release line at -`ghcr.io/durable-workflow/server:2.2.0`. Digest pinning is preferred for strict +`ghcr.io/durable-workflow/server:2.2.1`. Digest pinning is preferred for strict change control. The manifests expect you to provide: diff --git a/k8s/helm/durable-workflow/Chart.yaml b/k8s/helm/durable-workflow/Chart.yaml index c6487425..da98083c 100644 --- a/k8s/helm/durable-workflow/Chart.yaml +++ b/k8s/helm/durable-workflow/Chart.yaml @@ -5,11 +5,11 @@ type: application # The chart's own semver version. Bumped on every chart release; treated as # independent of the server image version (appVersion). Breaking-change rules # for this version live in docs/helm-upgrading.md alongside the chart. -version: 0.1.77 +version: 0.1.78 # The immutable Durable Workflow Server identity this chart release packages. # The onboarding default in values.yaml and appVersion are generated from the # checked-in source release record. -appVersion: "2.2.0" +appVersion: "2.2.1" kubeVersion: ">=1.27.0-0" home: https://durable-workflow.github.io/docs/2.0/deployment sources: @@ -30,7 +30,7 @@ annotations: # exact commit that most recently changed the packaged chart. org.opencontainers.image.source: https://github.com/durable-workflow/server dev.durable-workflow.source-revision: "unreleased" - dev.durable-workflow.image-reference: "docker.io/durableworkflow/server:2.2.0" + dev.durable-workflow.image-reference: "docker.io/durableworkflow/server:2.2.1" artifacthub.io/license: MIT artifacthub.io/category: integration-delivery # Free-form changelog for the current chart release shown by Artifact Hub. diff --git a/k8s/helm/durable-workflow/README.md b/k8s/helm/durable-workflow/README.md index d6f6545d..c857760e 100644 --- a/k8s/helm/durable-workflow/README.md +++ b/k8s/helm/durable-workflow/README.md @@ -63,7 +63,7 @@ helm install durable-workflow ./k8s/helm/durable-workflow \ ```yaml image: - tag: "2.2.0" + tag: "2.2.1" # Pin a digest in production: # digest: "sha256:abc123..." # memoPayloadStorage: "raw-json-v1" # Required for a digest or custom image. diff --git a/k8s/helm/durable-workflow/ci/existing-secrets-values.yaml b/k8s/helm/durable-workflow/ci/existing-secrets-values.yaml index 5d5f03c5..c54ecfde 100644 --- a/k8s/helm/durable-workflow/ci/existing-secrets-values.yaml +++ b/k8s/helm/durable-workflow/ci/existing-secrets-values.yaml @@ -1,7 +1,7 @@ # CI fixture: GitOps / externally-managed-secret path. The chart consumes # existing Secrets and renders no Secret resources of its own. image: - tag: "2.2.0" + tag: "2.2.1" externalDatabase: connection: pgsql diff --git a/k8s/helm/durable-workflow/ci/ingress-and-hpa-values.yaml b/k8s/helm/durable-workflow/ci/ingress-and-hpa-values.yaml index 7366fac3..b6064ff5 100644 --- a/k8s/helm/durable-workflow/ci/ingress-and-hpa-values.yaml +++ b/k8s/helm/durable-workflow/ci/ingress-and-hpa-values.yaml @@ -1,6 +1,6 @@ # CI fixture: ingress + autoscaling enabled. Exercises optional templates. image: - tag: "2.2.0" + tag: "2.2.1" externalDatabase: connection: mysql diff --git a/k8s/helm/durable-workflow/ci/inline-secrets-values.yaml b/k8s/helm/durable-workflow/ci/inline-secrets-values.yaml index 6d13a9be..ad403c2c 100644 --- a/k8s/helm/durable-workflow/ci/inline-secrets-values.yaml +++ b/k8s/helm/durable-workflow/ci/inline-secrets-values.yaml @@ -2,7 +2,7 @@ # chart's render path is exercised end-to-end. Real deployments should use # existingSecret instead. image: - tag: "2.2.0" + tag: "2.2.1" externalDatabase: connection: mysql diff --git a/k8s/helm/durable-workflow/templates/_helpers.tpl b/k8s/helm/durable-workflow/templates/_helpers.tpl index 1516e526..c624de0e 100644 --- a/k8s/helm/durable-workflow/templates/_helpers.tpl +++ b/k8s/helm/durable-workflow/templates/_helpers.tpl @@ -88,7 +88,7 @@ resolved by an explicit capability declaration or an existing workload marker. {{- define "durable-workflow.memoPayloadStorageForImage" -}} {{- $image := toString . -}} {{- $normalized := regexReplaceAll "^index\\.docker\\.io/" $image "docker.io/" -}} -{{- if eq $normalized "docker.io/durableworkflow/server:2.2.0" -}} +{{- if eq $normalized "docker.io/durableworkflow/server:2.2.1" -}} dual-v1 {{- else if regexMatch "^docker\\.io/durableworkflow/server:2\\.0\\.0-rc\\.[0-9]+$" $normalized -}} {{- $releaseCandidate := atoi (regexFind "[0-9]+$" $normalized) -}} diff --git a/k8s/helm/durable-workflow/values.yaml b/k8s/helm/durable-workflow/values.yaml index b55c15dc..22d167de 100644 --- a/k8s/helm/durable-workflow/values.yaml +++ b/k8s/helm/durable-workflow/values.yaml @@ -21,7 +21,7 @@ image: registry: docker.io repository: durableworkflow/server # Generated by scripts/ci/sync-source-release.mjs. Do not edit this default. - tag: "2.2.0" + tag: "2.2.1" # Optional digest pin. When set, takes precedence over tag for change control. # Example: "sha256:abc123..." digest: "" diff --git a/k8s/helm/examples/values-dev.yaml b/k8s/helm/examples/values-dev.yaml index ac1f6439..5b3cfbd5 100644 --- a/k8s/helm/examples/values-dev.yaml +++ b/k8s/helm/examples/values-dev.yaml @@ -3,7 +3,7 @@ # shape in production. image: - tag: "2.2.0" + tag: "2.2.1" externalDatabase: connection: mysql diff --git a/k8s/helm/examples/values-external-secrets-operator.yaml b/k8s/helm/examples/values-external-secrets-operator.yaml index f9ec5beb..24d852e9 100644 --- a/k8s/helm/examples/values-external-secrets-operator.yaml +++ b/k8s/helm/examples/values-external-secrets-operator.yaml @@ -5,7 +5,7 @@ # concern. image: - tag: "2.2.0" + tag: "2.2.1" externalDatabase: connection: pgsql diff --git a/k8s/helm/examples/values-production-existing-secrets.yaml b/k8s/helm/examples/values-production-existing-secrets.yaml index 06936248..946ed456 100644 --- a/k8s/helm/examples/values-production-existing-secrets.yaml +++ b/k8s/helm/examples/values-production-existing-secrets.yaml @@ -10,7 +10,7 @@ image: repository: durable-workflow/server # Pin a digest in production for change-control auditability. digest: "" # e.g. "sha256:abc123..." - tag: "2.2.0" + tag: "2.2.1" externalDatabase: connection: pgsql diff --git a/k8s/migration-job.yaml b/k8s/migration-job.yaml index 868ffa8a..b7dc0088 100644 --- a/k8s/migration-job.yaml +++ b/k8s/migration-job.yaml @@ -13,7 +13,7 @@ spec: restartPolicy: OnFailure containers: - name: migrate - image: durableworkflow/server:2.2.0 + image: durableworkflow/server:2.2.1 command: ["server-entrypoint"] args: ["server-bootstrap"] envFrom: diff --git a/k8s/scheduler-cronjob.yaml b/k8s/scheduler-cronjob.yaml index 91b9fa61..1bcd6a6a 100644 --- a/k8s/scheduler-cronjob.yaml +++ b/k8s/scheduler-cronjob.yaml @@ -24,7 +24,7 @@ spec: restartPolicy: Never containers: - name: scheduler - image: durableworkflow/server:2.2.0 + image: durableworkflow/server:2.2.1 command: ["server-entrypoint"] args: ["sh", "-c", "php artisan schedule:evaluate --limit=100 --json; php artisan activity:timeout-enforce --limit=100; if php artisan list --raw | grep -q '^external-payloads:cleanup '; then php artisan external-payloads:cleanup --limit=100 --json; fi; php artisan history:prune --limit=100"] envFrom: diff --git a/k8s/secret.yaml b/k8s/secret.yaml index 96287437..1f54411e 100644 --- a/k8s/secret.yaml +++ b/k8s/secret.yaml @@ -12,7 +12,7 @@ metadata: app.kubernetes.io/name: durable-workflow data: APP_NAME: "Durable Workflow Server" - APP_VERSION: "2.2.0" + APP_VERSION: "2.2.1" APP_ENV: production APP_DEBUG: "false" DB_CONNECTION: mysql diff --git a/k8s/server-deployment.yaml b/k8s/server-deployment.yaml index 9ba93f9c..4e2cf0d4 100644 --- a/k8s/server-deployment.yaml +++ b/k8s/server-deployment.yaml @@ -23,7 +23,7 @@ spec: spec: containers: - name: server - image: durableworkflow/server:2.2.0 + image: durableworkflow/server:2.2.1 ports: - containerPort: 8080 name: http diff --git a/k8s/worker-deployment.yaml b/k8s/worker-deployment.yaml index 0e147211..1c08961f 100644 --- a/k8s/worker-deployment.yaml +++ b/k8s/worker-deployment.yaml @@ -19,7 +19,7 @@ spec: spec: containers: - name: worker - image: durableworkflow/server:2.2.0 + image: durableworkflow/server:2.2.1 command: ["server-entrypoint"] args: ["php", "artisan", "queue:work", "--sleep=1", "--tries=3", "--max-time=3600"] envFrom: diff --git a/resources/platform-protocol-specs/worker-protocol-api.openapi.yaml b/resources/platform-protocol-specs/worker-protocol-api.openapi.yaml index 7d1e2784..dbf5d409 100644 --- a/resources/platform-protocol-specs/worker-protocol-api.openapi.yaml +++ b/resources/platform-protocol-specs/worker-protocol-api.openapi.yaml @@ -1,7 +1,7 @@ openapi: 3.1.0 info: title: durable-workflow.v2.worker-protocol-api - version: "18" + version: "19" summary: Durable Workflow worker-plane HTTP+JSON API description: > Normative OpenAPI specification for the worker-plane HTTP+JSON API: @@ -208,6 +208,7 @@ paths: "400": { $ref: "#/components/responses/WorkerError" } "409": { $ref: "#/components/responses/WorkerRegistrationFailure" } "422": { $ref: "#/components/responses/WorkerError" } + "503": { $ref: "#/components/responses/WorkerServiceUnavailable" } /worker/registrations/{workerId}: delete: operationId: deregisterWorker @@ -242,6 +243,7 @@ paths: responses: "200": { $ref: "#/components/responses/WorkerEnvelope" } "404": { $ref: "#/components/responses/WorkerError" } + "503": { $ref: "#/components/responses/WorkerServiceUnavailable" } /worker/sessions: post: operationId: createWorkerSession @@ -312,6 +314,7 @@ paths: "429": { $ref: "#/components/responses/WorkerPollBackpressure" } "409": { $ref: "#/components/responses/WorkflowTaskPollConflict" } "422": { $ref: "#/components/responses/WorkerError" } + "503": { $ref: "#/components/responses/WorkerServiceUnavailable" } /worker/workflow-tasks/{taskId}/history: post: operationId: getWorkflowTaskHistoryPage @@ -399,6 +402,7 @@ paths: "409": { $ref: "#/components/responses/WorkerError" } "429": { $ref: "#/components/responses/WorkerPollBackpressure" } "422": { $ref: "#/components/responses/WorkerError" } + "503": { $ref: "#/components/responses/WorkerServiceUnavailable" } /worker/update-validation-tasks/{taskId}/approve: post: operationId: approveUpdateValidationTask @@ -446,6 +450,7 @@ paths: responses: "200": { $ref: "#/components/responses/WorkerEnvelope" } "429": { $ref: "#/components/responses/WorkerPollBackpressure" } + "503": { $ref: "#/components/responses/WorkerServiceUnavailable" } /worker/query-tasks/{queryTaskId}/complete: post: operationId: completeQueryTask @@ -504,6 +509,7 @@ paths: - $ref: "#/components/schemas/EmptyPoll" - $ref: "#/components/schemas/DrainingPoll" "429": { $ref: "#/components/responses/WorkerPollBackpressure" } + "503": { $ref: "#/components/responses/WorkerServiceUnavailable" } /worker/activity-tasks/{taskId}/heartbeat: post: operationId: heartbeatActivityTask @@ -598,6 +604,25 @@ components: content: application/json: schema: { $ref: "#/components/schemas/WorkerError" } + WorkerServiceUnavailable: + description: > + Temporary database connection loss returns backend_unavailable with an + unknown operation outcome. Retry with bounded backoff and the identical + worker, queue and poll_request_id; task=null does not prove that no task + was claimed. Registration and heartbeat likewise do not acknowledge or + reject an operation whose result is unknown. Other 503 errors retain + their own contracts and are not automatically retryable. + headers: + X-Durable-Workflow-Protocol-Version: + schema: { $ref: "#/components/schemas/AdvertisedWorkerProtocolVersion" } + Retry-After: + schema: { type: integer, minimum: 1 } + content: + application/json: + schema: + anyOf: + - $ref: "#/components/schemas/WorkerBackendUnavailable" + - $ref: "#/components/schemas/WorkerError" WorkerPollBackpressure: description: The node has no capacity for another held long poll; retry after the advertised delay. headers: @@ -736,6 +761,31 @@ components: reason: { type: [string, "null"] } remediation: { type: [string, "null"] } server_capabilities: { $ref: "#/components/schemas/WorkerServerCapabilities" } + WorkerBackendUnavailable: + allOf: + - $ref: "#/components/schemas/WorkerError" + - type: object + required: [reason, operation, outcome, worker_id, task_queue, retryable, retry_after_seconds] + properties: + reason: { type: string, const: backend_unavailable } + operation: + type: string + enum: [register_worker, heartbeat_worker, poll_workflow_task, poll_activity_task, poll_query_task, poll_update_validation_task] + outcome: { type: string, const: unknown } + worker_id: { type: [string, "null"], minLength: 1, maxLength: 255 } + task_queue: { type: [string, "null"], minLength: 1, maxLength: 255 } + retryable: { type: boolean } + retry_after_seconds: { type: integer, minimum: 1 } + if: + properties: + operation: { enum: [poll_workflow_task, poll_activity_task, poll_query_task, poll_update_validation_task] } + then: + required: [task, poll_status, poll_request_id, retry_same_poll_request_id] + properties: + task: { type: "null" } + poll_status: { type: string, const: backend_unavailable } + poll_request_id: { type: [string, "null"], minLength: 1, maxLength: 255 } + retry_same_poll_request_id: { type: boolean, const: true } WorkerPollBackpressure: allOf: - $ref: "#/components/schemas/WorkerError" diff --git a/resources/release/source-release.json b/resources/release/source-release.json index ea02bfa2..761722c0 100644 --- a/resources/release/source-release.json +++ b/resources/release/source-release.json @@ -1,9 +1,9 @@ { "schema": "durable-workflow.server.source-release/v1", "server": { - "version": "2.2.0" + "version": "2.2.1" }, "helm_chart": { - "version": "0.1.77" + "version": "0.1.78" } } diff --git a/scripts/k8s-kind-smoke.sh b/scripts/k8s-kind-smoke.sh index 95184d68..3f846aa5 100755 --- a/scripts/k8s-kind-smoke.sh +++ b/scripts/k8s-kind-smoke.sh @@ -7,7 +7,7 @@ cluster="${K8S_SMOKE_CLUSTER:-durable-workflow-server-smoke}" image="${K8S_SMOKE_IMAGE:-durableworkflow/server:k8s-smoke}" # Generated by scripts/ci/sync-source-release.mjs so the smoke replaces the # same default shipped by the public manifests. -manifest_image="durableworkflow/server:2.2.0" +manifest_image="durableworkflow/server:2.2.1" kind_node_image="${K8S_SMOKE_KIND_NODE_IMAGE:-kindest/node:v1.29.4}" artifact_dir="${K8S_SMOKE_ARTIFACT_DIR:-/tmp/durable-workflow-k8s-kind-smoke-artifacts}" rendered_dir="${artifact_dir}/rendered-manifests" diff --git a/tests/Feature/WorkerDatabaseUnavailableTest.php b/tests/Feature/WorkerDatabaseUnavailableTest.php new file mode 100644 index 00000000..8a5b2a70 --- /dev/null +++ b/tests/Feature/WorkerDatabaseUnavailableTest.php @@ -0,0 +1,191 @@ +createNamespace('default'); + $this->registerWorker(workerId: 'database-worker', taskQueue: 'database-queue', + supportedWorkflowTypes: ['ExampleWorkflow'], supportedActivityTypes: ['ExampleActivity']); + + DB::connection()->beforeExecuting(function (string $query): void { + if ($this->databaseUnavailable && str_contains($query, $this->failureQuery)) { + $exception = new PDOException('SQLSTATE[HY000] [2002] Connection refused'); + $exception->errorInfo = ['HY000', 2002, 'private database connection refused']; + throw $exception; + } + }); + } + + #[DataProvider('pollPaths')] + public function test_database_loss_preserves_the_logical_poll_identity(string $path): void + { + $this->databaseUnavailable = true; + $response = $this->postJson($path, [ + 'worker_id' => 'database-worker', 'task_queue' => 'database-queue', + 'poll_request_id' => 'same-logical-poll', 'timeout_seconds' => 0, + ], $this->workerHeaders()); + + $response->assertStatus(503)->assertHeader(WorkerProtocol::HEADER, WorkerProtocol::VERSION) + ->assertHeader('Retry-After', '1')->assertJsonPath('reason', 'backend_unavailable') + ->assertJsonPath('poll_status', 'backend_unavailable')->assertJsonPath('task', null) + ->assertJsonPath('outcome', 'unknown')->assertJsonPath('retryable', true) + ->assertJsonPath('poll_request_id', 'same-logical-poll') + ->assertJsonPath('retry_same_poll_request_id', true); + $this->assertStringNotContainsString('private database', $response->getContent()); + $this->assertStringNotContainsString('SQLSTATE', $response->getContent()); + OpenApiSchema::fromFile(base_path('resources/platform-protocol-specs/worker-protocol-api.openapi.yaml')) + ->assertReferenceMatches('#/components/schemas/WorkerBackendUnavailable', json_decode($response->getContent())); + + $this->databaseUnavailable = false; + $this->assertSame(1, WorkerRegistration::query()->where('worker_id', 'database-worker')->count()); + } + + public static function pollPaths(): array + { + return [ + ['/api/worker/workflow-tasks/poll'], + ['/api/worker/activity-tasks/poll'], + ['/api/worker/query-tasks/poll'], + ['/api/worker/update-validation-tasks/poll'], + ]; + } + + public function test_worker_heartbeat_does_not_claim_an_acknowledgement_during_database_loss(): void + { + $this->databaseUnavailable = true; + $this->postJson('/api/worker/heartbeat', ['worker_id' => 'database-worker'], $this->workerHeaders()) + ->assertStatus(503)->assertJsonPath('reason', 'backend_unavailable') + ->assertJsonPath('operation', 'heartbeat_worker')->assertJsonPath('worker_id', 'database-worker') + ->assertJsonPath('retryable', true)->assertJsonPath('outcome', 'unknown') + ->assertJsonMissingPath('acknowledged'); + } + + public function test_connection_loss_in_a_bootstrap_query_keeps_the_retry_contract(): void + { + $this->failureQuery = 'migrations'; + $this->databaseUnavailable = true; + + $this->postJson('/api/worker/workflow-tasks/poll', [ + 'worker_id' => 'database-worker', 'task_queue' => 'database-queue', + 'poll_request_id' => 'same-logical-poll', 'timeout_seconds' => 0, + ], $this->workerHeaders())->assertStatus(503) + ->assertJsonPath('reason', 'backend_unavailable') + ->assertJsonPath('poll_request_id', 'same-logical-poll'); + } + + public function test_invalid_authentication_remains_unauthorized_during_database_loss(): void + { + config(['server.auth.driver' => 'token', 'server.auth.token' => 'test-secret-token']); + $this->databaseUnavailable = true; + + $this->postJson('/api/worker/workflow-tasks/poll', [ + 'worker_id' => 'database-worker', 'task_queue' => 'database-queue', + 'poll_request_id' => 'same-logical-poll', 'timeout_seconds' => 0, + ], $this->workerHeaders() + ['Authorization' => 'Bearer invalid-token']) + ->assertStatus(401)->assertJsonMissingPath('retryable'); + } + + public function test_runtime_credential_lookup_loss_retries_without_authenticating_the_request(): void + { + $token = 'disposable-worker-token'; + RuntimeCredential::query()->create([ + 'id' => 'outage-worker', 'subject' => 'test-worker', 'roles' => ['worker'], + 'tenant' => 'default', 'token_prefix' => RuntimeCredential::prefixFor($token), + 'token_hash' => RuntimeCredential::hashToken($token), + ]); + config([ + 'server.auth.driver' => 'token', 'server.auth.token' => null, + 'server.auth.runtime_credentials.enabled' => true, + ]); + $this->failureQuery = 'runtime_credentials'; + $this->databaseUnavailable = true; + $request = ['worker_id' => 'database-worker']; + $headers = $this->workerHeaders() + ['Authorization' => 'Bearer '.$token]; + $originalHeartbeat = WorkerRegistration::query()->firstOrFail()->last_heartbeat_at; + + $this->postJson('/api/worker/heartbeat', $request, $headers) + ->assertStatus(503)->assertJsonPath('reason', 'backend_unavailable') + ->assertJsonPath('outcome', 'unknown')->assertJsonMissingPath('acknowledged'); + $this->assertTrue($originalHeartbeat->equalTo(WorkerRegistration::query()->firstOrFail()->last_heartbeat_at)); + + $this->databaseUnavailable = false; + $this->postJson('/api/worker/heartbeat', $request, $headers)->assertOk(); + $this->postJson('/api/worker/heartbeat', $request, + $this->workerHeaders() + ['Authorization' => 'Bearer invalid-token']) + ->assertStatus(401)->assertJsonMissingPath('retryable'); + } + + public function test_registration_outcome_is_unknown_not_falsely_rejected(): void + { + $this->databaseUnavailable = true; + $this->postJson('/api/worker/register', [ + 'worker_id' => 'database-worker', 'task_queue' => 'database-queue', 'runtime' => 'php', + 'capability_manifest' => $this->portableWorkerAffinityRefusalManifest(), + ], $this->workerHeaders()) + ->assertStatus(503)->assertJsonPath('reason', 'backend_unavailable') + ->assertJsonPath('operation', 'register_worker')->assertJsonPath('worker_id', 'database-worker') + ->assertJsonPath('retryable', true)->assertJsonPath('outcome', 'unknown') + ->assertJsonMissingPath('registered'); + } + + #[DataProvider('bootstrapErrors')] + public function test_bootstrap_admission_distinguishes_connection_loss_from_configuration_failure(int $code, string $reason): void + { + $exception = new PDOException('private database diagnostic'); + $exception->errorInfo = ['HY000', $code, 'private database diagnostic']; + $connection = Mockery::mock(Connection::class); + $connection->shouldReceive('getPdo')->andThrow($exception); + $manager = DB::getFacadeRoot(); + DB::shouldReceive('connection')->andReturn($connection); + + try { + $response = $this->postJson('/api/worker/workflow-tasks/poll', [ + 'worker_id' => 'database-worker', 'task_queue' => 'database-queue', + 'poll_request_id' => 'bootstrap-poll', 'timeout_seconds' => 0, + ], $this->workerHeaders()); + $response->assertStatus(503)->assertJsonPath('reason', $reason); + $this->assertStringNotContainsString('private database', $response->getContent()); + if ($reason === 'backend_unavailable') { + $response->assertJsonPath('poll_request_id', 'bootstrap-poll') + ->assertJsonPath('retry_same_poll_request_id', true); + } else { + $response->assertJsonMissingPath('retryable'); + } + } finally { + DB::swap($manager); + } + } + + public static function bootstrapErrors(): array + { + return [ + [2002, 'backend_unavailable'], + [1045, 'workflow_v2_blocked'], + [1049, 'workflow_v2_blocked'], + ]; + } +} diff --git a/tests/Unit/BackendUnavailableTest.php b/tests/Unit/BackendUnavailableTest.php new file mode 100644 index 00000000..38c7c4eb --- /dev/null +++ b/tests/Unit/BackendUnavailableTest.php @@ -0,0 +1,51 @@ +errorInfo = [$state, $code, 'diagnostic']; + + $this->assertSame($retryable, BackendUnavailable::is($exception)); + $this->assertSame($retryable, BackendUnavailable::is(new QueryException('mysql', 'select ?', ['value'], $exception))); + } + + public static function driverErrors(): array + { + return [ + 'mysql refused' => ['HY000', 2002, true], + 'mysql host unavailable' => ['HY000', 2003, true], + 'mysql gone away' => ['HY000', 2006, true], + 'mysql connection lost' => ['HY000', 2013, true], + 'mysql extended connection lost' => ['HY000', 2055, true], + 'postgres shutdown' => ['57P01', 7, true], + 'postgres recovery' => ['57P03', 7, true], + 'connection failure' => ['08006', 7, true], + 'unknown transaction result' => ['08007', 7, true], + 'access denied' => ['HY000', 1045, false], + 'missing database' => ['HY000', 1049, false], + 'disk full' => ['HY000', 1114, false], + 'deadlock' => ['40001', 1213, false], + 'schema error' => ['42S02', 1146, false], + 'unique violation' => ['23000', 1062, false], + 'wrong sqlstate' => ['42000', 2002, false], + ]; + } + + public function test_error_text_cannot_classify_a_failure(): void + { + $this->assertFalse(BackendUnavailable::is(new RuntimeException('SQLSTATE[HY000] [2002] Connection refused'))); + $this->assertFalse(BackendUnavailable::is(new PDOException('SQLSTATE[HY000] [2002] Connection refused'))); + } +} diff --git a/tests/Unit/WorkerProtocolOpenApiContractTest.php b/tests/Unit/WorkerProtocolOpenApiContractTest.php index d3b5dda2..3298f1ef 100644 --- a/tests/Unit/WorkerProtocolOpenApiContractTest.php +++ b/tests/Unit/WorkerProtocolOpenApiContractTest.php @@ -96,6 +96,18 @@ public function test_worker_poll_routes_publish_typed_backpressure(): void ); } + public function test_worker_lifecycle_and_poll_routes_describe_unknown_database_outcomes(): void + { + foreach (['register', 'heartbeat', 'workflow-tasks/poll', 'activity-tasks/poll', 'query-tasks/poll', 'update-validation-tasks/poll'] as $path) { + $this->assertSame('#/components/responses/WorkerServiceUnavailable', + $this->spec['paths']['/worker/'.$path]['post']['responses']['503']['$ref'] ?? null); + } + $contract = $this->spec['components']['schemas']['WorkerBackendUnavailable']['allOf'][1]; + $this->assertSame('unknown', $contract['properties']['outcome']['const']); + $this->assertContains('poll_request_id', $contract['then']['required']); + $this->assertSame(true, $contract['then']['properties']['retry_same_poll_request_id']['const']); + } + public function test_portable_worker_affinity_is_machine_described_at_protocol_1_18(): void { $contract = $this->spec['x-durable-workflow-portable-worker-affinity-contract'];