diff --git a/.env.example b/.env.example index 1b7c784e..b3be4b56 100644 --- a/.env.example +++ b/.env.example @@ -56,6 +56,15 @@ DW_EXTERNAL_PAYLOAD_MAX_OBJECTS_PER_NAMESPACE= DW_EXTERNAL_PAYLOAD_HARD_MAX_BYTES_PER_NAMESPACE= DW_EXTERNAL_PAYLOAD_HARD_MAX_OBJECTS_PER_NAMESPACE= DW_EXTERNAL_PAYLOAD_NAMESPACE_OVERRIDES={} +# Shared object storage for namespaces whose external payload policy uses s3. +# Access key and secret may be omitted when the runtime has an IAM role. +DW_EXTERNAL_PAYLOAD_S3_ACCESS_KEY_ID= +DW_EXTERNAL_PAYLOAD_S3_SECRET_ACCESS_KEY= +DW_EXTERNAL_PAYLOAD_S3_SESSION_TOKEN= +DW_EXTERNAL_PAYLOAD_S3_REGION=us-east-1 +DW_EXTERNAL_PAYLOAD_S3_BUCKET= +DW_EXTERNAL_PAYLOAD_S3_ENDPOINT= +DW_EXTERNAL_PAYLOAD_S3_USE_PATH_STYLE_ENDPOINT=false DW_DEFAULT_NAMESPACE=default # # Cluster discovery advertises the current node's deployment shape and process diff --git a/.github/workflows/phpunit-feature.yml b/.github/workflows/phpunit-feature.yml index dad78652..00fc05f7 100644 --- a/.github/workflows/phpunit-feature.yml +++ b/.github/workflows/phpunit-feature.yml @@ -165,15 +165,42 @@ jobs: && apt-get install -y --no-install-recommends git \ && rm -rf /var/lib/apt/lists/* DOCKERFILE + docker network create \ + "dw-server-feature-${{ github.run_id }}-${{ github.run_attempt }}-${{ github.job }}" + docker run --detach \ + --name "dw-server-minio-${{ github.run_id }}-${{ github.run_attempt }}-${{ github.job }}" \ + --network "dw-server-feature-${{ github.run_id }}-${{ github.run_attempt }}-${{ github.job }}" \ + --env MINIO_ROOT_USER=dw-minio-access \ + --env MINIO_ROOT_PASSWORD=dw-minio-secret-key \ + minio/minio@sha256:14cea493d9a34af32f524e538b8346cf79f3321eff8e708c1e2960462bd8936e \ + server /data --address :9000 tar \ --exclude=vendor \ --exclude='*/vendor' \ --exclude=build \ --exclude='*/build' \ -cf - . \ - | docker run --rm -i -e CORPUS_BASE_REF -w /app --entrypoint sh durable-workflow-server-corpus-validator \ + | docker run --rm -i \ + --network "dw-server-feature-${{ github.run_id }}-${{ github.run_attempt }}-${{ github.job }}" \ + -e CORPUS_BASE_REF \ + -e DW_TEST_S3_ENDPOINT="http://dw-server-minio-${{ github.run_id }}-${{ github.run_attempt }}-${{ github.job }}:9000" \ + -e DW_TEST_S3_ACCESS_KEY_ID=dw-minio-access \ + -e DW_TEST_S3_SECRET_ACCESS_KEY=dw-minio-secret-key \ + -w /app \ + --entrypoint sh \ + durable-workflow-server-corpus-validator \ -lc 'set -eu; tar --no-same-owner -xf - -C /app; cp -a /app/workflow-package /workflow; php scripts/ci/prepare-release-workflow-composer-metadata.php; composer update durable-workflow/workflow --with-dependencies --minimal-changes --no-install --no-scripts --no-autoloader --no-interaction --no-progress --prefer-dist; composer install --no-interaction --no-progress --prefer-dist; git --version; vendor/bin/phpunit --version; git rev-parse --verify "$CORPUS_BASE_REF^{commit}"; php scripts/ci/check-worker-openapi-evolution.php "$CORPUS_BASE_REF"; vendor/bin/phpunit tests/Feature tests/Unit/NexusContractTest.php tests/Unit/CodecRegressionCorpusTest.php tests/Unit/PhpunitFeatureWorkflowContractTest.php tests/Unit/OpenApiDocumentEvolutionTest.php --no-progress --colors=never; python3 scripts/ci/validate-regression-corpus.py --base-ref "$CORPUS_BASE_REF" --verify-counterfactual' + - name: Clean up S3-compatible test resources + if: always() + run: | + docker rm -f \ + "dw-server-minio-${{ github.run_id }}-${{ github.run_attempt }}-${{ github.job }}" \ + >/dev/null 2>&1 || true + docker network rm \ + "dw-server-feature-${{ github.run_id }}-${{ github.run_attempt }}-${{ github.job }}" \ + >/dev/null 2>&1 || true + qualification: name: Feature source qualification needs: [preflight, feature] diff --git a/README.md b/README.md index 798e4f4a..9eaa477e 100644 --- a/README.md +++ b/README.md @@ -75,6 +75,9 @@ contract, so a workflow in one language can dispatch activities to another. Server supports SQLite for a single-node runtime and MySQL or PostgreSQL for shared durable state. Multi-node deployments use shared Redis for queue and coordination state. The database remains authoritative for workflow history. +Namespaces using external payloads in a multi-node deployment should use the +built-in [S3-compatible shared storage](docs/contracts/external-payload-storage.md#self-hosted-backing-storage) +instead of node-local files. ## Capabilities diff --git a/app/Http/Controllers/Api/HealthController.php b/app/Http/Controllers/Api/HealthController.php index 5e4f9608..8709fae8 100644 --- a/app/Http/Controllers/Api/HealthController.php +++ b/app/Http/Controllers/Api/HealthController.php @@ -12,11 +12,11 @@ use App\Support\ControlPlaneProtocol; use App\Support\ControlPlaneRequestContract; use App\Support\CoordinationHealthContract; -use App\Support\FilesystemDiskAvailability; use App\Support\HeartbeatRuntimeContract; use App\Support\LegacyV1ProjectionContract; use App\Support\MessageStreamsContract; use App\Support\MigrationRuntimeContract; +use App\Support\NamespaceExternalPayloadStorage; use App\Support\NamespaceRuntimeContract; use App\Support\NexusContract; use App\Support\PayloadCodecContract; @@ -63,6 +63,7 @@ class HealthController public function __construct( private readonly ServerReadiness $readiness, private readonly TaskQueueBuildIdRolloutSnapshot $buildIdRollouts, + private readonly NamespaceExternalPayloadStorage $externalPayloadStorage, ) {} public function check(): JsonResponse @@ -343,8 +344,8 @@ private function externalPayloadStoragePolicy(?WorkflowNamespace $ns): array $driver = $this->stringOrNull($policy['driver'] ?? null); $enabled = $policy !== [] && ($policy['enabled'] ?? true) !== false; $threshold = $policy['threshold_bytes'] ?? config('server.limits.max_payload_bytes', 2 * 1024 * 1024); - $config = is_array($policy['config'] ?? null) ? $policy['config'] : []; - $resolvedDriver = $enabled && $this->externalPayloadStorageResolvable($driver, $config); + $configurationError = $this->externalPayloadStorage->configurationErrorForPolicy($policy); + $resolvedDriver = $enabled && $this->externalPayloadStorage->policyResolvable($policy); return [ 'schema' => RuntimeExternalPayloadReference::SCHEMA, @@ -352,6 +353,7 @@ private function externalPayloadStoragePolicy(?WorkflowNamespace $ns): array 'configured' => $policy !== [], 'enabled' => $enabled, 'status' => $this->externalPayloadStorageStatus($policy, $enabled, $resolvedDriver), + 'configuration_error' => $enabled && ! $resolvedDriver ? $configurationError : null, 'threshold_bytes' => (int) $threshold, 'transport' => RuntimeExternalPayloadReference::transportManifest(), 'provider_details_exposed' => false, @@ -379,33 +381,6 @@ private function externalPayloadStorageStatus(array $policy, bool $enabled, bool return $resolved ? 'available' : 'driver_unavailable'; } - /** - * @param array $config - */ - private function externalPayloadStorageResolvable(?string $driver, array $config): bool - { - if ($driver === 'local') { - return true; - } - - if (! in_array($driver, ['s3', 'gcs', 'azure', 'custom'], true)) { - return false; - } - - $disk = $this->stringOrNull($config['disk'] ?? null); - $bucket = $this->stringOrNull( - $config['bucket'] - ?? $config['container'] - ?? $config['name'] - ?? null, - ); - $scheme = $driver === 'custom' - ? $this->stringOrNull($config['scheme'] ?? null) - : $driver; - - return FilesystemDiskAvailability::configured($disk) && $bucket !== null && $scheme !== null; - } - private function stringOrNull(mixed $value): ?string { return is_string($value) && $value !== '' ? $value : null; diff --git a/app/Http/Controllers/Api/StorageController.php b/app/Http/Controllers/Api/StorageController.php index e289cf16..395ac980 100644 --- a/app/Http/Controllers/Api/StorageController.php +++ b/app/Http/Controllers/Api/StorageController.php @@ -59,7 +59,10 @@ public function test(Request $request): JsonResponse 'The server can persist this storage policy, but the configured storage driver is not available in this runtime.', $namespace, $driver, - ['supported_diagnostic_drivers' => ['local', 's3', 'gcs', 'azure', 'custom']], + [ + 'configuration_error' => $this->externalPayloadStorage->configurationErrorFor($namespace), + 'supported_diagnostic_drivers' => ['local', 's3', 'gcs', 'azure', 'custom'], + ], ); } diff --git a/app/Support/FilesystemDiskAvailability.php b/app/Support/FilesystemDiskAvailability.php index df8b59c8..bca1b308 100644 --- a/app/Support/FilesystemDiskAvailability.php +++ b/app/Support/FilesystemDiskAvailability.php @@ -2,18 +2,96 @@ namespace App\Support; +use League\Flysystem\AwsS3V3\AwsS3V3Adapter; + class FilesystemDiskAvailability { public static function configured(mixed $disk): bool + { + return self::configurationError($disk) === null; + } + + public static function configurationError(mixed $disk): ?string + { + if (! is_string($disk) || $disk === '') { + return 'filesystem_disk_name_missing'; + } + + $config = self::configuration($disk); + if ($config === null) { + return 'filesystem_disk_not_configured'; + } + + if (($config['driver'] ?? null) !== 's3') { + return null; + } + + if (! class_exists(AwsS3V3Adapter::class)) { + return 's3_adapter_unavailable'; + } + + if (self::stringOrNull($config['bucket'] ?? null) === null) { + return 's3_bucket_missing'; + } + + if (self::stringOrNull($config['region'] ?? null) === null) { + return 's3_region_missing'; + } + + $key = self::stringOrNull($config['key'] ?? null); + $secret = self::stringOrNull($config['secret'] ?? null); + $token = self::stringOrNull($config['token'] ?? null); + + if (($key === null) !== ($secret === null) || ($token !== null && ($key === null || $secret === null))) { + return 's3_credentials_incomplete'; + } + + return null; + } + + public static function bucket(mixed $disk): ?string + { + if (! is_string($disk) || $disk === '') { + return null; + } + + return self::stringOrNull(self::configuration($disk)['bucket'] ?? null); + } + + public static function driver(mixed $disk): ?string { if (! is_string($disk) || $disk === '') { - return false; + return null; } + return self::stringOrNull(self::configuration($disk)['driver'] ?? null); + } + + /** + * @return array|null + */ + private static function configuration(string $disk): ?array + { $configuredDisks = config('filesystems.disks'); - return is_array($configuredDisks) - && array_key_exists($disk, $configuredDisks) - && is_array($configuredDisks[$disk]); + if (! is_array($configuredDisks) + || ! array_key_exists($disk, $configuredDisks) + || ! is_array($configuredDisks[$disk]) + ) { + return null; + } + + return $configuredDisks[$disk]; + } + + private static function stringOrNull(mixed $value): ?string + { + if (! is_string($value)) { + return null; + } + + $value = trim($value); + + return $value === '' ? null : $value; } } diff --git a/app/Support/NamespaceExternalPayloadStorage.php b/app/Support/NamespaceExternalPayloadStorage.php index 30c4ba2b..89b08592 100644 --- a/app/Support/NamespaceExternalPayloadStorage.php +++ b/app/Support/NamespaceExternalPayloadStorage.php @@ -34,26 +34,16 @@ public function untrackedDriverFor(?string $namespace): ?RuntimeExternalPayloadS } if (in_array($driver, ['s3', 'gcs', 'azure', 'custom'], true)) { - $disk = $policy['config']['disk'] ?? null; - $bucket = $policy['config']['bucket'] - ?? $policy['config']['container'] - ?? $policy['config']['name'] - ?? null; - $scheme = $driver === 'custom' - ? ($policy['config']['scheme'] ?? null) - : $driver; - - if (! FilesystemDiskAvailability::configured($disk) - || ! is_string($bucket) || $bucket === '' - || ! is_string($scheme) || $scheme === '' - ) { + $filesystem = $this->filesystemPolicy($policy, $driver); + + if ($filesystem['error'] !== null) { return null; } return $this->guard(new FilesystemExternalPayloadStorage( - disk: $disk, - scheme: $scheme, - bucket: $bucket, + disk: $filesystem['disk'], + scheme: $filesystem['scheme'], + bucket: $filesystem['bucket'], prefix: $this->prefix($policy), )); } @@ -61,6 +51,44 @@ public function untrackedDriverFor(?string $namespace): ?RuntimeExternalPayloadS return null; } + public function configurationErrorFor(?string $namespace): ?string + { + $namespace = $namespace ?: (string) config('server.default_namespace', 'default'); + + return $this->configurationErrorForPolicy($this->policyFor($namespace)); + } + + /** + * @param array $policy + */ + public function configurationErrorForPolicy(array $policy): ?string + { + if ($policy === [] || ($policy['enabled'] ?? true) === false) { + return null; + } + + $driver = $policy['driver'] ?? null; + if ($driver === 'local') { + return null; + } + + if (! is_string($driver) || ! in_array($driver, ['s3', 'gcs', 'azure', 'custom'], true)) { + return 'external_payload_storage_driver_unsupported'; + } + + return $this->filesystemPolicy($policy, $driver)['error']; + } + + /** + * @param array $policy + */ + public function policyResolvable(array $policy): bool + { + return $policy !== [] + && ($policy['enabled'] ?? true) !== false + && $this->configurationErrorForPolicy($policy) === null; + } + public function thresholdBytesFor(?string $namespace): ?int { $namespace = $namespace ?: (string) config('server.default_namespace', 'default'); @@ -119,6 +147,49 @@ private function prefix(array $policy): string return trim($prefix, '/').'/'; } + /** + * @param array $policy + * @return array{disk: string, bucket: string, scheme: string, error: ?string} + */ + private function filesystemPolicy(array $policy, string $driver): array + { + $config = is_array($policy['config'] ?? null) ? $policy['config'] : []; + $disk = $config['disk'] ?? null; + + if ((! is_string($disk) || $disk === '') && $driver === 's3') { + $disk = (string) config('server.external_payload_transport.s3_disk', 'external-payload-s3'); + } + + $diskError = FilesystemDiskAvailability::configurationError($disk); + $bucket = $config['bucket'] + ?? $config['container'] + ?? $config['name'] + ?? ($driver === 's3' ? FilesystemDiskAvailability::bucket($disk) : null); + $scheme = $driver === 'custom' ? ($config['scheme'] ?? null) : $driver; + + $error = $diskError; + if ($error === null && (! is_string($bucket) || $bucket === '')) { + $error = 'external_payload_storage_bucket_missing'; + } + if ($error === null && (! is_string($scheme) || $scheme === '')) { + $error = 'external_payload_storage_scheme_missing'; + } + if ($error === null + && $driver === 's3' + && FilesystemDiskAvailability::driver($disk) === 's3' + && ! hash_equals((string) FilesystemDiskAvailability::bucket($disk), (string) $bucket) + ) { + $error = 's3_bucket_mismatch'; + } + + return [ + 'disk' => is_string($disk) ? $disk : '', + 'bucket' => is_string($bucket) ? $bucket : '', + 'scheme' => is_string($scheme) ? $scheme : '', + 'error' => $error, + ]; + } + private function guard(RuntimeExternalPayloadStorageDriver $driver): RuntimeExternalPayloadStorageDriver { return new GuardedExternalPayloadStorage($driver); diff --git a/composer.json b/composer.json index e7568fb4..a5c5ff30 100644 --- a/composer.json +++ b/composer.json @@ -8,7 +8,8 @@ "apache/avro": "^1.12", "durable-workflow/workflow": "2.0.3", "laravel/framework": "^13.0", - "laravel/tinker": "^3.0" + "laravel/tinker": "^3.0", + "league/flysystem-aws-s3-v3": "^3.35.3" }, "require-dev": { "fakerphp/faker": "^1.23", @@ -47,7 +48,7 @@ }, "extra": { "durable-workflow": { - "product-train": "2.0.3" + "product-train": "2.1.0" }, "laravel": { "dont-discover": [] diff --git a/composer.lock b/composer.lock index 286a2585..7558b687 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "9eec1cee595983587c9e39b8d694f9ef", + "content-hash": "602d0f4ca90fb7c8f2515804046dab3f", "packages": [ { "name": "apache/avro", @@ -68,6 +68,157 @@ }, "time": "2025-09-05T07:25:14+00:00" }, + { + "name": "aws/aws-crt-php", + "version": "v1.2.7", + "source": { + "type": "git", + "url": "https://github.com/awslabs/aws-crt-php.git", + "reference": "d71d9906c7bb63a28295447ba12e74723bd3730e" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/awslabs/aws-crt-php/zipball/d71d9906c7bb63a28295447ba12e74723bd3730e", + "reference": "d71d9906c7bb63a28295447ba12e74723bd3730e", + "shasum": "" + }, + "require": { + "php": ">=5.5" + }, + "require-dev": { + "phpunit/phpunit": "^4.8.35||^5.6.3||^9.5", + "yoast/phpunit-polyfills": "^1.0" + }, + "suggest": { + "ext-awscrt": "Make sure you install awscrt native extension to use any of the functionality." + }, + "type": "library", + "autoload": { + "classmap": [ + "src/" + ] + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "Apache-2.0" + ], + "authors": [ + { + "name": "AWS SDK Common Runtime Team", + "email": "aws-sdk-common-runtime@amazon.com" + } + ], + "description": "AWS Common Runtime for PHP", + "homepage": "https://github.com/awslabs/aws-crt-php", + "keywords": [ + "amazon", + "aws", + "crt", + "sdk" + ], + "support": { + "issues": "https://github.com/awslabs/aws-crt-php/issues", + "source": "https://github.com/awslabs/aws-crt-php/tree/v1.2.7" + }, + "time": "2024-10-18T22:15:13+00:00" + }, + { + "name": "aws/aws-sdk-php", + "version": "3.394.7", + "source": { + "type": "git", + "url": "https://github.com/aws/aws-sdk-php.git", + "reference": "40a8717706fd789cd3e8c72a2e8e93ee4455b391" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/aws/aws-sdk-php/zipball/40a8717706fd789cd3e8c72a2e8e93ee4455b391", + "reference": "40a8717706fd789cd3e8c72a2e8e93ee4455b391", + "shasum": "" + }, + "require": { + "aws/aws-crt-php": "^1.2.3", + "ext-json": "*", + "ext-pcre": "*", + "ext-simplexml": "*", + "guzzlehttp/guzzle": "^7.8.2 || ^8.0", + "guzzlehttp/promises": "^2.0.3 || ^3.0", + "guzzlehttp/psr7": "^2.6.3 || ^3.0", + "mtdowling/jmespath.php": "^2.9.1", + "php": ">=8.1", + "psr/http-message": "^1.0 || ^2.0", + "symfony/filesystem": "^v5.4.45 || ^v6.4.3 || ^v7.1.0 || ^v8.0.0" + }, + "require-dev": { + "andrewsville/php-token-reflection": "^1.4", + "aws/aws-php-sns-message-validator": "~1.0", + "behat/behat": "~3.0", + "composer/composer": "^2.7.8", + "dms/phpunit-arraysubset-asserts": "^v0.5.0", + "doctrine/cache": "~1.4", + "ext-dom": "*", + "ext-openssl": "*", + "ext-sockets": "*", + "phpunit/phpunit": "^10.0", + "psr/cache": "^2.0 || ^3.0", + "psr/simple-cache": "^2.0 || ^3.0", + "sebastian/comparator": "^1.2.3 || ^4.0 || ^5.0", + "yoast/phpunit-polyfills": "^2.0" + }, + "suggest": { + "aws/aws-php-sns-message-validator": "To validate incoming SNS notifications", + "doctrine/cache": "To use the DoctrineCacheAdapter", + "ext-curl": "To send requests using cURL", + "ext-openssl": "Allows working with CloudFront private distributions and verifying received SNS messages", + "ext-pcntl": "To use client-side monitoring", + "ext-sockets": "To use client-side monitoring" + }, + "type": "library", + "extra": { + "branch-alias": { + "dev-master": "3.0-dev" + } + }, + "autoload": { + "files": [ + "src/functions.php" + ], + "psr-4": { + "Aws\\": "src/" + }, + "exclude-from-classmap": [ + "src/data/" + ] + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "Apache-2.0" + ], + "authors": [ + { + "name": "Amazon Web Services", + "homepage": "https://aws.amazon.com" + } + ], + "description": "AWS SDK for PHP - Use Amazon Web Services in your PHP project", + "homepage": "https://aws.amazon.com/sdk-for-php", + "keywords": [ + "amazon", + "aws", + "cloud", + "dynamodb", + "ec2", + "glacier", + "s3", + "sdk" + ], + "support": { + "forum": "https://github.com/aws/aws-sdk-php/discussions", + "issues": "https://github.com/aws/aws-sdk-php/issues", + "source": "https://github.com/aws/aws-sdk-php/tree/3.394.7" + }, + "time": "2026-09-02T18:07:40+00:00" + }, { "name": "brick/math", "version": "0.18.0", @@ -779,22 +930,22 @@ }, { "name": "guzzlehttp/guzzle", - "version": "7.15.2", + "version": "7.15.5", "source": { "type": "git", "url": "https://github.com/guzzle/guzzle.git", - "reference": "744101956d78b7c1384d0cbf379db13e859167bf" + "reference": "ee80339fd9177ba44c49cdb653ff02a4d1106b9a" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/guzzle/guzzle/zipball/744101956d78b7c1384d0cbf379db13e859167bf", - "reference": "744101956d78b7c1384d0cbf379db13e859167bf", + "url": "https://api.github.com/repos/guzzle/guzzle/zipball/ee80339fd9177ba44c49cdb653ff02a4d1106b9a", + "reference": "ee80339fd9177ba44c49cdb653ff02a4d1106b9a", "shasum": "" }, "require": { "ext-json": "*", - "guzzlehttp/promises": "^2.5.1", - "guzzlehttp/psr7": "^2.13", + "guzzlehttp/promises": "^2.5.3", + "guzzlehttp/psr7": "^2.13.1", "php": "^7.2.5 || ^8.0", "psr/http-client": "^1.0", "symfony/deprecation-contracts": "^2.5 || ^3.0", @@ -887,7 +1038,7 @@ ], "support": { "issues": "https://github.com/guzzle/guzzle/issues", - "source": "https://github.com/guzzle/guzzle/tree/7.15.2" + "source": "https://github.com/guzzle/guzzle/tree/7.15.5" }, "funding": [ { @@ -903,20 +1054,20 @@ "type": "tidelift" } ], - "time": "2026-07-26T23:23:20+00:00" + "time": "2026-08-24T09:21:06+00:00" }, { "name": "guzzlehttp/promises", - "version": "2.5.1", + "version": "2.5.3", "source": { "type": "git", "url": "https://github.com/guzzle/promises.git", - "reference": "9ad1e4fc607446a055b95870c7f668e93b5cff29" + "reference": "cde49999552d185d64715fe9c1f77a2aadd2f9f1" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/guzzle/promises/zipball/9ad1e4fc607446a055b95870c7f668e93b5cff29", - "reference": "9ad1e4fc607446a055b95870c7f668e93b5cff29", + "url": "https://api.github.com/repos/guzzle/promises/zipball/cde49999552d185d64715fe9c1f77a2aadd2f9f1", + "reference": "cde49999552d185d64715fe9c1f77a2aadd2f9f1", "shasum": "" }, "require": { @@ -971,7 +1122,7 @@ ], "support": { "issues": "https://github.com/guzzle/promises/issues", - "source": "https://github.com/guzzle/promises/tree/2.5.1" + "source": "https://github.com/guzzle/promises/tree/2.5.3" }, "funding": [ { @@ -987,20 +1138,20 @@ "type": "tidelift" } ], - "time": "2026-07-08T15:48:39+00:00" + "time": "2026-08-24T09:11:28+00:00" }, { "name": "guzzlehttp/psr7", - "version": "2.13.0", + "version": "2.13.1", "source": { "type": "git", "url": "https://github.com/guzzle/psr7.git", - "reference": "dad89620b7a6edb60c15858442eb2e408b45d8f4" + "reference": "95e7828100de18b4e269fb1703be530082d5166d" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/guzzle/psr7/zipball/dad89620b7a6edb60c15858442eb2e408b45d8f4", - "reference": "dad89620b7a6edb60c15858442eb2e408b45d8f4", + "url": "https://api.github.com/repos/guzzle/psr7/zipball/95e7828100de18b4e269fb1703be530082d5166d", + "reference": "95e7828100de18b4e269fb1703be530082d5166d", "shasum": "" }, "require": { @@ -1090,7 +1241,7 @@ ], "support": { "issues": "https://github.com/guzzle/psr7/issues", - "source": "https://github.com/guzzle/psr7/tree/2.13.0" + "source": "https://github.com/guzzle/psr7/tree/2.13.1" }, "funding": [ { @@ -1106,7 +1257,7 @@ "type": "tidelift" } ], - "time": "2026-07-16T22:23:49+00:00" + "time": "2026-08-24T09:13:11+00:00" }, { "name": "guzzlehttp/uri-template", @@ -1801,16 +1952,16 @@ }, { "name": "league/flysystem", - "version": "3.35.2", + "version": "3.36.0", "source": { "type": "git", "url": "https://github.com/thephpleague/flysystem.git", - "reference": "b277b5dc3d56650b68904117124e79c851e12376" + "reference": "f7fb152932f30072d573510cbd4dd657d6475b25" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/thephpleague/flysystem/zipball/b277b5dc3d56650b68904117124e79c851e12376", - "reference": "b277b5dc3d56650b68904117124e79c851e12376", + "url": "https://api.github.com/repos/thephpleague/flysystem/zipball/f7fb152932f30072d573510cbd4dd657d6475b25", + "reference": "f7fb152932f30072d573510cbd4dd657d6475b25", "shasum": "" }, "require": { @@ -1878,22 +2029,77 @@ ], "support": { "issues": "https://github.com/thephpleague/flysystem/issues", - "source": "https://github.com/thephpleague/flysystem/tree/3.35.2" + "source": "https://github.com/thephpleague/flysystem/tree/3.36.0" + }, + "time": "2026-09-02T08:00:27+00:00" + }, + { + "name": "league/flysystem-aws-s3-v3", + "version": "3.35.3", + "source": { + "type": "git", + "url": "https://github.com/thephpleague/flysystem-aws-s3-v3.git", + "reference": "b03780cb97585ee7e48977f40ed599b33b751634" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/thephpleague/flysystem-aws-s3-v3/zipball/b03780cb97585ee7e48977f40ed599b33b751634", + "reference": "b03780cb97585ee7e48977f40ed599b33b751634", + "shasum": "" + }, + "require": { + "aws/aws-sdk-php": "^3.371.5", + "league/flysystem": "^3.10.0", + "league/mime-type-detection": "^1.0.0", + "php": "^8.0.2" + }, + "conflict": { + "guzzlehttp/guzzle": "<7.0", + "guzzlehttp/ringphp": "<1.1.1" + }, + "type": "library", + "autoload": { + "psr-4": { + "League\\Flysystem\\AwsS3V3\\": "" + } + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "authors": [ + { + "name": "Frank de Jonge", + "email": "info@frankdejonge.nl" + } + ], + "description": "AWS S3 filesystem adapter for Flysystem.", + "keywords": [ + "Flysystem", + "aws", + "file", + "files", + "filesystem", + "s3", + "storage" + ], + "support": { + "source": "https://github.com/thephpleague/flysystem-aws-s3-v3/tree/3.35.3" }, - "time": "2026-07-06T14:42:07+00:00" + "time": "2026-08-08T16:19:23+00:00" }, { "name": "league/flysystem-local", - "version": "3.31.0", + "version": "3.35.3", "source": { "type": "git", "url": "https://github.com/thephpleague/flysystem-local.git", - "reference": "2f669db18a4c20c755c2bb7d3a7b0b2340488079" + "reference": "a099b24dce160f3b2239043d13d47c4a1a214ea4" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/thephpleague/flysystem-local/zipball/2f669db18a4c20c755c2bb7d3a7b0b2340488079", - "reference": "2f669db18a4c20c755c2bb7d3a7b0b2340488079", + "url": "https://api.github.com/repos/thephpleague/flysystem-local/zipball/a099b24dce160f3b2239043d13d47c4a1a214ea4", + "reference": "a099b24dce160f3b2239043d13d47c4a1a214ea4", "shasum": "" }, "require": { @@ -1927,9 +2133,9 @@ "local" ], "support": { - "source": "https://github.com/thephpleague/flysystem-local/tree/3.31.0" + "source": "https://github.com/thephpleague/flysystem-local/tree/3.35.3" }, - "time": "2026-01-23T15:30:45+00:00" + "time": "2026-08-12T13:29:21+00:00" }, { "name": "league/mime-type-detection", @@ -2272,6 +2478,72 @@ ], "time": "2026-01-02T08:56:05+00:00" }, + { + "name": "mtdowling/jmespath.php", + "version": "2.9.2", + "source": { + "type": "git", + "url": "https://github.com/jmespath/jmespath.php.git", + "reference": "2157c5e50e813ec6a96c1eed3be7f64a20fb32a8" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/jmespath/jmespath.php/zipball/2157c5e50e813ec6a96c1eed3be7f64a20fb32a8", + "reference": "2157c5e50e813ec6a96c1eed3be7f64a20fb32a8", + "shasum": "" + }, + "require": { + "php": "^7.2.5 || ^8.0", + "symfony/polyfill-mbstring": "^1.17" + }, + "require-dev": { + "composer/xdebug-handler": "^3.0.3", + "phpunit/phpunit": "^8.5.52" + }, + "bin": [ + "bin/jp.php" + ], + "type": "library", + "extra": { + "branch-alias": { + "dev-master": "2.9-dev" + } + }, + "autoload": { + "files": [ + "src/JmesPath.php" + ], + "psr-4": { + "JmesPath\\": "src/" + } + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "authors": [ + { + "name": "Graham Campbell", + "email": "hello@gjcampbell.co.uk", + "homepage": "https://github.com/GrahamCampbell" + }, + { + "name": "Michael Dowling", + "email": "mtdowling@gmail.com", + "homepage": "https://github.com/mtdowling" + } + ], + "description": "Declaratively specify how to extract elements from a JSON document", + "keywords": [ + "json", + "jsonpath" + ], + "support": { + "issues": "https://github.com/jmespath/jmespath.php/issues", + "source": "https://github.com/jmespath/jmespath.php/tree/2.9.2" + }, + "time": "2026-07-06T18:56:19+00:00" + }, { "name": "nesbot/carbon", "version": "3.13.1", @@ -4080,6 +4352,76 @@ ], "time": "2026-06-05T06:23:12+00:00" }, + { + "name": "symfony/filesystem", + "version": "v7.4.18", + "source": { + "type": "git", + "url": "https://github.com/symfony/filesystem.git", + "reference": "90d412aa5277c6819db39e7605aa46b1019e3232" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/symfony/filesystem/zipball/90d412aa5277c6819db39e7605aa46b1019e3232", + "reference": "90d412aa5277c6819db39e7605aa46b1019e3232", + "shasum": "" + }, + "require": { + "php": ">=8.2", + "symfony/polyfill-ctype": "~1.8", + "symfony/polyfill-mbstring": "~1.8" + }, + "require-dev": { + "symfony/process": "^6.4|^7.0|^8.0" + }, + "type": "library", + "autoload": { + "psr-4": { + "Symfony\\Component\\Filesystem\\": "" + }, + "exclude-from-classmap": [ + "/Tests/" + ] + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "authors": [ + { + "name": "Fabien Potencier", + "email": "fabien@symfony.com" + }, + { + "name": "Symfony Community", + "homepage": "https://symfony.com/contributors" + } + ], + "description": "Provides basic utilities for the filesystem", + "homepage": "https://symfony.com", + "support": { + "source": "https://github.com/symfony/filesystem/tree/v7.4.18" + }, + "funding": [ + { + "url": "https://symfony.com/sponsor", + "type": "custom" + }, + { + "url": "https://github.com/fabpot", + "type": "github" + }, + { + "url": "https://github.com/nicolas-grekas", + "type": "github" + }, + { + "url": "https://tidelift.com/funding/github/packagist/symfony/symfony", + "type": "tidelift" + } + ], + "time": "2026-08-23T10:03:40+00:00" + }, { "name": "symfony/finder", "version": "v7.4.14", @@ -5433,16 +5775,16 @@ }, { "name": "symfony/process", - "version": "v7.4.13", + "version": "v7.4.18", "source": { "type": "git", "url": "https://github.com/symfony/process.git", - "reference": "f5804be144caceb570f6747519999636b664f24c" + "reference": "058d17fc284cce14efb2385783b55014a461b176" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/process/zipball/f5804be144caceb570f6747519999636b664f24c", - "reference": "f5804be144caceb570f6747519999636b664f24c", + "url": "https://api.github.com/repos/symfony/process/zipball/058d17fc284cce14efb2385783b55014a461b176", + "reference": "058d17fc284cce14efb2385783b55014a461b176", "shasum": "" }, "require": { @@ -5474,7 +5816,7 @@ "description": "Executes commands in sub-processes", "homepage": "https://symfony.com", "support": { - "source": "https://github.com/symfony/process/tree/v7.4.13" + "source": "https://github.com/symfony/process/tree/v7.4.18" }, "funding": [ { @@ -5494,7 +5836,7 @@ "type": "tidelift" } ], - "time": "2026-05-23T16:05:06+00:00" + "time": "2026-08-21T17:40:08+00:00" }, { "name": "symfony/routing", diff --git a/config/dw-contract.php b/config/dw-contract.php index a257c435..1b2ab436 100644 --- a/config/dw-contract.php +++ b/config/dw-contract.php @@ -624,6 +624,48 @@ 'since' => '2.0.3', 'legacy' => 'WORKFLOW_SERVER_EXTERNAL_PAYLOAD_NAMESPACE_OVERRIDES', ], + 'DW_EXTERNAL_PAYLOAD_S3_ACCESS_KEY_ID' => [ + 'description' => 'Optional access-key ID for the runtime-managed S3-compatible external-payload disk. May be omitted when workload identity supplies credentials.', + 'default' => '(unset)', + 'since' => '2.0.4', + 'legacy' => 'WORKFLOW_SERVER_EXTERNAL_PAYLOAD_S3_ACCESS_KEY_ID', + ], + 'DW_EXTERNAL_PAYLOAD_S3_SECRET_ACCESS_KEY' => [ + 'description' => 'Optional secret access key for the runtime-managed S3-compatible external-payload disk. Must be set with the access-key ID.', + 'default' => '(unset)', + 'since' => '2.0.4', + 'legacy' => 'WORKFLOW_SERVER_EXTERNAL_PAYLOAD_S3_SECRET_ACCESS_KEY', + ], + 'DW_EXTERNAL_PAYLOAD_S3_SESSION_TOKEN' => [ + 'description' => 'Optional session token for temporary credentials on the runtime-managed S3-compatible external-payload disk.', + 'default' => '(unset)', + 'since' => '2.0.4', + 'legacy' => 'WORKFLOW_SERVER_EXTERNAL_PAYLOAD_S3_SESSION_TOKEN', + ], + 'DW_EXTERNAL_PAYLOAD_S3_REGION' => [ + 'description' => 'Region used by the runtime-managed S3-compatible external-payload disk.', + 'default' => 'us-east-1', + 'since' => '2.0.4', + 'legacy' => 'WORKFLOW_SERVER_EXTERNAL_PAYLOAD_S3_REGION', + ], + 'DW_EXTERNAL_PAYLOAD_S3_BUCKET' => [ + 'description' => 'Bucket used by the runtime-managed S3-compatible external-payload disk.', + 'default' => '(unset)', + 'since' => '2.0.4', + 'legacy' => 'WORKFLOW_SERVER_EXTERNAL_PAYLOAD_S3_BUCKET', + ], + 'DW_EXTERNAL_PAYLOAD_S3_ENDPOINT' => [ + 'description' => 'Optional HTTPS endpoint for an S3-compatible external-payload service.', + 'default' => '(unset)', + 'since' => '2.0.4', + 'legacy' => 'WORKFLOW_SERVER_EXTERNAL_PAYLOAD_S3_ENDPOINT', + ], + 'DW_EXTERNAL_PAYLOAD_S3_USE_PATH_STYLE_ENDPOINT' => [ + 'description' => 'Use path-style bucket addressing for the runtime-managed S3-compatible external-payload service.', + 'default' => 'false', + 'since' => '2.0.4', + 'legacy' => 'WORKFLOW_SERVER_EXTERNAL_PAYLOAD_S3_USE_PATH_STYLE_ENDPOINT', + ], 'DW_MAX_MEMO_BYTES' => [ 'description' => 'Maximum serialized bytes for a workflow memo.', 'default' => '262144', diff --git a/config/filesystems.php b/config/filesystems.php new file mode 100644 index 00000000..64ed396a --- /dev/null +++ b/config/filesystems.php @@ -0,0 +1,67 @@ + env('FILESYSTEM_DISK', 'local'), + + 'disks' => [ + 'local' => [ + 'driver' => 'local', + 'root' => storage_path('app/private'), + 'serve' => true, + 'throw' => false, + 'report' => false, + ], + + 'public' => [ + 'driver' => 'local', + 'root' => storage_path('app/public'), + 'url' => rtrim((string) env('APP_URL'), '/').'/storage', + 'visibility' => 'public', + 'throw' => false, + 'report' => false, + ], + + 'external-payload-s3' => [ + 'driver' => 's3', + 'key' => EnvAuditor::env( + 'DW_EXTERNAL_PAYLOAD_S3_ACCESS_KEY_ID', + 'WORKFLOW_SERVER_EXTERNAL_PAYLOAD_S3_ACCESS_KEY_ID', + ), + 'secret' => EnvAuditor::env( + 'DW_EXTERNAL_PAYLOAD_S3_SECRET_ACCESS_KEY', + 'WORKFLOW_SERVER_EXTERNAL_PAYLOAD_S3_SECRET_ACCESS_KEY', + ), + 'token' => EnvAuditor::env( + 'DW_EXTERNAL_PAYLOAD_S3_SESSION_TOKEN', + 'WORKFLOW_SERVER_EXTERNAL_PAYLOAD_S3_SESSION_TOKEN', + ), + 'region' => EnvAuditor::env( + 'DW_EXTERNAL_PAYLOAD_S3_REGION', + 'WORKFLOW_SERVER_EXTERNAL_PAYLOAD_S3_REGION', + 'us-east-1', + ), + 'bucket' => EnvAuditor::env( + 'DW_EXTERNAL_PAYLOAD_S3_BUCKET', + 'WORKFLOW_SERVER_EXTERNAL_PAYLOAD_S3_BUCKET', + ), + 'endpoint' => EnvAuditor::env( + 'DW_EXTERNAL_PAYLOAD_S3_ENDPOINT', + 'WORKFLOW_SERVER_EXTERNAL_PAYLOAD_S3_ENDPOINT', + ), + 'use_path_style_endpoint' => EnvAuditor::env( + 'DW_EXTERNAL_PAYLOAD_S3_USE_PATH_STYLE_ENDPOINT', + 'WORKFLOW_SERVER_EXTERNAL_PAYLOAD_S3_USE_PATH_STYLE_ENDPOINT', + false, + ), + 'visibility' => 'private', + 'throw' => true, + 'report' => true, + ], + ], + + 'links' => [ + public_path('storage') => storage_path('app/public'), + ], +]; diff --git a/config/server.php b/config/server.php index ae255504..7a88f870 100644 --- a/config/server.php +++ b/config/server.php @@ -480,6 +480,7 @@ */ 'external_payload_transport' => [ + 's3_disk' => 'external-payload-s3', 'max_payload_bytes' => (int) EnvAuditor::env( 'DW_EXTERNAL_PAYLOAD_MAX_BYTES', 'WORKFLOW_SERVER_EXTERNAL_PAYLOAD_MAX_BYTES', diff --git a/docker-compose.dedicated-matching.yml b/docker-compose.dedicated-matching.yml index 0c8d4d9c..c83c303d 100644 --- a/docker-compose.dedicated-matching.yml +++ b/docker-compose.dedicated-matching.yml @@ -32,13 +32,13 @@ name: durable-workflow-server # daemon reports `shape: dedicated`. # Generated by scripts/ci/sync-source-release.mjs. Do not edit the fallback. -x-server-image: &server-image ${DW_SERVER_IMAGE:-durableworkflow/server:${DW_SERVER_TAG:-2.0.3}} +x-server-image: &server-image ${DW_SERVER_IMAGE:-durableworkflow/server:${DW_SERVER_TAG:-2.1.0}} x-server-environment: &server-environment APP_NAME: "Durable Workflow Server" APP_ENV: ${APP_ENV:-local} DW_SERVER_KEY: ${DW_SERVER_KEY:-} - APP_VERSION: ${APP_VERSION:-${DW_SERVER_TAG:-2.0.3}} + APP_VERSION: ${APP_VERSION:-${DW_SERVER_TAG:-2.1.0}} APP_DEBUG: ${APP_DEBUG:-false} DB_CONNECTION: mysql DB_HOST: mysql diff --git a/docker-compose.memo-rolling.yml b/docker-compose.memo-rolling.yml index 8ec229de..12942616 100644 --- a/docker-compose.memo-rolling.yml +++ b/docker-compose.memo-rolling.yml @@ -49,14 +49,14 @@ services: command: ["server-bootstrap"] environment: <<: *runtime-environment - APP_VERSION: ${APP_VERSION:-2.0.3} + APP_VERSION: ${APP_VERSION:-2.1.0} successor: image: ${DW_MEMO_SUCCESSOR_IMAGE:-durable-workflow/server-memo-rolling:local} ports: !override [] environment: <<: *runtime-environment - APP_VERSION: ${APP_VERSION:-2.0.3} + APP_VERSION: ${APP_VERSION:-2.1.0} DW_SERVER_ID: memo-successor DW_SERVER_TOPOLOGY_SHAPE: standalone_server DW_SERVER_PROCESS_CLASS: server_http_node diff --git a/docker-compose.published.yml b/docker-compose.published.yml index 9978eabc..9e5993a6 100644 --- a/docker-compose.published.yml +++ b/docker-compose.published.yml @@ -1,13 +1,13 @@ name: durable-workflow-server # Generated by scripts/ci/sync-source-release.mjs. Do not edit the fallback. -x-server-image: &server-image ${DW_SERVER_IMAGE:-durableworkflow/server:${DW_SERVER_TAG:-2.0.3}} +x-server-image: &server-image ${DW_SERVER_IMAGE:-durableworkflow/server:${DW_SERVER_TAG:-2.1.0}} x-server-environment: &server-environment APP_NAME: "Durable Workflow Server" APP_ENV: ${APP_ENV:-local} DW_SERVER_KEY: ${DW_SERVER_KEY:-} - APP_VERSION: ${APP_VERSION:-${DW_SERVER_TAG:-2.0.3}} + APP_VERSION: ${APP_VERSION:-${DW_SERVER_TAG:-2.1.0}} APP_DEBUG: ${APP_DEBUG:-false} LOG_CHANNEL: ${LOG_CHANNEL:-stderr} LOG_LEVEL: ${LOG_LEVEL:-info} @@ -29,6 +29,13 @@ x-server-environment: &server-environment DW_OPERATOR_TOKEN: ${DW_OPERATOR_TOKEN:-} DW_ADMIN_TOKEN: ${DW_ADMIN_TOKEN:-} DW_AUTH_BACKWARD_COMPATIBLE: ${DW_AUTH_BACKWARD_COMPATIBLE:-true} + DW_EXTERNAL_PAYLOAD_S3_ACCESS_KEY_ID: ${DW_EXTERNAL_PAYLOAD_S3_ACCESS_KEY_ID:-} + DW_EXTERNAL_PAYLOAD_S3_SECRET_ACCESS_KEY: ${DW_EXTERNAL_PAYLOAD_S3_SECRET_ACCESS_KEY:-} + DW_EXTERNAL_PAYLOAD_S3_SESSION_TOKEN: ${DW_EXTERNAL_PAYLOAD_S3_SESSION_TOKEN:-} + DW_EXTERNAL_PAYLOAD_S3_REGION: ${DW_EXTERNAL_PAYLOAD_S3_REGION:-us-east-1} + DW_EXTERNAL_PAYLOAD_S3_BUCKET: ${DW_EXTERNAL_PAYLOAD_S3_BUCKET:-} + DW_EXTERNAL_PAYLOAD_S3_ENDPOINT: ${DW_EXTERNAL_PAYLOAD_S3_ENDPOINT:-} + DW_EXTERNAL_PAYLOAD_S3_USE_PATH_STYLE_ENDPOINT: ${DW_EXTERNAL_PAYLOAD_S3_USE_PATH_STYLE_ENDPOINT:-false} services: bootstrap: diff --git a/docker-compose.small-cluster.yml b/docker-compose.small-cluster.yml index b086849f..920e5861 100644 --- a/docker-compose.small-cluster.yml +++ b/docker-compose.small-cluster.yml @@ -12,7 +12,7 @@ x-server-build: &server-build x-server-environment: &server-environment APP_NAME: "Durable Workflow Server" APP_ENV: testing - APP_VERSION: ${APP_VERSION:-2.0.3} + APP_VERSION: ${APP_VERSION:-2.1.0} APP_DEBUG: "false" DW_SERVER_KEY: ${DW_SERVER_KEY:-base64:5Zt4nUhlCm3DD0nLXZJQdHiwPfb56yGo9gNV/g3jYbY=} DB_CONNECTION: ${DW_SMALL_CLUSTER_DB:-mysql} @@ -29,6 +29,13 @@ x-server-environment: &server-environment DW_PRINCIPAL_TOKENS: ${DW_PRINCIPAL_TOKENS:-${WORKFLOW_SERVER_PRINCIPAL_TOKENS:-}} DW_RUNTIME_CREDENTIALS_ENABLED: ${DW_RUNTIME_CREDENTIALS_ENABLED:-${WORKFLOW_SERVER_RUNTIME_CREDENTIALS_ENABLED:-false}} DW_AUTH_BACKWARD_COMPATIBLE: "true" + DW_EXTERNAL_PAYLOAD_S3_ACCESS_KEY_ID: ${DW_EXTERNAL_PAYLOAD_S3_ACCESS_KEY_ID:-} + DW_EXTERNAL_PAYLOAD_S3_SECRET_ACCESS_KEY: ${DW_EXTERNAL_PAYLOAD_S3_SECRET_ACCESS_KEY:-} + DW_EXTERNAL_PAYLOAD_S3_SESSION_TOKEN: ${DW_EXTERNAL_PAYLOAD_S3_SESSION_TOKEN:-} + DW_EXTERNAL_PAYLOAD_S3_REGION: ${DW_EXTERNAL_PAYLOAD_S3_REGION:-us-east-1} + DW_EXTERNAL_PAYLOAD_S3_BUCKET: ${DW_EXTERNAL_PAYLOAD_S3_BUCKET:-} + DW_EXTERNAL_PAYLOAD_S3_ENDPOINT: ${DW_EXTERNAL_PAYLOAD_S3_ENDPOINT:-} + DW_EXTERNAL_PAYLOAD_S3_USE_PATH_STYLE_ENDPOINT: ${DW_EXTERNAL_PAYLOAD_S3_USE_PATH_STYLE_ENDPOINT:-false} DW_WORKER_POLL_TIMEOUT: "1" DW_WORKER_POLL_INTERVAL_MS: "100" DW_ENV_AUDIT_STRICT: "1" diff --git a/docker-compose.yml b/docker-compose.yml index d496f917..4f49671c 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -15,7 +15,7 @@ services: DW_SERVER_KEY: "${DW_SERVER_KEY:-}" DW_SERVER_TOPOLOGY_SHAPE: standalone_server DW_SERVER_PROCESS_CLASS: server_http_node - APP_VERSION: "${APP_VERSION:-2.0.3}" + APP_VERSION: "${APP_VERSION:-2.1.0}" APP_DEBUG: "false" DB_CONNECTION: mysql DB_HOST: mysql @@ -35,6 +35,13 @@ services: DW_OPERATOR_TOKEN: "${DW_OPERATOR_TOKEN:-${WORKFLOW_SERVER_OPERATOR_TOKEN:-}}" DW_ADMIN_TOKEN: "${DW_ADMIN_TOKEN:-${WORKFLOW_SERVER_ADMIN_TOKEN:-}}" DW_AUTH_BACKWARD_COMPATIBLE: "${DW_AUTH_BACKWARD_COMPATIBLE:-${WORKFLOW_SERVER_AUTH_BACKWARD_COMPATIBLE:-true}}" + DW_EXTERNAL_PAYLOAD_S3_ACCESS_KEY_ID: "${DW_EXTERNAL_PAYLOAD_S3_ACCESS_KEY_ID:-}" + DW_EXTERNAL_PAYLOAD_S3_SECRET_ACCESS_KEY: "${DW_EXTERNAL_PAYLOAD_S3_SECRET_ACCESS_KEY:-}" + DW_EXTERNAL_PAYLOAD_S3_SESSION_TOKEN: "${DW_EXTERNAL_PAYLOAD_S3_SESSION_TOKEN:-}" + DW_EXTERNAL_PAYLOAD_S3_REGION: "${DW_EXTERNAL_PAYLOAD_S3_REGION:-us-east-1}" + DW_EXTERNAL_PAYLOAD_S3_BUCKET: "${DW_EXTERNAL_PAYLOAD_S3_BUCKET:-}" + DW_EXTERNAL_PAYLOAD_S3_ENDPOINT: "${DW_EXTERNAL_PAYLOAD_S3_ENDPOINT:-}" + DW_EXTERNAL_PAYLOAD_S3_USE_PATH_STYLE_ENDPOINT: "${DW_EXTERNAL_PAYLOAD_S3_USE_PATH_STYLE_ENDPOINT:-false}" DW_METRICS_WORKFLOW_TASK_FAILURE_TYPE_LIMIT: "${DW_METRICS_WORKFLOW_TASK_FAILURE_TYPE_LIMIT:-${WORKFLOW_SERVER_METRICS_WORKFLOW_TASK_FAILURE_TYPE_LIMIT:-20}}" depends_on: mysql: @@ -55,7 +62,7 @@ services: APP_NAME: "Durable Workflow Server" APP_ENV: local DW_SERVER_KEY: "${DW_SERVER_KEY:-}" - APP_VERSION: "${APP_VERSION:-2.0.3}" + APP_VERSION: "${APP_VERSION:-2.1.0}" APP_DEBUG: "false" DB_CONNECTION: mysql DB_HOST: mysql @@ -75,6 +82,13 @@ services: DW_OPERATOR_TOKEN: "${DW_OPERATOR_TOKEN:-${WORKFLOW_SERVER_OPERATOR_TOKEN:-}}" DW_ADMIN_TOKEN: "${DW_ADMIN_TOKEN:-${WORKFLOW_SERVER_ADMIN_TOKEN:-}}" DW_AUTH_BACKWARD_COMPATIBLE: "${DW_AUTH_BACKWARD_COMPATIBLE:-${WORKFLOW_SERVER_AUTH_BACKWARD_COMPATIBLE:-true}}" + DW_EXTERNAL_PAYLOAD_S3_ACCESS_KEY_ID: "${DW_EXTERNAL_PAYLOAD_S3_ACCESS_KEY_ID:-}" + DW_EXTERNAL_PAYLOAD_S3_SECRET_ACCESS_KEY: "${DW_EXTERNAL_PAYLOAD_S3_SECRET_ACCESS_KEY:-}" + DW_EXTERNAL_PAYLOAD_S3_SESSION_TOKEN: "${DW_EXTERNAL_PAYLOAD_S3_SESSION_TOKEN:-}" + DW_EXTERNAL_PAYLOAD_S3_REGION: "${DW_EXTERNAL_PAYLOAD_S3_REGION:-us-east-1}" + DW_EXTERNAL_PAYLOAD_S3_BUCKET: "${DW_EXTERNAL_PAYLOAD_S3_BUCKET:-}" + DW_EXTERNAL_PAYLOAD_S3_ENDPOINT: "${DW_EXTERNAL_PAYLOAD_S3_ENDPOINT:-}" + DW_EXTERNAL_PAYLOAD_S3_USE_PATH_STYLE_ENDPOINT: "${DW_EXTERNAL_PAYLOAD_S3_USE_PATH_STYLE_ENDPOINT:-false}" DW_METRICS_WORKFLOW_TASK_FAILURE_TYPE_LIMIT: "${DW_METRICS_WORKFLOW_TASK_FAILURE_TYPE_LIMIT:-${WORKFLOW_SERVER_METRICS_WORKFLOW_TASK_FAILURE_TYPE_LIMIT:-20}}" depends_on: bootstrap: @@ -109,7 +123,7 @@ services: DW_SERVER_KEY: "${DW_SERVER_KEY:-}" DW_SERVER_TOPOLOGY_SHAPE: standalone_server DW_SERVER_PROCESS_CLASS: worker_node - APP_VERSION: "${APP_VERSION:-2.0.3}" + APP_VERSION: "${APP_VERSION:-2.1.0}" DB_CONNECTION: mysql DB_HOST: mysql DB_PORT: 3306 @@ -119,6 +133,13 @@ services: REDIS_HOST: redis QUEUE_CONNECTION: redis CACHE_STORE: redis + DW_EXTERNAL_PAYLOAD_S3_ACCESS_KEY_ID: "${DW_EXTERNAL_PAYLOAD_S3_ACCESS_KEY_ID:-}" + DW_EXTERNAL_PAYLOAD_S3_SECRET_ACCESS_KEY: "${DW_EXTERNAL_PAYLOAD_S3_SECRET_ACCESS_KEY:-}" + DW_EXTERNAL_PAYLOAD_S3_SESSION_TOKEN: "${DW_EXTERNAL_PAYLOAD_S3_SESSION_TOKEN:-}" + DW_EXTERNAL_PAYLOAD_S3_REGION: "${DW_EXTERNAL_PAYLOAD_S3_REGION:-us-east-1}" + DW_EXTERNAL_PAYLOAD_S3_BUCKET: "${DW_EXTERNAL_PAYLOAD_S3_BUCKET:-}" + DW_EXTERNAL_PAYLOAD_S3_ENDPOINT: "${DW_EXTERNAL_PAYLOAD_S3_ENDPOINT:-}" + DW_EXTERNAL_PAYLOAD_S3_USE_PATH_STYLE_ENDPOINT: "${DW_EXTERNAL_PAYLOAD_S3_USE_PATH_STYLE_ENDPOINT:-false}" depends_on: bootstrap: condition: service_completed_successfully @@ -155,7 +176,7 @@ services: DW_SERVER_KEY: "${DW_SERVER_KEY:-}" DW_SERVER_TOPOLOGY_SHAPE: standalone_server DW_SERVER_PROCESS_CLASS: scheduler_node - APP_VERSION: "${APP_VERSION:-2.0.3}" + APP_VERSION: "${APP_VERSION:-2.1.0}" DB_CONNECTION: mysql DB_HOST: mysql DB_PORT: 3306 @@ -165,6 +186,13 @@ services: REDIS_HOST: redis QUEUE_CONNECTION: redis CACHE_STORE: redis + DW_EXTERNAL_PAYLOAD_S3_ACCESS_KEY_ID: "${DW_EXTERNAL_PAYLOAD_S3_ACCESS_KEY_ID:-}" + DW_EXTERNAL_PAYLOAD_S3_SECRET_ACCESS_KEY: "${DW_EXTERNAL_PAYLOAD_S3_SECRET_ACCESS_KEY:-}" + DW_EXTERNAL_PAYLOAD_S3_SESSION_TOKEN: "${DW_EXTERNAL_PAYLOAD_S3_SESSION_TOKEN:-}" + DW_EXTERNAL_PAYLOAD_S3_REGION: "${DW_EXTERNAL_PAYLOAD_S3_REGION:-us-east-1}" + DW_EXTERNAL_PAYLOAD_S3_BUCKET: "${DW_EXTERNAL_PAYLOAD_S3_BUCKET:-}" + DW_EXTERNAL_PAYLOAD_S3_ENDPOINT: "${DW_EXTERNAL_PAYLOAD_S3_ENDPOINT:-}" + DW_EXTERNAL_PAYLOAD_S3_USE_PATH_STYLE_ENDPOINT: "${DW_EXTERNAL_PAYLOAD_S3_USE_PATH_STYLE_ENDPOINT:-false}" depends_on: bootstrap: condition: service_completed_successfully diff --git a/docs/contracts/external-payload-storage.md b/docs/contracts/external-payload-storage.md index 1f288de8..d1ffe55c 100644 --- a/docs/contracts/external-payload-storage.md +++ b/docs/contracts/external-payload-storage.md @@ -59,6 +59,58 @@ publishes the request timeout. Fetch responses use private, short-lived, immutable caching; SDK caches must be bounded and cannot delete runtime-owned objects. +## Self-hosted backing storage + +Node-local storage is suitable for a single-node development runtime. A +multi-node runtime must place external payload bytes on storage reachable from +every HTTP, queue, scheduler, and maintenance process. The published Server +image includes an S3-compatible adapter for that purpose. + +Set the following environment on every Server process: + +```dotenv +DW_EXTERNAL_PAYLOAD_S3_ACCESS_KEY_ID=example-access-key +DW_EXTERNAL_PAYLOAD_S3_SECRET_ACCESS_KEY=example-secret-key +DW_EXTERNAL_PAYLOAD_S3_REGION=us-east-1 +DW_EXTERNAL_PAYLOAD_S3_BUCKET=durable-workflow-payloads +DW_EXTERNAL_PAYLOAD_S3_ENDPOINT=https://objects.example.com +DW_EXTERNAL_PAYLOAD_S3_USE_PATH_STYLE_ENDPOINT=false +``` + +`DW_EXTERNAL_PAYLOAD_S3_ENDPOINT` is optional for AWS S3. Access key and secret +may both be omitted when the Server workload receives credentials from an IAM +role. Temporary credentials can also set +`DW_EXTERNAL_PAYLOAD_S3_SESSION_TOKEN`. Custom S3-compatible services commonly +require their HTTPS endpoint and path-style addressing. + +Enable the policy with an administrator credential after the bucket exists: + +```bash +curl -X PUT http://localhost:8080/api/namespaces/default/external-storage \ + -H "Authorization: Bearer $DW_ADMIN_TOKEN" \ + -H "X-Durable-Workflow-Control-Plane-Version: 2" \ + -H "Content-Type: application/json" \ + --data '{ + "driver": "s3", + "enabled": true, + "threshold_bytes": 2097152, + "config": {"prefix": "namespaces/default/"} + }' +``` + +The namespace policy contains no object-store credential. Server resolves the +fixed `external-payload-s3` disk from process configuration and uses the +configured bucket when the policy omits one. If a policy names a bucket, it +must match the configured disk bucket. + +Before serving workload traffic, call `POST /api/storage/test` for the +namespace. `GET /api/cluster/info` reports `driver_unavailable` and a bounded +`configuration_error` such as `s3_bucket_missing`, +`s3_credentials_incomplete`, or `s3_bucket_mismatch` without returning +credentials, endpoints, bucket names, or object references. Production +operators remain responsible for object-store durability, access policy, +backup, retention, and recovery validation. + ## State, expiry, and retention An upload starts as unclaimed and expires after the advertised abandoned-upload diff --git a/docs/server-reference.md b/docs/server-reference.md index 27e2304a..7b0f5066 100644 --- a/docs/server-reference.md +++ b/docs/server-reference.md @@ -628,10 +628,13 @@ is needed. The storage diagnostic writes, reads, verifies, and deletes small and large test payloads through the namespace's configured policy. It supports `local` and -configured-disk `s3`, `gcs`, and `azure` policies; it returns -`storage_driver_unavailable` when the namespace only stores provider metadata -and the current server runtime has no filesystem disk configured for that -provider. +the built-in S3-compatible disk as well as configured-disk `gcs`, `azure`, and +`custom` policies. It returns `storage_driver_unavailable` with a non-secret +`configuration_error` when the selected disk is incomplete or unavailable. +Configure every Server process with the same `DW_EXTERNAL_PAYLOAD_S3_*` +environment and follow the +[external-payload backing-storage setup](contracts/external-payload-storage.md#self-hosted-backing-storage) +before enabling an S3 namespace policy. Every non-health, non-discovery control-plane endpoint must send `X-Durable-Workflow-Control-Plane-Version: 2` on the request. That @@ -1535,6 +1538,13 @@ every operator-facing variable the server honors. | `DW_EXTERNAL_PAYLOAD_MAX_BYTES` | `67108864` | Max encoded bytes accepted by the authenticated runtime external-payload transport. | | `DW_EXTERNAL_PAYLOAD_REQUEST_TIMEOUT` | `30` | Advertised upload/fetch request-timeout budget in seconds. | | `DW_EXTERNAL_PAYLOAD_UPLOAD_EXPIRY` | `3600` | Seconds an unclaimed uploaded reference remains valid. | +| `DW_EXTERNAL_PAYLOAD_S3_ACCESS_KEY_ID` | (unset) | Optional access-key ID for the S3-compatible external-payload disk; omit with its secret when workload identity supplies credentials. | +| `DW_EXTERNAL_PAYLOAD_S3_SECRET_ACCESS_KEY` | (unset) | Optional secret access key for the S3-compatible external-payload disk; set it together with the access-key ID. | +| `DW_EXTERNAL_PAYLOAD_S3_SESSION_TOKEN` | (unset) | Optional session token used with temporary S3-compatible credentials. | +| `DW_EXTERNAL_PAYLOAD_S3_REGION` | `us-east-1` | Region used by the S3-compatible external-payload disk. | +| `DW_EXTERNAL_PAYLOAD_S3_BUCKET` | (unset) | Bucket used by the S3-compatible external-payload disk. | +| `DW_EXTERNAL_PAYLOAD_S3_ENDPOINT` | (unset) | Optional HTTPS endpoint for an S3-compatible external-payload service. | +| `DW_EXTERNAL_PAYLOAD_S3_USE_PATH_STYLE_ENDPOINT` | `false` | Use path-style bucket addressing for the S3-compatible external-payload service. | | `DW_MAX_MEMO_BYTES` | `262144` | Max serialized bytes for a workflow memo. | | `DW_MAX_SEARCH_ATTRIBUTES` | `100` | Max search attributes per workflow. | | `DW_MAX_PENDING_ACTIVITIES` | `2000` | Max pending activities per run. | diff --git a/k8s/README.md b/k8s/README.md index 68d8940c..7d942c2d 100644 --- a/k8s/README.md +++ b/k8s/README.md @@ -13,7 +13,7 @@ The checked-in manifests are synchronized with the repository's stable source release and pin its Docker Hub tag: ```text -durableworkflow/server:2.0.3 +durableworkflow/server:2.1.0 ``` Before production use, patch every workload image to the exact published tag or @@ -21,15 +21,15 @@ digest you intend to run: ```bash kubectl set image -n durable-workflow deploy/durable-workflow-server \ - server=durableworkflow/server:2.0.3 + server=durableworkflow/server:2.1.0 kubectl set image -n durable-workflow deploy/durable-workflow-worker \ - worker=durableworkflow/server:2.0.3 + worker=durableworkflow/server:2.1.0 kubectl set image -n durable-workflow cronjob/durable-workflow-scheduler \ - scheduler=durableworkflow/server:2.0.3 + scheduler=durableworkflow/server:2.1.0 ``` GitHub Container Registry publishes the same release line at -`ghcr.io/durable-workflow/server:2.0.3`. Digest pinning is preferred for strict +`ghcr.io/durable-workflow/server:2.1.0`. Digest pinning is preferred for strict change control. The manifests expect you to provide: diff --git a/k8s/helm/durable-workflow/Chart.yaml b/k8s/helm/durable-workflow/Chart.yaml index 3998596b..09147072 100644 --- a/k8s/helm/durable-workflow/Chart.yaml +++ b/k8s/helm/durable-workflow/Chart.yaml @@ -5,11 +5,11 @@ type: application # The chart's own semver version. Bumped on every chart release; treated as # independent of the server image version (appVersion). Breaking-change rules # for this version live in docs/helm-upgrading.md alongside the chart. -version: 0.1.74 +version: 0.1.75 # The immutable Durable Workflow Server identity this chart release packages. # The onboarding default in values.yaml and appVersion are generated from the # checked-in source release record. -appVersion: "2.0.3" +appVersion: "2.1.0" kubeVersion: ">=1.27.0-0" home: https://durable-workflow.github.io/docs/2.0/deployment sources: @@ -30,7 +30,7 @@ annotations: # exact commit that most recently changed the packaged chart. org.opencontainers.image.source: https://github.com/durable-workflow/server dev.durable-workflow.source-revision: "unreleased" - dev.durable-workflow.image-reference: "docker.io/durableworkflow/server:2.0.3" + dev.durable-workflow.image-reference: "docker.io/durableworkflow/server:2.1.0" artifacthub.io/license: MIT artifacthub.io/category: integration-delivery # Free-form changelog for the current chart release shown by Artifact Hub. diff --git a/k8s/helm/durable-workflow/README.md b/k8s/helm/durable-workflow/README.md index d9f2abc1..f300a164 100644 --- a/k8s/helm/durable-workflow/README.md +++ b/k8s/helm/durable-workflow/README.md @@ -63,7 +63,7 @@ helm install durable-workflow ./k8s/helm/durable-workflow \ ```yaml image: - tag: "2.0.3" + tag: "2.1.0" # Pin a digest in production: # digest: "sha256:abc123..." # memoPayloadStorage: "raw-json-v1" # Required for a digest or custom image. diff --git a/k8s/helm/durable-workflow/ci/existing-secrets-values.yaml b/k8s/helm/durable-workflow/ci/existing-secrets-values.yaml index 37cec9bc..471651eb 100644 --- a/k8s/helm/durable-workflow/ci/existing-secrets-values.yaml +++ b/k8s/helm/durable-workflow/ci/existing-secrets-values.yaml @@ -1,7 +1,7 @@ # CI fixture: GitOps / externally-managed-secret path. The chart consumes # existing Secrets and renders no Secret resources of its own. image: - tag: "2.0.3" + tag: "2.1.0" externalDatabase: connection: pgsql diff --git a/k8s/helm/durable-workflow/ci/ingress-and-hpa-values.yaml b/k8s/helm/durable-workflow/ci/ingress-and-hpa-values.yaml index 32a14392..40fffadd 100644 --- a/k8s/helm/durable-workflow/ci/ingress-and-hpa-values.yaml +++ b/k8s/helm/durable-workflow/ci/ingress-and-hpa-values.yaml @@ -1,6 +1,6 @@ # CI fixture: ingress + autoscaling enabled. Exercises optional templates. image: - tag: "2.0.3" + tag: "2.1.0" externalDatabase: connection: mysql diff --git a/k8s/helm/durable-workflow/ci/inline-secrets-values.yaml b/k8s/helm/durable-workflow/ci/inline-secrets-values.yaml index 8f3e2f98..be9286ee 100644 --- a/k8s/helm/durable-workflow/ci/inline-secrets-values.yaml +++ b/k8s/helm/durable-workflow/ci/inline-secrets-values.yaml @@ -2,7 +2,7 @@ # chart's render path is exercised end-to-end. Real deployments should use # existingSecret instead. image: - tag: "2.0.3" + tag: "2.1.0" externalDatabase: connection: mysql diff --git a/k8s/helm/durable-workflow/templates/_helpers.tpl b/k8s/helm/durable-workflow/templates/_helpers.tpl index cb296837..59eb1728 100644 --- a/k8s/helm/durable-workflow/templates/_helpers.tpl +++ b/k8s/helm/durable-workflow/templates/_helpers.tpl @@ -88,7 +88,7 @@ resolved by an explicit capability declaration or an existing workload marker. {{- define "durable-workflow.memoPayloadStorageForImage" -}} {{- $image := toString . -}} {{- $normalized := regexReplaceAll "^index\\.docker\\.io/" $image "docker.io/" -}} -{{- if eq $normalized "docker.io/durableworkflow/server:2.0.3" -}} +{{- if eq $normalized "docker.io/durableworkflow/server:2.1.0" -}} dual-v1 {{- else if regexMatch "^docker\\.io/durableworkflow/server:2\\.0\\.0-rc\\.[0-9]+$" $normalized -}} {{- $releaseCandidate := atoi (regexFind "[0-9]+$" $normalized) -}} diff --git a/k8s/helm/durable-workflow/values.yaml b/k8s/helm/durable-workflow/values.yaml index 34d45a72..5fd7ca86 100644 --- a/k8s/helm/durable-workflow/values.yaml +++ b/k8s/helm/durable-workflow/values.yaml @@ -21,7 +21,7 @@ image: registry: docker.io repository: durableworkflow/server # Generated by scripts/ci/sync-source-release.mjs. Do not edit this default. - tag: "2.0.3" + tag: "2.1.0" # Optional digest pin. When set, takes precedence over tag for change control. # Example: "sha256:abc123..." digest: "" diff --git a/k8s/helm/examples/values-dev.yaml b/k8s/helm/examples/values-dev.yaml index 1544b1b4..139db1ee 100644 --- a/k8s/helm/examples/values-dev.yaml +++ b/k8s/helm/examples/values-dev.yaml @@ -3,7 +3,7 @@ # shape in production. image: - tag: "2.0.3" + tag: "2.1.0" externalDatabase: connection: mysql diff --git a/k8s/helm/examples/values-external-secrets-operator.yaml b/k8s/helm/examples/values-external-secrets-operator.yaml index c9f6e812..d7690e6f 100644 --- a/k8s/helm/examples/values-external-secrets-operator.yaml +++ b/k8s/helm/examples/values-external-secrets-operator.yaml @@ -5,7 +5,7 @@ # concern. image: - tag: "2.0.3" + tag: "2.1.0" externalDatabase: connection: pgsql diff --git a/k8s/helm/examples/values-production-existing-secrets.yaml b/k8s/helm/examples/values-production-existing-secrets.yaml index a46a8a75..4a1dfa03 100644 --- a/k8s/helm/examples/values-production-existing-secrets.yaml +++ b/k8s/helm/examples/values-production-existing-secrets.yaml @@ -10,7 +10,7 @@ image: repository: durable-workflow/server # Pin a digest in production for change-control auditability. digest: "" # e.g. "sha256:abc123..." - tag: "2.0.3" + tag: "2.1.0" externalDatabase: connection: pgsql diff --git a/k8s/migration-job.yaml b/k8s/migration-job.yaml index 1be2c78b..9780a079 100644 --- a/k8s/migration-job.yaml +++ b/k8s/migration-job.yaml @@ -13,7 +13,7 @@ spec: restartPolicy: OnFailure containers: - name: migrate - image: durableworkflow/server:2.0.3 + image: durableworkflow/server:2.1.0 command: ["server-entrypoint"] args: ["server-bootstrap"] envFrom: diff --git a/k8s/scheduler-cronjob.yaml b/k8s/scheduler-cronjob.yaml index 5ac90d03..16a0ca77 100644 --- a/k8s/scheduler-cronjob.yaml +++ b/k8s/scheduler-cronjob.yaml @@ -24,7 +24,7 @@ spec: restartPolicy: Never containers: - name: scheduler - image: durableworkflow/server:2.0.3 + image: durableworkflow/server:2.1.0 command: ["server-entrypoint"] args: ["sh", "-c", "php artisan schedule:evaluate --limit=100 --json; php artisan activity:timeout-enforce --limit=100; if php artisan list --raw | grep -q '^external-payloads:cleanup '; then php artisan external-payloads:cleanup --limit=100 --json; fi; php artisan history:prune --limit=100"] envFrom: diff --git a/k8s/secret.yaml b/k8s/secret.yaml index 33b03e89..e227ea70 100644 --- a/k8s/secret.yaml +++ b/k8s/secret.yaml @@ -12,7 +12,7 @@ metadata: app.kubernetes.io/name: durable-workflow data: APP_NAME: "Durable Workflow Server" - APP_VERSION: "2.0.3" + APP_VERSION: "2.1.0" APP_ENV: production APP_DEBUG: "false" DB_CONNECTION: mysql diff --git a/k8s/server-deployment.yaml b/k8s/server-deployment.yaml index 422eea63..72732428 100644 --- a/k8s/server-deployment.yaml +++ b/k8s/server-deployment.yaml @@ -23,7 +23,7 @@ spec: spec: containers: - name: server - image: durableworkflow/server:2.0.3 + image: durableworkflow/server:2.1.0 ports: - containerPort: 8080 name: http diff --git a/k8s/worker-deployment.yaml b/k8s/worker-deployment.yaml index 74fdc5cf..93627171 100644 --- a/k8s/worker-deployment.yaml +++ b/k8s/worker-deployment.yaml @@ -19,7 +19,7 @@ spec: spec: containers: - name: worker - image: durableworkflow/server:2.0.3 + image: durableworkflow/server:2.1.0 command: ["server-entrypoint"] args: ["php", "artisan", "queue:work", "--sleep=1", "--tries=3", "--max-time=3600"] envFrom: diff --git a/resources/release/source-release.json b/resources/release/source-release.json index f6f98c78..13f16177 100644 --- a/resources/release/source-release.json +++ b/resources/release/source-release.json @@ -1,9 +1,9 @@ { "schema": "durable-workflow.server.source-release/v1", "server": { - "version": "2.0.3" + "version": "2.1.0" }, "helm_chart": { - "version": "0.1.74" + "version": "0.1.75" } } diff --git a/scripts/k8s-kind-smoke.sh b/scripts/k8s-kind-smoke.sh index 69dddf14..963b084f 100755 --- a/scripts/k8s-kind-smoke.sh +++ b/scripts/k8s-kind-smoke.sh @@ -7,7 +7,7 @@ cluster="${K8S_SMOKE_CLUSTER:-durable-workflow-server-smoke}" image="${K8S_SMOKE_IMAGE:-durableworkflow/server:k8s-smoke}" # Generated by scripts/ci/sync-source-release.mjs so the smoke replaces the # same default shipped by the public manifests. -manifest_image="durableworkflow/server:2.0.3" +manifest_image="durableworkflow/server:2.1.0" kind_node_image="${K8S_SMOKE_KIND_NODE_IMAGE:-kindest/node:v1.29.4}" artifact_dir="${K8S_SMOKE_ARTIFACT_DIR:-/tmp/durable-workflow-k8s-kind-smoke-artifacts}" rendered_dir="${artifact_dir}/rendered-manifests" diff --git a/tests/Feature/ClusterInfoTest.php b/tests/Feature/ClusterInfoTest.php index a9ddba70..53015b8c 100644 --- a/tests/Feature/ClusterInfoTest.php +++ b/tests/Feature/ClusterInfoTest.php @@ -2052,6 +2052,7 @@ public function test_it_exposes_namespace_external_payload_storage_policy_path() ->assertJsonPath('namespace.external_payload_storage.configured', true) ->assertJsonPath('namespace.external_payload_storage.enabled', true) ->assertJsonPath('namespace.external_payload_storage.status', 'available') + ->assertJsonPath('namespace.external_payload_storage.configuration_error', null) ->assertJsonPath('namespace.external_payload_storage.threshold_bytes', 1024) ->assertJsonPath('namespace.external_payload_storage.provider_details_exposed', false) ->assertJsonPath('namespace.external_payload_storage.transport.version', 1) @@ -2087,6 +2088,10 @@ public function test_cluster_info_reports_unknown_object_storage_disk_unavailabl ->assertJsonPath('namespace.external_payload_storage.configured', true) ->assertJsonPath('namespace.external_payload_storage.enabled', true) ->assertJsonPath('namespace.external_payload_storage.status', 'driver_unavailable') + ->assertJsonPath( + 'namespace.external_payload_storage.configuration_error', + 'filesystem_disk_not_configured', + ) ->assertJsonPath('namespace.external_payload_storage.provider_details_exposed', false) ->assertJsonMissingPath('namespace.external_payload_storage.driver') ->assertJsonMissingPath('namespace.external_payload_storage.reference_uri_scheme') diff --git a/tests/Feature/ExternalPayloadStorageTest.php b/tests/Feature/ExternalPayloadStorageTest.php index b233fe5b..c1485547 100644 --- a/tests/Feature/ExternalPayloadStorageTest.php +++ b/tests/Feature/ExternalPayloadStorageTest.php @@ -216,6 +216,7 @@ public function test_storage_diagnostic_reports_unconfigured_and_unavailable_dri ])->assertStatus(422) ->assertJsonPath('reason', 'storage_driver_unavailable') ->assertJsonPath('driver', 's3') + ->assertJsonPath('configuration_error', 's3_bucket_missing') ->assertJsonPath('supported_diagnostic_drivers.0', 'local'); WorkflowNamespace::where('name', 'default')->update([ @@ -234,7 +235,8 @@ public function test_storage_diagnostic_reports_unconfigured_and_unavailable_dri 'large_payload_bytes' => 64, ])->assertStatus(422) ->assertJsonPath('reason', 'storage_driver_unavailable') - ->assertJsonPath('driver', 's3'); + ->assertJsonPath('driver', 's3') + ->assertJsonPath('configuration_error', 'filesystem_disk_not_configured'); } public function test_storage_diagnostic_reports_disabled_policy(): void diff --git a/tests/Feature/S3ExternalPayloadProcessTest.php b/tests/Feature/S3ExternalPayloadProcessTest.php new file mode 100644 index 00000000..ce404910 --- /dev/null +++ b/tests/Feature/S3ExternalPayloadProcessTest.php @@ -0,0 +1,196 @@ +markTestSkipped('Set DW_TEST_S3_ENDPOINT to run the real S3-compatible process test.'); + } + + $this->endpoint = $endpoint; + $this->key = $this->requiredEnv('DW_TEST_S3_ACCESS_KEY_ID'); + $this->secret = $this->requiredEnv('DW_TEST_S3_SECRET_ACCESS_KEY'); + $this->bucket = 'dw-external-payload-'.bin2hex(random_bytes(6)); + $database = tempnam(sys_get_temp_dir(), 'dw-s3-process-'); + + if ($database === false) { + self::fail('Unable to create the process-boundary SQLite database.'); + } + $this->database = $database; + + $this->s3 = new S3Client([ + 'version' => 'latest', + 'region' => 'us-east-1', + 'endpoint' => $this->endpoint, + 'use_path_style_endpoint' => true, + 'credentials' => ['key' => $this->key, 'secret' => $this->secret], + 'http' => ['connect_timeout' => 1, 'timeout' => 5], + ]); + + $this->waitForObjectStore(); + $this->s3->createBucket(['Bucket' => $this->bucket]); + $this->runProcess([PHP_BINARY, 'artisan', 'migrate:fresh', '--force']); + } + + protected function tearDown(): void + { + if (isset($this->s3, $this->bucket)) { + try { + $objects = $this->s3->listObjectsV2(['Bucket' => $this->bucket]); + foreach ($objects['Contents'] ?? [] as $object) { + $this->s3->deleteObject(['Bucket' => $this->bucket, 'Key' => $object['Key']]); + } + $this->s3->deleteBucket(['Bucket' => $this->bucket]); + } catch (AwsException) { + // The assertion failure is more useful than cleanup noise. + } + } + + if (isset($this->database) && is_file($this->database)) { + unlink($this->database); + } + + parent::tearDown(); + } + + public function test_runtime_payload_survives_a_fresh_application_process_and_validates_integrity(): void + { + $payload = "durable\0workflow\xffs3-process-boundary"; + $payloadHash = hash('sha256', $payload); + $payloadEnv = ['DW_TEST_EXTERNAL_PAYLOAD_BYTES' => base64_encode($payload)]; + + $write = $this->runProbe('write', $payloadEnv); + self::assertMatchesRegularExpression('/^DW_S3_PROBE_REFERENCE=/m', $write->getOutput()); + + preg_match('/^DW_S3_PROBE_REFERENCE=(.+)$/m', $write->getOutput(), $matches); + $reference = trim($matches[1]); + $referenceEnv = $payloadEnv + ['DW_TEST_EXTERNAL_PAYLOAD_REFERENCE' => $reference]; + $key = 'process-boundary/avro/'.substr($payloadHash, 0, 2).'/'.$payloadHash; + + self::assertTrue($this->s3->doesObjectExistV2($this->bucket, $key)); + + $read = $this->runProbe('read', $referenceEnv); + self::assertStringContainsString('DW_S3_PROBE_SHA256='.$payloadHash, $read->getOutput()); + + $corruptPayload = substr($payload, 0, -1).($payload[-1] === 'x' ? 'y' : 'x'); + $this->s3->putObject(['Bucket' => $this->bucket, 'Key' => $key, 'Body' => $corruptPayload]); + + $corruptRead = $this->runProbe('read', $referenceEnv, false); + self::assertSame(2, $corruptRead->getExitCode()); + self::assertStringContainsString( + 'DW_S3_PROBE_ERROR=external_payload_integrity_mismatch', + $corruptRead->getErrorOutput(), + ); + + $this->s3->putObject(['Bucket' => $this->bucket, 'Key' => $key, 'Body' => $payload]); + + $delete = $this->runProbe('delete', $referenceEnv); + self::assertStringContainsString('DW_S3_PROBE_DELETED=1', $delete->getOutput()); + self::assertFalse($this->s3->doesObjectExistV2($this->bucket, $key)); + } + + /** + * @param array $environment + */ + private function runProbe(string $operation, array $environment = [], bool $mustSucceed = true): Process + { + return $this->runProcess( + [PHP_BINARY, 'tests/Support/S3ExternalPayloadProcess.php', $operation], + $environment, + $mustSucceed, + ); + } + + /** + * @param list $command + * @param array $environment + */ + private function runProcess(array $command, array $environment = [], bool $mustSucceed = true): Process + { + $process = new Process($command, dirname(__DIR__, 2), $environment + $this->environment()); + $process->setTimeout(120); + $process->run(); + + if ($mustSucceed) { + self::assertTrue($process->isSuccessful(), $process->getErrorOutput().$process->getOutput()); + } + + return $process; + } + + /** + * @return array + */ + private function environment(): array + { + return [ + 'APP_ENV' => 'testing', + 'APP_KEY' => 'base64:dGVzdGluZy10ZXN0aW5nLXRlc3RpbmctdGVzdGluZzEyMzQ1Ng==', + 'CACHE_STORE' => 'array', + 'DB_CONNECTION' => 'sqlite', + 'DB_DATABASE' => $this->database, + 'DW_AUTH_DRIVER' => 'none', + 'DW_EXTERNAL_PAYLOAD_S3_ACCESS_KEY_ID' => $this->key, + 'DW_EXTERNAL_PAYLOAD_S3_SECRET_ACCESS_KEY' => $this->secret, + 'DW_EXTERNAL_PAYLOAD_S3_REGION' => 'us-east-1', + 'DW_EXTERNAL_PAYLOAD_S3_BUCKET' => $this->bucket, + 'DW_EXTERNAL_PAYLOAD_S3_ENDPOINT' => $this->endpoint, + 'DW_EXTERNAL_PAYLOAD_S3_USE_PATH_STYLE_ENDPOINT' => 'true', + 'LOG_CHANNEL' => 'single', + 'QUEUE_CONNECTION' => 'database', + ]; + } + + private function requiredEnv(string $name): string + { + $value = getenv($name); + + if (! is_string($value) || $value === '') { + self::fail("Missing required environment variable {$name}."); + } + + return $value; + } + + private function waitForObjectStore(): void + { + $deadline = microtime(true) + 30; + $lastError = null; + + do { + try { + $this->s3->listBuckets(); + + return; + } catch (AwsException $exception) { + $lastError = $exception; + usleep(250_000); + } + } while (microtime(true) < $deadline); + + self::fail('S3-compatible service did not become ready: '.($lastError?->getMessage() ?? 'unknown error')); + } +} diff --git a/tests/Support/S3ExternalPayloadProcess.php b/tests/Support/S3ExternalPayloadProcess.php new file mode 100644 index 00000000..4068d84f --- /dev/null +++ b/tests/Support/S3ExternalPayloadProcess.php @@ -0,0 +1,102 @@ +make(Kernel::class); +$kernel->bootstrap(); + +$requiredEnv = static function (string $name): string { + $value = getenv($name); + + if (! is_string($value) || $value === '') { + throw new RuntimeException("Missing required environment variable {$name}."); + } + + return $value; +}; + +$reference = static function () use ($requiredEnv): array { + $decoded = base64_decode($requiredEnv('DW_TEST_EXTERNAL_PAYLOAD_REFERENCE'), true); + $value = is_string($decoded) ? json_decode($decoded, true) : null; + + if (! is_array($value)) { + throw new RuntimeException('External payload reference is invalid.'); + } + + return $value; +}; + +try { + $operation = $argv[1] ?? ''; + $namespace = 's3-process-boundary'; + $registry = $app->make(RuntimeExternalPayloadRegistry::class); + + if ($operation === 'write') { + $payload = base64_decode($requiredEnv('DW_TEST_EXTERNAL_PAYLOAD_BYTES'), true); + if (! is_string($payload)) { + throw new RuntimeException('External payload bytes are invalid.'); + } + + WorkflowNamespace::query()->updateOrCreate( + ['name' => $namespace], + [ + 'description' => 'S3 process-boundary qualification', + 'retention_days' => 1, + 'status' => 'active', + 'external_payload_storage' => [ + 'driver' => 's3', + 'enabled' => true, + 'threshold_bytes' => 1, + 'config' => ['prefix' => 'process-boundary/'], + ], + ], + ); + + $result = $registry->upload($namespace, $payload, 'avro', hash('sha256', $payload)); + echo 'DW_S3_PROBE_REFERENCE='.base64_encode(json_encode($result, JSON_THROW_ON_ERROR)).PHP_EOL; + + exit(0); + } + + if ($operation === 'read') { + $payload = base64_decode($requiredEnv('DW_TEST_EXTERNAL_PAYLOAD_BYTES'), true); + if (! is_string($payload)) { + throw new RuntimeException('External payload bytes are invalid.'); + } + + $result = $registry->fetch($namespace, $reference()); + if (! hash_equals($payload, $result['data'])) { + throw new RuntimeException('Fetched external payload does not match the expected bytes.'); + } + + echo 'DW_S3_PROBE_SHA256='.hash('sha256', $result['data']).PHP_EOL; + + exit(0); + } + + if ($operation === 'delete') { + echo 'DW_S3_PROBE_DELETED='.$registry->deleteForNamespace($namespace).PHP_EOL; + + exit(0); + } + + throw new RuntimeException('Expected write, read, or delete operation.'); +} catch (RuntimeExternalPayloadException $exception) { + fwrite(STDERR, 'DW_S3_PROBE_ERROR='.$exception->reason.PHP_EOL); + + exit(2); +} catch (Throwable $exception) { + fwrite(STDERR, 'DW_S3_PROBE_ERROR='.$exception::class.': '.$exception->getMessage().PHP_EOL); + + exit(1); +} diff --git a/tests/Unit/PhpunitFeatureWorkflowContractTest.php b/tests/Unit/PhpunitFeatureWorkflowContractTest.php index 62b6687a..14c48570 100644 --- a/tests/Unit/PhpunitFeatureWorkflowContractTest.php +++ b/tests/Unit/PhpunitFeatureWorkflowContractTest.php @@ -119,7 +119,8 @@ public function test_regression_corpus_validation_uses_git_capable_ci_and_an_exa 'docker build --target base -t durable-workflow-server-phpunit-base .', 'FROM durable-workflow-server-phpunit-base', 'apt-get install -y --no-install-recommends git', - 'docker run --rm -i -e CORPUS_BASE_REF', + 'docker run --rm -i', + '-e CORPUS_BASE_REF', 'tar --no-same-owner -xf - -C /app', 'composer install --no-interaction --no-progress --prefer-dist', 'git --version',