From 55d64903ee9baf3941f2e3ed3a2a574a2d1cdc41 Mon Sep 17 00:00:00 2001 From: Steve Thomas Date: Tue, 19 May 2026 11:42:37 +1000 Subject: [PATCH 1/2] feat: sync IVS Real-Time recording configuration MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds a `sync:recording` command that provisions the S3 + IVS Real-Time resources needed for composite recording of IVS Real-Time stages to S3. Originally authored by Seth Sharp against codinglabsau/yolo#21 (targeted at the 1.x branch). Duplicated here so it can ship on yolo-alpha while LP is still mid-migration. Namespace + pint sweeps applied to match the yolo-alpha codebase. ### yolo.yml ```yaml aws: ivs: recording: real_time: true ``` Omitting `recording` skips all recording steps — no change for existing deployments. ### sync ```bash yolo-alpha sync:recording production ``` After running, three values are printed for the app's .env: - AWS_IVS_REALTIME_RECORDINGS_BUCKET - AWS_IVS_STORAGE_CONFIGURATION_ARN - AWS_IVS_ENCODER_CONFIGURATION_ARN ### What sync:recording provisions | Step | What it creates | |---|---| | SyncIvsRealtimeRecordingBucketStep | S3 bucket; policy allows `ivs-composite.{aws.region}.amazonaws.com` to write with `bucket-owner-full-control`, grants MediaConvert role read access. Ownership set to BucketOwnerPreferred | | SyncIvsStorageConfigurationStep | IVS Real-Time StorageConfiguration pointing at that bucket | | SyncIvsEncoderConfigurationStep | IVS Real-Time EncoderConfiguration (720p30 @ 2.5 Mbps) | All run in the single aws.region. Each step is idempotent (returns SYNCED when the resource already exists) and skips gracefully when aws.ivs.recording.real_time is not set. ### IAM permissions for the user running yolo-alpha | Policy | Why | |---|---| | AmazonS3FullAccess | Create recording bucket + bucket policy/ownership | | IVSFullAccess | Create StorageConfiguration and EncoderConfiguration | ### Notes - Bucket name auto-generated via Helpers::keyedResourceName() to avoid global S3 name collisions - BucketOwnerPreferred ownership is required because IVS writes objects with the bucket-owner-full-control ACL - No IAM roles are provisioned for recording (sync:iam unchanged) - IVS composite recording outputs TS-based HLS — directly compatible with the existing MediaConvert pipeline, no FFmpeg/pre-processing step required Co-Authored-By: Seth Sharp <58869086+SethSharp@users.noreply.github.com> Co-Authored-By: Claude Opus 4.7 (1M context) --- README.md | 1 + docs/reference/manifest.md | 27 +++++ src/Aws.php | 6 + src/Commands/SyncRecordingCommand.php | 25 ++++ src/Concerns/RegistersAws.php | 2 + src/Manifest.php | 5 + .../SyncIvsEncoderConfigurationStep.php | 65 +++++++++++ .../SyncIvsRealtimeRecordingBucketStep.php | 107 ++++++++++++++++++ .../SyncIvsStorageConfigurationStep.php | 62 ++++++++++ src/Yolo.php | 1 + tests/Unit/PathsTest.php | 4 + 11 files changed, 305 insertions(+) create mode 100644 src/Commands/SyncRecordingCommand.php create mode 100644 src/Steps/Recording/SyncIvsEncoderConfigurationStep.php create mode 100644 src/Steps/Recording/SyncIvsRealtimeRecordingBucketStep.php create mode 100644 src/Steps/Recording/SyncIvsStorageConfigurationStep.php diff --git a/README.md b/README.md index facbce4..d890bae 100644 --- a/README.md +++ b/README.md @@ -154,6 +154,7 @@ The full list of available sync commands are: - `yolo-alpha sync:ci ` prepares the continuous integration pipeline - `yolo-alpha sync:iam ` prepares necessary permissions - `yolo-alpha sync:logging ` prepares observability infrastructure (e.g. IVS state-change events) +- `yolo-alpha sync:recording ` prepares IVS Real-Time composite recording infrastructure (S3 bucket, StorageConfiguration, EncoderConfiguration) > [!TIP] > All sync commands support a `--dry-run` argument; this is a great starting point to see what resources will be created diff --git a/docs/reference/manifest.md b/docs/reference/manifest.md index 9b5adfc..e18a08d 100644 --- a/docs/reference/manifest.md +++ b/docs/reference/manifest.md @@ -117,6 +117,33 @@ aws: `logging` toggles the EventBridge → CloudWatch pipeline; `log-retention-days` overrides the log retention. +#### IVS Real-Time recording + +Enable S3 composite recording for IVS Real-Time stages: + +```yaml +aws: + ivs: + recording: + real_time: true +``` + +Setting `recording.real_time` to `true` provisions the S3 bucket, IVS `StorageConfiguration`, and `EncoderConfiguration` required for composite recording. + +| Key | Description | +|---|---| +| `recording.real_time` | Set to `true` to enable IVS Real-Time composite recording provisioning. Provisions an auto-named S3 bucket (`yolo-{env}-{app}-ivs-realtime-recordings`), a `StorageConfiguration` pointing to that bucket, and an `EncoderConfiguration` (720p30). | + +After running `sync:recording`, three values are printed for the app's `.env`: + +| Env var | Description | +|---|---| +| `AWS_IVS_REALTIME_RECORDINGS_BUCKET` | Name of the S3 bucket IVS writes recordings to | +| `AWS_IVS_STORAGE_CONFIGURATION_ARN` | ARN passed to `createStage` for automatic participant recording | +| `AWS_IVS_ENCODER_CONFIGURATION_ARN` | ARN passed to `startComposition` to define video resolution and bitrate | + +Omitting `recording` entirely skips all recording steps without affecting existing resources. + ### `mysqldump` Enable scheduled MySQL backups via `mysqldump`. diff --git a/src/Aws.php b/src/Aws.php index ab73140..896b16e 100644 --- a/src/Aws.php +++ b/src/Aws.php @@ -16,6 +16,7 @@ use Aws\CodeDeploy\CodeDeployClient; use Aws\AutoScaling\AutoScalingClient; use Aws\EventBridge\EventBridgeClient; +use Aws\IVSRealTime\IVSRealTimeClient; use Aws\CloudWatchLogs\CloudWatchLogsClient; use Aws\ElasticLoadBalancingV2\ElasticLoadBalancingV2Client; @@ -113,6 +114,11 @@ public static function iam(): IamClient return Helpers::app('iam'); } + public static function ivsRealTime(): IVSRealTimeClient + { + return Helpers::app('ivsRealTime'); + } + public static function rds(): RdsClient { return Helpers::app('rds'); diff --git a/src/Commands/SyncRecordingCommand.php b/src/Commands/SyncRecordingCommand.php new file mode 100644 index 0000000..05bc77d --- /dev/null +++ b/src/Commands/SyncRecordingCommand.php @@ -0,0 +1,25 @@ +setName('sync:recording') + ->addArgument('environment', InputArgument::REQUIRED, 'The environment name') + ->addOption('dry-run', null, null, 'Run the command without making changes') + ->addOption('no-progress', null, null, 'Hide the progress output') + ->setDescription('Sync the IVS recording resources for the given environment'); + } +} diff --git a/src/Concerns/RegistersAws.php b/src/Concerns/RegistersAws.php index 1cf38b7..09f75cd 100644 --- a/src/Concerns/RegistersAws.php +++ b/src/Concerns/RegistersAws.php @@ -20,6 +20,7 @@ use Aws\CodeDeploy\CodeDeployClient; use Aws\AutoScaling\AutoScalingClient; use Aws\EventBridge\EventBridgeClient; +use Aws\IVSRealTime\IVSRealTimeClient; use Aws\Credentials\CredentialProvider; use GuzzleHttp\Exception\ConnectException; use Codinglabs\YoloAlpha\Enums\ServerGroup; @@ -48,6 +49,7 @@ protected function registerAwsServices(): void Helpers::app()->singleton('eventBridge', fn () => new EventBridgeClient($arguments)); Helpers::app()->singleton('elasticLoadBalancingV2', fn () => new ElasticLoadBalancingV2Client($arguments)); Helpers::app()->singleton('iam', fn () => new IamClient($arguments)); + Helpers::app()->singleton('ivsRealTime', fn () => new IVSRealTimeClient($arguments)); Helpers::app()->singleton('rds', fn () => new RdsClient($arguments)); Helpers::app()->singleton('route53', fn () => new Route53Client($arguments)); Helpers::app()->singleton('s3', fn () => new S3Client($arguments)); diff --git a/src/Manifest.php b/src/Manifest.php index ee3f604..ddaa8ea 100644 --- a/src/Manifest.php +++ b/src/Manifest.php @@ -100,6 +100,11 @@ public static function ivsEnabled(): bool || static::get('aws.ivs.logging') === true; } + public static function ivsRealtimeRecordingEnabled(): bool + { + return ! empty(static::get('aws.ivs.recording.real_time')); + } + /** * @return arraylistEncoderConfigurations(); + $all = $response['encoderConfigurations']; + while ($nextToken = $response['nextToken'] ?? null) { + $response = Aws::ivsRealTime()->listEncoderConfigurations(['nextToken' => $nextToken]); + $all = array_merge($all, $response['encoderConfigurations']); + } + + $existing = collect($all)->first(fn ($config) => $config['name'] === $name); + + if ($existing) { + note(sprintf('IVS EncoderConfiguration ARN: %s', $existing['arn'])); + note(sprintf('Set AWS_IVS_ENCODER_CONFIGURATION_ARN=%s', $existing['arn'])); + + return StepResult::SYNCED; + } + + if (! Arr::get($options, 'dry-run')) { + $result = Aws::ivsRealTime()->createEncoderConfiguration([ + 'name' => $name, + 'video' => [ + 'width' => 1280, + 'height' => 720, + 'framerate' => 30, + 'bitrate' => 2500000, + ], + 'tags' => [ + 'yolo:environment' => Helpers::app('environment'), + 'Name' => $name, + ], + ]); + + $arn = $result['encoderConfiguration']['arn']; + + note(sprintf('IVS EncoderConfiguration ARN: %s', $arn)); + note(sprintf('Set AWS_IVS_ENCODER_CONFIGURATION_ARN=%s', $arn)); + + return StepResult::CREATED; + } + + return StepResult::WOULD_CREATE; + } +} diff --git a/src/Steps/Recording/SyncIvsRealtimeRecordingBucketStep.php b/src/Steps/Recording/SyncIvsRealtimeRecordingBucketStep.php new file mode 100644 index 0000000..35bc979 --- /dev/null +++ b/src/Steps/Recording/SyncIvsRealtimeRecordingBucketStep.php @@ -0,0 +1,107 @@ +putBucketPolicy($bucket); + } + + note(sprintf('IVS recordings bucket: %s', $bucket)); + note(sprintf('Set AWS_IVS_REALTIME_RECORDINGS_BUCKET=%s', $bucket)); + + return StepResult::SYNCED; + } catch (ResourceDoesNotExistException) { + if (! Arr::get($options, 'dry-run')) { + Aws::s3()->createBucket(['Bucket' => $bucket]); + Aws::s3()->waitUntil('BucketExists', ['Bucket' => $bucket]); + Aws::s3()->putBucketTagging([ + 'Bucket' => $bucket, + 'Tagging' => [...Aws::tags(['Name' => $bucket], wrap: 'TagSet')], + ]); + $this->putBucketPolicy($bucket); + + note(sprintf('IVS recordings bucket: %s', $bucket)); + note(sprintf('Set AWS_IVS_REALTIME_RECORDINGS_BUCKET=%s', $bucket)); + + return StepResult::CREATED; + } + + return StepResult::WOULD_CREATE; + } + } + + protected function putBucketPolicy(string $bucket): void + { + $region = Manifest::get('aws.region'); + $accountId = Aws::accountId(); + $mediaConvertRoleName = Helpers::keyedResourceName(Iam::MEDIA_CONVERT_ROLE); + $mediaConvertRoleArn = "arn:aws:iam::{$accountId}:role/{$mediaConvertRoleName}"; + + // IVS Real-Time writes via ivs-composite.{region}.amazonaws.com, not ivs.amazonaws.com. + // It requires PutObjectAcl with bucket-owner-full-control so ownership transfers to the + // bucket owner, allowing same-account services (e.g. MediaConvert) to read the objects. + Aws::s3()->putBucketPolicy([ + 'Bucket' => $bucket, + 'Policy' => json_encode([ + 'Version' => '2012-10-17', + 'Statement' => [ + [ + 'Sid' => 'IVSRealtimeRecording', + 'Effect' => 'Allow', + 'Principal' => ['Service' => "ivs-composite.{$region}.amazonaws.com"], + 'Action' => ['s3:PutObject', 's3:PutObjectAcl'], + 'Resource' => "arn:aws:s3:::{$bucket}/*", + 'Condition' => [ + 'StringEquals' => ['s3:x-amz-acl' => 'bucket-owner-full-control'], + 'Bool' => ['aws:SecureTransport' => 'true'], + ], + ], + [ + 'Sid' => 'MediaConvertRead', + 'Effect' => 'Allow', + 'Principal' => ['AWS' => $mediaConvertRoleArn], + 'Action' => ['s3:GetObject', 's3:GetObjectVersion', 's3:ListBucket'], + 'Resource' => ["arn:aws:s3:::{$bucket}", "arn:aws:s3:::{$bucket}/*"], + ], + ], + ]), + ]); + + Aws::s3()->putBucketOwnershipControls([ + 'Bucket' => $bucket, + 'OwnershipControls' => [ + 'Rules' => [['ObjectOwnership' => 'BucketOwnerPreferred']], + ], + ]); + } + + public static function bucketName(): string + { + return Helpers::keyedResourceName('ivs-realtime-recordings'); + } +} diff --git a/src/Steps/Recording/SyncIvsStorageConfigurationStep.php b/src/Steps/Recording/SyncIvsStorageConfigurationStep.php new file mode 100644 index 0000000..5185b13 --- /dev/null +++ b/src/Steps/Recording/SyncIvsStorageConfigurationStep.php @@ -0,0 +1,62 @@ +listStorageConfigurations(); + $all = $response['storageConfigurations']; + while ($nextToken = $response['nextToken'] ?? null) { + $response = Aws::ivsRealTime()->listStorageConfigurations(['nextToken' => $nextToken]); + $all = array_merge($all, $response['storageConfigurations']); + } + $existing = collect($all)->first(fn ($config) => Arr::get($config, 's3.bucketName') === $bucket); + + if ($existing) { + note(sprintf('IVS StorageConfiguration ARN: %s', $existing['arn'])); + note(sprintf('Set AWS_IVS_STORAGE_CONFIGURATION_ARN=%s', $existing['arn'])); + + return StepResult::SYNCED; + } + + if (! Arr::get($options, 'dry-run')) { + $result = Aws::ivsRealTime()->createStorageConfiguration([ + 'name' => $name, + 's3' => [ + 'bucketName' => $bucket, + ], + 'tags' => [ + 'yolo:environment' => Helpers::app('environment'), + 'Name' => $name, + ], + ]); + + $arn = $result['storageConfiguration']['arn']; + + note(sprintf('IVS StorageConfiguration ARN: %s', $arn)); + note(sprintf('Set AWS_IVS_STORAGE_CONFIGURATION_ARN=%s', $arn)); + + return StepResult::CREATED; + } + + return StepResult::WOULD_CREATE; + } +} diff --git a/src/Yolo.php b/src/Yolo.php index 81d611f..de3d83a 100644 --- a/src/Yolo.php +++ b/src/Yolo.php @@ -46,6 +46,7 @@ class Yolo Commands\SyncCiCommand::class, Commands\SyncIamCommand::class, Commands\SyncLoggingCommand::class, + Commands\SyncRecordingCommand::class, ]; public function __construct() diff --git a/tests/Unit/PathsTest.php b/tests/Unit/PathsTest.php index cd3f7d7..9576fc0 100644 --- a/tests/Unit/PathsTest.php +++ b/tests/Unit/PathsTest.php @@ -35,11 +35,15 @@ }); it('builds yolo dir for aws instances', function () { + writeManifest([]); + expect(Paths::yoloDir()) ->toBe('/home/ubuntu/yolo/yolo-testing-my-app'); }); it('builds log dir for aws instances', function () { + writeManifest([]); + expect(Paths::logDir()) ->toBe('/var/log/yolo/yolo-testing-my-app'); }); From 588cb47793f51bc77beae5635441b1c137fe06df Mon Sep 17 00:00:00 2001 From: Steve Thomas Date: Tue, 19 May 2026 11:49:33 +1000 Subject: [PATCH 2/2] test: per-process temp dir to fix parallel pest flakiness MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit All parallel pest processes shared the same /tmp/yolo-test directory and the same yolo.yml inside it. Process A wrote its manifest, process B overwrote it, process A read the wrong file — surfacing as intermittent IntegrityCheck / TypeError failures. Pre-existing latent bug; got unlucky with parallel ordering as ManifestTest grew, exposing the race. Tests still passed deterministically when run serially (which is why the alpha extraction + pint sweep CI runs went green). Use TEST_TOKEN (paratest sets this per worker) with getmypid() fallback to give each process its own temp dir. Verified: 5 consecutive --parallel --processes=4 runs all 54/54. Co-Authored-By: Claude Opus 4.7 (1M context) --- tests/Pest.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/Pest.php b/tests/Pest.php index cf0b9f9..b5bf9c2 100644 --- a/tests/Pest.php +++ b/tests/Pest.php @@ -27,7 +27,7 @@ | */ -$tempDir = sys_get_temp_dir() . '/yolo-test'; +$tempDir = sys_get_temp_dir() . '/yolo-test-' . (getenv('TEST_TOKEN') ?: getmypid()); @mkdir($tempDir, 0755, true);