feat(integ-runner): unstable resolved-snapshots writes each deployed stack's resolved template to the snapshot

packages/@aws-cdk/integ-runner/lib/cli.ts

@@ -223,6 +223,7 @@ async function run(options: ReturnType<typeof parseCliArgs>) {
         caBundlePath: options.caBundlePath,
         roleArn: options.roleArn,
         allowDeleteFailures: options.allowDeleteFailures,
+        resolvedSnapshots: options.unstable.includes('resolved-snapshots'),
       });
       testsSucceeded = success;
 

packages/@aws-cdk/integ-runner/lib/engines/cdk-interface.ts

@@ -1,5 +1,5 @@
 import type { DefaultCdkOptions, DestroyOptions as BaseDestroyOptions, DeployOptions as BaseDeployOptions } from '@aws-cdk/cloud-assembly-schema/lib/integ-tests';
-import type { DeploymentMethod, ResourceDeleteFailure } from '@aws-cdk/toolkit-lib';
+import type { DeploymentMethod, EvaluatedStack, ResourceDeleteFailure } from '@aws-cdk/toolkit-lib';
 
 /**
  * Events emitted during watch mode
@@ -143,4 +143,15 @@ export interface ICdk {
    * cdk watch
    */
   watch(options: DeployOptions, events?: WatchEvents): Promise<void>;
+
+  /**
+   * Evaluate the templates of deployed stacks against the deployed stacks
+   */
+  evaluate(options: EvaluateOptions): Promise<EvaluatedStack[]>;
+}
+
+/**
+ * Options for evaluating deployed stacks
+ */
+export interface EvaluateOptions extends DefaultCdkOptions {
 }

packages/@aws-cdk/integ-runner/lib/engines/toolkit-lib.ts

@@ -1,11 +1,11 @@
 import * as path from 'node:path';
 import { UNKNOWN_REGION } from '@aws-cdk/cloud-assembly-api';
 import type { DefaultCdkOptions } from '@aws-cdk/cloud-assembly-schema/lib/integ-tests';
-import type { ICloudAssemblySource, IIoHost, IoMessage, IoRequest, IReadableCloudAssembly, NonInteractiveIoHostProps, StackSelector } from '@aws-cdk/toolkit-lib';
+import type { EvaluatedStack, ICloudAssemblySource, IIoHost, IoMessage, IoRequest, IReadableCloudAssembly, NonInteractiveIoHostProps, StackSelector } from '@aws-cdk/toolkit-lib';
 import { BaseCredentials, ExpandStackSelection, MemoryContext, NonInteractiveIoHost, StackSelectionStrategy, Toolkit } from '@aws-cdk/toolkit-lib';
 import chalk from 'chalk';
 import * as fs from 'fs-extra';
-import type { CdkDeployResult, CxOptions, DeployOptions, DestroyOptions, ICdk, ListOptions, SynthOptions, WatchEvents, WatchOptions } from './cdk-interface';
+import type { CdkDeployResult, CxOptions, DeployOptions, DestroyOptions, EvaluateOptions, ICdk, ListOptions, SynthOptions, WatchEvents, WatchOptions } from './cdk-interface';
 import { ProxyAgentProvider } from './proxy-agent';
 
 export interface ToolkitLibEngineOptions {
@@ -130,6 +130,7 @@ export class ToolkitLibRunnerEngine implements ICdk {
 
     const toolkit = new Toolkit({
       ioHost: this.showOutput ? this.ioHost : new NoopIoHost(),
+      unstableFeatures: ['evaluate'],
       sdkConfig: {
         baseCredentials: BaseCredentials.awsCliCompatible({
           profile,
@@ -244,6 +245,18 @@ export class ToolkitLibRunnerEngine implements ICdk {
     }
   }
 
+  /**
+   * Evaluates the templates of the deployed stacks
+   */
+  public async evaluate(options: EvaluateOptions): Promise<EvaluatedStack[]> {
+    const toolkit = this.getOrCreateToolkit(options);
+    const cx = await this.cx(options);
+    const result = await toolkit.evaluate(cx, {
+      stacks: this.stackSelector(options),
+    });
+    return result.stacks;
+  }
+
   /**
    * Destroys the CDK app
    */

packages/@aws-cdk/integ-runner/lib/runner/cdk-test-app.ts

@@ -4,7 +4,7 @@ import { AVAILABILITY_ZONE_FALLBACK_CONTEXT_KEY, UNKNOWN_REGION } from '@aws-cdk
 import { type TestCase, type DefaultCdkOptions, RequireApproval } from '@aws-cdk/cloud-assembly-schema';
 import * as fs from 'fs-extra';
 import { IntegTestSuite, LegacyIntegTestSuite } from './integ-test-suite';
-import type { DeployOptions, DestroyOptions, ICdk, WatchEvents } from '../engines/cdk-interface';
+import type { DeployOptions, DestroyOptions, EvaluateOptions, ICdk, WatchEvents } from '../engines/cdk-interface';
 import * as logger from '../logger';
 import * as recommendedFlagsFile from '../recommended-feature-flags.json';
 import { flatten } from '../utils';
@@ -393,6 +393,24 @@ export class CdkTestApp {
     });
   }
 
+  public async evaluate(evaluateArgs: Omit<EvaluateOptions, 'app' | 'profile' | 'output'>) {
+    if (!this.hasValidRegion) {
+      throw new Error('CdkTestApp.evaluate: not initialized with a valid region');
+    }
+
+    return this.cdk.evaluate({
+      ...DEFAULT_ARGS,
+      ...evaluateArgs,
+      app: this.appCommand,
+      output: this.outputDirectory,
+      profile: this.profile,
+      context: {
+        ...this.getContext('deployment'),
+        ...evaluateArgs?.context,
+      },
+    });
+  }
+
   public async destroy(destroyArgs: Omit<DestroyOptions, 'app' | 'force' | 'profile' | 'output'>) {
     if (!this.hasValidRegion) {
       throw new Error('CdkTestApp.destroy: not initialized with a valid region');

packages/@aws-cdk/integ-runner/lib/runner/integ-test-runner.ts

@@ -11,6 +11,8 @@ import { chunks, exec, execWithSubShell, promiseWithResolvers, renderCommand } f
 import type { CdkTestAppOptions, LegacyEnableLookups } from './cdk-test-app';
 import { CdkTestApp, DEFAULT_ARGS } from './cdk-test-app';
 import type { IntegTest } from './integration-tests';
+import type { ResolvedSnapshot } from './resolved-snapshot';
+import { resolvedSnapshot, writeResolvedSnapshots } from './resolved-snapshot';
 import type { DestructiveChange, AssertionResults, AssertionResult } from '../workers/common';
 import { DiagnosticReason, formatAssertionResults, formatError } from '../workers/common';
 
@@ -108,6 +110,14 @@ export interface RunOptions extends CommonOptions {
    * @default false
    */
   readonly allowDeleteFailures?: boolean;
+
+  /**
+   * Write each deployed test stack's resolved template to `<stackName>.resolved.json` in the snapshot
+   * (unstable feature `resolved-snapshots`)
+   *
+   * @default false
+   */
+  readonly resolvedSnapshots?: boolean;
 }
 
 type RunnerOptions = Omit<CdkTestAppOptions, 'outputDirectoryNameTemplate'> & {
@@ -167,6 +177,7 @@ export class IntegTestRunner {
     }
 
     const clean = runOptions.clean ?? true;
+    const resolvedSnapshots: ResolvedSnapshot[] = [];
     try {
       for (const [testCaseName, actualTestCase] of Object.entries(actualTestSuite.testSuite)) {
         const updateWorkflowEnabled = (runOptions.updateWorkflow ?? false)
@@ -191,6 +202,14 @@ export class IntegTestRunner {
               allowDeleteFailures,
               runOptions.updateFromTags,
             );
+            if (runOptions.resolvedSnapshots) {
+              // while the stacks are still deployed
+              const evaluated = await actual.evaluate({
+                stacks: actualTestCase.stacks,
+                context: actualTestCase.cdkCommandOptions?.deploy?.args?.context,
+              });
+              resolvedSnapshots.push(...evaluated.map(resolvedSnapshot));
+            }
           } finally {
             if (!runOptions.dryRun && clean && (actualTestCase.cdkCommandOptions?.destroy?.enabled ?? true)) {
               await this.destroyTestAndAssertions(actual, actualTestCase, {
@@ -207,7 +226,9 @@ export class IntegTestRunner {
       // only create the snapshot if there are no failed assertion results
       // (i.e. no failures)
       if (!Object.values(assertionResults ?? {}).some(result => result.status === 'fail')) {
-        await this.createSnapshot(actualTestSuite.enableLookups);
+        const snapshot = await this.createSnapshot(actualTestSuite.enableLookups);
+        // after the snapshot is synthesized, which replaces the snapshot directory
+        await writeResolvedSnapshots(snapshot.outputDirectory, resolvedSnapshots);
       }
     } finally {
       actual.cleanup();
@@ -216,9 +237,10 @@ export class IntegTestRunner {
     return assertionResults;
   }
 
-  private async createSnapshot(enableLookups: LegacyEnableLookups) {
+  private async createSnapshot(enableLookups: LegacyEnableLookups): Promise<CdkTestApp> {
     const expected = await CdkTestApp.forGoldenSnapshot(this.appOptions);
     await expected.synthForGoldenSnapshot(this.destructiveChanges ?? [], enableLookups);
+    return expected;
   }
 
   /**

packages/@aws-cdk/integ-runner/lib/runner/resolved-snapshot.ts

@@ -0,0 +1,141 @@
+import * as path from 'path';
+import type { EvaluatedStack, UnresolvedExpression } from '@aws-cdk/toolkit-lib';
+import * as fs from 'fs-extra';
+
+/**
+ * Resource types whose settings are account-wide: the deployed value is whichever stack set it last
+ */
+const ACCOUNT_WIDE_TYPES = new Set(['AWS::ApiGateway::Account']);
+
+/**
+ * Shortest generated value that is also replaced inside longer strings.
+ * Shorter ones (e.g. API Gateway resource ids) are only replaced when they are the whole value.
+ */
+const MIN_EMBEDDED_LENGTH = 8;
+
+/**
+ * Parts of physical ids are only replaced if they contain a digit, like generated names and ids do:
+ * other parts are words such as a WAF scope (`REGIONAL`) that appear elsewhere with their own meaning
+ */
+const looksGenerated = (value: string) => /\d/.test(value);
+
+/**
+ * A deployed stack's template, with the values CloudFormation and the services chose for this deployment
+ * replaced by placeholders, so it is the same for every deployment of the same app
+ */
+export interface ResolvedSnapshot {
+  readonly version: '1';
+  readonly stackName: string;
+  /** The template's resources, evaluated against the deployed stack */
+  readonly resources: { [logicalId: string]: any };
+  readonly outputs?: { [logicalId: string]: any };
+  /** Expressions that could not be evaluated */
+  readonly unresolved: UnresolvedExpression[];
+  /** Resources with account-wide settings, whose deployed values depend on what else is deployed in the account */
+  readonly accountWide?: string[];
+  readonly nestedStacks?: { [logicalId: string]: ResolvedSnapshot };
+}
+
+/**
+ * Make a stack's evaluated template independent of the deployment it was evaluated against.
+ *
+ * Replaces, in every string value:
+ * - each generated physical id (one that doesn't appear in the resource's own properties) with `${LogicalId}`;
+ *   the name at the end of a URL or ARN physical id with
+ *   `${LogicalId.Name}`, and each part of a `|`-separated physical id with `${LogicalId[n]}`
+ * - the stack id, account, region and partition with `${AWS::StackId}`, `${AWS::AccountId}`, `${AWS::Region}`
+ *   and `${AWS::Partition}` (taken from the stack id)
+ */
+export function resolvedSnapshot(stack: EvaluatedStack): ResolvedSnapshot {
+  const replacements = new Map<string, string>();
+  const add = (value: string | undefined, placeholder: string) => {
+    if (value && !replacements.has(value)) {
+      replacements.set(value, placeholder);
+    }
+  };
+  // most specific first: when two values are equal, the first one added wins
+  add(stack.stackId, '${AWS::StackId}');
+  // Names the template chooses (e.g. `StageName: prod`, `BucketName: my-bucket`) appear in the resource's own
+  // properties and are the same in every deployment; CloudFormation-generated names and ids never do
+  const generated = Object.entries(stack.physicalResourceIds)
+    .filter(([logicalId, physicalId]) => !stringsIn(stack.template.Resources?.[logicalId]?.Properties).has(physicalId));
+  for (const [logicalId, physicalId] of generated) {
+    add(physicalId, `\${${logicalId}}`);
+  }
+  for (const [logicalId, physicalId] of generated) {
+    const parts = physicalId.split('|');
+    if (parts.length > 1) {
+      parts.forEach((part, i) => looksGenerated(part) && add(part, `\${${logicalId}[${i}]}`));
+    }
+    const name = physicalId.replace(/\/$/, '').split(/[/:]/).pop();
+    if (name && name !== physicalId && looksGenerated(name)) {
+      add(name, `\${${logicalId}.Name}`);
+    }
+  }
+  const [, partition, , region, account] = stack.stackId.split(':');
+  add(account, '${AWS::AccountId}');
+  add(region, '${AWS::Region}');
+
+  const embedded = [...replacements.keys()].filter((v) => v.length >= MIN_EMBEDDED_LENGTH).sort((a, b) => b.length - a.length);
+  const embeddedPattern = embedded.length > 0 ? new RegExp(embedded.map(escapeRegExp).join('|'), 'g') : undefined;
+  const replace = (value: string) => {
+    const whole = replacements.get(value);
+    if (whole !== undefined) {
+      return whole;
+    }
+    const replaced = embeddedPattern ? value.replace(embeddedPattern, (match) => replacements.get(match)!) : value;
+    return partition ? replaced.split(`arn:${partition}:`).join('arn:${AWS::Partition}:') : replaced;
+  };
+
+  const resources = mapStrings(stack.template.Resources ?? {}, replace);
+  const accountWide = Object.entries<any>(stack.template.Resources ?? {})
+    .filter(([, r]) => ACCOUNT_WIDE_TYPES.has(r?.Type))
+    .map(([logicalId]) => logicalId);
+
+  return {
+    version: '1',
+    stackName: stack.stackName,
+    resources,
+    ...(stack.template.Outputs ? { outputs: mapStrings(stack.template.Outputs, replace) } : {}),
+    unresolved: stack.unresolved,
+    ...(accountWide.length > 0 ? { accountWide } : {}),
+    ...(stack.nestedStacks ? {
+      nestedStacks: Object.fromEntries(Object.entries(stack.nestedStacks).map(([id, nested]) => [id, resolvedSnapshot(nested)])),
+    } : {}),
+  };
+}
+
+/**
+ * Write each stack's resolved snapshot to `<stackName>.resolved.json` in the snapshot directory
+ */
+export async function writeResolvedSnapshots(snapshotDirectory: string, snapshots: ResolvedSnapshot[]): Promise<void> {
+  for (const snapshot of snapshots) {
+    await fs.writeJson(path.join(snapshotDirectory, `${snapshot.stackName}.resolved.json`), snapshot, { spaces: 1 });
+  }
+}
+
+function mapStrings(value: any, fn: (s: string) => string): any {
+  if (typeof value === 'string') {
+    return fn(value);
+  }
+  if (Array.isArray(value)) {
+    return value.map((x) => mapStrings(x, fn));
+  }
+  if (value && typeof value === 'object') {
+    return Object.fromEntries(Object.entries(value).map(([k, x]) => [k, mapStrings(x, fn)]));
+  }
+  return value;
+}
+
+function stringsIn(value: any, into = new Set<string>()): Set<string> {
+  if (typeof value === 'string') {
+    into.add(value);
+  } else if (value && typeof value === 'object') {
+    Object.values(value).forEach((x) => stringsIn(x, into));
+  }
+  return into;
+}
+
+function escapeRegExp(s: string): string {
+  return s.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
+}

packages/@aws-cdk/integ-runner/lib/unstable-features.ts

@@ -28,6 +28,11 @@ export interface UnstableFeature {
  * Registry of all unstable features
  */
 export const UNSTABLE_FEATURES: Record<string, UnstableFeature> = {
+  'resolved-snapshots': {
+    status: FeatureStatus.SUPPORTED,
+    deprecationMessage: '',
+    removalMessage: '',
+  },
   'deprecated-cli-engine': {
     status: FeatureStatus.REMOVED,
     deprecationMessage: 'You have opted-in to use the deprecated CLI engine which is scheduled to be removed in January 2026. If you have encountered blockers while using the new default engine, please let us know by opening an issue: https://github.com/aws/aws-cdk-cli/issues/new/choose\n\nTo use the new default engine, remove the `--unstable=deprecated-cli-engine` option.',

packages/@aws-cdk/integ-runner/lib/workers/common.ts

@@ -222,6 +222,13 @@ export interface IntegTestOptions {
    */
   readonly allowDeleteFailures?: boolean;
 
+  /**
+   * Write each deployed test stack's resolved template to the snapshot (unstable feature `resolved-snapshots`)
+   *
+   * @default false
+   */
+  readonly resolvedSnapshots?: boolean;
+
   /**
    * In unit test mode, leave the synth output directories, don't clean them.
    *

packages/@aws-cdk/integ-runner/lib/workers/extract/extract_worker.ts

@@ -71,6 +71,7 @@ export async function integTestWorker(request: IntegTestBatchRequest): Promise<I
           verbosity,
           roleArn: request.roleArn,
           allowDeleteFailures: request.allowDeleteFailures,
+          resolvedSnapshots: request.resolvedSnapshots,
         });
         if (results && Object.values(results).some(result => result.status === 'fail')) {
           failures.push(testInfo);

packages/@aws-cdk/integ-runner/lib/workers/integ-test-worker.ts

@@ -168,6 +168,7 @@ export async function runIntegrationTestsInParallel(
         caBundlePath: options.caBundlePath,
         roleArn: options.roleArn,
         allowDeleteFailures: options.allowDeleteFailures,
+        resolvedSnapshots: options.resolvedSnapshots,
       }], {
         on: (diagnostic: Diagnostic) => {
           printResults(diagnostic);