diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..7c372f3 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,9 @@ +# Line endings that must not depend on the checkout's core.autocrlf (Windows runners convert to +# CRLF by default): bash cannot run a script with CRLF, and the samples shim's tests run +# tests/azure/samples-shim/az through Git Bash on Windows. +*.sh text eol=lf +tests/azure/samples-shim/az text eol=lf +# cmd.exe shims keep CRLF everywhere (tests/azure/samples-shim/quote.test.ts checks it). +*.cmd text eol=crlf +# The L2 matrix files are checked for LF (tests/azure/matrix/schema.test.ts). +tests/azure/matrix/*.yaml text eol=lf diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS new file mode 100644 index 0000000..77f79f1 --- /dev/null +++ b/.github/CODEOWNERS @@ -0,0 +1,40 @@ +# Code owners: GitHub requests their review on any pull request that touches these paths. +# +# The Azure tool, with its tests, fixtures, CI and image parts: the Azure team (smurf) and the +# repository's maintainer. Files shared with the AWS and Snowflake tools (the lifecycle tool, the +# launcher, the wizard, the Dockerfile, the README) are left out on purpose, so that AWS and +# Snowflake changes do not request Azure reviewers. + +# The tool +/src/lib/azure/ @localstack/smurf @HarshCasper +/src/tools/localstack-azure-client.ts @localstack/smurf @HarshCasper +/src/tools-tests/localstack-azure-client.test.ts @localstack/smurf @HarshCasper +/src/lib/wizard/azure-steps.ts @localstack/smurf @HarshCasper +/src/lib/wizard/azure-steps.test.ts @localstack/smurf @HarshCasper +/src/cli/azure-addons.ts @localstack/smurf @HarshCasper +/src/cli/azure-addons.test.ts @localstack/smurf @HarshCasper + +# Tests, fixtures and sample data +/tests/azure/ @localstack/smurf @HarshCasper +/tests/fixtures/azure/ @localstack/smurf @HarshCasper +/tests/mcp/azure-offline.spec.mjs @localstack/smurf @HarshCasper +/tests/mcp/evals-gemini-azure.spec.mjs @localstack/smurf @HarshCasper +/data/sample-azure/ @localstack/smurf @HarshCasper +/data/evals/gemini-azure.json @localstack/smurf @HarshCasper +/jest.azure-live.config.js @localstack/smurf @HarshCasper + +# CI and scripts +/.github/workflows/azure-live.yml @localstack/smurf @HarshCasper +/.github/workflows/azure-weekly.yml @localstack/smurf @HarshCasper +/.github/actions/azure-live-setup/ @localstack/smurf @HarshCasper +/scripts/ci/azure-emulator-up.sh @localstack/smurf @HarshCasper +/scripts/ci/scan-for-secret.mjs @localstack/smurf @HarshCasper +/scripts/ci/merge-op-catalogue.cjs @localstack/smurf @HarshCasper +/scripts/install-azure-extensions.mjs @localstack/smurf @HarshCasper +/scripts/gen-az-file-args.py @localstack/smurf @HarshCasper + +# The image's Azure layers (the Azure CLI, its extensions and Bicep) +/docker/azure-extensions.txt @localstack/smurf @HarshCasper +/docker/image-size.json @localstack/smurf @HarshCasper +/tests/docker/image-size.mjs @localstack/smurf @HarshCasper +/tests/docker/l5-image-assertions.sh @localstack/smurf @HarshCasper diff --git a/.github/actions/azure-live-setup/action.yml b/.github/actions/azure-live-setup/action.yml new file mode 100644 index 0000000..246c960 --- /dev/null +++ b/.github/actions/azure-live-setup/action.yml @@ -0,0 +1,45 @@ +name: Azure live setup +description: >- + Node, the build, the pinned azure-cli in its own venv, the curated extensions and the + pinned Bicep. The calling job sets LOCALSTACK_AZ_PATH and + LOCALSTACK_AZ_BICEP_PATH to /home/runner/az/bin/python3 and /home/runner/bicep-bin/bicep. +inputs: + az-version: + description: azure-cli version (the image pins 2.90.0; DR3 also runs the minimum) + default: "2.90.0" + extensions: + description: install the curated extensions (docker/azure-extensions.txt) + default: "true" +runs: + using: composite + steps: + - uses: actions/setup-node@v4 + with: + node-version: 22.x + + - shell: bash + run: yarn install --frozen-lockfile && yarn build + + - name: Pinned azure-cli in its own venv (never the runner's /usr/bin/az) + shell: bash + run: | + python3 -m venv ~/az + if [ "${{ inputs.az-version }}" = "latest" ]; then + ~/az/bin/pip install --quiet azure-cli + else + ~/az/bin/pip install --quiet "azure-cli==${{ inputs.az-version }}" + fi + ~/az/bin/python3 -c "import azure.cli.core as c; print('azure-cli-core', c.__version__)" + + - name: Curated extensions + if: inputs.extensions == 'true' + shell: bash + run: node scripts/install-azure-extensions.mjs --az-python ~/az/bin/python3 --dir ~/.localstack/azure/mcp-extensions + + - name: Pinned Bicep v0.47.16, sha256-checked + shell: bash + run: | + mkdir -p ~/bicep-bin + curl -fsSL -o ~/bicep-bin/bicep https://github.com/Azure/bicep/releases/download/v0.47.16/bicep-linux-x64 + echo "64c345a58e0c3e48b1bc98a4e62d6b3adb1d238281297de3400aeafb2697aa5a ${HOME}/bicep-bin/bicep" | sha256sum -c - + chmod +x ~/bicep-bin/bicep diff --git a/.github/workflows/azure-live.yml b/.github/workflows/azure-live.yml new file mode 100644 index 0000000..3ff1cd1 --- /dev/null +++ b/.github/workflows/azure-live.yml @@ -0,0 +1,251 @@ +name: Azure live tests + +# The Azure tool against a real LocalStack Azure emulator: the L2 matrix +# subset, L3 egress and L1 (the harness stage) in one job; the L4 samples subset in its own +# job with its own emulator. Path-filtered, so AWS-only PRs do not pay for the emulator. +on: + pull_request: + branches: [main] + paths: + - "src/lib/azure/**" + - "src/lib/cli/**" + - "src/tools/localstack-azure-client.ts" + - "src/tools/localstack-management.ts" + # The Azure tool imports core/ (analytics, the response builder); L1 runs dist/cli.js. + - "src/core/**" + - "src/cli/**" + - "src/lib/docker/**" + - "src/lib/localstack/**" + - "tests/azure/**" + # Read by the live suites: the leak commands, the Bicep fixtures, L1's deployments. + - "tests/fixtures/azure/**" + - "data/sample-azure/**" + - "tests/docker/validate-image.mjs" + - "docker/azure-extensions.txt" + - "scripts/install-azure-extensions.mjs" + - "scripts/ci/**" + - "jest.azure-live.config.js" + - ".github/actions/azure-live-setup/**" + - ".github/workflows/azure-live.yml" + push: + branches: [main] + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: azure-live-${{ github.ref }} + cancel-in-progress: true + +env: + AZ_VERSION: "2.90.0" + # The samples repo's main (2026-09-23). The samples corpus (tests/fixtures/azure/corpus) was + # built from 4193d67, whose sample scripts are byte-identical to this commit's; only + # run-samples.sh differs (it registers two more Bicep samples). + SAMPLES_COMMIT: "5ae698478bd4810b619469959b73c61f326bc569" + +jobs: + live: + # Secrets are not available to PRs from forks. + if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository + runs-on: ubuntu-latest + timeout-minutes: 120 + env: + LOCALSTACK_AUTH_TOKEN: ${{ secrets.LOCALSTACK_AUTH_TOKEN }} + # Job level, so every step (the installer included) uses the pinned az and Bicep, + # never the runner's /usr/bin/az. + LOCALSTACK_AZ_PATH: /home/runner/az/bin/python3 + LOCALSTACK_AZ_BICEP_PATH: /home/runner/bicep-bin/bicep + AZURE_CI_EMULATOR_CONTAINER: ls-azure-ci + MCP_ANALYTICS_DISABLED: "1" + steps: + - uses: actions/checkout@v4 + + # Node, the build, the pinned az (own venv), the curated extensions, the pinned Bicep. + - uses: ./.github/actions/azure-live-setup + with: + az-version: ${{ env.AZ_VERSION }} + + # Before the emulator starts, so that "~/.azure unchanged" has a baseline even when the + # start fails. + - name: "~/.azure fingerprint (before)" + run: node tests/azure/tools/azure-home-fingerprint.mjs > "$RUNNER_TEMP/azure-home-before.json" + + - name: Start the job's own emulator + run: bash scripts/ci/azure-emulator-up.sh + + - name: L2 matrix subset + run: AZURE_LIVE=1 npx jest -c jest.azure-live.config.js --selectProjects matrix-subset --runInBand + env: + AZURE_MATRIX_BACKING: "1" + + # Last against this emulator: its stop case stops the container and starts it again. + - name: L3 egress and the home guard + run: AZURE_LIVE=1 npx jest -c jest.azure-live.config.js --selectProjects egress --runInBand + env: + AZURE_EGRESS_CI: "1" + AZURE_EGRESS_REPORT: test-results/egress-steps.jsonl + + - name: Emulator logs, then remove the job's emulator (L1 starts its own) + if: always() + run: | + mkdir -p emulator-logs + docker logs "$AZURE_CI_EMULATOR_CONTAINER" > emulator-logs/emulator-live.log 2>&1 || true + docker rm -f "$AZURE_CI_EMULATOR_CONTAINER" || true + + - name: L1 through the npx path (its own start, scenario, stop) + run: node tests/docker/validate-image.mjs -- node dist/cli.js + env: + HARNESS_ONLY: azure + HARNESS_TOKEN_REAL: "1" + HARNESS_AZURE_BICEP: "1" + + - name: "~/.azure unchanged" + if: always() + run: node tests/azure/tools/azure-home-fingerprint.mjs --compare "$RUNNER_TEMP/azure-home-before.json" + + - name: Scan logs and results for the token before any upload + id: scan + if: always() + run: node scripts/ci/scan-for-secret.mjs --env LOCALSTACK_AUTH_TOKEN emulator-logs test-results + + # Never upload when the scan failed: the artifacts would carry the token. + - name: Upload logs and results on failure + if: failure() && steps.scan.outcome == 'success' + uses: actions/upload-artifact@v4 + with: + name: azure-live-logs + path: | + emulator-logs/ + test-results/ + retention-days: 7 + + - name: Upload the operation catalogue (for the portal) + if: always() && steps.scan.outcome == 'success' + uses: actions/upload-artifact@v4 + with: + name: azure-op-catalogue + path: test-results/azure-op-catalogue.json + if-no-files-found: ignore + retention-days: 30 + + # Every run, not only failed ones: the per-case results show which known gaps still + # fail and which started passing (tests/azure/matrix/README.md). + - name: Upload the L2 case results + if: always() && steps.scan.outcome == 'success' + uses: actions/upload-artifact@v4 + with: + name: azure-matrix-results + path: test-results/azure-matrix.jsonl + if-no-files-found: ignore + retention-days: 30 + + samples: + name: L4 samples subset + if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository + runs-on: ubuntu-latest + timeout-minutes: 90 + env: + LOCALSTACK_AUTH_TOKEN: ${{ secrets.LOCALSTACK_AUTH_TOKEN }} + LOCALSTACK_AZ_PATH: /home/runner/az/bin/python3 + LOCALSTACK_AZ_BICEP_PATH: /home/runner/bicep-bin/bicep + AZURE_CI_EMULATOR_CONTAINER: ls-azure-samples + MCP_ANALYTICS_DISABLED: "1" + steps: + - uses: actions/checkout@v4 + + - uses: ./.github/actions/azure-live-setup + with: + az-version: ${{ env.AZ_VERSION }} + + - name: Samples repo at the pinned commit + uses: actions/checkout@v4 + with: + repository: localstack/localstack-azure-samples + ref: ${{ env.SAMPLES_COMMIT }} + path: samples-repo + + # What the three PR samples call besides az (tests/azure/samples-shim/README.md). + - name: JDK 17 for servicebus/java (Maven is preinstalled) + uses: actions/setup-java@v4 + with: + distribution: temurin + java-version: "17" + + - name: Python with azure-eventhub for the Event Hubs validate.sh scripts + run: | + python3 -m venv "$RUNNER_TEMP/samples-py" + "$RUNNER_TEMP/samples-py/bin/pip" install --quiet azure-eventhub + echo "PYTHON_BIN=$RUNNER_TEMP/samples-py/bin/python" >> "$GITHUB_ENV" + + # Before the emulator starts, so that "~/.azure unchanged" has a baseline even when the + # start fails. + - name: "~/.azure fingerprint (before)" + run: node tests/azure/tools/azure-home-fingerprint.mjs > "$RUNNER_TEMP/azure-home-before.json" + + - name: Start the job's own emulator + run: bash scripts/ci/azure-emulator-up.sh + + - name: L4 samples subset through the az shim + run: AZURE_LIVE=1 npx jest -c jest.azure-live.config.js --selectProjects samples-subset --runInBand + env: + AZURE_SAMPLES_DIR: ${{ github.workspace }}/samples-repo + AZURE_SAMPLES_COMMIT: ${{ env.SAMPLES_COMMIT }} + AZURE_SAMPLES_RESULTS_DIR: ${{ github.workspace }}/test-results/samples + + - name: Emulator logs, then remove the job's emulator + if: always() + run: | + mkdir -p emulator-logs + docker logs "$AZURE_CI_EMULATOR_CONTAINER" > emulator-logs/emulator-samples.log 2>&1 || true + docker rm -f "$AZURE_CI_EMULATOR_CONTAINER" || true + + - name: "~/.azure unchanged" + if: always() + run: node tests/azure/tools/azure-home-fingerprint.mjs --compare "$RUNNER_TEMP/azure-home-before.json" + + - name: Scan logs and results for the token before any upload + id: scan + if: always() + run: node scripts/ci/scan-for-secret.mjs --env LOCALSTACK_AUTH_TOKEN emulator-logs test-results + + - name: Upload logs and results on failure + if: failure() && steps.scan.outcome == 'success' + uses: actions/upload-artifact@v4 + with: + name: azure-samples-logs + path: | + emulator-logs/ + test-results/ + retention-days: 7 + + alert: + name: Tracking issue on failure (main) + # A PR's failure shows on the PR, to its author and its code-owner reviewers. A failure on + # main has no PR, so it opens or updates an issue instead, as azure-weekly.yml does. + needs: [live, samples] + if: failure() && github.event_name == 'push' + runs-on: ubuntu-latest + permissions: + issues: write + steps: + - name: Open or update the tracking issue + env: + GH_TOKEN: ${{ github.token }} + RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + # The Azure code owners (.github/CODEOWNERS): every alert mentions them, and a new + # issue is assigned to ASSIGNEE. + OWNERS: "@localstack/smurf @HarshCasper" + ASSIGNEE: HarshCasper + run: | + title="Azure live tests failing on main" + existing=$(gh issue list --repo "$GITHUB_REPOSITORY" --state open --search "$title in:title" --json number --jq '.[0].number' --limit 100) + body="The Azure live tests failed on main at ${GITHUB_SHA::7}: $RUN_URL (jobs: live, samples). cc $OWNERS" + if [ -n "$existing" ]; then + gh issue comment "$existing" --repo "$GITHUB_REPOSITORY" --body "$body" + else + url=$(gh issue create --repo "$GITHUB_REPOSITORY" --title "$title" --body "$body") + # Assigned separately, so that an assignee GitHub refuses cannot lose the alert. + gh issue edit "$url" --repo "$GITHUB_REPOSITORY" --add-assignee "$ASSIGNEE" || echo "::warning::could not assign $ASSIGNEE to $url" + fi diff --git a/.github/workflows/azure-weekly.yml b/.github/workflows/azure-weekly.yml new file mode 100644 index 0000000..4e00424 --- /dev/null +++ b/.github/workflows/azure-weekly.yml @@ -0,0 +1,354 @@ +name: Azure weekly + +# The heavy Azure suites: the L2 matrix sharded x4, L3 in full with the +# internal-network job, L4 over all samples, the drift gates DR1-DR8 and the az version +# matrix (DR3). A summary job opens or updates a tracking issue on failure. E2 calls a +# model API, so it runs locally only (tests/azure/evals/README.md). +on: + schedule: + - cron: "0 7 * * 1" # Mondays 07:00 UTC + workflow_dispatch: + repository_dispatch: + # samples-changed: for localstack-azure-samples to send when its samples change; + # azure-emulator-release: for an emulator release hook, when the platform sends one. + types: [samples-changed, azure-emulator-release] + +permissions: + contents: read + +env: + AZ_VERSION: "2.90.0" + AZ_MIN_VERSION: "2.85.0" + SAMPLES_COMMIT: "5ae698478bd4810b619469959b73c61f326bc569" + # LOCALSTACK_AUTH_TOKEN is set per step: only on the emulator starts, the suites and the scans + # (a scan without it has nothing to look for, and would pass unscanned files). + LOCALSTACK_AZ_PATH: /home/runner/az/bin/python3 + LOCALSTACK_AZ_BICEP_PATH: /home/runner/bicep-bin/bicep + AZURE_CI_EMULATOR_CONTAINER: ls-azure-ci + MCP_ANALYTICS_DISABLED: "1" + +jobs: + matrix: + name: L2 matrix (shard ${{ matrix.shard }}/4) + runs-on: ubuntu-latest + timeout-minutes: 180 + strategy: + fail-fast: false + matrix: + shard: [1, 2, 3, 4] + steps: + - uses: actions/checkout@v4 + - uses: ./.github/actions/azure-live-setup + with: + az-version: ${{ env.AZ_VERSION }} + - name: Start this shard's own emulator + run: bash scripts/ci/azure-emulator-up.sh + env: + LOCALSTACK_AUTH_TOKEN: ${{ secrets.LOCALSTACK_AUTH_TOKEN }} + - name: The full matrix, this shard's cases + run: AZURE_LIVE=1 npx jest -c jest.azure-live.config.js --selectProjects matrix-full --runInBand + env: + LOCALSTACK_AUTH_TOKEN: ${{ secrets.LOCALSTACK_AUTH_TOKEN }} + AZURE_MATRIX_SHARD: ${{ matrix.shard }}/4 + AZURE_MATRIX_BACKING: "1" + AZURE_OP_CATALOGUE_OUT: test-results/azure-op-catalogue-${{ matrix.shard }}.json + - name: Emulator logs + if: always() + run: mkdir -p emulator-logs && docker logs "$AZURE_CI_EMULATOR_CONTAINER" > emulator-logs/emulator.log 2>&1 || true + - name: Scan for the token before any upload + id: scan + if: always() + run: node scripts/ci/scan-for-secret.mjs emulator-logs test-results + env: + LOCALSTACK_AUTH_TOKEN: ${{ secrets.LOCALSTACK_AUTH_TOKEN }} + - uses: actions/upload-artifact@v4 + if: always() && steps.scan.outcome == 'success' + with: + name: azure-matrix-shard-${{ matrix.shard }} + path: | + test-results/ + emulator-logs/ + retention-days: 7 + + # One catalogue for the portal's inventory gate, kept longer than the shards: the + # portal's own weekly runs at the same hour and reads it by artifact name. + catalogue: + name: Merged operation catalogue (for the portal) + needs: matrix + if: always() + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version: 22.x + - uses: actions/download-artifact@v4 + with: + pattern: azure-matrix-shard-* + path: shards + - name: Merge the shard catalogues (best result per operation) + run: node scripts/ci/merge-op-catalogue.cjs --out catalogue/azure-op-catalogue.json shards + - uses: actions/upload-artifact@v4 + with: + name: azure-op-catalogue + path: catalogue/azure-op-catalogue.json + retention-days: 30 + + egress: + name: L3 egress (guard records, home guard, leak replay) + runs-on: ubuntu-latest + timeout-minutes: 90 + steps: + - uses: actions/checkout@v4 + - uses: ./.github/actions/azure-live-setup + with: + az-version: ${{ env.AZ_VERSION }} + - run: bash scripts/ci/azure-emulator-up.sh + env: + LOCALSTACK_AUTH_TOKEN: ${{ secrets.LOCALSTACK_AUTH_TOKEN }} + - name: L3 in full, including the stop-between-calls case and the leak replay + run: AZURE_LIVE=1 npx jest -c jest.azure-live.config.js --selectProjects egress --runInBand + env: + LOCALSTACK_AUTH_TOKEN: ${{ secrets.LOCALSTACK_AUTH_TOKEN }} + AZURE_EGRESS_CI: "1" + AZURE_EGRESS_REPORT: test-results/egress-steps.jsonl + - name: Emulator logs, then remove the job's emulator + if: always() + run: | + mkdir -p emulator-logs + docker logs "$AZURE_CI_EMULATOR_CONTAINER" > emulator-logs/emulator.log 2>&1 || true + docker rm -f "$AZURE_CI_EMULATOR_CONTAINER" || true + - id: scan + if: always() + run: node scripts/ci/scan-for-secret.mjs --env LOCALSTACK_AUTH_TOKEN emulator-logs test-results + env: + LOCALSTACK_AUTH_TOKEN: ${{ secrets.LOCALSTACK_AUTH_TOKEN }} + - uses: actions/upload-artifact@v4 + if: failure() && steps.scan.outcome == 'success' + with: + name: azure-egress-logs + path: | + test-results/ + emulator-logs/ + retention-days: 7 + + egress-internal-network: + name: L3 internal network (no route to the internet) + runs-on: ubuntu-latest + timeout-minutes: 90 + env: + EGRESS_INTERNAL_OUT: ${{ github.workspace }}/egress-internal + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version: 22.x + - name: Build the server image under test + run: docker build -t localstack/localstack-mcp-server:ci . + - run: docker pull --quiet localstack/localstack-azure:latest + # Feasibility first: licence activation while dual-homed, and reachability on the + # internal network (tests/azure/egress-internal/README.md). + - name: Dual-homed emulator, MCP server on an internal network only + run: bash tests/azure/egress-internal/run.sh + env: + LOCALSTACK_AUTH_TOKEN: ${{ secrets.LOCALSTACK_AUTH_TOKEN }} + MCP_SERVER_IMAGE: localstack/localstack-mcp-server:ci + # run.sh has already replaced any token in these files. + - uses: actions/upload-artifact@v4 + if: failure() + with: + name: egress-internal + path: egress-internal/ + retention-days: 7 + + samples-all: + name: L4 all samples + runs-on: ubuntu-latest + timeout-minutes: 240 + env: + AZURE_CI_EMULATOR_CONTAINER: ls-azure-samples + steps: + - uses: actions/checkout@v4 + - uses: ./.github/actions/azure-live-setup + with: + az-version: ${{ env.AZ_VERSION }} + - name: Samples repo at the pinned commit + uses: actions/checkout@v4 + with: + repository: localstack/localstack-azure-samples + ref: ${{ env.SAMPLES_COMMIT }} + path: samples-repo + # The samples repo's own tool list (its .github/workflows/run-samples.yml), without az: + # the shim is the samples' az. + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + - uses: actions/setup-dotnet@v4 + with: + dotnet-version: "10.0" + - uses: actions/setup-java@v4 + with: + distribution: temurin + java-version: "25" + - uses: hashicorp/setup-terraform@v3 + with: + terraform_version: "1.5.0" + terraform_wrapper: false + - name: System packages (jq, zip, the MySQL, PostgreSQL and SQL Server clients) + run: | + sudo apt-get update -q + sudo apt-get install -y jq zip unixodbc-dev libsnappy-dev default-mysql-client postgresql-client + sudo rm -f /etc/apt/sources.list.d/microsoft-prod.list + curl -fsSL https://packages.microsoft.com/keys/microsoft.asc | sudo tee /etc/apt/trusted.gpg.d/microsoft.asc >/dev/null + curl -fsSL "https://packages.microsoft.com/config/ubuntu/$(lsb_release -rs)/prod.list" | sudo tee /etc/apt/sources.list.d/mssql-release.list >/dev/null + sudo apt-get update -q + sudo ACCEPT_EULA=Y apt-get install -y msodbcsql18 mssql-tools18 + echo "/opt/mssql-tools18/bin" >> "$GITHUB_PATH" + - name: The samples' Python requirements + run: | + python -m venv "$RUNNER_TEMP/samples-py" + "$RUNNER_TEMP/samples-py/bin/pip" install --quiet -r samples-repo/requirements-dev.txt + echo "PYTHON_BIN=$RUNNER_TEMP/samples-py/bin/python" >> "$GITHUB_ENV" + # Before the emulator starts, so that "~/.azure unchanged" has a baseline even when the + # start fails. + - name: "~/.azure fingerprint (before)" + run: node tests/azure/tools/azure-home-fingerprint.mjs > "$RUNNER_TEMP/azure-home-before.json" + - run: bash scripts/ci/azure-emulator-up.sh + env: + LOCALSTACK_AUTH_TOKEN: ${{ secrets.LOCALSTACK_AUTH_TOKEN }} + - name: Every sample's own deploy and test commands through the az shim + run: AZURE_LIVE=1 npx jest -c jest.azure-live.config.js --selectProjects samples-all --runInBand + env: + LOCALSTACK_AUTH_TOKEN: ${{ secrets.LOCALSTACK_AUTH_TOKEN }} + AZURE_SAMPLES_DIR: ${{ github.workspace }}/samples-repo + AZURE_SAMPLES_COMMIT: ${{ env.SAMPLES_COMMIT }} + AZURE_SAMPLES_RESULTS_DIR: ${{ github.workspace }}/test-results/samples + AZURE_SAMPLES_CI_REWRITE: "1" + - name: Emulator logs, then remove the job's emulator + if: always() + run: | + mkdir -p emulator-logs + docker logs "$AZURE_CI_EMULATOR_CONTAINER" > emulator-logs/emulator.log 2>&1 || true + docker rm -f "$AZURE_CI_EMULATOR_CONTAINER" || true + - name: "~/.azure unchanged" + if: always() + run: node tests/azure/tools/azure-home-fingerprint.mjs --compare "$RUNNER_TEMP/azure-home-before.json" + - id: scan + if: always() + run: node scripts/ci/scan-for-secret.mjs --env LOCALSTACK_AUTH_TOKEN emulator-logs test-results + env: + LOCALSTACK_AUTH_TOKEN: ${{ secrets.LOCALSTACK_AUTH_TOKEN }} + - uses: actions/upload-artifact@v4 + if: failure() && steps.scan.outcome == 'success' + with: + name: azure-samples-logs + path: | + test-results/ + emulator-logs/ + retention-days: 7 + + drift: + name: Drift gates DR1-DR8 + runs-on: ubuntu-latest + timeout-minutes: 90 + steps: + - uses: actions/checkout@v4 + - uses: ./.github/actions/azure-live-setup + with: + az-version: ${{ env.AZ_VERSION }} + - name: The samples repo's current commit (DR8) + run: git clone --quiet --depth 1 https://github.com/localstack/localstack-azure-samples.git "$RUNNER_TEMP/samples-latest" + - run: bash scripts/ci/azure-emulator-up.sh + env: + LOCALSTACK_AUTH_TOKEN: ${{ secrets.LOCALSTACK_AUTH_TOKEN }} + - name: DR1-DR8 + run: AZURE_LIVE=1 npx jest -c jest.azure-live.config.js --selectProjects drift --runInBand + env: + LOCALSTACK_AUTH_TOKEN: ${{ secrets.LOCALSTACK_AUTH_TOKEN }} + AZURE_DR4: "1" + AZURE_SAMPLES_DIR: ${{ runner.temp }}/samples-latest + - name: DR4 — the generated file-argument table is current for the pinned az + run: | + export AZURE_CONFIG_DIR="$RUNNER_TEMP/gen-az-config" AZURE_EXTENSION_DIR="$HOME/.localstack/azure/mcp-extensions" + ~/az/bin/python3 scripts/gen-az-file-args.py > "$RUNNER_TEMP/az-file-args.json" + node -e " + const a = require(process.argv[1]).commands, b = require('./src/lib/azure/az-file-args.generated.json').commands; + const diff = [...new Set([...Object.keys(a), ...Object.keys(b)])].filter((k) => JSON.stringify(a[k]) !== JSON.stringify(b[k])); + if (diff.length) { console.error('az-file-args.generated.json is stale for', diff.length, 'commands:', diff.slice(0, 20).join(', ')); process.exit(1); } + console.log('file-argument table current'); + " "$RUNNER_TEMP/az-file-args.json" + - name: Emulator logs + if: always() + run: mkdir -p emulator-logs && docker logs "$AZURE_CI_EMULATOR_CONTAINER" > emulator-logs/emulator.log 2>&1 || true + - id: scan + if: always() + run: node scripts/ci/scan-for-secret.mjs emulator-logs test-results + env: + LOCALSTACK_AUTH_TOKEN: ${{ secrets.LOCALSTACK_AUTH_TOKEN }} + - uses: actions/upload-artifact@v4 + if: always() && steps.scan.outcome == 'success' + with: + name: azure-drift-reports + path: | + test-results/ + emulator-logs/ + retention-days: 30 + + az-versions: + name: DR3 — L1 with az ${{ matrix.az }} + runs-on: ubuntu-latest + timeout-minutes: 60 + strategy: + fail-fast: false + matrix: + az: ["2.85.0", "latest"] + steps: + - uses: actions/checkout@v4 + - uses: ./.github/actions/azure-live-setup + with: + az-version: ${{ matrix.az }} + extensions: "false" + - run: bash scripts/ci/azure-emulator-up.sh + env: + LOCALSTACK_AUTH_TOKEN: ${{ secrets.LOCALSTACK_AUTH_TOKEN }} + - name: DR3 — the resolved version + run: AZURE_LIVE=1 npx jest -c jest.azure-live.config.js --selectProjects drift -t "DR3" --runInBand + env: + LOCALSTACK_AUTH_TOKEN: ${{ secrets.LOCALSTACK_AUTH_TOKEN }} + AZ_EXPECTED_VERSION: ${{ matrix.az != 'latest' && matrix.az || '' }} + # The emulator above is not the harness's own, so L1 runs its scenario on it and + # skips its stop step (it stops only what it started). + - name: L1 (scenario) + run: node tests/docker/validate-image.mjs -- node dist/cli.js + env: + LOCALSTACK_AUTH_TOKEN: ${{ secrets.LOCALSTACK_AUTH_TOKEN }} + HARNESS_ONLY: azure + HARNESS_TOKEN_REAL: "1" + + summary: + name: Tracking issue on failure + needs: [matrix, egress, egress-internal-network, samples-all, drift, az-versions] + if: failure() + runs-on: ubuntu-latest + permissions: + issues: write + steps: + - name: Open or update the tracking issue + env: + GH_TOKEN: ${{ github.token }} + RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + # The Azure code owners (.github/CODEOWNERS): every alert mentions them, and a new + # issue is assigned to ASSIGNEE. + OWNERS: "@localstack/smurf @HarshCasper" + ASSIGNEE: HarshCasper + run: | + title="Azure weekly suite failing" + existing=$(gh issue list --repo "$GITHUB_REPOSITORY" --state open --search "$title in:title" --json number --jq '.[0].number' --limit 100) + body="The Azure weekly run failed: $RUN_URL (jobs: matrix, egress, egress-internal-network, samples-all, drift, az-versions). cc $OWNERS" + if [ -n "$existing" ]; then + gh issue comment "$existing" --repo "$GITHUB_REPOSITORY" --body "$body" + else + url=$(gh issue create --repo "$GITHUB_REPOSITORY" --title "$title" --body "$body") + # Assigned separately, so that an assignee GitHub refuses cannot lose the alert. + gh issue edit "$url" --repo "$GITHUB_REPOSITORY" --add-assignee "$ASSIGNEE" || echo "::warning::could not assign $ASSIGNEE to $url" + fi diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c118b08..0b245a9 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -9,34 +9,102 @@ on: jobs: build: - runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, windows-latest, macos-latest] + runs-on: ${{ matrix.os }} steps: - name: Checkout code uses: actions/checkout@v4 + with: + # The prettier step diffs against origin/main. + fetch-depth: 0 - - name: Use Node.js 20 - uses: actions/setup-node@v2 + - name: Use Node.js 22 + uses: actions/setup-node@v4 with: node-version: 22.x - name: Install dependencies - run: yarn + run: yarn install --frozen-lockfile - name: Audit production dependencies + if: matrix.os == 'ubuntu-latest' run: npx audit-ci --high --skip-dev - - name: Type-check - run: npx tsc --noEmit + - name: Type-check (src and the test code) + run: npx tsc --noEmit && npx tsc --noEmit -p tsconfig.tests.json - - name: Unit tests - run: yarn test + - name: Prettier on the changed files + # Only the changed files: prettier already fails at HEAD on localstack.client.ts. + if: matrix.os == 'ubuntu-latest' && github.event_name == 'pull_request' + shell: bash + run: | + files=$(git diff --name-only --diff-filter=d origin/main... -- '*.ts' '*.mjs') + if [ -n "$files" ]; then npx prettier --check $files; fi + + - name: Unit tests, with the Azure coverage gate + # The threshold is the gate. + shell: bash + run: >- + yarn test --ci --coverage + --collectCoverageFrom='src/lib/azure/**/*.ts' + --collectCoverageFrom='!src/lib/azure/**/*.test.ts' + --collectCoverageFrom='!src/lib/azure/testing/**' + --collectCoverageFrom='src/lib/cli/**/*.ts' + --collectCoverageFrom='!src/lib/cli/**/*.test.ts' + --collectCoverageFrom='src/tools/localstack-azure-client.ts' + --coverageThreshold='{"global":{"lines":90}}' - name: Build the project run: yarn build + - name: Install az (macOS, when the image lacks it) + if: matrix.os == 'macos-latest' + run: command -v az || brew install azure-cli + + - name: Pinned Bicep for the spawn smoke (ubuntu, macOS) + if: matrix.os != 'windows-latest' + shell: bash + run: | + case "$(uname -s)-$(uname -m)" in + Linux-x86_64) asset=bicep-linux-x64; sum=64c345a58e0c3e48b1bc98a4e62d6b3adb1d238281297de3400aeafb2697aa5a ;; + Linux-aarch64) asset=bicep-linux-arm64; sum=4406214cc274cfac7c821552aec2178b80aec637d91ed8b244282964c1cf24e3 ;; + Darwin-arm64) asset=bicep-osx-arm64; sum=68046a084c88503cf6bd11dacf2a1c4ffcb7e3ac9c6b310d295e024af21bbea4 ;; + Darwin-x86_64) asset=bicep-osx-x64; sum=8ba5771b5261413d88583829f2ea24509eb65b06d899620c17283ecb60d5ca73 ;; + *) echo "no pinned Bicep for $(uname -s)-$(uname -m)"; exit 0 ;; + esac + mkdir -p "$HOME/bicep-bin" + curl -fsSL -o "$HOME/bicep-bin/bicep" "https://github.com/Azure/bicep/releases/download/v0.47.16/$asset" + echo "$sum $HOME/bicep-bin/bicep" | shasum -a 256 -c - + chmod +x "$HOME/bicep-bin/bicep" + echo "LOCALSTACK_AZ_BICEP_PATH=$HOME/bicep-bin/bicep" >> "$GITHUB_ENV" + + - name: az spawn smoke (the runner's real az, no emulator) + run: npx jest src/lib/azure/spawn.smoke + env: + AZ_SMOKE: "1" + + - name: az spawn smoke with a pip az (Windows) + # The only job with the pip layout. + if: matrix.os == 'windows-latest' + shell: bash + run: | + python -m venv "$RUNNER_TEMP/az" + "$RUNNER_TEMP/az/Scripts/pip" install --quiet azure-cli==2.90.0 + npx jest src/lib/azure/spawn.smoke + env: + AZ_SMOKE: "1" + LOCALSTACK_AZ_PATH: ${{ runner.temp }}\az\Scripts\python.exe + mcp-direct-tests: - runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, windows-latest] + runs-on: ${{ matrix.os }} env: LOCALSTACK_AUTH_TOKEN: ${{ secrets.LOCALSTACK_AUTH_TOKEN }} @@ -50,7 +118,7 @@ jobs: node-version: 22.x - name: Install dependencies - run: yarn + run: yarn install --frozen-lockfile - name: Run MCP Server Tester run: yarn test:mcp:direct diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 339607c..ede65b0 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -9,7 +9,10 @@ on: - "src/**" - "package.json" - "yarn.lock" + - "docker/**" + - "data/sample-azure/**" - "tests/docker/**" + - "scripts/ci/azure-emulator-up.sh" - ".github/workflows/docker.yml" push: branches: [main] @@ -18,6 +21,9 @@ on: env: IMAGE_NAME: localstack/localstack-mcp-server + # The image's pins (Dockerfile ARG AZURE_CLI_VERSION, the Bicep stage); L5 checks them. + AZ_EXPECTED: "2.90.0" + BICEP_EXPECTED: "0.47.16" jobs: smoke: @@ -36,8 +42,8 @@ jobs: context: . load: true tags: ${{ env.IMAGE_NAME }}:ci - cache-from: type=gha - cache-to: type=gha,mode=max + cache-from: type=gha,scope=amd64 + cache-to: type=gha,mode=max,scope=amd64 - name: Use Node.js 22 uses: actions/setup-node@v4 @@ -46,18 +52,48 @@ jobs: - name: Smoke test MCP startup env: - HARNESS_SKIP: prompt,docs,status,start,aws,logs,state,cloudpods,appinspector,chaos,iam,replicator,ephemeral,deploy,deploy-cdk,extensions,restart,stop,snowflake + HARNESS_SKIP: prompt,docs,status,start,aws,logs,state,cloudpods,appinspector,chaos,iam,replicator,ephemeral,deploy,deploy-cdk,extensions,restart,stop,snowflake,azure run: | node tests/docker/validate-image.mjs -- \ docker run -i --rm \ ${{ env.IMAGE_NAME }}:ci - - name: Integration test Docker runtime + # L5. The absence checks run before any az command in the container. + - name: L5 image assertions (az, extensions, Bicep, no build leftovers) + run: | + docker run --rm --entrypoint /bin/sh \ + -v "$PWD/docker/azure-extensions.txt:/l5/azure-extensions.txt:ro" \ + -v "$PWD/tests/docker/l5-image-assertions.sh:/l5/assert.sh:ro" \ + -e AZ_EXPECTED -e BICEP_EXPECTED \ + ${{ env.IMAGE_NAME }}:ci /l5/assert.sh + + # The stage images for the size gate: all hits in this job's builder cache, which the + # build above filled (a run step gets no gha cache token, so none is named here). + - name: Build the stage images for the size gate + run: | + for stage in runtime-base runtime-az runtime-bicep; do + docker buildx build --load --target "$stage" -t "${IMAGE_NAME}:stage-$stage" . + done + + - name: L5 size gate (compressed) + run: | + node tests/docker/image-size.mjs \ + --base "${IMAGE_NAME}:stage-runtime-base" \ + --az "${IMAGE_NAME}:stage-runtime-az" \ + --bicep "${IMAGE_NAME}:stage-runtime-bicep" \ + --final "${IMAGE_NAME}:ci" \ + --record docker/image-size.json + + # The whole scenario in two runs with the same token, whose licence must cover AWS, + # Snowflake and Azure: AWS and Snowflake, then the Azure stage. The Azure stage starts its + # own emulator (bound to 0.0.0.0, so the forwarder's host.docker.internal path reaches it), + # and runs even when the first run fails. + - name: Integration test Docker runtime (AWS and Snowflake) if: github.event_name == 'push' || github.event.pull_request.head.repo.full_name == github.repository env: LOCALSTACK_AUTH_TOKEN: ${{ secrets.LOCALSTACK_AUTH_TOKEN }} HARNESS_TOKEN_REAL: ${{ secrets.LOCALSTACK_AUTH_TOKEN != '' && '1' || '0' }} - HARNESS_SKIP: cloudpods,ephemeral,replicator,chaos + HARNESS_SKIP: cloudpods,ephemeral,replicator,chaos,azure run: | npm ci --prefix data/sample-cdk node tests/docker/validate-image.mjs -- \ @@ -71,9 +107,162 @@ jobs: -v "$PWD/data:/work/data" \ ${{ env.IMAGE_NAME }}:ci + - name: Integration test Docker runtime (the Azure stage) + if: ${{ !cancelled() && (github.event_name == 'push' || github.event.pull_request.head.repo.full_name == github.repository) }} + env: + LOCALSTACK_AUTH_TOKEN: ${{ secrets.LOCALSTACK_AUTH_TOKEN }} + HARNESS_TOKEN_REAL: ${{ secrets.LOCALSTACK_AUTH_TOKEN != '' && '1' || '0' }} + HARNESS_ONLY: azure + HARNESS_AZURE_EXTENSIONS: "1" + HARNESS_AZURE_FORWARDER: "1" + HARNESS_AZURE_BICEP: "1" + run: | + node tests/docker/validate-image.mjs -- \ + docker run -i --rm \ + -v /var/run/docker.sock:/var/run/docker.sock \ + --add-host host.docker.internal:host-gateway \ + -e LOCALSTACK_AUTH_TOKEN \ + -e LOCALSTACK_HOSTNAME=host.docker.internal \ + -e LOCALSTACK_AZ_TEST_ENVELOPE=1 \ + -v "$PWD/data:/work/data" \ + ${{ env.IMAGE_NAME }}:ci + + # L5 on a Linux engine against an emulator published on 127.0.0.1 only, as lstk and + # `localstack start` publish it: host.docker.internal cannot reach it, so the forwarder + # must fall back to the container's IP. Also restarts the emulator mid-session, and + # checks the hard error for a missing LOCALSTACK_AZ_BICEP_PATH. + azure-loopback: + name: L5 Azure (127.0.0.1-published emulator, restart) + needs: smoke + if: github.event_name == 'push' || github.event.pull_request.head.repo.full_name == github.repository + runs-on: ubuntu-latest + timeout-minutes: 45 + env: + LOCALSTACK_AUTH_TOKEN: ${{ secrets.LOCALSTACK_AUTH_TOKEN }} + AZURE_CI_EMULATOR_CONTAINER: ls-azure-l5 + AZURE_EMULATOR_LOG_DIR: emulator-logs + steps: + - uses: actions/checkout@v4 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Build image (cache hits after the smoke job) + uses: docker/build-push-action@v6 + with: + context: . + load: true + tags: ${{ env.IMAGE_NAME }}:ci + cache-from: type=gha,scope=amd64 + + - name: Use Node.js 22 + uses: actions/setup-node@v4 + with: + node-version: 22.x + + - name: Start this job's own Azure emulator on 127.0.0.1 + run: bash scripts/ci/azure-emulator-up.sh + + # This job's Bicep checks use a mounted binary instead of the bundled one, the way + # docs/DOCKER.md shows (the smoke job's integration run covers the bundled one). + - name: The pinned Bicep, sha256-checked, to mount over the bundled one + run: | + mkdir -p "$RUNNER_TEMP/bicep-mount" + curl -fsSL -o "$RUNNER_TEMP/bicep-mount/bicep" \ + "https://github.com/Azure/bicep/releases/download/v${BICEP_EXPECTED}/bicep-linux-x64" + echo "64c345a58e0c3e48b1bc98a4e62d6b3adb1d238281297de3400aeafb2697aa5a $RUNNER_TEMP/bicep-mount/bicep" | sha256sum -c - + chmod +x "$RUNNER_TEMP/bicep-mount/bicep" + + - name: Azure stage through the image (fallback forwarder path, mounted Bicep, restart) + env: + HARNESS_ONLY: azure + HARNESS_AZURE_EXTERNAL: "1" + HARNESS_AZURE_EXTENSIONS: "1" + HARNESS_AZURE_FORWARDER: "1" + HARNESS_AZURE_BICEP: "1" + HARNESS_AZURE_RESTART_CONTAINER: ls-azure-l5 + run: | + node tests/docker/validate-image.mjs -- \ + docker run -i --rm \ + -v /var/run/docker.sock:/var/run/docker.sock \ + --add-host host.docker.internal:host-gateway \ + -e LOCALSTACK_AUTH_TOKEN \ + -e MAIN_CONTAINER_NAME=ls-azure-l5 \ + -e LOCALSTACK_AZ_TEST_ENVELOPE=1 \ + -v "$RUNNER_TEMP/bicep-mount/bicep:/opt/bicep/bicep:ro" \ + -e LOCALSTACK_AZ_BICEP_PATH=/opt/bicep/bicep \ + -v "$PWD/data:/work/data" \ + ${{ env.IMAGE_NAME }}:ci + + - name: A missing LOCALSTACK_AZ_BICEP_PATH is a hard error + env: + HARNESS_ONLY: azure + HARNESS_AZURE_EXTERNAL: "1" + HARNESS_AZURE_BICEP_MISSING: "1" + run: | + node tests/docker/validate-image.mjs -- \ + docker run -i --rm \ + -v /var/run/docker.sock:/var/run/docker.sock \ + -e LOCALSTACK_AUTH_TOKEN \ + -e MAIN_CONTAINER_NAME=ls-azure-l5 \ + -e LOCALSTACK_AZ_BICEP_PATH=/nonexistent/bicep \ + -v "$PWD/data:/work/data" \ + ${{ env.IMAGE_NAME }}:ci + + - name: Emulator logs, then remove this job's container + if: always() + run: | + mkdir -p emulator-logs + docker logs ls-azure-l5 > emulator-logs/ls-azure-l5.log 2>&1 || true + docker rm -f ls-azure-l5 >/dev/null 2>&1 || true + + - name: Scan the logs for the token before upload + id: scan + if: failure() + run: node scripts/ci/scan-for-secret.mjs --env LOCALSTACK_AUTH_TOKEN emulator-logs + + - name: Upload emulator logs + if: failure() && steps.scan.outcome == 'success' + uses: actions/upload-artifact@v4 + with: + name: azure-loopback-emulator-logs + path: emulator-logs/ + retention-days: 7 + + # L5 on arm64 before anything is published: the push job below is the + # only other arm64 build, and it pushes. A native arm64 runner, so the az layers need + # no QEMU. + arm64-az: + name: L5 on arm64 + runs-on: ubuntu-24.04-arm + timeout-minutes: 60 + steps: + - uses: actions/checkout@v4 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Build image (linux/arm64, load locally) + uses: docker/build-push-action@v6 + with: + context: . + platforms: linux/arm64 + load: true + tags: ${{ env.IMAGE_NAME }}:ci-arm64 + cache-from: type=gha,scope=arm64 + cache-to: type=gha,mode=max,scope=arm64 + + - name: L5 image assertions (arm64) + run: | + docker run --rm --platform linux/arm64 --entrypoint /bin/sh \ + -v "$PWD/docker/azure-extensions.txt:/l5/azure-extensions.txt:ro" \ + -v "$PWD/tests/docker/l5-image-assertions.sh:/l5/assert.sh:ro" \ + -e AZ_EXPECTED -e BICEP_EXPECTED \ + ${{ env.IMAGE_NAME }}:ci-arm64 /l5/assert.sh + push: name: Build & Push (multi-arch) - needs: smoke + needs: [smoke, arm64-az] if: github.event_name == 'push' && (github.ref == 'refs/heads/main' || github.ref_type == 'tag') runs-on: ubuntu-latest environment: @@ -107,6 +296,8 @@ jobs: flavor: | latest=false + # Reads the per-architecture caches the smoke and arm64-az jobs wrote on this ref, + # so the arm64 half needs QEMU only for the layers that changed. - name: Build and push (linux/amd64, linux/arm64) uses: docker/build-push-action@v6 with: @@ -116,8 +307,10 @@ jobs: provenance: false tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} - cache-from: type=gha - cache-to: type=gha,mode=max + cache-from: | + type=gha,scope=amd64 + type=gha,scope=arm64 + cache-to: type=gha,mode=max,scope=multi - name: Sync README to Docker Hub overview if: github.ref == 'refs/heads/main' diff --git a/.gitignore b/.gitignore index b6fd557..6f3a8d5 100644 --- a/.gitignore +++ b/.gitignore @@ -7,6 +7,10 @@ xmcp-env.d.ts .mcpregistry* playwright-report/ test-results/ +# jest --coverage output (CI's unit step writes it; it holds absolute local paths) +coverage/ +# state exports written by the harness's state step (tests/docker/validate-image.mjs) +data/harness-state-*.zip terraform.tfstate* .terraform/ .terraform.lock.hcl diff --git a/.prettierignore b/.prettierignore index 68f01c3..1deaf8a 100644 --- a/.prettierignore +++ b/.prettierignore @@ -11,3 +11,11 @@ yarn.lock # Log files *.log + +# Generated by scripts/gen-az-file-args.py; the weekly drift job regenerates and compares it +src/lib/azure/az-file-args.generated.json + +# Generated fixtures: their generators define the format +tests/fixtures/azure/corpus/samples-az-corpus.json +tests/fixtures/azure/leak-commands.json +tests/fixtures/azure/stderr/index.json diff --git a/Dockerfile b/Dockerfile index a49381e..39240e7 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,20 @@ # syntax=docker/dockerfile:1 +# Declared globally so the Bicep stage can be picked per architecture (BuildKit sets it). +ARG TARGETARCH + +# The pinned Bicep CLI: one sha256-checked binary per +# architecture. `ADD --checksum` fails the build on any mismatch. +FROM scratch AS bicep-amd64 +ADD --checksum=sha256:64c345a58e0c3e48b1bc98a4e62d6b3adb1d238281297de3400aeafb2697aa5a --chmod=755 \ + https://github.com/Azure/bicep/releases/download/v0.47.16/bicep-linux-x64 /bicep + +FROM scratch AS bicep-arm64 +ADD --checksum=sha256:4406214cc274cfac7c821552aec2178b80aec637d91ed8b244282964c1cf24e3 --chmod=755 \ + https://github.com/Azure/bicep/releases/download/v0.47.16/bicep-linux-arm64 /bicep + +FROM bicep-${TARGETARCH} AS bicep + FROM node:22-bookworm-slim AS builder WORKDIR /app @@ -8,7 +23,9 @@ RUN yarn install --frozen-lockfile COPY . . RUN yarn build -FROM node:22-bookworm-slim AS runtime +# The runtime is built in chained stages so the size gate can measure the Azure layers +# alone: runtime-base -> runtime-az (layers A and B) -> runtime-bicep -> runtime. +FROM node:22-bookworm-slim AS runtime-base ENV DEBIAN_FRONTEND=noninteractive \ PYTHONDONTWRITEBYTECODE=1 \ PATH="/opt/venv/bin:${PATH}" @@ -39,6 +56,59 @@ RUN python3 -m venv /opt/venv \ \( -type d \( -name __pycache__ -o -name tests -o -name test \) -o -type f \( -name '*.pyc' -o -name '*.pyo' \) \) \ -prune -exec rm -rf '{}' + +FROM runtime-base AS runtime-az + +# Layer A: azure-cli in a venv of its own, so it cannot clash +# with the /opt/venv tools. Bytecode is stripped here (+41.6 MB compressed); the tool +# rebuilds what it needs in LOCALSTACK_AZ_PYCACHE_DIR, which saves ~0.7 s per call. +ARG AZURE_CLI_VERSION=2.90.0 +# The whole venv is stripped, not only lib/: pip also compiles scripts in bin/ (jp.py; L5 +# found /opt/az/bin/__pycache__). +RUN python3 -m venv /opt/az \ + && /opt/az/bin/pip install --no-cache-dir --no-compile "azure-cli==${AZURE_CLI_VERSION}" \ + && find /opt/az \ + \( -type d \( -name __pycache__ -o -name tests -o -name test \) -o -type f \( -name '*.pyc' -o -name '*.pyo' \) \) \ + -prune -exec rm -rf '{}' + \ + && ln -s /opt/az/bin/az /usr/local/bin/az + +# AZURE_EXTENSION_DIR serves the build-time `az extension add`. The tool's child gets its +# extension dir from LOCALSTACK_AZ_EXTENSION_DIR, because the child environment drops every +# plain AZURE_* variable. DOTNET_SYSTEM_GLOBALIZATION_INVARIANT lets the Linux +# Bicep binary run without ICU (with identical output). The tool's working directory is +# /work: mount your templates and files there. +ENV AZURE_EXTENSION_DIR=/opt/az-extensions \ + AZURE_CORE_COLLECT_TELEMETRY=no \ + LOCALSTACK_AZ_EXTENSION_DIR=/opt/az-extensions \ + LOCALSTACK_AZ_PYCACHE_DIR=/tmp/localstack-az-pycache \ + LOCALSTACK_AZ_WORKDIR=/work \ + DOTNET_SYSTEM_GLOBALIZATION_INVARIANT=1 + +# Layer B: the 26 curated extensions, pinned and fail-fast, stripped in the SAME RUN +# (70.0 MB unstripped, ~21 MB stripped). The pin list may arrive with CRLF +# line endings from a Windows checkout, so they are removed first. +COPY docker/azure-extensions.txt /tmp/azure-extensions.txt +RUN set -eu; \ + export AZURE_CONFIG_DIR=/tmp/az-build-config; \ + tr -d '\r' < /tmp/azure-extensions.txt > /tmp/azure-extensions.lf; \ + while read -r name version flag || [ -n "${name:-}" ]; do \ + case "$name" in ''|'#'*) continue;; esac; \ + preview=""; [ "${flag:-}" = preview ] && preview="--allow-preview true"; \ + az extension add --name "$name" --version "$version" $preview --yes --only-show-errors; \ + done < /tmp/azure-extensions.lf; \ + find /opt/az-extensions \( -type d -name __pycache__ -o -type f -name '*.pyc' \) -prune -exec rm -rf '{}' +; \ + rm -f /opt/az-extensions/*/*.whl; \ + rm -rf /tmp/tmp* "$AZURE_CONFIG_DIR" /tmp/azure-extensions.txt /tmp/azure-extensions.lf; \ + AZURE_CONFIG_DIR=/tmp/az-manifest-config az extension list -o json > /opt/az-extensions/manifest.json; \ + rm -rf /tmp/az-manifest-config; \ + mkdir -p /work + +FROM runtime-az AS runtime-bicep + +# The bundled Bicep CLI: about +110 MB compressed on amd64. +COPY --from=bicep /bicep /usr/local/bin/bicep + +FROM runtime-bicep AS runtime + RUN npm install -g aws-cdk@2.1133.0 aws-cdk-local \ && npm cache clean --force @@ -70,6 +140,10 @@ RUN mkdir -p /tmp/dockerode-deps \ COPY --from=builder /app/dist ./dist COPY --from=builder /app/package.json ./package.json +# `az version` runs in a throwaway config dir created and removed in the same command: a +# prefix assignment would leave /tmp/tmp.* behind. `bicep --version` +# extracts its .NET bundle and creates a /tmp/.bicep cache; both are removed too, so L5's +# absence checks (no /root/.azure, no /tmp/tmp*) hold and the build leaves nothing behind. RUN set -eux; \ terraform version; \ tflocal --version; \ @@ -77,11 +151,14 @@ RUN set -eux; \ command -v samlocal; \ cdklocal --version; \ snow --version; \ + d=$(mktemp -d); AZURE_CONFIG_DIR="$d" az version; rm -rf "$d"; \ + bicep --version; \ + rm -rf /root/.net /tmp/.net /tmp/.bicep /root/.bicep; \ node dist/cli.js version; \ node -e "require('dockerode'); console.log('dockerode ok')" LABEL org.opencontainers.image.title="LocalStack MCP Server" \ - org.opencontainers.image.description="Self-contained MCP server for managing LocalStack" \ + org.opencontainers.image.description="Self-contained MCP server for managing LocalStack (AWS, Snowflake and Azure)" \ org.opencontainers.image.source="https://github.com/localstack/localstack-mcp-server" \ org.opencontainers.image.licenses="Apache-2.0" diff --git a/README.md b/README.md index 4b22ac0..ca457b3 100644 --- a/README.md +++ b/README.md @@ -35,6 +35,8 @@ Once the server is configured, talk to LocalStack through your agent in natural - "My Lambda calls are failing. Read the LocalStack logs, find the permission errors, and generate an IAM policy that fixes them." - "Inject 500ms of latency into DynamoDB and confirm my retry logic still works." - "Search the LocalStack docs for how to enable S3 event notifications and summarize the steps." +- "Start the LocalStack Azure emulator, create a resource group and a storage account in westeurope, and upload `./data/sample.csv` to a new container." +- "Using the Azure emulator, create a Key Vault, store a secret called `db-password`, and show me how my app would read it." ## How it works @@ -42,7 +44,8 @@ The server connects MCP-compatible apps directly to your local LocalStack enviro This server eliminates custom scripts and manual LocalStack management. Your agent can: -- Start, stop, restart, and monitor LocalStack for AWS container status with built-in auth. +- Start, stop, restart, and monitor the LocalStack for AWS, Snowflake and Azure emulators with built-in auth. +- Run Azure CLI (`az`) commands against the local LocalStack for Azure emulator, never real Azure. - Deploy CDK, Terraform, and SAM projects with automatic configuration detection. - Search LocalStack documentation for guides, API references, and configuration details. - Parse logs, catch errors, and auto-generate IAM policies from violations. @@ -62,22 +65,23 @@ This server provides your AI with dedicated tools for managing your LocalStack e > [!NOTE] > All tools in this MCP server require `LOCALSTACK_AUTH_TOKEN`. -| Tool Name | Description | Key Features | -| :-------------------------------------------------------------------------------- | :------------------------------------------------------------------------- | :------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| [`localstack-management`](./src/tools/localstack-management.ts) | Manages LocalStack runtime operations for AWS and Snowflake stacks | - Execute start, stop, restart, and status checks
- Integrate LocalStack authentication tokens
- Inject custom environment variables
- Verify real-time status and perform health monitoring | -| [`localstack-deployer`](./src/tools/localstack-deployer.ts) | Handles infrastructure deployment to LocalStack for AWS environments | - Automatically run CDK, Terraform, and SAM tooling to deploy infrastructure locally
- Enable parameterized deployments with variable support
- Process and present deployment results
- Requires you to have [`cdklocal`](https://github.com/localstack/aws-cdk-local), [`tflocal`](https://github.com/localstack/terraform-local), or [`samlocal`](https://github.com/localstack/aws-sam-cli-local) installed in your system path | -| [`localstack-logs-analysis`](./src/tools/localstack-logs-analysis.ts) | Analyzes LocalStack for AWS logs for troubleshooting and insights | - Offer multiple analysis options including summaries, errors, requests, and raw data
- Filter by specific services and operations
- Generate API call metrics and failure breakdowns
- Group errors intelligently and identify patterns | -| [`localstack-iam-policy-analyzer`](./src/tools/localstack-iam-policy-analyzer.ts) | Handles IAM policy management and violation remediation | - Set IAM enforcement levels including `enforced`, `soft`, and `disabled` modes
- Search logs for permission-related violations
- Generate IAM policies automatically from detected access failures
- Requires a valid LocalStack Auth Token | -| [`localstack-chaos-injector`](./src/tools/localstack-chaos-injector.ts) | Injects and manages chaos experiment faults for system resilience testing | - Inject, add, remove, and clear service fault rules
- Configure network latency effects
- Comprehensive fault targeting by service, region, and operation
- Built-in workflow guidance for chaos experiments
- Requires a valid LocalStack Auth Token | -| [`localstack-cloud-pods`](./src/tools/localstack-cloud-pods.ts) | Manages remote LocalStack Cloud Pods for development workflows | - Save current state as a Cloud Pod
- Load previously saved Cloud Pods instantly
- Delete Cloud Pods from remote cloud-backed storage
- Use this for managed remote state snapshots, not local export/import files
- Requires a valid LocalStack Auth Token | -| [`localstack-state-management`](./src/tools/localstack-state-management.ts) | Manages local file-based LocalStack state export/import workflows | - Export LocalStack state to a local file on disk through the LocalStack State REST API
- Import LocalStack state from a local file
- Inspect current LocalStack state as JSON metamodel data
- Reset all state or only selected services
- Supports service-level granularity for export, reset, and inspect
- Use this for local disk workflows; use Cloud Pods for remote cloud-backed snapshots
- Requires a valid LocalStack Auth Token | -| [`localstack-extensions`](./src/tools/localstack-extensions.ts) | Installs, uninstalls, lists, and discovers LocalStack Extensions | - Manage installed extensions (`list`, `install`, `uninstall`) inside the running container
- Browse the LocalStack Extensions marketplace (`available`)
- Requires a valid LocalStack Auth Token | -| [`localstack-ephemeral-instances`](./src/tools/localstack-ephemeral-instances.ts) | Manages cloud-hosted LocalStack Ephemeral Instances | - Create temporary cloud-hosted LocalStack instances and get an endpoint URL
- List available ephemeral instances, fetch logs, and delete instances
- Supports lifetime, extension preload, Cloud Pod preload, and custom env vars on create
- Requires a valid LocalStack Auth Token | -| [`localstack-aws-client`](./src/tools/localstack-aws-client.ts) | Runs AWS CLI commands inside the LocalStack for AWS container | - Executes commands via `awslocal` inside the running container
- Sanitizes commands to block shell chaining
- Auto-detects LocalStack coverage errors and links to docs | -| [`localstack-aws-replicator`](./src/tools/localstack-aws-replicator.ts) | Replicates external AWS resources into a running LocalStack instance | - Start single-resource replication jobs with a resource type and identifier or ARN
- Start batch replication jobs, such as SSM parameters under a path prefix
- Poll job status by job ID and list existing jobs
- List resource types supported by the running Replicator extension
- Reads source AWS credentials from the MCP server environment and supports optional target account or region overrides | -| [`localstack-app-inspector`](./src/tools/localstack-app-inspector.ts) | Inspects LocalStack application traces, spans, events, and IAM evaluations | - Enable or disable App Inspector for the running LocalStack instance
- List and inspect traces to understand AWS service-to-service flows
- Drill into spans, events, payload metadata, and IAM policy evaluation events
- Filter by service, region, operation, resource, ARN, status, and time range
- Requires a valid LocalStack Auth Token and the App Inspector feature in the connected LocalStack license | -| [`localstack-docs`](./src/tools/localstack-docs.ts) | Searches LocalStack documentation through CrawlChat | - Queries LocalStack docs through a public CrawlChat collection
- Returns focused snippets with source links only
- Helps answer coverage, configuration, and setup questions without requiring LocalStack runtime | -| [`localstack-snowflake-client`](./src/tools/localstack-snowflake-client.ts) | Runs SQL against the LocalStack Snowflake emulator through the `snow` CLI | - Execute SELECT, DDL (CREATE/DROP), DML (INSERT/UPDATE/DELETE), and SHOW/DESCRIBE statements from a query string or a `.sql` file
- Check the Snowflake connection before running queries
- Set optional database, schema, warehouse, and role context per query
- Requires the Snowflake CLI (`snow`) and a valid LocalStack Auth Token | +| Tool Name | Description | Key Features | +| :-------------------------------------------------------------------------------- | :------------------------------------------------------------------------- | :------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| [`localstack-management`](./src/tools/localstack-management.ts) | Manages LocalStack runtime operations for AWS, Snowflake and Azure stacks | - Execute start, stop, restart, and status checks
- Integrate LocalStack authentication tokens
- Inject custom environment variables
- Verify real-time status and perform health monitoring | +| [`localstack-deployer`](./src/tools/localstack-deployer.ts) | Handles infrastructure deployment to LocalStack for AWS environments | - Automatically run CDK, Terraform, and SAM tooling to deploy infrastructure locally
- Enable parameterized deployments with variable support
- Process and present deployment results
- Requires you to have [`cdklocal`](https://github.com/localstack/aws-cdk-local), [`tflocal`](https://github.com/localstack/terraform-local), or [`samlocal`](https://github.com/localstack/aws-sam-cli-local) installed in your system path | +| [`localstack-logs-analysis`](./src/tools/localstack-logs-analysis.ts) | Analyzes LocalStack for AWS logs for troubleshooting and insights | - Offer multiple analysis options including summaries, errors, requests, and raw data
- Filter by specific services and operations
- Generate API call metrics and failure breakdowns
- Group errors intelligently and identify patterns | +| [`localstack-iam-policy-analyzer`](./src/tools/localstack-iam-policy-analyzer.ts) | Handles IAM policy management and violation remediation | - Set IAM enforcement levels including `enforced`, `soft`, and `disabled` modes
- Search logs for permission-related violations
- Generate IAM policies automatically from detected access failures
- Requires a valid LocalStack Auth Token | +| [`localstack-chaos-injector`](./src/tools/localstack-chaos-injector.ts) | Injects and manages chaos experiment faults for system resilience testing | - Inject, add, remove, and clear service fault rules
- Configure network latency effects
- Comprehensive fault targeting by service, region, and operation
- Built-in workflow guidance for chaos experiments
- Requires a valid LocalStack Auth Token | +| [`localstack-cloud-pods`](./src/tools/localstack-cloud-pods.ts) | Manages remote LocalStack Cloud Pods for development workflows | - Save current state as a Cloud Pod
- Load previously saved Cloud Pods instantly
- Delete Cloud Pods from remote cloud-backed storage
- Use this for managed remote state snapshots, not local export/import files
- Requires a valid LocalStack Auth Token | +| [`localstack-state-management`](./src/tools/localstack-state-management.ts) | Manages local file-based LocalStack state export/import workflows | - Export LocalStack state to a local file on disk through the LocalStack State REST API
- Import LocalStack state from a local file
- Inspect current LocalStack state as JSON metamodel data
- Reset all state or only selected services
- Supports service-level granularity for export, reset, and inspect
- Use this for local disk workflows; use Cloud Pods for remote cloud-backed snapshots
- Requires a valid LocalStack Auth Token | +| [`localstack-extensions`](./src/tools/localstack-extensions.ts) | Installs, uninstalls, lists, and discovers LocalStack Extensions | - Manage installed extensions (`list`, `install`, `uninstall`) inside the running container
- Browse the LocalStack Extensions marketplace (`available`)
- Requires a valid LocalStack Auth Token | +| [`localstack-ephemeral-instances`](./src/tools/localstack-ephemeral-instances.ts) | Manages cloud-hosted LocalStack Ephemeral Instances | - Create temporary cloud-hosted LocalStack instances and get an endpoint URL
- List available ephemeral instances, fetch logs, and delete instances
- Supports lifetime, extension preload, Cloud Pod preload, and custom env vars on create
- Requires a valid LocalStack Auth Token | +| [`localstack-aws-client`](./src/tools/localstack-aws-client.ts) | Runs AWS CLI commands inside the LocalStack for AWS container | - Executes commands via `awslocal` inside the running container
- Sanitizes commands to block shell chaining
- Auto-detects LocalStack coverage errors and links to docs | +| [`localstack-azure-client`](./src/tools/localstack-azure-client.ts) | Runs Azure CLI (`az`) commands against the LocalStack for Azure emulator | - Runs the host's `az` in an isolated CLI profile that is logged in to the emulator only
- Refuses shell syntax, CLI-profile changes, logins, extension installs and commands that open a browser, shell or tunnel
- Rewrites absolute `management.azure.com` URLs to relative ones and blocks every other outbound host
- Keeps file arguments inside the working directory; returns hints when the emulator does not implement an operation
- Requires the Azure CLI (`az` 2.85+) and a valid LocalStack Auth Token | +| [`localstack-aws-replicator`](./src/tools/localstack-aws-replicator.ts) | Replicates external AWS resources into a running LocalStack instance | - Start single-resource replication jobs with a resource type and identifier or ARN
- Start batch replication jobs, such as SSM parameters under a path prefix
- Poll job status by job ID and list existing jobs
- List resource types supported by the running Replicator extension
- Reads source AWS credentials from the MCP server environment and supports optional target account or region overrides | +| [`localstack-app-inspector`](./src/tools/localstack-app-inspector.ts) | Inspects LocalStack application traces, spans, events, and IAM evaluations | - Enable or disable App Inspector for the running LocalStack instance
- List and inspect traces to understand AWS service-to-service flows
- Drill into spans, events, payload metadata, and IAM policy evaluation events
- Filter by service, region, operation, resource, ARN, status, and time range
- Requires a valid LocalStack Auth Token and the App Inspector feature in the connected LocalStack license | +| [`localstack-docs`](./src/tools/localstack-docs.ts) | Searches LocalStack documentation through CrawlChat | - Queries LocalStack docs through a public CrawlChat collection
- Returns focused snippets with source links only
- Helps answer coverage, configuration, and setup questions without requiring LocalStack runtime | +| [`localstack-snowflake-client`](./src/tools/localstack-snowflake-client.ts) | Runs SQL against the LocalStack Snowflake emulator through the `snow` CLI | - Execute SELECT, DDL (CREATE/DROP), DML (INSERT/UPDATE/DELETE), and SHOW/DESCRIBE statements from a query string or a `.sql` file
- Check the Snowflake connection before running queries
- Set optional database, schema, warehouse, and role context per query
- Requires the Snowflake CLI (`snow`) and a valid LocalStack Auth Token | ## Prompts @@ -124,9 +128,35 @@ Run `npx -y @localstack/localstack-mcp-server init --help` for all options. - Docker installed and running. The MCP server manages the LocalStack container directly through the Docker API. - [`cdklocal`](https://github.com/localstack/aws-cdk-local), [`tflocal`](https://github.com/localstack/terraform-local), or [`samlocal`](https://github.com/localstack/aws-sam-cli-local) installed in your system path if you want to deploy CDK, Terraform, or SAM projects - Snowflake CLI (`snow`) installed in your system path if you want to use the Snowflake tool +- Azure CLI (`az` 2.85 or newer) installed in your system path if you want to use the Azure tool, plus the tool's add-ons: see [Setting up the Azure tool](#setting-up-the-azure-tool) - A [valid LocalStack Auth Token](https://docs.localstack.cloud/aws/getting-started/auth-token/) configured as `LOCALSTACK_AUTH_TOKEN` (**required for all MCP tools**) - [Node.js v20](https://nodejs.org/en/download/) or higher installed in your system path +The Docker image bundles `cdklocal`, `tflocal`, `samlocal`, the Snowflake CLI and the Azure CLI with the Azure tool's add-ons, so with `--method docker` you do not install those. + +### Setting up the Azure tool + +The Azure tool runs the Azure CLI on your machine against the LocalStack Azure emulator, the way the Snowflake tool runs the Snowflake CLI. The emulator does not include the Azure CLI, and the setup wizard does not install it, so install it yourself: + +1. **The Azure CLI** (`az` 2.85 or newer), following the [official documentation](https://learn.microsoft.com/cli/azure/install-azure-cli). Installation options: + - Windows: `winget install --exact --id Microsoft.AzureCLI` + - macOS: `brew install azure-cli` + - Debian or Ubuntu: `curl -sL https://aka.ms/InstallAzureCLIDeb | sudo bash` + + If you already have it, `az --version` shows which version, and `az upgrade` updates it. + +2. **The tool's add-ons**: the Azure CLI extensions it supports, and the Bicep CLI for `.bicep` templates: + + ```bash + npx -y @localstack/localstack-mcp-server install-azure-addons + ``` + + This installs the 26 pinned extensions into `~/.localstack/azure/mcp-extensions` and the pinned, sha256-checked Bicep into `~/.localstack/azure/bin`. Skip either part with `--no-extensions` or `--no-bicep`. A `bicep` already on your `PATH` works too (for example `winget install -e --id Microsoft.Bicep`, or `brew install azure/bicep/bicep`). + + Without the add-ons, the core `az` commands still work. A command that needs an extension, or a `.bicep` deployment without Bicep, answers with this command. + +The tool keeps its own Azure CLI profile, logged in to the emulator only, and never changes yours (`~/.azure`). If `az` is missing when the tool is used, its answer says how to install it, as the Snowflake tool's does for `snow`. + ### Run with npx Add the following to your MCP client's configuration file (e.g., `~/.cursor/mcp.json`). This configuration uses `npx` to run the server, which will automatically download and install the package if needed. The server manages LocalStack through your Docker daemon; any deployment CLIs used by tools run from your host PATH. @@ -206,21 +236,52 @@ See **[docs/DOCKER.md](./docs/DOCKER.md)** for the run command, MCP client confi ## LocalStack configuration -| Variable Name | Description | Default Value | -| -------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ---------------------------------- | -| `LOCALSTACK_AUTH_TOKEN` (**required**) | The LocalStack Auth Token to use for the MCP server | None | -| `MAIN_CONTAINER_NAME` | The explicit LocalStack container name to use for Docker-based tools. When unset, the server auto-detects standard LocalStack containers such as `localstack-main` and `localstack-aws`, then LocalStack Docker images. | Auto-detect | -| `LOCALSTACK_IMAGE_NAME` / `IMAGE_NAME` | Docker image the `start` action launches for the AWS stack. `LOCALSTACK_IMAGE_NAME` wins when both are set (beware: bare `IMAGE_NAME` is a common CI variable). | `localstack/localstack-pro:latest` | -| `LOCALSTACK_VOLUME_DIR` | Host directory mounted at `/var/lib/localstack` in the LocalStack container. Defaults to the same per-OS cache path the LocalStack CLI used (state carries over); inside Docker it defaults to the `localstack-mcp` named volume. | Per-OS cache dir | -| `GATEWAY_LISTEN` | Override the gateway port bindings for `start` (comma-separated `[host]:port`). When set, the implicit `443` binding is skipped. | `:4566,:443` | -| `DOCKER_HOST` | Docker daemon endpoint used for all container operations (`unix://`, `npipe://`, `tcp://`). | Platform default socket | -| `DOCKER_SOCK` | Host path of the Docker socket mounted into the LocalStack container (for Lambda). Validated to exist when set explicitly. | `/var/run/docker.sock` | -| `EXTERNAL_SERVICE_PORTS_START` / `_END` | Inclusive host port range published for external services on `start`. Validated (1–65535, start ≤ end). | `4510` / `4560` | -| `MCP_ANALYTICS_DISABLED` | Disable MCP analytics when set to `1` | `0` | -| `APP_INSPECTOR` | Set to `1` in the LocalStack container environment to enable App Inspector by default across restarts. The MCP tool can also toggle App Inspector at runtime with `set-status`. | `0` | -| `AWS_ACCESS_KEY_ID` (**required for AWS Replicator tool**) | Source AWS access key used by AWS Replicator to read external AWS resources | None | -| `AWS_SECRET_ACCESS_KEY` (**required for AWS Replicator tool**) | Source AWS secret access key used by AWS Replicator to read external AWS resources | None | -| `AWS_DEFAULT_REGION` (**required for AWS Replicator tool**) | Source AWS region used by AWS Replicator | None | +| Variable Name | Description | Default Value | +| ----------------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -------------------------------------------------------- | +| `LOCALSTACK_AUTH_TOKEN` (**required**) | The LocalStack Auth Token to use for the MCP server | None | +| `MAIN_CONTAINER_NAME` | The explicit LocalStack container name to use for Docker-based tools. When unset, the server auto-detects standard LocalStack containers such as `localstack-main` and `localstack-aws`, then LocalStack Docker images. | Auto-detect | +| `LOCALSTACK_IMAGE_NAME` / `IMAGE_NAME` | Docker image the `start` action launches for the AWS stack. `LOCALSTACK_IMAGE_NAME` wins when both are set (beware: bare `IMAGE_NAME` is a common CI variable). | `localstack/localstack-pro:latest` | +| `LOCALSTACK_VOLUME_DIR` | Host directory mounted at `/var/lib/localstack` in the LocalStack container. Defaults to the same per-OS cache path the LocalStack CLI used (state carries over); inside Docker it defaults to the `localstack-mcp` named volume. The Azure emulator needs a host directory here to deploy Function Apps and Web Apps. | Per-OS cache dir | +| `GATEWAY_LISTEN` | Override the gateway port bindings for `start` (comma-separated `[host]:port`). When set, the implicit `443` binding is skipped. | `:4566,:443` | +| `DOCKER_HOST` | Docker daemon endpoint used for all container operations (`unix://`, `npipe://`, `tcp://`). | Platform default socket | +| `DOCKER_SOCK` | Host path of the Docker socket mounted into the LocalStack container (for Lambda). Validated to exist when set explicitly. | `/var/run/docker.sock` | +| `EXTERNAL_SERVICE_PORTS_START` / `_END` | Inclusive host port range published for external services on `start`. Validated (1–65535, start ≤ end). | `4510` / `4560` | +| `MCP_ANALYTICS_DISABLED` | Disable MCP analytics when set to `1` | `0` | +| `APP_INSPECTOR` | Set to `1` in the LocalStack container environment to enable App Inspector by default across restarts. The MCP tool can also toggle App Inspector at runtime with `set-status`. | `0` | +| `AWS_ACCESS_KEY_ID` (**required for AWS Replicator tool**) | Source AWS access key used by AWS Replicator to read external AWS resources | None | +| `AWS_SECRET_ACCESS_KEY` (**required for AWS Replicator tool**) | Source AWS secret access key used by AWS Replicator to read external AWS resources | None | +| `AWS_DEFAULT_REGION` (**required for AWS Replicator tool**) | Source AWS region used by AWS Replicator | None | +| `LOCALSTACK_AZURE_IMAGE_NAME` | Docker image the `start` action launches for `service: azure`. | `localstack/localstack-azure:latest` | +| `LOCALSTACK_AZURE_PORT` | Gateway port of the Azure emulator the Azure tool talks to (for example `4666` for a port-shifted emulator). | `LOCALSTACK_PORT`, then `4566` | +| `LOCALSTACK_AZURE_ENDPOINT` | ARM endpoint override. It must use a local name (`localhost.localstack.cloud` or a subdomain, `localhost`, `127.0.0.1`, `::1`); remote endpoints are refused. | `https://azure.localhost.localstack.cloud:` | +| `LOCALSTACK_AZ_PATH` | Explicit Azure CLI launcher (`az`, `az.cmd`) or its Python. | `PATH` lookup | +| `LOCALSTACK_AZ_CONFIG_DIR` | The Azure tool's own isolated Azure CLI profile. It may not be, lie inside, or contain your `~/.azure`. | `~/.localstack/azure/mcp-config-` | +| `LOCALSTACK_AZ_EXTENSION_DIR` | Azure CLI extension directory for the tool (automatic extension installs are off). | `~/.localstack/azure/mcp-extensions` | +| `LOCALSTACK_AZ_BICEP_PATH` | An explicit Bicep binary: an absolute path to a file named `bicep` or `bicep.exe`. Otherwise `~/.localstack/azure/bin`, then `PATH` (never `~/.azure/bin`). | Auto-detect | +| `LOCALSTACK_AZ_BICEP_ENV` | Comma-separated server environment variables a `.bicepparam` may read with `readEnvironmentVariable()`. Nothing else reaches Bicep; the auth token and `AZURE_*`, `ARM_*`, `BICEP_*`, proxy and path variables are never passed. | None | +| `LOCALSTACK_AZ_WORKDIR` | Directory that the Azure tool's file arguments must stay inside; also `az`'s working directory. | The server's working directory | +| `LOCALSTACK_AZ_TIMEOUT_SECONDS` | Per-command timeout of the Azure tool (5–3600). | `300` | +| `LOCALSTACK_AZ_MAX_OUTPUT_CHARS` / `LOCALSTACK_AZ_MAX_HELP_CHARS` | Output and help-page limits of the Azure tool. | `30000` / `30000` | +| `LOCALSTACK_AZ_DENYLIST_FILE` | A file of extra `az` command prefixes the tool refuses, one per line (`#` comments allowed). | None | +| `LOCALSTACK_AZ_EGRESS_GUARD` | `0` turns off the Azure tool's egress guard (debugging only; every answer then says so). | `1` | +| `LOCALSTACK_AZ_PYCACHE_DIR` | Bytecode cache for Azure CLI installs without `.pyc` files (the Docker image sets it). | None | +| `LOCALSTACK_AZ_RUNNER` | `worker` (experimental) keeps warm Azure CLI processes and runs each command in one of them: about 5x faster per call on Linux. Same isolation as the default; a worker is replaced after 200 commands or any change to the tool's CLI profile. | `host` (a fresh `az` process per command) | +| `LOCALSTACK_AZURE_FORWARD_TARGET` | Docker image only: where the loopback forwarder relays the Azure emulator's ports. | `host.docker.internal`, then the emulator's container IP | + +### How the Azure tool stays local + +`localstack-azure-client` runs the Azure CLI installed on your machine, but it never uses your own Azure CLI login and never reaches real Azure: + +- **Its own CLI profile.** `az` runs with `AZURE_CONFIG_DIR` set to the tool's profile (`~/.localstack/azure/mcp-config-`), registered with a `LocalStack` cloud and a dummy login. It also gets a private home and temp directory inside that profile, so commands that write into `~` (SSH keys, kubeconfig, certificates) never touch yours. +- **A command policy.** Shell syntax, logins, cloud and config changes, extension installs, `upgrade`, and commands that open a browser, shell, tunnel or run Docker on your machine are refused. File arguments must stay inside `LOCALSTACK_AZ_WORKDIR` (or the private home), and never reach `~/.azure`, `~/.ssh`, `~/.kube` or `~/.docker`. +- **An allow-list environment.** `az` gets a minimal environment; your `AZURE_*`, `ARM_*`, proxy and CA variables are never passed on. +- **An egress guard.** Every connection `az` or Bicep makes goes through a local proxy that allows only the emulator's names (`*.localhost.localstack.cloud`, `localhost`). Anything else is refused. A few calls `az` makes on its own, such as update checks, Bicep's module index and the Application Insights region map that `functionapp create` downloads, are refused quietly and `az` carries on without them; any other refused host stops the command at once, and the answer names it. Absolute `https://management.azure.com/...` URLs given to `rest` are rewritten to relative ones. + +Long-running creates wait until the resource is ready. Claude Desktop stops waiting for a tool call after about 60 seconds; with that client, ask for `--no-wait` and poll with `show`. Other clients (Claude Code, most IDEs) wait much longer. + +File arguments (templates, uploads, downloads) must be inside the tool's working directory, which is the server's own working directory unless you set `LOCALSTACK_AZ_WORKDIR`. GUI clients such as Claude Desktop start the server in their own folder, so add `"LOCALSTACK_AZ_WORKDIR": ""` to the entry's `env` to work with local files. + +The other tools with the Azure emulator: `localstack-management` manages it (`service: "azure"`), and `localstack-logs-analysis` shows its raw log (`analysisType: "logs"`); the summary, errors and requests analyses read AWS logs only. ### Migration notes (CLI-free lifecycle) diff --git a/data/evals/gemini-azure.json b/data/evals/gemini-azure.json new file mode 100644 index 0000000..447e2fe --- /dev/null +++ b/data/evals/gemini-azure.json @@ -0,0 +1,61 @@ +{ + "name": "localstack-mcp-gemini-azure", + "description": "Scenario-based Gemini evals for the Azure tools (E1): a pinned-command case for localstack-azure-client and a management status case for service azure. A separate dataset from gemini-comprehensive.json, because the AWS and Azure emulators cannot share port 4566 in one run.", + "cases": [ + { + "id": "scenario-azure-client-group-list", + "description": "localstack-azure-client: run a pinned Azure CLI command.", + "mode": "mcp_host", + "scenario": "Using the LocalStack Azure client tool, run the Azure CLI command \"group list\" against the LocalStack Azure emulator. Pass the command string exactly as group list, with no az prefix and no extra flags. Then tell me which resource groups exist.", + "mcpHostConfig": { + "provider": "google", + "model": "gemini-2.5-flash", + "temperature": 0 + }, + "iterations": 5, + "accuracyThreshold": 0.8, + "expect": { + "toolsTriggered": { + "calls": [ + { + "name": "localstack-azure-client", + "arguments": { + "command": "group list" + }, + "required": true + } + ], + "order": "any" + } + } + }, + { + "id": "scenario-management-azure-status", + "description": "localstack-management: status check of the Azure emulator.", + "mode": "mcp_host", + "scenario": "Check the current status of the LocalStack Azure emulator using the LocalStack management tool with the azure service. Report whether it is running and ready.", + "mcpHostConfig": { + "provider": "google", + "model": "gemini-2.5-flash", + "temperature": 0 + }, + "iterations": 5, + "accuracyThreshold": 0.8, + "expect": { + "toolsTriggered": { + "calls": [ + { + "name": "localstack-management", + "arguments": { + "action": "status", + "service": "azure" + }, + "required": true + } + ], + "order": "any" + } + } + } + ] +} diff --git a/data/sample-azure/hello.txt b/data/sample-azure/hello.txt new file mode 100644 index 0000000..6b85822 --- /dev/null +++ b/data/sample-azure/hello.txt @@ -0,0 +1 @@ +hello from the LocalStack MCP server Azure stage diff --git a/data/sample-azure/main.bicep b/data/sample-azure/main.bicep new file mode 100644 index 0000000..0df4703 --- /dev/null +++ b/data/sample-azure/main.bicep @@ -0,0 +1,11 @@ +// A deployment for the Docker harness's Bicep checks. It +// creates one user-assigned identity named by the parameter, so the check can see that +// Bicep compiled the file, the parameter arrived and the resource exists. (The emulator +// returns no deployment outputs, so an output alone would prove nothing.) +param identityName string +param location string = 'westeurope' + +resource identity 'Microsoft.ManagedIdentity/userAssignedIdentities@2023-01-31' = { + name: identityName + location: location +} diff --git a/data/sample-azure/main.bicepparam b/data/sample-azure/main.bicepparam new file mode 100644 index 0000000..2498afb --- /dev/null +++ b/data/sample-azure/main.bicepparam @@ -0,0 +1,3 @@ +using './main.bicep' + +param identityName = 'l5-from-bicepparam' diff --git a/docker/azure-extensions.txt b/docker/azure-extensions.txt new file mode 100644 index 0000000..4b7d3cd --- /dev/null +++ b/docker/azure-extensions.txt @@ -0,0 +1,46 @@ +# Curated Azure CLI extensions for the localstack-azure-client tool. +# +# Format: one extension per line, " [preview]". +# "preview" marks a preview-only build, installed with --allow-preview true. +# Lines starting with '#' and blank lines are ignored. No trailing comments: the Docker build +# reads this file with `while read -r name version flag`. +# +# Which extensions: measured against azure-cli 2.90.0, an extension is curated when its commands reach emulator-served providers and +# it overrides no core command. Excluded because they override core commands: aks-preview, authV2, +# containerapp, cosmosdb-preview, keyvault-preview, storage-blob-preview, storage-preview. +# +# Versions: as installed against azure-cli 2.90.0 (2026-09-27), except +# application-insights, pinned to the stable 1.2.3 instead of the preview 2.0.0b2 that +# --allow-preview picked. +# +# DR4 checks weekly that this set installs and loads with the pinned az, and reports newer +# releases. Installed by scripts/install-azure-extensions.mjs (CI, `install-azure-addons`, developers) +# and by the Docker image build. src/lib/azure/extension-map.ts maps the extensions' command +# groups; its test keeps the two lists in sync. + +acrcssc 1.0.0b8 preview +acrquery 1.0.1 +acrtransfer 2.0.0 +application-insights 1.2.3 +azure-firewall 2.2.1 +bastion 1.4.3 +cdn 1.0.0b3 preview +dns-resolver 1.2.0 +documentdb 1.0.0 +edge-action 1.0.0b4 preview +eventgrid 1.0.0b2 preview +express-route-cross-connection 1.0.0 +fleet 1.11.1 +front-door 2.3.0 +ip-group 1.0.1 +k8s-configuration 2.3.0 +k8s-extension 1.9.1 +monitor-control-service 1.2.0 +nsp 1.1.0 +resource-graph 2.1.1 +scheduled-query 1.0.0b2 preview +staticwebapp 1.0.1 +virtual-network-manager 3.0.2 +virtual-network-tap 1.0.0b2 preview +virtual-wan 1.0.1 +webapp 0.4.0 diff --git a/docker/image-size.json b/docker/image-size.json new file mode 100644 index 0000000..0e75f6a --- /dev/null +++ b/docker/image-size.json @@ -0,0 +1,8 @@ +{ + "method": "docker save | gzip -6 (node zlib), bytes", + "base": 231524437, + "azLayers": 62153935, + "bicepLayer": 110109646, + "final": 417698790, + "measuredAt": "2026-09-27" +} diff --git a/docs/DOCKER.md b/docs/DOCKER.md index 0bff983..bb512f1 100644 --- a/docs/DOCKER.md +++ b/docs/DOCKER.md @@ -1,10 +1,10 @@ # Running the LocalStack MCP Server in Docker The published image bundles everything the server shells out to — Terraform + -`tflocal`, AWS CDK + `cdklocal`, AWS SAM + `samlocal`, and the Snowflake `snow` -CLI — so the **only dependency on your machine is Docker itself**. LocalStack -lifecycle, logs, and `awslocal` run through the Docker Engine API and LocalStack's -REST APIs. +`tflocal`, AWS CDK + `cdklocal`, AWS SAM + `samlocal`, the Snowflake `snow` CLI, +and for the Azure tool the Azure CLI, its curated extensions and the Bicep CLI — so +the **only dependency on your machine is Docker itself**. LocalStack lifecycle, logs, +and `awslocal` run through the Docker Engine API and LocalStack's REST APIs. The image is multi-arch (`linux/amd64` and `linux/arm64`). @@ -36,6 +36,12 @@ so no host directory needs to be mounted or path-mirrored. To keep state in a ho directory instead, set `-e LOCALSTACK_VOLUME_DIR=/absolute/host/path` (the path is interpreted by the **host** daemon). +For the LocalStack for Azure emulator, set `LOCALSTACK_VOLUME_DIR` if you deploy Function Apps or +Web Apps: the emulator runs them in containers of their own and shares their files from its state +folder, which it can do only from a host directory. On the named volume, `az functionapp deploy` +and `az webapp deploy` fail with an internal server error, and `localstack-management start` +(service `azure`) says so. + > **Upgrading from an older image?** Previous versions required a one-to-one cache > mount plus `XDG_CACHE_HOME`. Old configs keep working: when `XDG_CACHE_HOME` is > set, the server keeps using `$XDG_CACHE_HOME/localstack/volume` for state, so @@ -116,6 +122,84 @@ Terraform, SAM, and CDK receive the LocalStack endpoint automatically when to path-style S3 inside the Docker image, so the single `s3.host.docker.internal` alias covers bootstrap asset uploads. +## Azure (`localstack-azure-client`) + +The image contains everything the Azure tool runs: + +| What | Where | Version | +| ------------------------------------------------------------------- | ------------------------------------------------- | -------------------- | +| Azure CLI, in a virtualenv of its own | `/opt/az` (`az` on `PATH`) | 2.90.0 | +| The 26 curated Azure CLI extensions (`docker/azure-extensions.txt`) | `/opt/az-extensions` (`manifest.json` lists them) | pinned per extension | +| Bicep CLI | `/usr/local/bin/bicep` | 0.47.16 | + +Start the Azure emulator with the `localstack-management` tool (`action: start`, +`service: azure`), or use one that is already running. No extra flags are needed +beyond the quick-start ones; an Azure-only setup needs just these: + +```bash +docker run -i --rm \ + -v /var/run/docker.sock:/var/run/docker.sock \ + --add-host host.docker.internal:host-gateway \ + -e LOCALSTACK_AUTH_TOKEN="" \ + -v "$PWD/infra:/work/infra" \ + localstack/localstack-mcp-server:latest +``` + +**Files go under `/work`.** The Azure tool's working directory in the image is +`/work`, and file arguments (`--template-file`, `--file`, `@body.json`, …) must stay +inside it. Mount what `az` should read there and use paths relative to it: with the +mount above, `deployment group create -g demo --template-file infra/main.bicep`. To +keep the same-path project mount shown for IaC deploys, point the tool at it with +`-e LOCALSTACK_AZ_WORKDIR=/Users/you/projects`. + +**How the tool reaches the emulator.** Inside the container, the emulator is not on +`127.0.0.1`, yet `az`'s endpoint and the tool's health checks expect it there. So, +in Docker only, the server starts a small **loopback forwarder** on the first Azure +call: it listens on `127.0.0.1` on the emulator's gateway port and its external +service ports, and relays each connection unchanged (TLS stays end to end). It picks +its target in this order: + +1. `LOCALSTACK_AZURE_FORWARD_TARGET`, when set; +2. `host.docker.internal` (hence the `--add-host` flag on Linux); +3. the emulator container's own IP, found through the mounted Docker socket (both + containers must share a Docker network; the default bridge counts). + +Under `--network host`, `127.0.0.1` already answers and no forwarder starts. If the +emulator restarts and comes back elsewhere, the next call finds it again. + +> **Linux caveat.** On a Linux Docker engine, `host.docker.internal` is the bridge +> gateway (`172.17.0.1`), which cannot reach an emulator published on `127.0.0.1` +> only — the way `lstk start` and `localstack start` publish it. The forwarder then +> falls back to the emulator container's IP (step 3), which works when the socket is +> mounted and both containers are on the same network. Otherwise, run the MCP server +> with `--network host`, or attach both to a shared network and set +> `-e LOCALSTACK_AZURE_FORWARD_TARGET=`. An emulator that +> this server starts itself listens on all interfaces and needs none of this. Docker +> Desktop (Mac and Windows) is not affected. + +**A different Bicep binary.** Mount it and name it explicitly; a path that does not +exist is a hard error, never a silent fallback: + +``` +-v /path/to/dir/bicep:/opt/bicep/bicep:ro -e LOCALSTACK_AZ_BICEP_PATH=/opt/bicep/bicep +``` + +**Parameter files that read environment variables.** `az`, and so Bicep, runs with a +clean environment, so a `.bicepparam` that uses `readEnvironmentVariable('DB_PASSWORD')` +sees nothing unless you list the name. Pass the variable into the container and list it: + +``` +-e DB_PASSWORD -e LOCALSTACK_AZ_BICEP_ENV=DB_PASSWORD +``` + +Only listed names pass. The auth token and variables that steer `az` (`AZURE_*`, +`ARM_*`, `BICEP_*`, proxies, paths) are never passed, even when listed. + +**Extensions.** Only the curated extensions are installed, and automatic installs +are off, so an extension command outside the list fails at once with a hint naming +the extension. To add your own, mount a directory with the extensions you need and +set `-e LOCALSTACK_AZ_EXTENSION_DIR=` (it replaces the curated set). + ## Known limitations - **Extra host aliases.** Include the aliases shown in the quick-start command. @@ -136,6 +220,8 @@ alias covers bootstrap asset uploads. | `LocalStack container not found` or `Could not find a running LocalStack container named "localstack-main"` | Set `MAIN_CONTAINER_NAME` if you use a custom LocalStack container name. | | State disappeared after upgrading the image | Old configs stored state under `$XDG_CACHE_HOME/localstack/volume` — keep that env var, or point `LOCALSTACK_VOLUME_DIR` at the old directory. | | MCP server containers pile up over time | Older images did not exit when the client disconnected. Pull the latest image, then remove strays with `docker ps -aq --filter ancestor=localstack/localstack-mcp-server \| xargs docker rm -f`. | +| The Azure tool reports the emulator is not running, but it is (Linux) | The emulator is published on `127.0.0.1` only: see the Linux caveat in the Azure section (mount the socket, share a network, or use `--network host`). | +| An Azure file argument is refused as outside the working directory | Mount the file under `/work` and use a path relative to it, or set `LOCALSTACK_AZ_WORKDIR` to the directory you mounted. | ## Validating an image yourself @@ -162,3 +248,17 @@ Use `HARNESS_SKIP` to skip scenarios, for example: ```bash HARNESS_SKIP=docs,cloudpods,ephemeral,replicator ``` + +The Azure stage takes extra switches (the header of `validate-image.mjs` lists them +all). The image-specific checks need the server started with +`-e LOCALSTACK_AZ_TEST_ENVELOPE=1`: + +```bash +HARNESS_AZURE_EXTENSIONS=1 # an extension command reaches the emulator; a non-curated one fails fast +HARNESS_AZURE_FORWARDER=1 # the traffic went through the loopback forwarder +HARNESS_AZURE_BICEP=1 # a .bicep and a .bicepparam deployment +HARNESS_AZURE_EXTERNAL=1 # the emulator is managed outside the harness: no start or stop +``` + +The harness stops or restarts only a container that its own `start` created, so it +never stops an emulator you already had running. diff --git a/jest.azure-live.config.js b/jest.azure-live.config.js new file mode 100644 index 0000000..23de5f5 --- /dev/null +++ b/jest.azure-live.config.js @@ -0,0 +1,60 @@ +// Live Azure suites. They need a running LocalStack +// Azure emulator, so `yarn test` never runs them (jest.config.js ignores *.live.test.ts). +// +// AZURE_LIVE=1 npx jest -c jest.azure-live.config.js --selectProjects matrix-subset egress +// +// Jest projects select files; the matrix and samples subsets are chosen by the env var +// each project's setup file sets, and every live file skips itself unless AZURE_LIVE=1. +const shared = { + preset: "ts-jest", + testEnvironment: "node", + transform: { "^.+\\.tsx?$": ["ts-jest", { tsconfig: "tsconfig.tests.json" }] }, + // Closes the egress guard after each file, so Jest exits. + setupFilesAfterEnv: ["/tests/azure/live/teardown.ts"], +}; + +module.exports = { + // A live az call can take minutes (storage accounts on a busy emulator: ~100 s). Jest + // reads testTimeout from the top level only; inside projects[] it is ignored and every + // test would get the 5 s default. + testTimeout: 900_000, + projects: [ + { + ...shared, + displayName: "matrix-subset", + testMatch: ["/tests/azure/matrix.live.test.ts"], + setupFiles: ["/tests/azure/live/setup-matrix-pr.js"], + }, + { + ...shared, + displayName: "matrix-full", + testMatch: ["/tests/azure/matrix.live.test.ts"], + setupFiles: ["/tests/azure/live/setup-matrix-full.js"], + }, + { + ...shared, + displayName: "egress", + testMatch: [ + "/tests/azure/egress.live.test.ts", + "/tests/azure/home-guard.live.test.ts", + ], + }, + { + ...shared, + displayName: "samples-subset", + testMatch: ["/tests/azure/samples-replay.live.test.ts"], + setupFiles: ["/tests/azure/live/setup-samples-pr.js"], + }, + { + ...shared, + displayName: "samples-all", + testMatch: ["/tests/azure/samples-replay.live.test.ts"], + setupFiles: ["/tests/azure/live/setup-samples-all.js"], + }, + { + ...shared, + displayName: "drift", + testMatch: ["/tests/azure/drift/*.live.test.ts"], + }, + ], +}; diff --git a/jest.config.js b/jest.config.js index 8694a35..2732ec6 100644 --- a/jest.config.js +++ b/jest.config.js @@ -2,4 +2,13 @@ module.exports = { preset: "ts-jest", testEnvironment: "node", testMatch: ["**/?(*.)+(spec|test).[jt]s?(x)"], + // Live suites need a running emulator; they run only through + // jest.azure-live.config.js. + testPathIgnorePatterns: ["/node_modules/", "\\.live\\.test\\.ts$"], + // The Azure runner tests spawn real processes; on a loaded machine the 5 s default + // starved unrelated timing-based tests (seen in full-suite runs). + testTimeout: 30000, + transform: { + "^.+\\.tsx?$": ["ts-jest", { tsconfig: "tsconfig.tests.json" }], + }, }; diff --git a/manifest.json b/manifest.json index 75afc29..a7d0e69 100644 --- a/manifest.json +++ b/manifest.json @@ -67,6 +67,10 @@ "name": "localstack-aws-client", "description": "Runs AWS CLI commands inside the running LocalStack container" }, + { + "name": "localstack-azure-client", + "description": "Runs Azure CLI (az) commands against the local LocalStack for Azure emulator, never real Azure" + }, { "name": "localstack-aws-replicator", "description": "Replicates external AWS resources into a running LocalStack instance using the AWS Replicator HTTP API" @@ -156,6 +160,12 @@ "arguments": ["description"], "text": "Please query the LocalStack container for ${arguments.description}." }, + { + "name": "localstack-azure-client", + "description": "Runs Azure CLI commands against the LocalStack Azure emulator", + "arguments": ["description"], + "text": "Please query the LocalStack Azure emulator for ${arguments.description}." + }, { "name": "aws-replicator-start", "description": "Start an AWS Replicator job", diff --git a/package.json b/package.json index 5878b32..dd531e8 100644 --- a/package.json +++ b/package.json @@ -14,8 +14,9 @@ "prepack": "yarn build", "format": "prettier --write .", "test": "jest", - "test:mcp:direct": "yarn build && playwright test -c playwright.config.mjs tests/mcp/direct.spec.mjs", + "test:mcp:direct": "yarn build && playwright test -c playwright.config.mjs tests/mcp/direct.spec.mjs tests/mcp/azure-offline.spec.mjs", "test:mcp:evals": "yarn build && playwright test -c playwright.config.mjs tests/mcp/evals-gemini.spec.mjs", + "test:mcp:evals:azure": "yarn build && playwright test -c playwright.config.mjs tests/mcp/evals-gemini-azure.spec.mjs", "test:mcp": "yarn test:mcp:direct && yarn test:mcp:evals" }, "dependencies": { @@ -24,6 +25,7 @@ "dockerode": "^5.0.1" }, "devDependencies": { + "@anthropic-ai/sdk": "^0.128.0", "@clack/prompts": "^1.5.1", "@gleanwork/mcp-server-tester": "1.0.0-beta.6", "@playwright/test": "^1.58.2", @@ -40,6 +42,7 @@ "ts-jest": "^29.4.1", "typescript": "^5.9.2", "xmcp": "^0.6.13", + "yaml": "2.9.1", "zod": "4.3.6" }, "resolutions": { diff --git a/playwright.config.mjs b/playwright.config.mjs index f1123b7..f6b5b34 100644 --- a/playwright.config.mjs +++ b/playwright.config.mjs @@ -1,10 +1,42 @@ import { defineConfig } from "@playwright/test"; +import { mkdirSync } from "node:fs"; +import net from "node:net"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; const mcpCommand = process.env.MCP_TEST_COMMAND || "node"; const mcpArgs = process.env.MCP_TEST_ARGS ? process.env.MCP_TEST_ARGS.split(" ").filter(Boolean) : ["dist/cli.js"]; +const sharedMcpConfig = { + transport: "stdio", + command: mcpCommand, + args: mcpArgs, + cwd: process.cwd(), + quiet: true, + connectTimeoutMs: 30000, + requestTimeoutMs: 300000, + callTimeoutMs: 300000, +}; + +// The Azure tool without an emulator. A port that is free right +// now keeps "no emulator" true even where one runs on 4566; 4766 and 4710-4790 are +// never picked because the OS chooses from its ephemeral range. +async function freePort() { + const server = net.createServer(); + await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)); + const { port } = server.address(); + await new Promise((resolve) => server.close(resolve)); + return port; +} +const azureOfflinePort = await freePort(); + +// The catalogue budget check measures the description filled with a 200-character workdir. +const workdirRoot = join(tmpdir(), "lsmcp-p2"); +const azureWorkdir = join(workdirRoot, "w".repeat(Math.max(1, 200 - workdirRoot.length - 1))); +mkdirSync(azureWorkdir, { recursive: true }); + export default defineConfig({ testDir: "./tests/mcp", timeout: 120000, @@ -23,16 +55,10 @@ export default defineConfig({ projects: [ { name: "localstack-mcp-server", + testIgnore: /azure-offline\.spec\.mjs$/, use: { mcpConfig: { - transport: "stdio", - command: mcpCommand, - args: mcpArgs, - cwd: process.cwd(), - quiet: true, - connectTimeoutMs: 30000, - requestTimeoutMs: 300000, - callTimeoutMs: 300000, + ...sharedMcpConfig, env: { ...process.env, LOCALSTACK_AUTH_TOKEN: process.env.LOCALSTACK_AUTH_TOKEN || "", @@ -40,5 +66,24 @@ export default defineConfig({ }, }, }, + { + // Its own env: a dummy token, and the Azure tool pointed at a + // port where nothing listens, with its own config dir. + name: "azure-offline", + testMatch: /azure-offline\.spec\.mjs$/, + use: { + mcpConfig: { + ...sharedMcpConfig, + env: { + ...process.env, + LOCALSTACK_AUTH_TOKEN: "ls-dummy-token-for-offline-tests", + LOCALSTACK_AZURE_PORT: String(azureOfflinePort), + LOCALSTACK_AZ_CONFIG_DIR: join(workdirRoot, "azure-config"), + LOCALSTACK_AZ_WORKDIR: azureWorkdir, + MCP_ANALYTICS_DISABLED: "1", + }, + }, + }, + }, ], }); diff --git a/scripts/ci/azure-emulator-up.sh b/scripts/ci/azure-emulator-up.sh new file mode 100644 index 0000000..a12afda --- /dev/null +++ b/scripts/ci/azure-emulator-up.sh @@ -0,0 +1,89 @@ +#!/usr/bin/env bash +# CI only: start this job's own LocalStack Azure emulator and wait until +# both the plain-HTTP gateway and the HTTPS ARM endpoint answer. It never runs on a +# developer machine, where an emulator may already be shared. +# +# AZURE_CI_EMULATOR_CONTAINER container name (default ls-azure-ci) +# LOCALSTACK_AZURE_IMAGE_NAME image (default localstack/localstack-azure:latest) +# ORYX_BUILD_IMAGE pinned Oryx build image +# LOCALSTACK_AUTH_TOKEN an Azure-entitled token (never printed) +set -euo pipefail + +if [ "${CI:-}" != "true" ]; then + echo "azure-emulator-up.sh: refusing to run outside CI (CI=true)" >&2 + exit 2 +fi + +name="${AZURE_CI_EMULATOR_CONTAINER:-ls-azure-ci}" +image="${LOCALSTACK_AZURE_IMAGE_NAME:-localstack/localstack-azure:latest}" +oryx="${ORYX_BUILD_IMAGE:-mcr.microsoft.com/oryx/build:github-actions-debian-bookworm-20260415.1}" +logs="${AZURE_EMULATOR_LOG_DIR:-emulator-logs}" +mkdir -p "$logs" +# The emulator shares files with the containers it starts for Function and Web Apps from its +# state folder, which must be a bind mount: without one, the image's own anonymous volume is +# used, and every app deployment fails with "Mount to /var/lib/localstack needs to be a bind +# mount". +volume="${AZURE_EMULATOR_VOLUME_DIR:-${RUNNER_TEMP:-/tmp}/$name-volume}" +mkdir -p "$volume" + +docker pull --quiet "$image" +# DNS_ADDRESS=0: with the internal DNS on, Cosmos hostnames resolved to the app +# container's own IP. The token goes in by name, never by value. +docker run -d --name "$name" \ + -p 127.0.0.1:4566:4566 \ + -p 127.0.0.1:4510-4559:4510-4559 \ + -e LOCALSTACK_AUTH_TOKEN \ + -e ACTIVATE_PRO=1 \ + -e DNS_ADDRESS=0 \ + -e DISABLE_EVENTS=1 \ + -e LS_LOG="${LS_LOG:-DEBUG}" \ + -e MAIN_CONTAINER_NAME="$name" \ + -e MSSQL_ACCEPT_EULA=Y \ + -e STREAM_DOCKER_LOGS=1 \ + -e FRONT_DOOR_CLASSIC_ALLOW_CREATE=1 \ + -e CDN_CLASSIC_ALLOW_CREATE=1 \ + -e LS_AZURE_ORYX_BUILD_IMAGE_TO_USE="$oryx" \ + -v "$volume:/var/lib/localstack" \ + -v /var/run/docker.sock:/var/run/docker.sock \ + "$image" >/dev/null + +# Poll health, not `localstack wait` (it greps logs and misses the marker under DEBUG). A +# container that exits, such as on a licence failure, fails the step at once with its reason. +healthy=0 +for _ in $(seq 1 150); do + if curl -fs http://127.0.0.1:4566/_localstack/health >/dev/null 2>&1; then healthy=1; break; fi + state=$(docker inspect -f '{{.State.Status}} {{.State.ExitCode}}' "$name" 2>/dev/null || echo "missing -") + case "$state" in + exited* | dead* | missing*) + docker logs "$name" >"$logs/emulator-start.log" 2>&1 || true + echo "the emulator stopped before it became healthy (status and exit code: $state); logs saved to $logs/ (scan them before upload)" >&2 + grep -E '^(License activation failed|Reason: .)' "$logs/emulator-start.log" >&2 || true + if [ "${state#* }" = 55 ]; then + echo "exit code 55 is a licence failure: LOCALSTACK_AUTH_TOKEN must belong to an account with the LocalStack for Azure emulator enabled" >&2 + fi + exit 1 + ;; + esac + sleep 2 +done +if [ "$healthy" != 1 ]; then + docker logs "$name" >"$logs/emulator-start.log" 2>&1 || true + echo "the emulator did not become healthy in 300 s; logs saved to $logs/ (scan them before upload)" >&2 + exit 1 +fi + +# HTTPS comes up seconds after plain HTTP. +https_ok=0 +for _ in $(seq 1 60); do + if curl -ksf --resolve azure.localhost.localstack.cloud:4566:127.0.0.1 \ + https://azure.localhost.localstack.cloud:4566/_localstack/health >/dev/null 2>&1; then + https_ok=1 + break + fi + sleep 2 +done +[ "$https_ok" = 1 ] || { echo "the HTTPS ARM endpoint did not answer in 120 s" >&2; exit 1; } + +# Warm the coverage list: it can be slow right after start. +curl -s -m 120 http://127.0.0.1:4566/_localstack/coverage >/dev/null || true +echo "LocalStack Azure emulator $name is up: $(curl -s http://127.0.0.1:4566/_localstack/health)" diff --git a/scripts/ci/merge-op-catalogue.cjs b/scripts/ci/merge-op-catalogue.cjs new file mode 100644 index 0000000..0c515a7 --- /dev/null +++ b/scripts/ci/merge-op-catalogue.cjs @@ -0,0 +1,111 @@ +#!/usr/bin/env node +/** + * Merges the L2 matrix shards' operation catalogues into one (the portal's inventory gate + * reads the result). The rule of tests/azure/matrix/README.md: per + * operation key, concatenate the cases and keep the best result (pass > gap_fixed > + * known_gap > fail > setup_failed > skipped). + * + * node scripts/ci/merge-op-catalogue.cjs --out ... + * + * A directory is searched recursively for azure-op-catalogue*.json files. + */ +"use strict"; +const fs = require("fs"); +const path = require("path"); + +const RESULT_RANK = { pass: 6, gap_fixed: 5, known_gap: 4, fail: 3, setup_failed: 2, skipped: 1 }; +const rank = (result) => RESULT_RANK[result] ?? 0; + +function mergeCatalogues(catalogues) { + if (catalogues.length === 0) throw new Error("no catalogues to merge"); + for (const c of catalogues) { + if (c.schemaVersion !== 1) + throw new Error(`unsupported catalogue schemaVersion ${c.schemaVersion}`); + } + const cases = new Map(); + const implemented = new Map(); + for (const c of catalogues) { + for (const [key, op] of Object.entries(c.operations ?? {})) { + cases.set(key, [...(cases.get(key) ?? []), ...(op.cases ?? [])]); + if (op.implemented !== null && op.implemented !== undefined && !implemented.has(key)) { + implemented.set(key, op.implemented); + } + } + } + const operations = {}; + const summary = { operations: 0 }; + for (const key of [...cases.keys()].sort()) { + const entries = cases.get(key); + const best = entries.reduce((a, b) => (rank(b.result) > rank(a.result) ? b : a)); + const passing = entries.filter((e) => e.result === "pass" || e.result === "gap_fixed"); + const verified = passing.reduce( + (latest, e) => (!latest || e.checked > latest.checked ? e : latest), + undefined + ); + operations[key] = { + implemented: implemented.has(key) ? implemented.get(key) : null, + result: best.result, + verified: verified ? verified.checked : null, + command: (verified ?? best).command, + cases: entries, + }; + summary.operations++; + summary[best.result] = (summary[best.result] ?? 0) + 1; + } + return { + schemaVersion: 1, + generatedAt: new Date().toISOString(), + run: catalogues + .map((c) => c.run) + .filter(Boolean) + .join(","), + matrix: [...new Set(catalogues.map((c) => c.matrix))].join(","), + shard: `merged ${catalogues.map((c) => c.shard).join(",")}`, + backing: catalogues.every((c) => c.backing === true), + coverageChecked: catalogues.some((c) => c.coverageChecked === true), + extensionsInstalled: [ + ...new Set(catalogues.flatMap((c) => c.extensionsInstalled ?? [])), + ].sort(), + summary, + operations, + }; +} + +function findCatalogues(target) { + const stat = fs.statSync(target); + if (stat.isFile()) return [target]; + return fs + .readdirSync(target, { withFileTypes: true }) + .flatMap((entry) => { + const p = path.join(target, entry.name); + if (entry.isDirectory()) return findCatalogues(p); + return /^azure-op-catalogue.*\.json$/.test(entry.name) ? [p] : []; + }) + .sort(); +} + +function main(argv) { + const outIndex = argv.indexOf("--out"); + if (outIndex < 0 || !argv[outIndex + 1]) { + console.error("usage: merge-op-catalogue.cjs --out ..."); + return 2; + } + const out = argv[outIndex + 1]; + const inputs = argv.filter((_, i) => i !== outIndex && i !== outIndex + 1); + const files = inputs.flatMap(findCatalogues); + if (files.length === 0) { + console.error("merge-op-catalogue: no azure-op-catalogue*.json found"); + return 1; + } + const merged = mergeCatalogues(files.map((f) => JSON.parse(fs.readFileSync(f, "utf8")))); + fs.mkdirSync(path.dirname(path.resolve(out)), { recursive: true }); + fs.writeFileSync(out, JSON.stringify(merged, null, 2) + "\n"); + console.log( + `merged ${files.length} catalogues, ${merged.summary.operations} operations -> ${out}` + ); + return 0; +} + +module.exports = { mergeCatalogues, findCatalogues, RESULT_RANK }; + +if (require.main === module) process.exitCode = main(process.argv.slice(2)); diff --git a/scripts/ci/scan-for-secret.mjs b/scripts/ci/scan-for-secret.mjs new file mode 100644 index 0000000..8637cbc --- /dev/null +++ b/scripts/ci/scan-for-secret.mjs @@ -0,0 +1,45 @@ +#!/usr/bin/env node +// CI: fail when any given file (or any file under a given directory) contains a secret, +// before logs and artifacts are uploaded. GitHub masks +// registered secrets in step logs, but not inside uploaded artifacts. The secret's value +// is read from the environment and never printed. +// +// node scripts/ci/scan-for-secret.mjs ... [--env LOCALSTACK_AUTH_TOKEN] +import { existsSync, readdirSync, readFileSync, statSync } from "node:fs"; +import { join } from "node:path"; + +const args = process.argv.slice(2); +const envIndex = args.indexOf("--env"); +const names = envIndex >= 0 ? args.splice(envIndex, 2)[1].split(",") : ["LOCALSTACK_AUTH_TOKEN"]; +const secrets = names + .map((name) => [name, process.env[name]]) + .filter(([, value]) => typeof value === "string" && value.length >= 8); + +if (secrets.length === 0) { + console.log(`scan-for-secret: none of ${names.join(", ")} is set; nothing to scan for`); + process.exit(0); +} + +const files = []; +const walk = (p) => { + if (!existsSync(p)) return; + const stat = statSync(p); + if (stat.isDirectory()) for (const entry of readdirSync(p)) walk(join(p, entry)); + else if (stat.isFile()) files.push(p); +}; +for (const target of args) walk(target); + +const hits = []; +for (const file of files) { + const content = readFileSync(file); + for (const [name, value] of secrets) { + if (content.includes(Buffer.from(value))) hits.push(`${file} (${name})`); + } +} + +if (hits.length) { + console.error(`scan-for-secret: a secret appears in ${hits.length} file(s); not uploading:`); + for (const hit of hits) console.error(` ${hit}`); + process.exit(1); +} +console.log(`scan-for-secret: ${files.length} file(s) clean of ${names.join(", ")}`); diff --git a/scripts/gen-az-file-args.py b/scripts/gen-az-file-args.py new file mode 100644 index 0000000..8ce855e --- /dev/null +++ b/scripts/gen-az-file-args.py @@ -0,0 +1,166 @@ +#!/usr/bin/env python3 +"""Generate the file-argument table the Azure command policy uses. + +Why this exists +--------------- +The policy's file rule must know which `az` arguments take a local path, so it can keep those +paths inside the workdir. A hand-maintained list drifts and misses short aliases and output +flags. Instead we load az's own command table and record, per command, every argument +that az itself treats as a file: + + * its `type` is `azure.cli.core.commands.parameters.file_type` (it calls os.path.expanduser); or + * its completer is argcomplete's FilesCompleter or DirectoriesCompleter. + +For each such argument we keep every `options_list` string (long and short), whether it is +"greedy" (nargs '+' or '*', so it consumes several path values), and whether it is positional +(the `acr build`/`acr run` source; the policy treats that specially). + +This does NOT capture path arguments that az fails to annotate (e.g. `storage blob +download-batch --destination`, `webapp deploy --src-path`); those live in a small reviewed +supplement inside policy.ts (EXTRA_FILE_ARGS), documented there. + +It also does NOT need to capture the arguments az opens a file for only when the value happens +to name an existing path -- the AAZ structured arguments (`--tags`, `--settings`, ... : their +`_decode_value` does `os.path.exists(...)` then `get_file_json/yaml`) and the +`validate_file_or_dict` arguments. There are ~1,500 of them, and az gives them no file marker. +Rather than list them, policy.ts mirrors az's own gate at runtime: any argument value that +resolves to an existing file must pass the workdir/protected check (the "safety net"). +So this table only needs the arguments that are ALWAYS a path (including write targets, which may +not exist yet and so must be checked unconditionally). + +How to run (offline; never touches the real ~/.azure) +----------------------------------------------------- + AZURE_CONFIG_DIR= \ + python scripts/gen-az-file-args.py > src/lib/azure/az-file-args.generated.json + +The image pins azure-cli 2.90 plus 26 curated extensions (docker/azure-extensions.txt). Whoever +regenerates on a pin move should install those extensions first, so extension +commands (cdn/afd, k8s-*, fleet, ...) are covered too. On the machine that produced the checked-in +copy only the built-in modules were present; the JSON metadata records that. +""" +import datetime +import json +import os +import platform +import sys +import tempfile + + +def _fail(message: str) -> "None": + sys.stderr.write(message + "\n") + raise SystemExit(1) + + +def _ensure_safe_config_dir() -> str: + """Never load az against the user's real profile: force a throwaway AZURE_CONFIG_DIR.""" + cfg = os.environ.get("AZURE_CONFIG_DIR", "") + base = os.path.basename(cfg.rstrip("\\/")) if cfg else "" + if not cfg or base == ".azure" or ".azure" in cfg.replace("\\", "/").split("/"): + # Refuse to run without an explicit, non-.azure config dir. + cfg = tempfile.mkdtemp(prefix="az-file-args-") + os.environ["AZURE_CONFIG_DIR"] = cfg + sys.stderr.write(f"AZURE_CONFIG_DIR was unsafe or unset; using throwaway {cfg}\n") + os.environ.setdefault("AZURE_CORE_COLLECT_TELEMETRY", "no") + # A closed loopback port, so any stray outbound call fails fast instead of reaching the network. + os.environ.setdefault("HTTPS_PROXY", "http://127.0.0.1:9") + os.environ.setdefault("HTTP_PROXY", "http://127.0.0.1:9") + return os.environ["AZURE_CONFIG_DIR"] + + +def main() -> "None": + _ensure_safe_config_dir() + + try: + from azure.cli.core import get_default_cli + from azure.cli.core.file_util import create_invoker_and_load_cmds_and_args + from azure.cli.core.commands.parameters import file_type + from argcomplete.completers import FilesCompleter, DirectoriesCompleter + except Exception as exc: # pragma: no cover - environment problem + _fail(f"could not import azure-cli: {exc}") + + import azure.cli.core as core + + cli = get_default_cli() + create_invoker_and_load_cmds_and_args(cli) + parser = cli.invocation.parser + + commands: "dict[str, dict]" = {} + file_arg_count = 0 + for name, subparser in parser.subparser_map.items(): + flags: "list[str]" = [] + greedy: "list[str]" = [] + positional_file = False + for action in subparser._actions: + is_file = action.type is file_type or isinstance( + getattr(action, "completer", None), (FilesCompleter, DirectoriesCompleter) + ) + if not is_file: + continue + file_arg_count += 1 + if not action.option_strings: + # A positional path (acr build/run source). The policy handles those by name. + positional_file = True + continue + multi = action.nargs in ("+", "*") + for option in action.option_strings: + if option not in flags: + flags.append(option) + if multi and option not in greedy: + greedy.append(option) + if flags or positional_file: + entry: "dict[str, object]" = {"flags": sorted(flags)} + if greedy: + entry["greedy"] = sorted(greedy) + if positional_file: + entry["positional"] = True + commands[name] = entry + + extensions = [] + try: + from azure.cli.core.extension import get_extensions + + extensions = sorted(e.name for e in get_extensions()) + except Exception: # pragma: no cover - best effort + extensions = [] + + document = { + "_metadata": { + "generator": "scripts/gen-az-file-args.py", + "description": ( + "Per-command file-taking arguments, from az's command table: every argument whose " + "type is file_type or whose completer completes files/directories, with all its " + "option strings. Consumed by src/lib/azure/policy.ts. Regenerated on an az pin move." + ), + "az_version": core.__version__, + "python_version": platform.python_version(), + "platform": sys.platform, + "generated_utc": datetime.datetime.now(datetime.timezone.utc) + .isoformat(timespec="seconds") + .replace("+00:00", "Z"), + "command_count": len(parser.subparser_map), + "file_command_count": len(commands), + "file_argument_count": file_arg_count, + "extensions_installed": extensions, + "note": ( + "The production image pins azure-cli 2.90 and 26 curated extensions " + "(docker/azure-extensions.txt). This table was generated with azure-cli " + f"{core.__version__} and the extensions listed in extensions_installed " + "(empty here means only built-in command modules were present). DR4 regenerates " + "it with the pinned az and the installed extensions. Path arguments az does not " + "annotate (e.g. storage blob download-batch --destination, webapp deploy " + "--src-path) are covered by EXTRA_FILE_ARGS in policy.ts, not by this file." + ), + }, + "commands": dict(sorted(commands.items())), + } + + sys.stdout.write(json.dumps(document, indent=2, ensure_ascii=False)) + sys.stdout.write("\n") + sys.stderr.write( + f"az {core.__version__}: {len(parser.subparser_map)} commands, " + f"{len(commands)} with file args, {file_arg_count} file arguments\n" + ) + + +if __name__ == "__main__": + main() diff --git a/scripts/install-azure-extensions.mjs b/scripts/install-azure-extensions.mjs new file mode 100644 index 0000000..0c1df34 --- /dev/null +++ b/scripts/install-azure-extensions.mjs @@ -0,0 +1,258 @@ +#!/usr/bin/env node +// Installs the curated Azure CLI extensions of docker/azure-extensions.txt into +// the extension dir the localstack-azure-client tool uses. One installer for CI, the init wizard +// and developers. +// +// node scripts/install-azure-extensions.mjs [--dir ] [--az | --az-python ] +// [--list ] [--dry-run] +// +// --dir target dir; default LOCALSTACK_AZ_EXTENSION_DIR, else ~/.localstack/azure/mcp-extensions +// --az the az executable to run (default: az on PATH). It is spawned without a shell, so on +// Windows a .cmd/.bat launcher is mapped to its Python (\..\python.exe) +// --az-python run ` -X utf8 -IBm azure.cli` instead (the MSI layout on Windows: +// "C:\Program Files\Microsoft SDKs\Azure\CLI2\python.exe") +// --list another pin list (default: docker/azure-extensions.txt) +// --dry-run print the commands and change nothing +// +// Every entry runs `az extension add --name --version [--allow-preview true] --upgrade +// --yes --only-show-errors` with AZURE_EXTENSION_DIR set to the target dir and AZURE_CONFIG_DIR set +// to a fresh temporary dir, so the user's own ~/.azure is never read or written. `--upgrade` makes +// a re-run converge on the pinned version: without it az keeps whatever version is installed and, +// under --only-show-errors, says nothing. The first failure stops the run. +// +// The pin-list parser and the installer are shared with the server, whose init wizard uses the +// same code: they are imported from src/lib/azure/extension-map.ts and extension-install.ts, +// which needs Node.js 22.18 or newer (built-in type stripping). +import * as fs from "node:fs"; +import * as os from "node:os"; +import * as path from "node:path"; +import { fileURLToPath } from "node:url"; + +const REPO_ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), ".."); +const CLI_PREFIX = ["-X", "utf8", "-IBm", "azure.cli"]; + +class ScriptError extends Error { + constructor(message, exitCode = 2) { + super(message); + this.exitCode = exitCode; + } +} + +function parseArgs(argv) { + const opts = { + dryRun: false, + dir: undefined, + az: undefined, + azPython: undefined, + list: undefined, + }; + for (let i = 0; i < argv.length; i++) { + const arg = argv[i]; + const value = () => { + const next = argv[++i]; + if (next === undefined || next.startsWith("--")) + throw new ScriptError(`${arg} needs a value`); + return next; + }; + switch (arg) { + case "--dry-run": + opts.dryRun = true; + break; + case "--dir": + opts.dir = value(); + break; + case "--az": + opts.az = value(); + break; + case "--az-python": + opts.azPython = value(); + break; + case "--list": + opts.list = value(); + break; + case "-h": + case "--help": + opts.help = true; + break; + default: + throw new ScriptError(`unknown argument ${arg} (see --help)`); + } + } + if (opts.az && opts.azPython) throw new ScriptError("pass --az or --az-python, not both"); + return opts; +} + +async function loadModules() { + try { + const map = await import("../src/lib/azure/extension-map.ts"); + const install = await import("../src/lib/azure/extension-install.ts"); + return { parsePinList: map.parsePinList, install }; + } catch (error) { + throw new ScriptError( + `cannot load src/lib/azure/extension-map.ts and extension-install.ts (${error.message}). ` + + "This script needs Node.js 22.18 or newer, which runs TypeScript files directly." + ); + } +} + +function targetDir(opts, install) { + const dir = opts.dir + ? path.resolve(opts.dir) + : install.defaultExtensionDir(process.env, os.homedir()); + // Never install into the user's own CLI profile: that would change their real `az`. + try { + install.assertNotInProfile(dir, { homedir: os.homedir(), env: process.env }); + } catch (error) { + throw new ScriptError(error.message); + } + return dir; +} + +function isWsl() { + if (process.platform !== "linux") return false; + if (process.env.WSL_DISTRO_NAME) return true; + try { + return /microsoft/i.test(fs.readFileSync("/proc/version", "utf8")); + } catch { + return false; + } +} + +function findOnPath(names) { + for (const dir of (process.env.PATH || "").split(path.delimiter)) { + if (!dir) continue; + for (const name of names) { + const candidate = path.join(dir, name); + try { + if (!fs.statSync(candidate).isFile()) continue; + if (process.platform !== "win32") fs.accessSync(candidate, fs.constants.X_OK); + return candidate; + } catch { + // not here + } + } + } + return undefined; +} + +// A .cmd/.bat launcher cannot be spawned without a shell (EINVAL), so run the Python it wraps. +// Both the MSI (CLI2\wbin\az.cmd) and pip (Scripts\az.cmd) keep python.exe one level up. +function pythonOfLauncher(launcher, dryRun) { + const python = path.join(path.dirname(launcher), "..", "python.exe"); + if (!dryRun && !fs.existsSync(python)) { + throw new ScriptError(`no python.exe next to ${launcher}; pass --az-python `); + } + return { file: path.normalize(python), prefix: CLI_PREFIX }; +} + +function resolveAz(opts) { + let az; + if (opts.azPython) { + az = { file: opts.azPython, prefix: CLI_PREFIX }; + } else if (opts.az) { + az = + process.platform === "win32" && /\.(cmd|bat)$/i.test(opts.az) + ? pythonOfLauncher(opts.az, opts.dryRun) + : { file: opts.az, prefix: [] }; + } else if (process.platform === "win32") { + const launcher = findOnPath(["az.cmd", "az.bat"]); + if (launcher) { + az = pythonOfLauncher(launcher, opts.dryRun); + } else { + for (const base of [process.env.ProgramFiles, process.env["ProgramFiles(x86)"]]) { + const python = base && path.join(base, "Microsoft SDKs", "Azure", "CLI2", "python.exe"); + if (python && fs.existsSync(python)) { + az = { file: python, prefix: CLI_PREFIX }; + break; + } + } + } + } else { + const found = findOnPath(["az"]); + if (found) az = { file: found, prefix: [] }; + } + if (!az) { + if (!opts.dryRun) { + throw new ScriptError("no Azure CLI found on PATH; pass --az or --az-python "); + } + console.log('# warning: no Azure CLI found on PATH; the commands below show "az"'); + return { file: "az", prefix: [] }; + } + // Under WSL a Windows az ignores AZURE_CONFIG_DIR and AZURE_EXTENSION_DIR (WSL does not + // forward them) and would install into the Windows user's real ~/.azure. + if (isWsl() && (az.file.startsWith("/mnt/") || /\.(exe|cmd|bat)$/i.test(az.file))) { + throw new ScriptError(`refusing ${az.file}: under WSL this is a Windows az; install az in WSL`); + } + return az; +} + +const quote = (arg) => (/[\s"']/.test(arg) || arg === "" ? `"${arg.replace(/"/g, '\\"')}"` : arg); + +async function main() { + const opts = parseArgs(process.argv.slice(2)); + if (opts.help) { + // The usage is the comment block at the top of this file. + const lines = fs + .readFileSync(fileURLToPath(import.meta.url), "utf8") + .split(/\r?\n/) + .slice(1); + const end = lines.findIndex((line) => !line.startsWith("//")); + console.log( + lines + .slice(0, end) + .map((line) => line.replace(/^\/\/ ?/, "")) + .join("\n") + ); + return 0; + } + const listFile = path.resolve( + opts.list || path.join(REPO_ROOT, "docker", "azure-extensions.txt") + ); + const { parsePinList, install } = await loadModules(); + let pins; + try { + pins = parsePinList(fs.readFileSync(listFile, "utf8")); + } catch (error) { + throw new ScriptError(`${listFile}: ${error.message}`); + } + if (pins.length === 0) throw new ScriptError(`${listFile}: no extensions listed`); + const dir = targetDir(opts, install); + const az = resolveAz(opts); + + if (opts.dryRun) { + console.log(`# dry run: ${pins.length} extensions from ${listFile}`); + console.log(`# AZURE_EXTENSION_DIR=${dir}`); + console.log("# AZURE_CONFIG_DIR="); + for (const pin of pins) { + console.log([az.file, ...az.prefix, ...install.extensionAddArgs(pin)].map(quote).join(" ")); + } + return 0; + } + + try { + await install.installExtensions({ + pins, + dir, + az, + run: install.spawnAz("inherit"), + onProgress: (index, total, pin) => + console.log( + `[${index + 1}/${total}] ${pin.name} ${pin.version}${pin.preview ? " (preview)" : ""}` + ), + }); + } catch (error) { + throw new ScriptError(error.message, error.exitCode ?? 1); + } + console.log(`installed ${pins.length} extensions into ${dir}`); + return 0; +} + +main().then( + (code) => { + process.exitCode = code; + }, + (error) => { + console.error(`install-azure-extensions: ${error.message}`); + process.exitCode = error instanceof ScriptError ? error.exitCode : 1; + } +); diff --git a/server.json b/server.json index 88bbe54..7438fe9 100644 --- a/server.json +++ b/server.json @@ -1,7 +1,7 @@ { "$schema": "https://static.modelcontextprotocol.io/schemas/2025-12-11/server.schema.json", "name": "io.github.localstack/localstack-mcp-server", - "description": "A LocalStack MCP Server providing essential tools for local cloud development & testing", + "description": "A LocalStack MCP Server providing essential tools for local cloud development & testing with AWS, Snowflake and Azure", "status": "active", "repository": { "url": "https://github.com/localstack/localstack-mcp-server", @@ -157,6 +157,125 @@ "format": "string", "isSecret": false, "name": "EXTERNAL_SERVICE_PORTS_END" + }, + { + "description": "Gateway port of the LocalStack Azure emulator used by localstack-azure-client (defaults to LOCALSTACK_PORT, then 4566).", + "isRequired": false, + "format": "string", + "isSecret": false, + "name": "LOCALSTACK_AZURE_PORT" + }, + { + "description": "ARM endpoint override for localstack-azure-client. Must use a local name (localhost.localstack.cloud or a subdomain, localhost, 127.0.0.1, ::1). Defaults to https://azure.localhost.localstack.cloud:.", + "isRequired": false, + "format": "string", + "isSecret": false, + "name": "LOCALSTACK_AZURE_ENDPOINT" + }, + { + "description": "Docker image the management start action launches for service: azure. Defaults to localstack/localstack-azure:latest.", + "isRequired": false, + "format": "string", + "isSecret": false, + "name": "LOCALSTACK_AZURE_IMAGE_NAME" + }, + { + "description": "Explicit path to the Azure CLI launcher (az, az.cmd) or its Python. Defaults to a PATH lookup.", + "isRequired": false, + "format": "string", + "isSecret": false, + "name": "LOCALSTACK_AZ_PATH" + }, + { + "description": "The Azure tool's isolated Azure CLI config dir, never your ~/.azure. Defaults to ~/.localstack/azure/mcp-config-.", + "isRequired": false, + "format": "string", + "isSecret": false, + "name": "LOCALSTACK_AZ_CONFIG_DIR" + }, + { + "description": "Azure CLI extension dir for the Azure tool. Defaults to ~/.localstack/azure/mcp-extensions (/opt/az-extensions in the Docker image).", + "isRequired": false, + "format": "string", + "isSecret": false, + "name": "LOCALSTACK_AZ_EXTENSION_DIR" + }, + { + "description": "Explicit Bicep binary for .bicep templates: an absolute path to a file named bicep or bicep.exe. Otherwise ~/.localstack/azure/bin, then PATH (never ~/.azure/bin).", + "isRequired": false, + "format": "string", + "isSecret": false, + "name": "LOCALSTACK_AZ_BICEP_PATH" + }, + { + "description": "Comma-separated server environment variables a .bicepparam may read with readEnvironmentVariable(). Nothing else reaches Bicep; the auth token and AZURE_*, ARM_*, BICEP_*, proxy and path variables are never passed.", + "isRequired": false, + "format": "string", + "isSecret": false, + "name": "LOCALSTACK_AZ_BICEP_ENV" + }, + { + "description": "Per-command timeout of the Azure tool, 5-3600 (default 300).", + "isRequired": false, + "format": "string", + "isSecret": false, + "name": "LOCALSTACK_AZ_TIMEOUT_SECONDS" + }, + { + "description": "Azure tool stdout truncation limit (default 30000).", + "isRequired": false, + "format": "string", + "isSecret": false, + "name": "LOCALSTACK_AZ_MAX_OUTPUT_CHARS" + }, + { + "description": "Azure tool help-page limit after re-flow (default 30000).", + "isRequired": false, + "format": "string", + "isSecret": false, + "name": "LOCALSTACK_AZ_MAX_HELP_CHARS" + }, + { + "description": "Directory the Azure tool's file arguments must stay inside, and az's working directory. Defaults to the server's working directory.", + "isRequired": false, + "format": "string", + "isSecret": false, + "name": "LOCALSTACK_AZ_WORKDIR" + }, + { + "description": "Set to 0 to turn off the Azure tool's egress guard (debugging only; every response then says so). Default 1.", + "isRequired": false, + "format": "string", + "isSecret": false, + "name": "LOCALSTACK_AZ_EGRESS_GUARD" + }, + { + "description": "File of extra az command prefixes the Azure tool refuses, one per line (# comments allowed).", + "isRequired": false, + "format": "string", + "isSecret": false, + "name": "LOCALSTACK_AZ_DENYLIST_FILE" + }, + { + "description": "Bytecode cache dir for Azure CLI installs without .pyc files (the Docker image sets it).", + "isRequired": false, + "format": "string", + "isSecret": false, + "name": "LOCALSTACK_AZ_PYCACHE_DIR" + }, + { + "description": "host (default) runs a fresh Azure CLI process per command; worker (experimental) runs commands in warm Azure CLI processes, with the same isolation.", + "isRequired": false, + "format": "string", + "isSecret": false, + "name": "LOCALSTACK_AZ_RUNNER" + }, + { + "description": "Docker image only: the host the loopback forwarder relays the Azure emulator's ports to. Defaults to host.docker.internal, then the emulator container's IP.", + "isRequired": false, + "format": "string", + "isSecret": false, + "name": "LOCALSTACK_AZURE_FORWARD_TARGET" } ] } diff --git a/src/cli/azure-addons.test.ts b/src/cli/azure-addons.test.ts new file mode 100644 index 0000000..98434c7 --- /dev/null +++ b/src/cli/azure-addons.test.ts @@ -0,0 +1,128 @@ +import * as fs from "fs"; +import * as os from "os"; +import * as path from "path"; +import type { RunAz } from "../lib/azure/extension-install"; +import { AZURE_EXTENSION_PINS } from "../lib/azure/extension-pins"; +import type { AzureStepDeps } from "../lib/wizard/azure-steps"; +import { runInstallAzureAddons } from "./azure-addons"; + +// `install-azure-addons`: what the Azure tool uses beside the Azure CLI itself. The user runs it, +// as they install the Snowflake CLI for the Snowflake tool; the setup wizard never does. az and +// the Bicep download are injected, and the home is a temporary directory, so the real +// ~/.localstack and ~/.azure are never touched. +describe("install-azure-addons", () => { + let home: string; + beforeEach(() => { + home = fs.mkdtempSync(path.join(os.tmpdir(), "lsmcp-azure-addons-")); + }); + afterEach(() => fs.rmSync(home, { recursive: true, force: true })); + + function setup(over: Partial = {}) { + const azCalls: string[][] = []; + const run: RunAz = async (_file, args) => { + azCalls.push(args); + return { status: 0 }; + }; + const deps: AzureStepDeps = { + platform: process.platform, + arch: process.arch, + homedir: home, + env: {}, + locate: () => ({ + file: "/usr/bin/python3", + prefixArgs: ["-X", "utf8", "-IBm", "azure.cli"], + installer: "pip", + }), + run, + pins: AZURE_EXTENSION_PINS, + musl: false, + installBicep: jest.fn(async () => ({ path: "/x/bicep", asset: "bicep-linux-x64" })), + ...over, + }; + const out: string[] = []; + const err: string[] = []; + const io = { deps, out: (l: string) => out.push(l), err: (l: string) => err.push(l) }; + return { deps, azCalls, out, err, io, text: () => [...out, ...err].join("\n") }; + } + + test("installs the 26 pinned extensions, then Bicep, and exits 0", async () => { + const s = setup(); + expect(await runInstallAzureAddons([], s.io)).toBe(0); + expect(s.azCalls).toHaveLength(26); + expect(s.deps.installBicep).toHaveBeenCalledTimes(1); + const dir = path.join(home, ".localstack", "azure", "mcp-extensions"); + expect(s.text()).toContain(`✓ 26 Azure CLI extensions in ${dir}`); + expect(s.text()).toMatch(/✓ Bicep \d+\.\d+\.\d+ .*sha256 checked/); + expect(s.err).toEqual([]); + }); + + test("without an Azure CLI it installs nothing and says how to install az, as for snow", async () => { + const s = setup({ + locate: () => { + throw new Error("no az"); + }, + }); + expect(await runInstallAzureAddons([], s.io)).toBe(1); + const said = s.err.join("\n"); + expect(said).toMatch(/The Azure CLI \(az\) was not found/); + expect(said).toContain("2.85 or newer"); + expect(said).toContain("winget install --exact --id Microsoft.AzureCLI"); + expect(said).toContain("brew install azure-cli"); + expect(said).toContain("curl -sL https://aka.ms/InstallAzureCLIDeb | sudo bash"); + expect(said).toContain("https://learn.microsoft.com/cli/azure/install-azure-cli"); + expect(said).toMatch(/run this command again/); + expect(s.azCalls).toEqual([]); + expect(s.deps.installBicep).not.toHaveBeenCalled(); + expect(fs.readdirSync(home)).toEqual([]); + }); + + test("--no-bicep installs only the extensions; --no-extensions only Bicep", async () => { + const onlyExtensions = setup(); + expect(await runInstallAzureAddons(["--no-bicep"], onlyExtensions.io)).toBe(0); + expect(onlyExtensions.azCalls).toHaveLength(26); + expect(onlyExtensions.deps.installBicep).not.toHaveBeenCalled(); + + const onlyBicep = setup(); + expect(await runInstallAzureAddons(["--no-extensions"], onlyBicep.io)).toBe(0); + expect(onlyBicep.azCalls).toEqual([]); + expect(onlyBicep.deps.installBicep).toHaveBeenCalledTimes(1); + }); + + test("--no-extensions installs Bicep without an Azure CLI: only the extensions need az", async () => { + const s = setup({ + locate: () => { + throw new Error("no az"); + }, + }); + expect(await runInstallAzureAddons(["--no-extensions"], s.io)).toBe(0); + expect(s.deps.installBicep).toHaveBeenCalledTimes(1); + expect(s.azCalls).toEqual([]); + expect(s.err).toEqual([]); + }); + + test("a failed step exits 1 and says which one, with the installer's message", async () => { + const s = setup({ + installBicep: jest.fn(async () => { + throw new Error("the downloaded bicep has sha256 abc, not the pinned def"); + }), + }); + expect(await runInstallAzureAddons([], s.io)).toBe(1); + expect(s.text()).toContain("✗ Bicep: the downloaded bicep has sha256 abc, not the pinned def"); + expect(s.azCalls).toHaveLength(26); + }); + + test("--help prints the usage and installs nothing", async () => { + const s = setup(); + expect(await runInstallAzureAddons(["--help"], s.io)).toBe(0); + expect(s.text()).toContain("install-azure-addons"); + expect(s.azCalls).toEqual([]); + expect(s.deps.installBicep).not.toHaveBeenCalled(); + }); + + test("an unknown option is an error and installs nothing", async () => { + const s = setup(); + expect(await runInstallAzureAddons(["--azure-extensions"], s.io)).toBe(1); + expect(s.err.join("\n")).toMatch(/Unknown option/); + expect(s.azCalls).toEqual([]); + }); +}); diff --git a/src/cli/azure-addons.ts b/src/cli/azure-addons.ts new file mode 100644 index 0000000..bfa9f32 --- /dev/null +++ b/src/cli/azure-addons.ts @@ -0,0 +1,105 @@ +import { parseArgs } from "util"; +import { AZURE_CLI_INSTALL_OPTIONS } from "../lib/azure/resolve-az"; +import { + defaultAzureStepDeps, + installAzureExtensionsStep, + installBicepStep, + type AzureStepDeps, + type AzureStepResult, +} from "../lib/wizard/azure-steps"; +import { HELP_TEXT } from "./help"; + +/** + * `install-azure-addons`: what the Azure tool uses beside the Azure CLI itself, installed when the + * user runs it. The setup wizard never installs a tool's CLI (it does not install the Snowflake + * CLI either): the README and the Azure tool's own answers name this command. It installs the + * pinned Azure CLI extensions into the tool's own extension dir and the pinned Bicep CLI into + * ~/.localstack/azure/bin; the user's own Azure CLI profile is never changed. + */ + +export interface AzureAddonsIo { + deps?: AzureStepDeps; + out?: (line: string) => void; + err?: (line: string) => void; +} + +const LABEL = { extensions: "Azure CLI extensions", bicep: "Bicep" } as const; + +function resultLine(result: AzureStepResult): string { + if (result.status === "installed") return `✓ ${result.detail}`; + return `${result.status === "skipped" ? "−" : "✗"} ${LABEL[result.step]}: ${result.detail}`; +} + +export async function runInstallAzureAddons( + argv: string[], + io: AzureAddonsIo = {} +): Promise { + const out = io.out ?? ((line: string) => console.log(line)); + const err = io.err ?? ((line: string) => console.error(line)); + + let values: { "no-extensions"?: boolean; "no-bicep"?: boolean; help?: boolean }; + try { + ({ values } = parseArgs({ + args: argv, + allowPositionals: false, + strict: true, + options: { + "no-extensions": { type: "boolean" }, + "no-bicep": { type: "boolean" }, + help: { type: "boolean", short: "h" }, + }, + })); + } catch (error) { + err(`Error: ${error instanceof Error ? error.message : String(error)}`); + err('Run "install-azure-addons --help" for usage.'); + return 1; + } + if (values.help) { + out(HELP_TEXT); + return 0; + } + const wantExtensions = !values["no-extensions"]; + const wantBicep = !values["no-bicep"]; + if (!wantExtensions && !wantBicep) { + out("Nothing to install (--no-extensions and --no-bicep)."); + return 0; + } + + const deps = io.deps ?? defaultAzureStepDeps(); + // Only the extensions need az: Bicep is a plain download, so `--no-extensions` works without it. + if (wantExtensions) { + try { + deps.locate(); + } catch { + err( + "The Azure CLI (az) was not found. The Azure tool runs it against the LocalStack Azure " + + `emulator, so install it first.\n\n${AZURE_CLI_INSTALL_OPTIONS}\n\nThen run this command again.` + ); + return 1; + } + } + + const results: AzureStepResult[] = []; + if (wantExtensions) { + out(`Installing the ${deps.pins.length} pinned Azure CLI extensions the Azure tool uses...`); + const result = await installAzureExtensionsStep(deps, (index, total, pin) => + out(` (${index + 1}/${total}) ${pin.name} ${pin.version}${pin.preview ? " (preview)" : ""}`) + ); + results.push(result); + out(resultLine(result)); + } + if (wantBicep) { + out("Downloading the pinned Bicep CLI..."); + const result = await installBicepStep(deps); + results.push(result); + out(resultLine(result)); + } + + const notDone = results.filter((result) => result.status !== "installed"); + if (notDone.length > 0) { + err(`Done with errors: ${notDone.map((r) => LABEL[r.step]).join(" and ")} (see above).`); + return 1; + } + out("Done. The Azure tool uses them from its next call."); + return 0; +} diff --git a/src/cli/help.test.ts b/src/cli/help.test.ts new file mode 100644 index 0000000..cc853e2 --- /dev/null +++ b/src/cli/help.test.ts @@ -0,0 +1,23 @@ +import { HELP_TEXT } from "./help"; + +// Setup is the same for every tool: `init` configures MCP clients and never installs a tool's +// CLI. The Azure tool's add-ons (its Azure CLI extensions and Bicep) have their own command, +// which the user runs, as they would `pip install snowflake-cli-labs` for the Snowflake tool. +describe("the CLI's help", () => { + const section = (from: string, to: string) => + HELP_TEXT.slice(HELP_TEXT.indexOf(from), HELP_TEXT.indexOf(to)); + + test("init's options include nothing for Azure", () => { + expect(HELP_TEXT).toContain("init options:"); + expect(section("init options:", "remove options:")).not.toMatch(/azure|bicep/i); + }); + + test("lists the install-azure-addons command and its options", () => { + expect(HELP_TEXT).toMatch( + /npx -y @localstack\/localstack-mcp-server install-azure-addons\s+Install the Azure CLI extensions and Bicep/ + ); + const addons = section("install-azure-addons options:", "Examples:"); + expect(addons).toContain("--no-extensions"); + expect(addons).toContain("--no-bicep"); + }); +}); diff --git a/src/cli/help.ts b/src/cli/help.ts index 816de0a..6909b2c 100644 --- a/src/cli/help.ts +++ b/src/cli/help.ts @@ -6,6 +6,9 @@ Usage: npx -y @localstack/localstack-mcp-server Start the MCP server (stdio) npx -y @localstack/localstack-mcp-server init Set up the server in your MCP clients npx -y @localstack/localstack-mcp-server remove Remove the server from your MCP clients + npx -y @localstack/localstack-mcp-server install-azure-addons + Install the Azure CLI extensions and Bicep + the Azure tool uses (needs the Azure CLI) init options: --method How the MCP server should run (default: npx) @@ -28,11 +31,21 @@ remove options: --client Clients to remove "localstack" from (default: all with an entry) --force, -y, --yes Don't ask for confirmation +install-azure-addons options: + --no-extensions Skip the 26 pinned Azure CLI extensions + (they go into ~/.localstack/azure/mcp-extensions) + --no-bicep Skip the pinned Bicep CLI (into ~/.localstack/azure/bin, + sha256-checked); a bicep on your PATH works as well + -h, --help Show this help + The Azure CLI itself (az 2.85 or newer) is yours to install, as the README says; + your own Azure CLI profile is never changed. The Docker image bundles all three. + Examples: npx -y @localstack/localstack-mcp-server init npx -y @localstack/localstack-mcp-server init --method npx --client cursor,claude-code npx -y @localstack/localstack-mcp-server init --method docker --client cursor --yes npx -y @localstack/localstack-mcp-server remove --client cursor + npx -y @localstack/localstack-mcp-server install-azure-addons The auth token is read from $LOCALSTACK_AUTH_TOKEN when --token is not given. Get yours at https://app.localstack.cloud/workspace/auth-tokens diff --git a/src/cli/index.ts b/src/cli/index.ts index f789b8d..d5c8340 100644 --- a/src/cli/index.ts +++ b/src/cli/index.ts @@ -1,7 +1,8 @@ /** * Launcher for the published bin. With no recognized subcommand it starts the * MCP server exactly as before (dist/stdio.js), so every existing client - * config keeps working. `init`/`remove` run the setup wizard. + * config keeps working. `init`/`remove` run the setup wizard, and `install-azure-addons` + * installs the Azure tool's Azure CLI extensions and Bicep. */ import * as fs from "fs"; import * as path from "path"; @@ -33,6 +34,11 @@ async function main(): Promise { process.exit(await runRemove(process.argv.slice(3))); break; } + case "install-azure-addons": { + const { runInstallAzureAddons } = await import("./azure-addons"); + process.exit(await runInstallAzureAddons(process.argv.slice(3))); + break; + } case "help": case "--help": case "-h": { diff --git a/src/core/analytics.ts b/src/core/analytics.ts index 522b50e..57ba0ea 100644 --- a/src/core/analytics.ts +++ b/src/core/analytics.ts @@ -19,6 +19,9 @@ const SHUTDOWN_TIMEOUT_MS = 1000; export const TOOL_ARG_ALLOWLIST: Record = { "localstack-aws-client": ["command"], + // Value-free by construction: the tool passes derived fields, never + // the command. Exit code and error class travel in the failure text's first line. + "localstack-azure-client": ["command_path", "flag_names", "policy_outcome"], "localstack-aws-replicator": [ "action", "replication_type", diff --git a/src/core/config.test.ts b/src/core/config.test.ts new file mode 100644 index 0000000..60fb50d --- /dev/null +++ b/src/core/config.test.ts @@ -0,0 +1,371 @@ +import { mkdirSync, mkdtempSync, rmSync, symlinkSync, writeFileSync } from "fs"; +import os from "os"; +import path from "path"; +import { + azureProtectedDirs, + canonicalPath, + getAzureConfig, + isInsideOrEqual, + type AzureConfigDeps, +} from "./config"; + +// Most cases use a fake POSIX home, so they run the same on +// every OS; the real-filesystem cases use a temp dir. +const HOME = "/home/user"; +const CWD = "/work/project"; +const posix: AzureConfigDeps = { + homedir: () => HOME, + cwd: () => CWD, + platform: "linux", + inDocker: false, + realpath: () => undefined, + isDirectory: (p) => p === CWD || p.startsWith("/work/") || p === HOME || p === "/", + isFile: (p) => p === "/etc/az-deny.txt", +}; + +const config = (env: NodeJS.ProcessEnv = {}, deps: AzureConfigDeps = posix) => + getAzureConfig(env, deps); + +describe("getAzureConfig: defaults", () => { + test("every value has its documented default", () => { + const c = config(); + expect(c).toMatchObject({ + port: 4566, + healthBaseUrl: "http://127.0.0.1:4566", + endpoint: "https://azure.localhost.localstack.cloud:4566", + endpointHost: "azure.localhost.localstack.cloud", + configDir: "/home/user/.localstack/azure/mcp-config-4566", + homeDir: "/home/user/.localstack/azure/mcp-config-4566/home", + tmpDir: "/home/user/.localstack/azure/mcp-config-4566/tmp", + extensionDir: "/home/user/.localstack/azure/mcp-extensions", + timeoutMs: 300_000, + maxOutputChars: 30000, + maxHelpChars: 30000, + workdir: CWD, + egressGuard: true, + runner: "host", + testEnvelope: false, + inDocker: false, + warnings: [], + errors: [], + }); + expect(c.azPath).toBeUndefined(); + expect(c.bicepPath).toBeUndefined(); + expect(c.denylistFile).toBeUndefined(); + expect(c.pycacheDir).toBeUndefined(); + expect(c.forwardTarget).toBeUndefined(); + }); + + test("LOCALSTACK_AZURE_PORT defaults to LOCALSTACK_PORT", () => { + const c = config({ LOCALSTACK_PORT: "4567" }); + expect(c.port).toBe(4567); + expect(c.endpoint).toBe("https://azure.localhost.localstack.cloud:4567"); + }); +}); + +describe("getAzureConfig: overrides", () => { + test("the port shift (4666) flows into the health URL, the ARM endpoint and the config dir", () => { + const c = config({ LOCALSTACK_PORT: "4566", LOCALSTACK_AZURE_PORT: "4666" }); + expect(c.port).toBe(4666); + expect(c.healthBaseUrl).toBe("http://127.0.0.1:4666"); + expect(c.endpoint).toBe("https://azure.localhost.localstack.cloud:4666"); + expect(c.configDir).toBe("/home/user/.localstack/azure/mcp-config-4666"); + }); + + test("two ports get two default config dirs", () => { + expect(config({ LOCALSTACK_AZURE_PORT: "4566" }).configDir).not.toBe( + config({ LOCALSTACK_AZURE_PORT: "4666" }).configDir + ); + }); + + test.each([ + ["https://localhost.localstack.cloud:4566", "https://localhost.localstack.cloud:4566"], + [ + "https://azure.localhost.localstack.cloud:4666/", + "https://azure.localhost.localstack.cloud:4666", + ], + ["https://LOCALHOST:4566", "https://localhost:4566"], + ["https://127.0.0.1:4566", "https://127.0.0.1:4566"], + ["https://[::1]:4566", "https://[::1]:4566"], + ])("a local endpoint override is accepted: %s", (value, expected) => { + const c = config({ LOCALSTACK_AZURE_ENDPOINT: value }); + expect(c.errors).toEqual([]); + expect(c.endpoint).toBe(expected); + }); + + test.each([ + "https://management.azure.com", + "https://my-emulator.example.com:4566", + "https://localhost.localstack.cloud.evil.com", + "https://evillocalhost.localstack.cloud", + "https://10.0.0.5:4566", + ])("a remote endpoint is refused: %s", (value) => { + const c = config({ LOCALSTACK_AZURE_ENDPOINT: value }); + expect(c.errors.join("\n")).toMatch(/must point at a local emulator/); + }); + + test.each([ + "http://localhost.localstack.cloud:4566", + "not a url", + "https://user:pw@localhost:4566", + ])("an endpoint that is not a plain https URL is refused: %s", (value) => { + expect(config({ LOCALSTACK_AZURE_ENDPOINT: value }).errors.join("\n")).toMatch( + /must be an https URL/ + ); + }); + + test("an endpoint with a path is refused", () => { + expect( + config({ LOCALSTACK_AZURE_ENDPOINT: "https://localhost:4566/arm" }).errors.join("\n") + ).toMatch(/bare origin/); + }); + + test("each path and value override", () => { + const c = config({ + LOCALSTACK_AZ_CONFIG_DIR: "~/lsaz/profile", + LOCALSTACK_AZ_EXTENSION_DIR: "/opt/az-extensions", + LOCALSTACK_AZ_PATH: "/usr/bin/az", + LOCALSTACK_AZ_BICEP_PATH: "/usr/local/bin/bicep", + LOCALSTACK_AZ_TIMEOUT_SECONDS: "60", + LOCALSTACK_AZ_MAX_OUTPUT_CHARS: "5000", + LOCALSTACK_AZ_MAX_HELP_CHARS: "8000", + LOCALSTACK_AZ_WORKDIR: "/work/other", + LOCALSTACK_AZ_EGRESS_GUARD: "0", + LOCALSTACK_AZ_DENYLIST_FILE: "/etc/az-deny.txt", + LOCALSTACK_AZ_PYCACHE_DIR: "/tmp/localstack-az-pycache", + LOCALSTACK_AZURE_FORWARD_TARGET: "host.docker.internal", + LOCALSTACK_AZ_TEST_ENVELOPE: "1", + }); + expect(c.errors).toEqual([]); + expect(c).toMatchObject({ + configDir: "/home/user/lsaz/profile", + homeDir: "/home/user/lsaz/profile/home", + extensionDir: "/opt/az-extensions", + azPath: "/usr/bin/az", + bicepPath: "/usr/local/bin/bicep", + timeoutMs: 60_000, + maxOutputChars: 5000, + maxHelpChars: 8000, + workdir: "/work/other", + egressGuard: false, + denylistFile: "/etc/az-deny.txt", + pycacheDir: "/tmp/localstack-az-pycache", + forwardTarget: "host.docker.internal", + testEnvelope: true, + }); + }); + + test("a relative workdir resolves against the server's cwd", () => { + expect(config({ LOCALSTACK_AZ_WORKDIR: "sub" }).workdir).toBe("/work/project/sub"); + }); +}); + +describe("getAzureConfig: invalid values fall back with a warning", () => { + test.each([ + ["LOCALSTACK_AZ_TIMEOUT_SECONDS", "4", "timeoutMs", 300_000], + ["LOCALSTACK_AZ_TIMEOUT_SECONDS", "3601", "timeoutMs", 300_000], + ["LOCALSTACK_AZ_TIMEOUT_SECONDS", "ten", "timeoutMs", 300_000], + ["LOCALSTACK_AZ_TIMEOUT_SECONDS", "1.5", "timeoutMs", 300_000], + ["LOCALSTACK_AZ_MAX_OUTPUT_CHARS", "999", "maxOutputChars", 30000], + ["LOCALSTACK_AZ_MAX_HELP_CHARS", "-5", "maxHelpChars", 30000], + ["LOCALSTACK_AZURE_PORT", "70000", "port", 4566], + ["LOCALSTACK_AZURE_PORT", "0", "port", 4566], + ] as const)("%s=%s", (name, value, key, expected) => { + const c = config({ [name]: value }); + expect(c[key]).toBe(expected); + expect(c.warnings.join("\n")).toContain(`${name}=${value}`); + expect(c.errors).toEqual([]); + }); + + test("the timeout bounds themselves are accepted", () => { + expect(config({ LOCALSTACK_AZ_TIMEOUT_SECONDS: "5" }).timeoutMs).toBe(5000); + expect(config({ LOCALSTACK_AZ_TIMEOUT_SECONDS: "3600" }).timeoutMs).toBe(3_600_000); + }); + + test("an unknown guard value keeps the guard on", () => { + const c = config({ LOCALSTACK_AZ_EGRESS_GUARD: "maybe" }); + expect(c.egressGuard).toBe(true); + expect(c.warnings).toHaveLength(1); + }); + + test("LOCALSTACK_AZ_RUNNER: host by default, worker on request, anything else warns", () => { + expect(config({}).runner).toBe("host"); + const worker = config({ LOCALSTACK_AZ_RUNNER: "Worker" }); + expect(worker.runner).toBe("worker"); + expect(worker.warnings).toEqual([]); + const other = config({ LOCALSTACK_AZ_RUNNER: "fast" }); + expect(other.runner).toBe("host"); + expect(other.warnings.join("\n")).toMatch(/not a runner; use host or worker/); + }); + + test("LOCALSTACK_AZ_BICEP_ENV: the variables Bicep may read, none by default", () => { + expect(config({}).bicepEnv).toEqual([]); + const c = config({ + LOCALSTACK_AZ_BICEP_ENV: " BACKEND_SECRET, PG_ADMIN_PASSWORD ,BACKEND_SECRET", + }); + expect(c.bicepEnv).toEqual(["BACKEND_SECRET", "PG_ADMIN_PASSWORD"]); + expect(c.warnings).toEqual([]); + }); + + test("LOCALSTACK_AZ_BICEP_ENV never passes the token or a variable that steers az, and says so", () => { + const c = config({ + LOCALSTACK_AZ_BICEP_ENV: + "LOCALSTACK_AUTH_TOKEN,AZURE_CLIENT_SECRET,ARM_CLIENT_ID,HTTPS_PROXY,https_proxy,PATH," + + "BICEP_TRACING_ENABLED,not-a-name,OK_VAR", + }); + expect(c.bicepEnv).toEqual(["OK_VAR"]); + const warned = c.warnings.join("\n"); + for (const name of [ + "LOCALSTACK_AUTH_TOKEN", + "AZURE_CLIENT_SECRET", + "HTTPS_PROXY", + "not-a-name", + ]) { + expect(warned).toContain(name); + } + }); + + test("an invalid LOCALSTACK_AZURE_PORT falls back to LOCALSTACK_PORT", () => { + expect(config({ LOCALSTACK_PORT: "4600", LOCALSTACK_AZURE_PORT: "x" }).port).toBe(4600); + }); +}); + +describe("getAzureConfig: the config dir never touches the real Azure CLI profile", () => { + test.each([ + ["~/.azure", /basename \.azure/], + ["/home/user/.azure/sub", /inside your Azure CLI profile/], + ["/data/profiles/.azure", /basename \.azure/], + ["/home/user", /contains your Azure CLI profile/], + ["/", /contains your Azure CLI profile/], + ])("LOCALSTACK_AZ_CONFIG_DIR=%s is refused", (value, reason) => { + const errors = config({ LOCALSTACK_AZ_CONFIG_DIR: value }).errors.join("\n"); + expect(errors).toMatch(reason); + expect(errors).toContain("LOCALSTACK_AZ_CONFIG_DIR"); + }); + + test("a dir inside the parent's AZURE_CONFIG_DIR is refused", () => { + const errors = config({ + AZURE_CONFIG_DIR: "/srv/azcfg", + LOCALSTACK_AZ_CONFIG_DIR: "/srv/azcfg/mcp", + }).errors.join("\n"); + expect(errors).toMatch(/inside AZURE_CONFIG_DIR/); + }); + + test("a dir under ~/.localstack is accepted", () => { + expect(config({ LOCALSTACK_AZ_CONFIG_DIR: "~/.localstack/azure/x" }).errors).toEqual([]); + }); + + test("the comparison is case-insensitive on Windows", () => { + const win: AzureConfigDeps = { + homedir: () => "C:\\Users\\me", + cwd: () => "C:\\work", + platform: "win32", + inDocker: false, + realpath: () => undefined, + isDirectory: () => true, + isFile: () => false, + }; + const c = getAzureConfig({ LOCALSTACK_AZ_CONFIG_DIR: "c:\\USERS\\ME\\.Azure\\x" }, win); + expect(c.errors.join("\n")).toMatch(/inside your Azure CLI profile/); + const ok = getAzureConfig({}, win); + expect(ok.errors).toEqual([]); + expect(ok.configDir).toBe("C:\\Users\\me\\.localstack\\azure\\mcp-config-4566"); + expect(ok.homeDir).toBe("C:\\Users\\me\\.localstack\\azure\\mcp-config-4566\\home"); + }); + + test("a symlink into ~/.azure is refused (real filesystem)", () => { + const root = mkdtempSync(path.join(os.tmpdir(), "lsaz-cfg-")); + try { + const home = path.join(root, "home"); + mkdirSync(path.join(home, ".azure"), { recursive: true }); + const link = path.join(root, "link"); + try { + symlinkSync(path.join(home, ".azure"), link, "junction"); + } catch { + return; // no symlink privilege on this machine: covered by the fake-fs cases + } + const c = getAzureConfig( + { LOCALSTACK_AZ_CONFIG_DIR: path.join(link, "profile") }, + { homedir: () => home, cwd: () => root, inDocker: false } + ); + expect(c.errors.join("\n")).toMatch(/inside your Azure CLI profile/); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); +}); + +describe("getAzureConfig: the workdir", () => { + test("a missing workdir is an error", () => { + expect(config({ LOCALSTACK_AZ_WORKDIR: "/nowhere" }).errors.join("\n")).toMatch( + /not an existing directory/ + ); + }); + + test("a workdir inside ~/.azure is an error", () => { + const deps = { ...posix, isDirectory: () => true }; + expect(config({ LOCALSTACK_AZ_WORKDIR: "/home/user/.azure" }, deps).errors.join("\n")).toMatch( + /inside an Azure CLI profile/ + ); + }); + + test("a workdir that contains the home directory is a warning, not an error", () => { + const c = config({ LOCALSTACK_AZ_WORKDIR: "/home/user" }); + expect(c.errors).toEqual([]); + expect(c.warnings.join("\n")).toMatch(/contains your home directory/); + }); + + test("a denylist file that does not exist is an error", () => { + expect(config({ LOCALSTACK_AZ_DENYLIST_FILE: "/etc/missing.txt" }).errors.join("\n")).toMatch( + /LOCALSTACK_AZ_DENYLIST_FILE/ + ); + }); + + test("a real temp workdir passes with the default filesystem checks", () => { + const root = mkdtempSync(path.join(os.tmpdir(), "lsaz-wd-")); + try { + writeFileSync(path.join(root, "deny.txt"), "storage account\n"); + const c = getAzureConfig( + { LOCALSTACK_AZ_WORKDIR: root, LOCALSTACK_AZ_DENYLIST_FILE: path.join(root, "deny.txt") }, + { homedir: () => path.join(root, "home"), inDocker: false } + ); + expect(c.errors).toEqual([]); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); +}); + +describe("protected directories and path helpers", () => { + test("azureProtectedDirs lists the real profile dirs, the parent's AZURE_CONFIG_DIR and the tool's config dir", () => { + const c = config(); + expect( + azureProtectedDirs( + c, + { AZURE_CONFIG_DIR: "/srv/azcfg" }, + { homedir: () => HOME, cwd: () => CWD, platform: "linux" } + ) + ).toEqual([ + "/home/user/.azure", + "/home/user/.ssh", + "/home/user/.kube", + "/home/user/.docker", + "/srv/azcfg", + "/home/user/.localstack/azure/mcp-config-4566", + ]); + }); + + test("isInsideOrEqual", () => { + expect(isInsideOrEqual("/a/b", "/a", "linux")).toBe(true); + expect(isInsideOrEqual("/a", "/a", "linux")).toBe(true); + expect(isInsideOrEqual("/ab", "/a", "linux")).toBe(false); + expect(isInsideOrEqual("/a/../b", "/a", "linux")).toBe(false); + expect(isInsideOrEqual("d:\\x", "c:\\", "win32")).toBe(false); + }); + + test("canonicalPath resolves the longest existing prefix and lower-cases on win32", () => { + const realpath = (p: string) => (p === "C:\\Link" ? "C:\\Users\\Me\\.azure" : undefined); + expect(canonicalPath("C:\\Link\\x\\y", "win32", realpath)).toBe("c:\\users\\me\\.azure\\x\\y"); + expect(canonicalPath("/A/b", "linux", () => undefined)).toBe("/A/b"); + }); +}); diff --git a/src/core/config.ts b/src/core/config.ts index b399325..eb73d90 100644 --- a/src/core/config.ts +++ b/src/core/config.ts @@ -1,3 +1,9 @@ +import { existsSync, realpathSync, statSync } from "fs"; +import os from "os"; +import path from "path"; +import type { AzureConfig } from "../lib/azure/types"; +import { isLocalHost } from "../lib/azure/local-hosts"; + export const LOCALSTACK_HOSTNAME = process.env.LOCALSTACK_HOSTNAME || "localhost"; export const LOCALSTACK_PORT = process.env.LOCALSTACK_PORT || 4566; export const LOCALSTACK_BASE_URL = `http://${LOCALSTACK_HOSTNAME}:${LOCALSTACK_PORT}`; @@ -9,3 +15,396 @@ export const DEFAULT_FETCH_TIMEOUT = 15000; export const DEFAULT_COMMAND_TIMEOUT = 300000; // 5 minutes export const DEFAULT_COMMAND_MAX_BUFFER = 1024 * 1024 * 10; // 10 MB export const IAM_CONFIG_ENDPOINT = "/_aws/iam/config"; + +// --------------------------------------------------------------------------- +// The Azure client. Unlike the constants above, this is a +// function of the environment, so tests can build any configuration without +// re-importing the module. +// --------------------------------------------------------------------------- + +const DEFAULT_AZURE_PORT = 4566; +const DEFAULT_AZ_TIMEOUT_SECONDS = 300; +const DEFAULT_AZ_MAX_CHARS = 30000; +/** The runner caps each stream at 10 MB, so a larger text cap would never apply. */ +const MAX_AZ_OUTPUT_CHARS = 10_000_000; + +export interface AzureConfigDeps { + homedir?: () => string; + cwd?: () => string; + platform?: NodeJS.Platform; + inDocker?: boolean; + /** The realpath of an existing path; undefined when it does not exist. */ + realpath?: (p: string) => string | undefined; + isDirectory?: (p: string) => boolean; + isFile?: (p: string) => boolean; +} + +function defaultRealpath(p: string): string | undefined { + try { + return realpathSync.native(p); + } catch { + return undefined; + } +} + +function statIs(p: string, kind: "dir" | "file"): boolean { + try { + const stat = statSync(p); + return kind === "dir" ? stat.isDirectory() : stat.isFile(); + } catch { + return false; + } +} + +/** + * A comparable form of a path: resolved, with its longest existing prefix put + * through realpath (a symlinked parent cannot hide `~/.azure`), and lower-cased on + * the case-insensitive platforms. + */ +export function canonicalPath( + p: string, + platform: NodeJS.Platform, + realpath: (p: string) => string | undefined = defaultRealpath +): string { + const pathApi = platform === "win32" ? path.win32 : path.posix; + let head = pathApi.resolve(p); + const tail: string[] = []; + for (;;) { + const real = realpath(head); + if (real !== undefined) { + head = real; + break; + } + const parent = pathApi.dirname(head); + if (parent === head) break; + tail.unshift(pathApi.basename(head)); + head = parent; + } + const joined = tail.length ? pathApi.join(head, ...tail) : head; + return platform === "win32" || platform === "darwin" ? joined.toLowerCase() : joined; +} + +/** `child` equals `parent` or lies inside it; both canonical. */ +export function isInsideOrEqual(child: string, parent: string, platform: NodeJS.Platform): boolean { + const pathApi = platform === "win32" ? path.win32 : path.posix; + const rel = pathApi.relative(parent, child); + return rel === "" || (!rel.startsWith("..") && !pathApi.isAbsolute(rel)); +} + +/** Expand a leading `~` (MCP client configs pass values without a shell), then resolve. */ +function resolveUserPath(value: string, home: string, cwd: string, platform: NodeJS.Platform) { + const pathApi = platform === "win32" ? path.win32 : path.posix; + const expanded = + value === "~" ? home : /^~[\\/]/.test(value) ? pathApi.join(home, value.slice(2)) : value; + return pathApi.resolve(cwd, expanded); +} + +function readInt( + env: NodeJS.ProcessEnv, + name: string, + fallback: number, + min: number, + max: number, + warnings: string[] +): number { + const raw = env[name]?.trim(); + if (!raw) return fallback; + const value = /^\d+$/.test(raw) ? Number(raw) : NaN; + if (!Number.isSafeInteger(value) || value < min || value > max) { + warnings.push(`${name}=${raw} is not a whole number from ${min} to ${max}; using ${fallback}.`); + return fallback; + } + return value; +} + +function readFlag(env: NodeJS.ProcessEnv, name: string, fallback: boolean, warnings: string[]) { + const raw = env[name]?.trim().toLowerCase(); + if (!raw) return fallback; + if (["1", "true", "yes", "on"].includes(raw)) return true; + if (["0", "false", "no", "off"].includes(raw)) return false; + warnings.push(`${name}=${env[name]} is not 0 or 1; using ${fallback ? 1 : 0}.`); + return fallback; +} + +// LOCALSTACK_AZ_BICEP_ENV may never pass these, even when listed: the auth token, and the +// variables that steer az, Bicep, Python, .NET or the network, the same families the child +// environment keeps out (child-env.ts). +const BICEP_ENV_DENIED_NAMES = new Set([ + "PATH", + "PATHEXT", + "HOME", + "USERPROFILE", + "HOMEDRIVE", + "HOMEPATH", + "APPDATA", + "LOCALAPPDATA", + "TEMP", + "TMP", + "TMPDIR", + "SYSTEMROOT", + "SYSTEMDRIVE", + "WINDIR", + "COMSPEC", + "HTTP_PROXY", + "HTTPS_PROXY", + "NO_PROXY", + "ALL_PROXY", + "REQUESTS_CA_BUNDLE", + "SSL_CERT_FILE", + "SSL_CERT_DIR", + "CURL_CA_BUNDLE", + "NODE_EXTRA_CA_CERTS", + "KUBECONFIG", + "DOCKER_COMMAND", + "DOCKER_HOST", + "GITHUB_ACTIONS", + "TF_BUILD", + "BROWSER", + "MSI_ENDPOINT", + "MSI_SECRET", +]); +const BICEP_ENV_DENIED_PREFIXES = [ + "LOCALSTACK_", // the auth token and this tool's own settings + "AZURE_", + "ARM_", + "IDENTITY_", + "BICEP_", + "DOTNET_", + "PYTHON", +]; + +/** + * LOCALSTACK_AZ_BICEP_ENV: the server variables a `.bicepparam` may read with + * `readEnvironmentVariable()`. az runs with an allow-list environment, so a variable reaches Bicep + * only when it is listed here: a planted parameter file can read nothing the user did not choose. + */ +function readBicepEnv(env: NodeJS.ProcessEnv, warnings: string[]): string[] { + const raw = env.LOCALSTACK_AZ_BICEP_ENV?.trim(); + if (!raw) return []; + const names: string[] = []; + const invalid: string[] = []; + const denied: string[] = []; + for (const name of raw.split(/[\s,]+/).filter(Boolean)) { + if (!/^[A-Za-z_][A-Za-z0-9_]*$/.test(name)) { + invalid.push(name); + continue; + } + const upper = name.toUpperCase(); + if ( + BICEP_ENV_DENIED_NAMES.has(upper) || + BICEP_ENV_DENIED_PREFIXES.some((prefix) => upper.startsWith(prefix)) + ) { + denied.push(name); + continue; + } + if (!names.includes(name)) names.push(name); + } + if (invalid.length > 0) { + warnings.push( + `LOCALSTACK_AZ_BICEP_ENV: ${invalid.join(", ")} ${invalid.length === 1 ? "is not a variable name" : "are not variable names"}; ignored.` + ); + } + if (denied.length > 0) { + warnings.push( + `LOCALSTACK_AZ_BICEP_ENV: ${denied.join(", ")} ${denied.length === 1 ? "is" : "are"} never passed to Bicep (the auth token and the variables that steer az, Bicep, Python, .NET or the network stay out); ignored.` + ); + } + return names; +} + +/** + * The Azure client's settings. Invalid numbers fall back + * to their defaults with a warning; values that would weaken containment (a remote + * endpoint, a config dir inside the user's real Azure CLI profile) are hard errors, + * and the tool refuses to run while any is present. + */ +export function getAzureConfig( + env: NodeJS.ProcessEnv = process.env, + deps: AzureConfigDeps = {} +): AzureConfig { + const platform = deps.platform ?? process.platform; + const pathApi = platform === "win32" ? path.win32 : path.posix; + const home = (deps.homedir ?? os.homedir)(); + const cwd = (deps.cwd ?? process.cwd)(); + const realpath = deps.realpath ?? defaultRealpath; + const isDirectory = deps.isDirectory ?? ((p: string) => statIs(p, "dir")); + const isFile = deps.isFile ?? ((p: string) => statIs(p, "file")); + const canon = (p: string) => canonicalPath(p, platform, realpath); + const inside = (child: string, parent: string) => + isInsideOrEqual(canon(child), canon(parent), platform); + const warnings: string[] = []; + const errors: string[] = []; + + const sharedPort = readInt(env, "LOCALSTACK_PORT", DEFAULT_AZURE_PORT, 1, 65535, warnings); + const port = readInt(env, "LOCALSTACK_AZURE_PORT", sharedPort, 1, 65535, warnings); + + let endpoint = `https://azure.localhost.localstack.cloud:${port}`; + const endpointOverride = env.LOCALSTACK_AZURE_ENDPOINT?.trim(); + if (endpointOverride) { + let url: URL | undefined; + try { + url = new URL(endpointOverride); + } catch { + url = undefined; + } + if (!url || url.protocol !== "https:" || url.username || url.password) { + errors.push( + `LOCALSTACK_AZURE_ENDPOINT must be an https URL such as https://azure.localhost.localstack.cloud:${port}; got "${endpointOverride}".` + ); + } else if (!isLocalHost(url.hostname)) { + errors.push( + `LOCALSTACK_AZURE_ENDPOINT must point at a local emulator (localhost.localstack.cloud or a subdomain, localhost, 127.0.0.1 or ::1); "${url.hostname}" is not local. The tool never talks to remote endpoints.` + ); + } else if ((url.pathname !== "/" && url.pathname !== "") || url.search || url.hash) { + errors.push( + `LOCALSTACK_AZURE_ENDPOINT must be a bare origin without a path, query or fragment; got "${endpointOverride}".` + ); + } else { + endpoint = `https://${url.host}`; + } + } + const endpointHost = new URL(endpoint).hostname.toLowerCase(); + + const realAzureDir = pathApi.join(home, ".azure"); + const parentAzureConfigDir = env.AZURE_CONFIG_DIR?.trim() + ? resolveUserPath(env.AZURE_CONFIG_DIR.trim(), home, cwd, platform) + : undefined; + + const configDirSetting = env.LOCALSTACK_AZ_CONFIG_DIR?.trim(); + const configDir = configDirSetting + ? resolveUserPath(configDirSetting, home, cwd, platform) + : pathApi.join(home, ".localstack", "azure", `mcp-config-${port}`); + // The bootstrap rewrites the config dir's cloud, config and login, so it must + // never be (or contain) the user's real Azure CLI profile. + const configDirProblem = + pathApi.basename(configDir).toLowerCase() === ".azure" + ? "has the basename .azure" + : inside(configDir, realAzureDir) + ? `lies inside your Azure CLI profile (${realAzureDir})` + : parentAzureConfigDir && inside(configDir, parentAzureConfigDir) + ? `lies inside AZURE_CONFIG_DIR (${parentAzureConfigDir})` + : inside(realAzureDir, configDir) + ? `contains your Azure CLI profile (${realAzureDir})` + : undefined; + if (configDirProblem) { + errors.push( + `The Azure tool's CLI config dir ${configDir} ${configDirProblem}. The tool keeps its own isolated profile so it never changes your Azure CLI login; set LOCALSTACK_AZ_CONFIG_DIR to another directory (the default is ~/.localstack/azure/mcp-config-).` + ); + } + + const extensionSetting = env.LOCALSTACK_AZ_EXTENSION_DIR?.trim(); + const extensionDir = extensionSetting + ? resolveUserPath(extensionSetting, home, cwd, platform) + : pathApi.join(home, ".localstack", "azure", "mcp-extensions"); + + const workdirSetting = env.LOCALSTACK_AZ_WORKDIR?.trim(); + const workdir = workdirSetting ? resolveUserPath(workdirSetting, home, cwd, platform) : cwd; + if (!isDirectory(workdir)) { + errors.push( + `The Azure tool's working directory ${workdir} is not an existing directory. Set LOCALSTACK_AZ_WORKDIR to the directory that holds your templates and files.` + ); + } else if ( + inside(workdir, realAzureDir) || + (parentAzureConfigDir && inside(workdir, parentAzureConfigDir)) || + inside(workdir, configDir) + ) { + errors.push( + `The Azure tool's working directory ${workdir} lies inside an Azure CLI profile. Set LOCALSTACK_AZ_WORKDIR to a project directory.` + ); + } else if (inside(home, workdir)) { + // Not an error: a client may start servers in the home directory. The file + // rule's protected directories still keep ~/.azure, ~/.ssh, ~/.kube and ~/.docker out. + warnings.push( + `The Azure tool's working directory is ${workdir}, which contains your home directory; commands can read and write files anywhere under it. Set LOCALSTACK_AZ_WORKDIR to a project directory.` + ); + } + + const optionalPath = (name: string) => { + const value = env[name]?.trim(); + return value ? resolveUserPath(value, home, cwd, platform) : undefined; + }; + const azPath = env.LOCALSTACK_AZ_PATH?.trim() || undefined; + const bicepPath = env.LOCALSTACK_AZ_BICEP_PATH?.trim() || undefined; + + const denylistFile = optionalPath("LOCALSTACK_AZ_DENYLIST_FILE"); + if (denylistFile && !isFile(denylistFile)) { + // A missing denylist must not silently weaken the user's own policy. + errors.push(`LOCALSTACK_AZ_DENYLIST_FILE ${denylistFile} does not exist or is not a file.`); + } + + // `worker` is opt-in: one warm az process per concurrent call. + const runnerSetting = env.LOCALSTACK_AZ_RUNNER?.trim().toLowerCase(); + let runner: AzureConfig["runner"] = "host"; + if (runnerSetting === "worker") { + runner = "worker"; + } else if (runnerSetting && runnerSetting !== "host") { + warnings.push( + `LOCALSTACK_AZ_RUNNER=${env.LOCALSTACK_AZ_RUNNER} is not a runner; use host or worker. Using host.` + ); + } + + const inDocker = deps.inDocker ?? existsSync("/.dockerenv"); + + return { + port, + healthBaseUrl: `http://127.0.0.1:${port}`, + endpoint, + endpointHost, + configDir, + homeDir: pathApi.join(configDir, "home"), + tmpDir: pathApi.join(configDir, "tmp"), + extensionDir, + azPath, + bicepPath, + bicepEnv: readBicepEnv(env, warnings), + timeoutMs: + readInt(env, "LOCALSTACK_AZ_TIMEOUT_SECONDS", DEFAULT_AZ_TIMEOUT_SECONDS, 5, 3600, warnings) * + 1000, + maxOutputChars: readInt( + env, + "LOCALSTACK_AZ_MAX_OUTPUT_CHARS", + DEFAULT_AZ_MAX_CHARS, + 1000, + MAX_AZ_OUTPUT_CHARS, + warnings + ), + maxHelpChars: readInt( + env, + "LOCALSTACK_AZ_MAX_HELP_CHARS", + DEFAULT_AZ_MAX_CHARS, + 1000, + MAX_AZ_OUTPUT_CHARS, + warnings + ), + workdir, + egressGuard: readFlag(env, "LOCALSTACK_AZ_EGRESS_GUARD", true, warnings), + denylistFile, + runner, + pycacheDir: optionalPath("LOCALSTACK_AZ_PYCACHE_DIR"), + forwardTarget: env.LOCALSTACK_AZURE_FORWARD_TARGET?.trim() || undefined, + testEnvelope: env.LOCALSTACK_AZ_TEST_ENVELOPE?.trim() === "1", + inDocker, + warnings, + errors, + }; +} + +/** + * The directories the file rule refuses even inside the workdir (see + * `PolicyOptions.protectedDirs`). + */ +export function azureProtectedDirs( + config: AzureConfig, + env: NodeJS.ProcessEnv = process.env, + deps: Pick = {} +): string[] { + const platform = deps.platform ?? process.platform; + const pathApi = platform === "win32" ? path.win32 : path.posix; + const home = (deps.homedir ?? os.homedir)(); + const cwd = (deps.cwd ?? process.cwd)(); + const dirs = [".azure", ".ssh", ".kube", ".docker"].map((d) => pathApi.join(home, d)); + const parent = env.AZURE_CONFIG_DIR?.trim(); + if (parent) dirs.push(resolveUserPath(parent, home, cwd, platform)); + dirs.push(config.configDir); + return dirs; +} diff --git a/src/core/preflight.test.ts b/src/core/preflight.test.ts index 1bf1fda..8a8c248 100644 --- a/src/core/preflight.test.ts +++ b/src/core/preflight.test.ts @@ -1,4 +1,9 @@ -import { requireDockerDaemon, requireLocalStackRunning } from "./preflight"; +import { + requireDockerDaemon, + requireLocalStackRunning, + requireStack, + resetStackDetectionCache, +} from "./preflight"; import { getGatewayHealth } from "../lib/localstack/localstack.utils"; import { DockerApiClient } from "../lib/docker/docker.client"; @@ -63,3 +68,159 @@ describe("requireDockerDaemon", () => { expect(result?.content[0].text).toMatch(/Docker daemon is not reachable/); }); }); + +describe("requireStack", () => { + const azureHealth = { reachable: true, ready: true, edition: "azure-alpha" }; + const awsHealth = { reachable: true, ready: true, edition: "pro" }; + + function mockContainer(container?: { image?: string; labels?: Record }) { + MockedDockerApiClient.mockImplementation( + () => + ({ + findLocalStackContainer: container + ? jest.fn().mockResolvedValue("id-1") + : jest.fn().mockRejectedValue(new Error("no container")), + inspectContainer: jest.fn().mockResolvedValue({ id: "id-1", ...container }), + }) as any + ); + } + + beforeEach(() => { + mockedGetGatewayHealth.mockReset(); + MockedDockerApiClient.mockReset(); + resetStackDetectionCache(); + }); + + test("passes when the edition matches", async () => { + mockedGetGatewayHealth.mockResolvedValue(awsHealth); + expect(await requireStack("aws", "localstack-aws-client")).toBeNull(); + }); + + test("refuses an AWS-only tool on the Azure emulator with the exact title and hint", async () => { + mockedGetGatewayHealth.mockResolvedValue(azureHealth); + const result = await requireStack("aws", "localstack-aws-client"); + const text = result?.content[0].text ?? ""; + expect(text).toMatch(/^❌ \*\*Wrong emulator for this tool\*\*/); + expect(text).toContain("`localstack-aws-client` works with the LocalStack AWS emulator"); + expect(text).toContain("is LocalStack Azure"); + expect(text).toContain("Use `localstack-azure-client`"); + expect(text).toContain("service: aws"); + }); + + test("appends a tool-specific hint", async () => { + mockedGetGatewayHealth.mockResolvedValue(azureHealth); + const result = await requireStack("aws", "localstack-logs-analysis", { + hint: "Use analysisType logs.", + }); + expect(result?.content[0].text).toMatch(/Use analysisType logs\.$/); + }); + + test("falls back to the container when the edition is unknown", async () => { + mockedGetGatewayHealth.mockResolvedValue({ reachable: true, ready: true, edition: "unknown" }); + mockContainer({ image: "localstack/localstack-azure:latest" }); + const result = await requireStack("aws", "localstack-aws-client"); + expect(result?.content[0].text).toContain("is LocalStack Azure"); + }); + + test("uses the container's Azure label for a bare image ID", async () => { + mockedGetGatewayHealth.mockResolvedValue({ reachable: true, ready: true }); + mockContainer({ image: "f91897f1de85", labels: { description: "LocalStack for Azure" } }); + expect(await requireStack("aws", "localstack-aws-client")).not.toBeNull(); + }); + + test("passes (fail-open) when neither the edition nor the container says which stack", async () => { + mockedGetGatewayHealth.mockResolvedValue({ reachable: true, ready: true }); + mockContainer(undefined); + expect(await requireStack("aws", "localstack-aws-client")).toBeNull(); + }); + + test("passes when the gateway is unreachable (the running-check preflight reports it)", async () => { + mockedGetGatewayHealth.mockResolvedValue({ reachable: false, ready: false }); + expect(await requireStack("aws", "localstack-aws-client")).toBeNull(); + }); + + test("two parallel calls share one health request (2 s cache)", async () => { + mockedGetGatewayHealth.mockResolvedValue(awsHealth); + await Promise.all([ + requireStack("aws", "localstack-aws-client"), + requireStack("aws", "localstack-deployer"), + ]); + expect(mockedGetGatewayHealth).toHaveBeenCalledTimes(1); + }); + + test("side by side: the Azure tool reads its own port, so both tools pass", async () => { + mockedGetGatewayHealth.mockImplementation(async (baseUrl?: string) => + baseUrl === "http://127.0.0.1:4666" ? azureHealth : awsHealth + ); + expect( + await requireStack("azure", "localstack-azure-client", { baseUrl: "http://127.0.0.1:4666" }) + ).toBeNull(); + expect(await requireStack("aws", "localstack-aws-client")).toBeNull(); + }); + + test("side by side with the ports swapped, both tools refuse", async () => { + mockedGetGatewayHealth.mockImplementation(async (baseUrl?: string) => + baseUrl === "http://127.0.0.1:4666" ? awsHealth : azureHealth + ); + const azure = await requireStack("azure", "localstack-azure-client", { + baseUrl: "http://127.0.0.1:4666", + }); + expect(azure?.content[0].text).toContain( + "the emulator at http://127.0.0.1:4666 is LocalStack AWS" + ); + expect(await requireStack("aws", "localstack-aws-client")).not.toBeNull(); + }); + + test("names the Snowflake emulator, not AWS, when a Snowflake image reports `pro`", async () => { + // The Snowflake emulator's health reports edition `pro`, + // as AWS does, so the Azure tool met it and said "is LocalStack AWS". + mockedGetGatewayHealth.mockResolvedValue(awsHealth); + mockContainer({ image: "localstack/snowflake:latest" }); + const text = (await requireStack("azure", "localstack-azure-client"))?.content[0].text ?? ""; + expect(text).toContain("is LocalStack Snowflake"); + expect(text).toContain("Use `localstack-snowflake-client`"); + }); + + test("lets AWS tools through on the Snowflake image, which serves AWS APIs too", async () => { + // The Snowflake docs run `lstk aws s3 mb` and MWAA against that emulator (Snowpipe, stages, + // storage integrations, Airflow), and document state export and Cloud Pods for it. + mockedGetGatewayHealth.mockResolvedValue(awsHealth); + mockContainer({ image: "localstack/snowflake:latest" }); + expect(await requireStack("aws", "localstack-aws-client")).toBeNull(); + expect(await requireStack("aws", "localstack-state-management")).toBeNull(); + }); + + test("still names AWS when the `pro` container is the AWS image", async () => { + mockedGetGatewayHealth.mockResolvedValue(awsHealth); + mockContainer({ image: "localstack/localstack-pro:latest" }); + const text = (await requireStack("azure", "localstack-azure-client"))?.content[0].text ?? ""; + expect(text).toContain("is LocalStack AWS"); + expect(text).toContain("Use `localstack-aws-client`"); + }); + + describe("the Snowflake client, whose edition reads like AWS's", () => { + test("is refused on the Azure emulator", async () => { + mockedGetGatewayHealth.mockResolvedValue(azureHealth); + expect(await requireStack("snowflake", "localstack-snowflake-client")).not.toBeNull(); + }); + + test("is refused on `pro` when the container is the AWS image", async () => { + mockedGetGatewayHealth.mockResolvedValue(awsHealth); + mockContainer({ image: "localstack/localstack-pro:latest" }); + const result = await requireStack("snowflake", "localstack-snowflake-client"); + expect(result?.content[0].text).toContain("is LocalStack AWS"); + }); + + test("is NOT refused on `pro` when the container is the Snowflake image", async () => { + mockedGetGatewayHealth.mockResolvedValue(awsHealth); + mockContainer({ image: "localstack/snowflake:latest" }); + expect(await requireStack("snowflake", "localstack-snowflake-client")).toBeNull(); + }); + + test("is NOT refused on `pro` when the container cannot be inspected", async () => { + mockedGetGatewayHealth.mockResolvedValue(awsHealth); + mockContainer(undefined); + expect(await requireStack("snowflake", "localstack-snowflake-client")).toBeNull(); + }); + }); +}); diff --git a/src/core/preflight.ts b/src/core/preflight.ts index 74d078d..4b1625b 100644 --- a/src/core/preflight.ts +++ b/src/core/preflight.ts @@ -1,11 +1,123 @@ import { ensureSnowflakeCli, getGatewayHealth } from "../lib/localstack/localstack.utils"; import { DockerApiClient } from "../lib/docker/docker.client"; import { checkProFeature, ProFeature } from "../lib/localstack/license-checker"; +import { + stackFromEdition, + stackFromImage, + type LocalStackStack, +} from "../lib/localstack/container-spec.logic"; import { LOCALSTACK_BASE_URL } from "./config"; import { ResponseBuilder } from "./response-builder"; +import { BootstrapError, ConfigDirInUseError } from "../lib/azure/bootstrap"; +import { getAzureEmulatorStatus } from "../lib/azure/emulator"; +import { AzResolveError } from "../lib/azure/resolve-az"; +import { azCli, azureConfig, ensureProfile, recordEmulatorSession } from "../lib/azure/services"; type ToolResponse = ReturnType; +const STACK_LABELS: Record = { + aws: "AWS", + snowflake: "Snowflake", + azure: "Azure", +}; + +const STACK_CLIENT_TOOLS: Record = { + aws: "localstack-aws-client", + snowflake: "localstack-snowflake-client", + azure: "localstack-azure-client", +}; + +/** + * Whether the Snowflake emulator's health `edition` tells it apart from AWS. + * It does not: localstack/snowflake:latest reports `pro`, as AWS does. + * So the Snowflake client's guard refuses only on clear evidence (a Snowflake image that + * reports `pro` is never turned away), and requireStack names the stack from the image. + */ +const SNOWFLAKE_EDITION_CONFIRMED = false; + +const STACK_DETECTION_CACHE_MS = 2000; +const detectionCache = new Map }>(); + +/** Reset the 2 s stack-detection cache (tests). */ +export function resetStackDetectionCache(): void { + detectionCache.clear(); +} + +/** Share one detection between the parallel preflights of a call. */ +function cachedDetection(key: string, detect: () => Promise): Promise { + const hit = detectionCache.get(key); + if (hit && Date.now() - hit.at < STACK_DETECTION_CACHE_MS) return hit.value as Promise; + const value = detect(); + detectionCache.set(key, { at: Date.now(), value }); + return value; +} + +interface ContainerStack { + stack?: LocalStackStack; + image?: string; +} + +/** The running container's stack from its image and labels (empty when unknown). */ +function detectContainerStack(): Promise { + return cachedDetection("container", async () => { + try { + const docker = new DockerApiClient(); + const metadata = await docker.inspectContainer(await docker.findLocalStackContainer()); + return { stack: stackFromImage(metadata.image, metadata.labels), image: metadata.image }; + } catch { + return {}; + } + }); +} + +/** + * Refuse a tool that meets the wrong emulator. + * + * The stack comes from the gateway's health `edition`, then from the running + * container's image and labels. When it is still unknown, the check passes + * (fail-open), so externally managed or remote setups keep working. An unreachable + * gateway also passes: `requireLocalStackRunning()` reports that case. + */ +export const requireStack = async ( + expected: LocalStackStack, + toolName: string, + opts: { baseUrl?: string; hint?: string } = {} +): Promise => { + const baseUrl = opts.baseUrl ?? LOCALSTACK_BASE_URL; + const health = await cachedDetection(`health:${baseUrl}`, () => getGatewayHealth(opts.baseUrl)); + if (!health.reachable) return null; + + let actual = stackFromEdition(health.edition); + let container: ContainerStack | undefined; + if (!actual) { + container = await detectContainerStack(); + actual = container.stack; + } + // The Snowflake emulator's health reports edition `pro` too, so an AWS reading is checked + // against the running container's image before a non-AWS tool is refused with it. AWS tools + // skip this: the Snowflake emulator serves AWS APIs as well (its docs create S3 buckets and + // MWAA environments on it), so it is not the wrong emulator for them. + if (actual === "aws" && expected !== "aws") { + container = container ?? (await detectContainerStack()); + if (container.stack === "snowflake") actual = "snowflake"; + } + if (!actual || actual === expected) return null; + + if (expected === "snowflake" && !SNOWFLAKE_EDITION_CONFIRMED && actual === "aws") { + container = container ?? (await detectContainerStack()); + if (!container.image || /\/snowflake(:|@|$)/.test(container.image)) return null; + } + + return ResponseBuilder.error( + "Wrong emulator for this tool", + `\`${toolName}\` works with the LocalStack ${STACK_LABELS[expected]} emulator, but the emulator at ` + + `${baseUrl} is LocalStack ${STACK_LABELS[actual]}. Use \`${STACK_CLIENT_TOOLS[actual]}\`, or stop it ` + + `and start the ${STACK_LABELS[expected]} emulator with \`localstack-management\` ` + + `(action: start, service: ${expected}).` + + (opts.hint ? `\n\n${opts.hint}` : "") + ); +}; + export const requireSnowflakeCli = async (): Promise => { const cliCheck = await ensureSnowflakeCli(); return cliCheck ? (cliCheck as ToolResponse) : null; @@ -68,3 +180,69 @@ export const requireLocalStackRunning = async (): Promise = } return null; }; + +// --------------------------------------------------------------------------- +// The Azure client. The handler runs them in this order: the config and the policy +// first (no emulator needed), then +// requireStack("azure"), requireAzureEmulatorRunning() and requireAzureCli() as one +// group, then the bootstrap. +// --------------------------------------------------------------------------- + +/** Hard configuration errors (a remote endpoint, a config dir inside ~/.azure, ...). */ +export const requireAzureConfig = (): ToolResponse | null => { + const { errors } = azureConfig(); + if (!errors.length) return null; + return ResponseBuilder.error( + "Azure Tool Configuration Error", + errors.map((e) => `- ${e}`).join("\n") + ); +}; + +export const requireAzureEmulatorRunning = async (): Promise => { + const status = await getAzureEmulatorStatus(azureConfig()); + recordEmulatorSession(status.sessionId); + return status.ok + ? null + : ResponseBuilder.error("LocalStack Azure Emulator Not Ready", status.message); +}; + +/** Resolves and probes `az` once per process (~0.17 s); the minimum is 2.85. */ +export const requireAzureCli = async (): Promise => { + try { + await azCli(); + return null; + } catch (error) { + if (error instanceof AzResolveError) { + const skipped = error.reasons.length + ? `\n\nSkipped candidates:\n${error.reasons.map((r) => `- ${r}`).join("\n")}` + : ""; + return ResponseBuilder.error("Azure CLI Not Available", error.message + skipped); + } + return ResponseBuilder.error( + "Azure CLI Not Available", + error instanceof Error ? error.message : String(error) + ); + } +}; + +/** + * Bootstraps the isolated CLI profile under its locks. A failed + * `az` step is an az failure, so the caller formats it with the error classes. + */ +export const requireAzureCliConfigured = async ( + formatBootstrapFailure: (error: BootstrapError) => ToolResponse +): Promise => { + try { + await ensureProfile(); + return null; + } catch (error) { + if (error instanceof BootstrapError) return formatBootstrapFailure(error); + if (error instanceof ConfigDirInUseError) { + return ResponseBuilder.error("Azure CLI Profile In Use", error.message); + } + return ResponseBuilder.error( + "Azure CLI Profile Setup Failed", + error instanceof Error ? error.message : String(error) + ); + } +}; diff --git a/src/lib/aws/aws-cli-sanitizer.ts b/src/lib/aws/aws-cli-sanitizer.ts index 9f961db..09548de 100644 --- a/src/lib/aws/aws-cli-sanitizer.ts +++ b/src/lib/aws/aws-cli-sanitizer.ts @@ -1,3 +1,9 @@ +import { splitCliArgs } from "../cli/argv"; + +// The tokenizer lives in src/lib/cli/argv.ts so the Azure client can share it. The +// AWS client calls it without options, which keeps its original behaviour. +export { splitCliArgs as splitAwsCliArgs } from "../cli/argv"; + export function sanitizeAwsCliCommand(rawCommand: string): string { const command = rawCommand.trim(); if (!command) { @@ -16,64 +22,7 @@ export function sanitizeAwsCliCommand(rawCommand: string): string { throw new Error("Command must start with an AWS service or built-in command."); } - splitAwsCliArgs(command); + splitCliArgs(command); return command; } - -export function splitAwsCliArgs(command: string): string[] { - const args: string[] = []; - let current = ""; - let quote: "'" | '"' | undefined; - - for (let index = 0; index < command.length; index++) { - const character = command[index]; - - if (character === "\n" || character === "\r" || character === "`") { - throw new Error("Command contains forbidden shell syntax."); - } - - if (character === "\\" && quote === '"') { - const escaped = command[index + 1]; - if (escaped === '"' || escaped === "\\") { - current += escaped; - index++; - } else { - current += character; - } - continue; - } - - if ((character === "'" || character === '"') && (!quote || quote === character)) { - quote = quote ? undefined : character; - continue; - } - - if (!quote) { - if ( - ";&|<>".includes(character) || - command.startsWith("$(", index) || - command.startsWith("${", index) - ) { - throw new Error("Command contains forbidden shell syntax."); - } - if (/\s/.test(character)) { - if (current) { - args.push(current); - current = ""; - } - continue; - } - } - - current += character; - } - - if (quote) { - throw new Error("Command contains an unterminated quote."); - } - if (current) { - args.push(current); - } - return args; -} diff --git a/src/lib/azure/az-file-args.generated.json b/src/lib/azure/az-file-args.generated.json new file mode 100644 index 0000000..53f4e1c --- /dev/null +++ b/src/lib/azure/az-file-args.generated.json @@ -0,0 +1,1292 @@ +{ + "_metadata": { + "generator": "scripts/gen-az-file-args.py", + "description": "Per-command file-taking arguments, from az's command table: every argument whose type is file_type or whose completer completes files/directories, with all its option strings. Consumed by src/lib/azure/policy.ts. Regenerated on an az pin move.", + "az_version": "2.90.0", + "python_version": "3.11.0rc1", + "platform": "linux", + "generated_utc": "2026-09-27T18:44:57Z", + "command_count": 7048, + "file_command_count": 189, + "file_argument_count": 255, + "extensions_installed": [ + "acrcssc", + "acrquery", + "acrtransfer", + "application-insights", + "azure-firewall", + "bastion", + "cdn", + "dns-resolver", + "documentdb", + "edge-action", + "eventgrid", + "express-route-cross-connection", + "fleet", + "front-door", + "ip-group", + "k8s-configuration", + "k8s-extension", + "monitor-control-service", + "nsp", + "resource-graph", + "scheduled-query", + "staticwebapp", + "virtual-network-manager", + "virtual-network-tap", + "virtual-wan", + "webapp" + ], + "note": "The production image pins azure-cli 2.90 and 26 curated extensions (docker/azure-extensions.txt). This table was generated with azure-cli 2.90.0 and the extensions listed in extensions_installed (empty here means only built-in command modules were present). DR4 regenerates it with the pinned az and the installed extensions. Path arguments az does not annotate (e.g. storage blob download-batch --destination, webapp deploy --src-path) are covered by EXTRA_FILE_ARGS in policy.ts, not by this file." + }, + "commands": { + "acr build": { + "flags": [], + "positional": true + }, + "acr helm push": { + "flags": [], + "positional": true + }, + "acr pack build": { + "flags": [], + "positional": true + }, + "acr run": { + "flags": [], + "positional": true + }, + "aks create": { + "flags": [ + "--ssh-key-value" + ] + }, + "aks get-credentials": { + "flags": [ + "--file", + "-f" + ] + }, + "aks namespace get-credentials": { + "flags": [ + "--file", + "-f" + ] + }, + "backup policy create": { + "flags": [ + "--policy" + ] + }, + "backup policy set": { + "flags": [ + "--policy" + ] + }, + "batch application package create": { + "flags": [ + "--package-file" + ] + }, + "batch job create": { + "flags": [ + "--json-file" + ] + }, + "batch job disable": { + "flags": [ + "--json-file" + ] + }, + "batch job reset": { + "flags": [ + "--json-file" + ] + }, + "batch job set": { + "flags": [ + "--json-file" + ] + }, + "batch job-schedule create": { + "flags": [ + "--json-file" + ] + }, + "batch job-schedule reset": { + "flags": [ + "--json-file" + ] + }, + "batch job-schedule set": { + "flags": [ + "--json-file" + ] + }, + "batch node delete": { + "flags": [ + "--json-file" + ] + }, + "batch node file download": { + "flags": [ + "--destination" + ] + }, + "batch node reboot": { + "flags": [ + "--json-file" + ] + }, + "batch node scheduling disable": { + "flags": [ + "--json-file" + ] + }, + "batch node service-logs upload": { + "flags": [ + "--json-file" + ] + }, + "batch node user create": { + "flags": [ + "--json-file" + ] + }, + "batch node user reset": { + "flags": [ + "--json-file" + ] + }, + "batch pool autoscale evaluate": { + "flags": [ + "--json-file" + ] + }, + "batch pool create": { + "flags": [ + "--json-file" + ] + }, + "batch pool reset": { + "flags": [ + "--json-file" + ] + }, + "batch pool set": { + "flags": [ + "--json-file" + ] + }, + "batch task create": { + "flags": [ + "--json-file" + ] + }, + "batch task file download": { + "flags": [ + "--destination" + ] + }, + "batch task reset": { + "flags": [ + "--json-file" + ] + }, + "batchai cluster create": { + "flags": [ + "--ssh-key", + "-k" + ] + }, + "batchai file-server create": { + "flags": [ + "--ssh-key", + "-k" + ] + }, + "bicep build": { + "flags": [ + "--file", + "--outdir", + "--outfile", + "-f" + ] + }, + "bicep build-params": { + "flags": [ + "--file", + "--outdir", + "--outfile", + "-f" + ] + }, + "bicep decompile": { + "flags": [ + "--file", + "-f" + ] + }, + "bicep decompile-params": { + "flags": [ + "--bicep-file", + "--file", + "--outdir", + "--outfile", + "-f" + ] + }, + "bicep format": { + "flags": [ + "--file", + "--outdir", + "--outfile", + "-f" + ] + }, + "bicep generate-params": { + "flags": [ + "--file", + "--outdir", + "--outfile", + "-f" + ] + }, + "bicep lint": { + "flags": [ + "--file", + "-f" + ] + }, + "bicep publish": { + "flags": [ + "--file", + "-f" + ] + }, + "bicep restore": { + "flags": [ + "--file", + "-f" + ] + }, + "bicep snapshot": { + "flags": [ + "--file", + "-f" + ] + }, + "containerapp create": { + "flags": [ + "--yaml" + ] + }, + "containerapp env dapr-component set": { + "flags": [ + "--yaml" + ] + }, + "containerapp env http-route-config create": { + "flags": [ + "--yaml" + ] + }, + "containerapp env http-route-config update": { + "flags": [ + "--yaml" + ] + }, + "containerapp job create": { + "flags": [ + "--yaml" + ] + }, + "containerapp job start": { + "flags": [ + "--yaml" + ] + }, + "containerapp job update": { + "flags": [ + "--yaml" + ] + }, + "containerapp revision copy": { + "flags": [ + "--yaml" + ] + }, + "containerapp update": { + "flags": [ + "--yaml" + ] + }, + "cosmosdb cassandra table create": { + "flags": [ + "--schema" + ] + }, + "cosmosdb cassandra table update": { + "flags": [ + "--schema" + ] + }, + "cosmosdb collection create": { + "flags": [ + "--cep", + "--full-text-policy", + "--indexing-policy", + "--vector-embeddings" + ] + }, + "cosmosdb collection update": { + "flags": [ + "--indexing-policy" + ] + }, + "cosmosdb fleetspace account create": { + "flags": [ + "--body", + "-b" + ] + }, + "cosmosdb fleetspace create": { + "flags": [ + "--body", + "-b" + ] + }, + "cosmosdb fleetspace update": { + "flags": [ + "--body", + "-b" + ] + }, + "cosmosdb gremlin graph create": { + "flags": [ + "--conflict-resolution-policy", + "--idx", + "-c" + ] + }, + "cosmosdb gremlin graph update": { + "flags": [ + "--idx" + ] + }, + "cosmosdb mongodb collection create": { + "flags": [ + "--idx" + ] + }, + "cosmosdb mongodb collection update": { + "flags": [ + "--idx" + ] + }, + "cosmosdb mongodb role definition create": { + "flags": [ + "--body", + "-b" + ] + }, + "cosmosdb mongodb role definition update": { + "flags": [ + "--body", + "-b" + ] + }, + "cosmosdb mongodb user definition create": { + "flags": [ + "--body", + "-b" + ] + }, + "cosmosdb mongodb user definition update": { + "flags": [ + "--body", + "-b" + ] + }, + "cosmosdb sql container create": { + "flags": [ + "--cep", + "--conflict-resolution-policy", + "--full-text-policy", + "--idx", + "--unique-key-policy", + "--vector-embeddings", + "-c", + "-u" + ] + }, + "cosmosdb sql container update": { + "flags": [ + "--full-text-policy", + "--idx", + "--vector-embeddings" + ] + }, + "cosmosdb sql role definition create": { + "flags": [ + "--body", + "-b" + ] + }, + "cosmosdb sql role definition update": { + "flags": [ + "--body", + "-b" + ] + }, + "cosmosdb sql stored-procedure create": { + "flags": [ + "--body", + "-b" + ] + }, + "cosmosdb sql stored-procedure update": { + "flags": [ + "--body", + "-b" + ] + }, + "cosmosdb sql trigger create": { + "flags": [ + "--body", + "-b" + ] + }, + "cosmosdb sql trigger update": { + "flags": [ + "--body", + "-b" + ] + }, + "cosmosdb sql user-defined-function create": { + "flags": [ + "--body", + "-b" + ] + }, + "cosmosdb sql user-defined-function update": { + "flags": [ + "--body", + "-b" + ] + }, + "deployment create": { + "flags": [ + "--parameters", + "--template-file", + "-f", + "-p" + ], + "greedy": [ + "--parameters", + "-p" + ] + }, + "deployment group create": { + "flags": [ + "--parameters", + "--template-file", + "-f", + "-p" + ], + "greedy": [ + "--parameters", + "-p" + ] + }, + "deployment group validate": { + "flags": [ + "--parameters", + "--template-file", + "-f", + "-p" + ], + "greedy": [ + "--parameters", + "-p" + ] + }, + "deployment group what-if": { + "flags": [ + "--parameters", + "--template-file", + "-f", + "-p" + ], + "greedy": [ + "--parameters", + "-p" + ] + }, + "deployment mg create": { + "flags": [ + "--parameters", + "--template-file", + "-f", + "-p" + ], + "greedy": [ + "--parameters", + "-p" + ] + }, + "deployment mg validate": { + "flags": [ + "--parameters", + "--template-file", + "-f", + "-p" + ], + "greedy": [ + "--parameters", + "-p" + ] + }, + "deployment mg what-if": { + "flags": [ + "--parameters", + "--template-file", + "-f", + "-p" + ], + "greedy": [ + "--parameters", + "-p" + ] + }, + "deployment sub create": { + "flags": [ + "--parameters", + "--template-file", + "-f", + "-p" + ], + "greedy": [ + "--parameters", + "-p" + ] + }, + "deployment sub validate": { + "flags": [ + "--parameters", + "--template-file", + "-f", + "-p" + ], + "greedy": [ + "--parameters", + "-p" + ] + }, + "deployment sub what-if": { + "flags": [ + "--parameters", + "--template-file", + "-f", + "-p" + ], + "greedy": [ + "--parameters", + "-p" + ] + }, + "deployment tenant create": { + "flags": [ + "--parameters", + "--template-file", + "-f", + "-p" + ], + "greedy": [ + "--parameters", + "-p" + ] + }, + "deployment tenant validate": { + "flags": [ + "--parameters", + "--template-file", + "-f", + "-p" + ], + "greedy": [ + "--parameters", + "-p" + ] + }, + "deployment tenant what-if": { + "flags": [ + "--parameters", + "--template-file", + "-f", + "-p" + ], + "greedy": [ + "--parameters", + "-p" + ] + }, + "deployment validate": { + "flags": [ + "--parameters", + "--template-file", + "-f", + "-p" + ], + "greedy": [ + "--parameters", + "-p" + ] + }, + "extension add": { + "flags": [ + "--source", + "-s" + ] + }, + "fleet get-credentials": { + "flags": [ + "--file", + "-f" + ] + }, + "fleet namespace get-credentials": { + "flags": [ + "--file" + ] + }, + "functionapp config ssl upload": { + "flags": [ + "--certificate-file" + ] + }, + "group deployment create": { + "flags": [ + "--parameters", + "--template-file", + "-f", + "-p" + ], + "greedy": [ + "--parameters", + "-p" + ] + }, + "group deployment validate": { + "flags": [ + "--parameters", + "--template-file", + "-f", + "-p" + ], + "greedy": [ + "--parameters", + "-p" + ] + }, + "hdinsight create": { + "flags": [ + "--cluster-configurations", + "--entra-uinfo", + "--entra-user-full-info", + "--private-link-config", + "--private-link-configurations" + ] + }, + "hdinsight credentials update": { + "flags": [ + "--entra-uinfo", + "--entra-user-full-info" + ] + }, + "iot dps certificate create": { + "flags": [ + "--path", + "-p" + ] + }, + "iot dps certificate update": { + "flags": [ + "--path", + "-p" + ] + }, + "iot dps certificate verify": { + "flags": [ + "--path", + "-p" + ] + }, + "iot hub certificate create": { + "flags": [ + "--path", + "-p" + ] + }, + "iot hub certificate update": { + "flags": [ + "--path", + "-p" + ] + }, + "iot hub certificate verify": { + "flags": [ + "--path", + "-p" + ] + }, + "keyvault certificate backup": { + "flags": [ + "--file", + "-f" + ] + }, + "keyvault certificate download": { + "flags": [ + "--file", + "-f" + ] + }, + "keyvault certificate import": { + "flags": [ + "--file", + "-f" + ] + }, + "keyvault certificate pending merge": { + "flags": [ + "--file", + "-f" + ] + }, + "keyvault certificate restore": { + "flags": [ + "--file", + "-f" + ] + }, + "keyvault ekm-connection create": { + "flags": [ + "--server-ca-certificate" + ], + "greedy": [ + "--server-ca-certificate" + ] + }, + "keyvault ekm-connection update": { + "flags": [ + "--server-ca-certificate" + ], + "greedy": [ + "--server-ca-certificate" + ] + }, + "keyvault key backup": { + "flags": [ + "--file", + "-f" + ] + }, + "keyvault key create": { + "flags": [ + "--policy" + ] + }, + "keyvault key download": { + "flags": [ + "--file", + "-f" + ] + }, + "keyvault key get-attestation": { + "flags": [ + "--file", + "-f" + ] + }, + "keyvault key import": { + "flags": [ + "--byok-file", + "--byok-string", + "--pem-file", + "--pem-string", + "--policy" + ] + }, + "keyvault key restore": { + "flags": [ + "--file", + "-f" + ] + }, + "keyvault key rotation-policy update": { + "flags": [ + "--value" + ] + }, + "keyvault key set-attributes": { + "flags": [ + "--policy" + ] + }, + "keyvault secret backup": { + "flags": [ + "--file", + "-f" + ] + }, + "keyvault secret download": { + "flags": [ + "--file", + "-f" + ] + }, + "keyvault secret restore": { + "flags": [ + "--file", + "-f" + ] + }, + "keyvault secret set": { + "flags": [ + "--file", + "-f" + ] + }, + "managedapp create": { + "flags": [ + "--parameters" + ] + }, + "managedapp definition create": { + "flags": [ + "--create-ui-definition", + "--main-template", + "-c", + "-t" + ] + }, + "managedapp definition update": { + "flags": [ + "--create-ui-definition", + "--main-template", + "-c", + "-t" + ] + }, + "network application-gateway create": { + "flags": [ + "--cert-file" + ] + }, + "network dns zone export": { + "flags": [ + "--file-name", + "-f" + ] + }, + "network dns zone import": { + "flags": [ + "--file-name", + "-f" + ] + }, + "network private-dns zone export": { + "flags": [ + "--file-name", + "-f" + ] + }, + "network private-dns zone import": { + "flags": [ + "--file-name", + "-f" + ] + }, + "postgres flexible-server migration create": { + "flags": [ + "--properties", + "-b" + ] + }, + "stack group create": { + "flags": [ + "--parameters", + "--template-file", + "-f", + "-p" + ], + "greedy": [ + "--parameters", + "-p" + ] + }, + "stack group validate": { + "flags": [ + "--parameters", + "--template-file", + "-f", + "-p" + ], + "greedy": [ + "--parameters", + "-p" + ] + }, + "stack mg create": { + "flags": [ + "--parameters", + "--template-file", + "-f", + "-p" + ], + "greedy": [ + "--parameters", + "-p" + ] + }, + "stack mg validate": { + "flags": [ + "--parameters", + "--template-file", + "-f", + "-p" + ], + "greedy": [ + "--parameters", + "-p" + ] + }, + "stack sub create": { + "flags": [ + "--parameters", + "--template-file", + "-f", + "-p" + ], + "greedy": [ + "--parameters", + "-p" + ] + }, + "stack sub validate": { + "flags": [ + "--parameters", + "--template-file", + "-f", + "-p" + ], + "greedy": [ + "--parameters", + "-p" + ] + }, + "stack-whatif group create": { + "flags": [ + "--parameters", + "--template-file", + "-f", + "-p" + ], + "greedy": [ + "--parameters", + "-p" + ] + }, + "stack-whatif mg create": { + "flags": [ + "--parameters", + "--template-file", + "-f", + "-p" + ], + "greedy": [ + "--parameters", + "-p" + ] + }, + "stack-whatif sub create": { + "flags": [ + "--parameters", + "--template-file", + "-f", + "-p" + ], + "greedy": [ + "--parameters", + "-p" + ] + }, + "storage account blob-inventory-policy create": { + "flags": [ + "--policy" + ] + }, + "storage account management-policy create": { + "flags": [ + "--policy" + ] + }, + "storage blob download": { + "flags": [ + "--file", + "-f" + ] + }, + "storage blob upload": { + "flags": [ + "--file", + "-f" + ] + }, + "storage file download": { + "flags": [ + "--dest" + ] + }, + "storage file upload": { + "flags": [ + "--source" + ] + }, + "storage fs file download": { + "flags": [ + "--destination", + "-d" + ] + }, + "synapse data-flow create": { + "flags": [ + "--file" + ] + }, + "synapse data-flow set": { + "flags": [ + "--file" + ] + }, + "synapse dataset create": { + "flags": [ + "--file" + ] + }, + "synapse dataset set": { + "flags": [ + "--file" + ] + }, + "synapse dataset update": { + "flags": [ + "--file" + ] + }, + "synapse kql-script create": { + "flags": [ + "--file", + "-f" + ] + }, + "synapse kql-script import": { + "flags": [ + "--file", + "-f" + ] + }, + "synapse link-connection create": { + "flags": [ + "--file" + ] + }, + "synapse link-connection edit-link-tables": { + "flags": [ + "--file", + "-f" + ] + }, + "synapse link-connection update": { + "flags": [ + "--file" + ] + }, + "synapse linked-service create": { + "flags": [ + "--file" + ] + }, + "synapse linked-service set": { + "flags": [ + "--file" + ] + }, + "synapse linked-service update": { + "flags": [ + "--file" + ] + }, + "synapse managed-private-endpoints create": { + "flags": [ + "--file" + ] + }, + "synapse notebook create": { + "flags": [ + "--file" + ] + }, + "synapse notebook import": { + "flags": [ + "--file" + ] + }, + "synapse notebook set": { + "flags": [ + "--file" + ] + }, + "synapse pipeline create": { + "flags": [ + "--file" + ] + }, + "synapse pipeline create-run": { + "flags": [ + "--parameters" + ] + }, + "synapse pipeline set": { + "flags": [ + "--file" + ] + }, + "synapse pipeline update": { + "flags": [ + "--file" + ] + }, + "synapse spark statement invoke": { + "flags": [ + "--code" + ] + }, + "synapse spark-job-definition create": { + "flags": [ + "--file" + ] + }, + "synapse spark-job-definition update": { + "flags": [ + "--file" + ] + }, + "synapse sql-script create": { + "flags": [ + "--file", + "-f" + ] + }, + "synapse sql-script import": { + "flags": [ + "--file", + "-f" + ] + }, + "synapse trigger create": { + "flags": [ + "--file" + ] + }, + "synapse trigger set": { + "flags": [ + "--file" + ] + }, + "synapse trigger update": { + "flags": [ + "--file" + ] + }, + "synapse workspace-package upload": { + "flags": [ + "--file", + "--package", + "-f" + ] + }, + "ts create": { + "flags": [ + "--template-file", + "--ui-form-definition", + "-f" + ] + }, + "ts update": { + "flags": [ + "--template-file", + "--ui-form-definition", + "-f" + ] + }, + "vm create": { + "flags": [ + "--custom-data", + "--secrets", + "--ssh-key-values", + "--user-data" + ], + "greedy": [ + "--secrets", + "--ssh-key-values" + ] + }, + "vm diagnostics set": { + "flags": [ + "--protected-settings", + "--settings" + ] + }, + "vm update": { + "flags": [ + "--user-data" + ] + }, + "vmss create": { + "flags": [ + "--custom-data", + "--secrets", + "--ssh-key-values", + "--user-data" + ], + "greedy": [ + "--secrets", + "--ssh-key-values" + ] + }, + "vmss diagnostics set": { + "flags": [ + "--protected-settings", + "--settings" + ] + }, + "vmss update": { + "flags": [ + "--custom-data", + "--user-data" + ] + }, + "webapp config ssl upload": { + "flags": [ + "--certificate-file" + ] + }, + "webapp log download": { + "flags": [ + "--log-file" + ] + } + } +} diff --git a/src/lib/azure/bicep-install.test.ts b/src/lib/azure/bicep-install.test.ts new file mode 100644 index 0000000..c97bbd6 --- /dev/null +++ b/src/lib/azure/bicep-install.test.ts @@ -0,0 +1,163 @@ +import { createHash } from "crypto"; +import * as fs from "fs"; +import * as os from "os"; +import * as path from "path"; +import { + BICEP_ASSETS, + BICEP_VERSION, + bicepAssetFor, + bicepInstallDir, + bicepInstallPath, + installBicep, + type BicepAsset, + type Download, +} from "./bicep-install"; + +// The Bicep step. Every install goes into a temporary home: the real +// ~/.localstack is never touched, and nothing is downloaded (the download is injected). + +const sha = (b: Buffer) => createHash("sha256").update(b).digest("hex"); +const bytes = + (s: string): Download => + async () => + (async function* () { + // two chunks, so the hash is computed across chunk boundaries + yield Buffer.from(s.slice(0, 3)); + yield Buffer.from(s.slice(3)); + })(); + +describe("bicepAssetFor", () => { + test.each([ + ["win32", "x64", false, "bicep-win-x64.exe"], + ["win32", "arm64", false, "bicep-win-arm64.exe"], + ["linux", "x64", false, "bicep-linux-x64"], + ["linux", "x64", true, "bicep-linux-musl-x64"], + ["linux", "arm64", false, "bicep-linux-arm64"], + ["darwin", "x64", false, "bicep-osx-x64"], + ["darwin", "arm64", false, "bicep-osx-arm64"], + ] as const)("%s/%s (musl %s) is %s, from the pinned release", (platform, arch, musl, asset) => { + const found = bicepAssetFor(platform, arch, { musl }); + expect(found?.asset).toBe(asset); + expect(found?.url).toBe( + `https://github.com/Azure/bicep/releases/download/v${BICEP_VERSION}/${asset}` + ); + expect(found?.sha256).toMatch(/^[0-9a-f]{64}$/); + }); + + test("no pinned build: undefined (the wizard then says so)", () => { + expect(bicepAssetFor("freebsd", "x64")).toBeUndefined(); + expect(bicepAssetFor("linux", "ia32")).toBeUndefined(); + expect(bicepAssetFor("linux", "arm64", { musl: true })).toBeUndefined(); + }); + + test("the Linux sha256 values equal the Dockerfile's ADD --checksum pins", () => { + const dockerfile = fs.readFileSync(path.join(__dirname, "../../../Dockerfile"), "utf8"); + const pins = [...dockerfile.matchAll(/--checksum=sha256:([0-9a-f]{64})[^\n]*\n\s*(\S+)/g)].map( + (m) => [m[2].split("/").pop(), m[1]] + ); + expect(Object.fromEntries(pins)).toEqual({ + "bicep-linux-x64": BICEP_ASSETS["linux-x64"].sha256, + "bicep-linux-arm64": BICEP_ASSETS["linux-arm64"].sha256, + }); + expect(dockerfile).toContain(`/download/v${BICEP_VERSION}/`); + }); +}); + +describe("the install target", () => { + test("is ~/.localstack/azure/bin, never ~/.azure/bin", () => { + for (const platform of ["win32", "linux", "darwin"] as const) { + const home = platform === "win32" ? "C:\\Users\\me" : "/home/me"; + const target = bicepInstallPath(home, platform); + const segments = target.split(/[\\/]/); + expect(segments).not.toContain(".azure"); + expect(segments.slice(-4)).toEqual([ + ".localstack", + "azure", + "bin", + platform === "win32" ? "bicep.exe" : "bicep", + ]); + expect(target.startsWith(bicepInstallDir(home, platform))).toBe(true); + } + }); +}); + +describe("installBicep", () => { + let home: string; + beforeEach(() => { + home = fs.mkdtempSync(path.join(os.tmpdir(), "lsmcp-bicep-")); + }); + afterEach(() => fs.rmSync(home, { recursive: true, force: true })); + + const platform = process.platform; + const key = `${platform}-${process.arch}`; + const table = (content: string): Record => ({ + [key]: { asset: "bicep-test", sha256: sha(Buffer.from(content)) }, + }); + const binDir = () => bicepInstallDir(home, platform); + + test("a matching download is installed at the target, executable, with nothing left over", async () => { + const result = await installBicep({ + platform, + arch: process.arch, + homedir: home, + download: bytes("fake-bicep-binary"), + assets: table("fake-bicep-binary"), + }); + expect(result.path).toBe(bicepInstallPath(home, platform)); + expect(fs.readFileSync(result.path, "utf8")).toBe("fake-bicep-binary"); + expect(fs.readdirSync(binDir())).toEqual([path.basename(result.path)]); + if (platform !== "win32") expect(fs.statSync(result.path).mode & 0o111).not.toBe(0); + }); + + test("a sha256 mismatch aborts and leaves no file", async () => { + await expect( + installBicep({ + platform, + arch: process.arch, + homedir: home, + download: bytes("tampered-binary"), + assets: table("the-real-binary"), + }) + ).rejects.toThrow(/sha256 .* not the pinned .*nothing was installed/); + expect(fs.readdirSync(binDir())).toEqual([]); + }); + + test("a mismatch leaves an existing Bicep untouched", async () => { + fs.mkdirSync(binDir(), { recursive: true }); + const target = bicepInstallPath(home, platform); + fs.writeFileSync(target, "the-old-bicep"); + await expect( + installBicep({ + platform, + arch: process.arch, + homedir: home, + download: bytes("tampered-binary"), + assets: table("the-real-binary"), + }) + ).rejects.toThrow(/sha256/); + expect(fs.readFileSync(target, "utf8")).toBe("the-old-bicep"); + expect(fs.readdirSync(binDir())).toEqual([path.basename(target)]); + }); + + test("a failed download leaves no file", async () => { + const failing: Download = async () => { + throw new Error("HTTP 503"); + }; + await expect( + installBicep({ + platform, + arch: process.arch, + homedir: home, + download: failing, + assets: table("x"), + }) + ).rejects.toThrow(/HTTP 503/); + expect(fs.readdirSync(binDir())).toEqual([]); + }); + + test("a platform without a pinned build is an error that names the way out", async () => { + await expect( + installBicep({ platform: "freebsd", arch: "x64", homedir: home, download: bytes("x") }) + ).rejects.toThrow(/no pinned Bicep .* LOCALSTACK_AZ_BICEP_PATH/); + }); +}); diff --git a/src/lib/azure/bicep-install.ts b/src/lib/azure/bicep-install.ts new file mode 100644 index 0000000..d1965fa --- /dev/null +++ b/src/lib/azure/bicep-install.ts @@ -0,0 +1,150 @@ +import { createHash, randomBytes } from "crypto"; +import * as fs from "fs"; +import * as path from "path"; +import { Readable, Transform } from "stream"; +import { pipeline } from "stream/promises"; + +/** + * The Bicep step of `install-azure-addons`: the pinned Bicep + * release for this OS and architecture, downloaded from GitHub and checked against the + * sha256 table below before it is moved into ~/.localstack/azure/bin, where the tool + * looks second (after LOCALSTACK_AZ_BICEP_PATH). Never agent-triggered, and never + * ~/.azure/bin, which belongs to the user's own `az bicep install`. + */ + +export const BICEP_VERSION = "0.47.16"; + +export interface BicepAsset { + asset: string; + sha256: string; +} + +/** Keyed `-` (Node's names); `linux-musl-x64` for musl libc (Alpine). */ +export const BICEP_ASSETS: Readonly> = { + "win32-x64": { + asset: "bicep-win-x64.exe", + sha256: "3f343ab1ce41feac156464adee3dc499cb6c197366fc731aed276192011d867c", + }, + "win32-arm64": { + asset: "bicep-win-arm64.exe", + sha256: "657e6aacc4e44d73674874f802d98396bf5e2e23530d160f275fbf9d67b7fde7", + }, + "linux-x64": { + asset: "bicep-linux-x64", + sha256: "64c345a58e0c3e48b1bc98a4e62d6b3adb1d238281297de3400aeafb2697aa5a", + }, + "linux-musl-x64": { + asset: "bicep-linux-musl-x64", + sha256: "ffa36eca49db30fb2d6a9b44cf9e49ef4946f0a26425714c8483f8f63cfff125", + }, + "linux-arm64": { + asset: "bicep-linux-arm64", + sha256: "4406214cc274cfac7c821552aec2178b80aec637d91ed8b244282964c1cf24e3", + }, + "darwin-x64": { + asset: "bicep-osx-x64", + sha256: "8ba5771b5261413d88583829f2ea24509eb65b06d899620c17283ecb60d5ca73", + }, + "darwin-arm64": { + asset: "bicep-osx-arm64", + sha256: "68046a084c88503cf6bd11dacf2a1c4ffcb7e3ac9c6b310d295e024af21bbea4", + }, +}; + +function assetKey(platform: NodeJS.Platform, arch: string, musl?: boolean): string { + return platform === "linux" && musl ? `linux-musl-${arch}` : `${platform}-${arch}`; +} + +function releaseUrl(asset: string): string { + return `https://github.com/Azure/bicep/releases/download/v${BICEP_VERSION}/${asset}`; +} + +export function bicepAssetFor( + platform: NodeJS.Platform, + arch: string, + opts: { musl?: boolean } = {} +): (BicepAsset & { url: string }) | undefined { + const asset = BICEP_ASSETS[assetKey(platform, arch, opts.musl)]; + return asset ? { ...asset, url: releaseUrl(asset.asset) } : undefined; +} + +/** ~/.localstack/azure/bin: the tool's own Bicep dir (resolveBicep's second choice). */ +export function bicepInstallDir(homedir: string, platform: NodeJS.Platform): string { + const api = platform === "win32" ? path.win32 : path.posix; + return api.join(homedir, ".localstack", "azure", "bin"); +} + +export function bicepInstallPath(homedir: string, platform: NodeJS.Platform): string { + const api = platform === "win32" ? path.win32 : path.posix; + return api.join(bicepInstallDir(homedir, platform), platform === "win32" ? "bicep.exe" : "bicep"); +} + +/** musl libc (Alpine): Node's report has no glibc runtime version there. */ +export function isMusl(): boolean { + if (process.platform !== "linux") return false; + const report = process.report?.getReport() as { header?: { glibcVersionRuntime?: string } }; + return !report?.header?.glibcVersionRuntime; +} + +export type Download = (url: string) => Promise>; + +/** The release asset, following GitHub's redirect to its object store. */ +export const httpsDownload: Download = async (url) => { + const response = await fetch(url, { redirect: "follow" }); + if (!response.ok || !response.body) { + throw new Error(`downloading ${url} failed: HTTP ${response.status}`); + } + return Readable.fromWeb(response.body as import("stream/web").ReadableStream); +}; + +/** + * Downloads the pinned asset into a temporary file next to the target, checks its + * sha256, and only then renames it into place. On any failure, a mismatch included, + * the temporary file is removed and nothing is installed. + */ +export async function installBicep(opts: { + platform: NodeJS.Platform; + arch: string; + homedir: string; + musl?: boolean; + download?: Download; + /** Tests: another sha256 table. */ + assets?: Readonly>; +}): Promise<{ path: string; asset: string }> { + const pinned = (opts.assets ?? BICEP_ASSETS)[assetKey(opts.platform, opts.arch, opts.musl)]; + if (!pinned) { + throw new Error( + `there is no pinned Bicep ${BICEP_VERSION} build for ${opts.platform}/${opts.arch}; install Bicep yourself and set LOCALSTACK_AZ_BICEP_PATH` + ); + } + const url = releaseUrl(pinned.asset); + const target = bicepInstallPath(opts.homedir, opts.platform); + const dir = path.dirname(target); + fs.mkdirSync(dir, { recursive: true }); + const temp = path.join(dir, `.bicep-download-${randomBytes(6).toString("hex")}`); + const hash = createHash("sha256"); + try { + const body = await (opts.download ?? httpsDownload)(url); + await pipeline( + Readable.from(body), + new Transform({ + transform(chunk: Buffer, _encoding, callback) { + hash.update(chunk); + callback(null, chunk); + }, + }), + fs.createWriteStream(temp, { mode: 0o755 }) + ); + const digest = hash.digest("hex"); + if (digest !== pinned.sha256) { + throw new Error( + `the downloaded ${pinned.asset} has sha256 ${digest}, not the pinned ${pinned.sha256}; nothing was installed` + ); + } + if (opts.platform !== "win32") fs.chmodSync(temp, 0o755); + fs.renameSync(temp, target); + return { path: target, asset: pinned.asset }; + } finally { + fs.rmSync(temp, { force: true }); + } +} diff --git a/src/lib/azure/bootstrap.test.ts b/src/lib/azure/bootstrap.test.ts new file mode 100644 index 0000000..903eb37 --- /dev/null +++ b/src/lib/azure/bootstrap.test.ts @@ -0,0 +1,474 @@ +import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "fs"; +import os from "os"; +import path from "path"; +import { + CLI_CONFIG, + ConfigDirInUseError, + BootstrapError, + DUMMY_LOGIN, + ensureAzureCliConfigured, + freshLeaseFor, + LEASE_DIR, + LOCK_DIR, + MARKER_FILE, + needsRebootstrap, + readMarker, + ReadWriteLock, + resetBootstrapState, + runInProfile, + runWithSelfHeal, + STALE_LOCK_MS, + versionCheckSeed, + versionsFromAzVersionJson, + versionsFromLocalVersionsJson, + VERSION_CHECK_FILE, + type BootstrapDeps, + type BootstrapTarget, +} from "./bootstrap"; +import type { AzRunner, AzRunOptions, AzRunResult } from "./types"; + +// With a fake runner that records argv sequences. + +const ENDPOINT = "https://azure.localhost.localstack.cloud:4566"; +const ok = (stdout = ""): AzRunResult => ({ + exitCode: 0, + stdout, + stderr: "", + timedOut: false, + aborted: false, + truncated: false, + durationMs: 1, + egress: { refused: [], upstream: [], housekeeping: [], allowed: 0 }, +}); +const fail = (stderr: string, exitCode = 1): AzRunResult => ({ ...ok(), exitCode, stderr }); + +type Responder = (argv: string[], call: number) => AzRunResult | Promise; + +function fakeRunner( + respond: Responder = (argv) => (argv[0] === "cloud" && argv[1] === "list" ? ok("") : ok()) +) { + const calls: string[][] = []; + const runner: AzRunner = { + run: jest.fn(async (argv: string[], _opts: AzRunOptions) => { + calls.push(argv); + return respond(argv, calls.length); + }), + }; + return { runner, calls }; +} + +let root: string; +let configDir: string; +let target: BootstrapTarget; + +beforeEach(() => { + resetBootstrapState(); + root = mkdtempSync(path.join(os.tmpdir(), "lsaz-boot-")); + configDir = path.join(root, "mcp-config-4566"); + target = { + configDir, + endpoint: ENDPOINT, + sessionId: "s-1", + azVersion: "2.87.0", + cwd: root, + timeoutMs: 60_000, + }; +}); + +afterEach(() => { + rmSync(root, { recursive: true, force: true }); +}); + +const LOCAL_VERSIONS = { "azure-cli": "2.87.0", core: "2.87.0", telemetry: "1.1.0" }; +const deps = (runner: AzRunner, over: Partial = {}): BootstrapDeps => ({ + runner, + readLocalVersions: async () => LOCAL_VERSIONS, + hostname: "test-host", + ...over, +}); + +describe("the five calls", () => { + test("exactly five calls, with the cloud JSON as one element and the CLI config values", async () => { + const { runner, calls } = fakeRunner(); + await ensureAzureCliConfigured(target, deps(runner)); + expect(calls).toEqual([ + ["cloud", "list", "--query", "[?name=='LocalStack'].name", "-o", "tsv"], + [ + "cloud", + "register", + "--name", + "LocalStack", + "--cloud-config", + expect.any(String), + "--only-show-errors", + ], + ["cloud", "set", "--name", "LocalStack", "--only-show-errors"], + ["config", "set", ...CLI_CONFIG, "--only-show-errors"], + DUMMY_LOGIN, + ]); + expect(JSON.parse(calls[1][5])).toEqual({ + endpoints: { + activeDirectory: ENDPOINT, + activeDirectoryResourceId: ENDPOINT, + activeDirectoryGraphResourceId: ENDPOINT, + management: `${ENDPOINT}/`, + microsoftGraphResourceId: `${ENDPOINT}/`, + resourceManager: `${ENDPOINT}/`, + logAnalyticsResourceId: ENDPOINT, + }, + }); + expect(CLI_CONFIG).toEqual([ + "core.instance_discovery=false", + "core.collect_telemetry=false", + "output.show_survey_link=no", + "extension.use_dynamic_install=no", + "core.error_recommendation=off", + "core.output=json", + "core.display_region_identified=false", + "bicep.use_binary_from_path=true", + "bicep.check_version=false", + "auto-upgrade.enable=no", + ]); + }); + + test("an existing LocalStack cloud (cloud list's stdout) is updated, not registered", async () => { + const { runner, calls } = fakeRunner((argv) => + argv[1] === "list" ? ok("LocalStack\r\n") : ok() + ); + await ensureAzureCliConfigured(target, deps(runner)); + expect(calls[1].slice(0, 2)).toEqual(["cloud", "update"]); + }); + + test("a failed cloud list means register", async () => { + const { runner, calls } = fakeRunner((argv) => + argv[1] === "list" ? fail("ERROR: boom") : ok() + ); + await ensureAzureCliConfigured(target, deps(runner)); + expect(calls[1].slice(0, 2)).toEqual(["cloud", "register"]); + }); + + test("a matching marker skips the bootstrap entirely", async () => { + const first = fakeRunner(); + await ensureAzureCliConfigured(target, deps(first.runner)); + const second = fakeRunner(); + await ensureAzureCliConfigured(target, deps(second.runner)); + expect(second.calls).toEqual([]); + }); + + test.each([ + ["the endpoint", { endpoint: "https://localhost.localstack.cloud:4566" }], + ["the emulator session", { sessionId: "s-2" }], + ["the az version", { azVersion: "2.90.0" }], + ])("a change of %s re-bootstraps", async (_label, change) => { + await ensureAzureCliConfigured(target, deps(fakeRunner().runner)); + const again = fakeRunner(); + await ensureAzureCliConfigured({ ...target, ...change }, deps(again.runner)); + expect(again.calls).toHaveLength(5); + }); +}); + +describe("the versionCheck.json seed", () => { + test("written BEFORE the first az call, with the exact keys and times", async () => { + let seenAtFirstCall: unknown; + const { runner } = fakeRunner((argv, n) => { + if (n === 1) + seenAtFirstCall = JSON.parse( + readFileSync(path.join(configDir, VERSION_CHECK_FILE), "utf8") + ); + return argv[1] === "list" ? ok("") : ok(); + }); + await ensureAzureCliConfigured(target, deps(runner)); + expect(seenAtFirstCall).toEqual({ + versions: { + "azure-cli": { local: "2.87.0", pypi: "2.87.0" }, + core: { local: "2.87.0", pypi: "2.87.0" }, + telemetry: { local: "1.1.0", pypi: "1.1.0" }, + }, + update_time: "2999-01-01 00:00:00.000000", + check_time: "2999-01-01 00:00:00.000000", + }); + }); + + test("re-seeded when the az version changes", async () => { + await ensureAzureCliConfigured(target, deps(fakeRunner().runner)); + const newer = { "azure-cli": "2.90.0", core: "2.90.0", telemetry: "1.1.0" }; + await ensureAzureCliConfigured( + { ...target, azVersion: "2.90.0" }, + deps(fakeRunner().runner, { readLocalVersions: async () => newer }) + ); + const seed = JSON.parse(readFileSync(path.join(configDir, VERSION_CHECK_FILE), "utf8")); + expect(seed.versions.core.local).toBe("2.90.0"); + }); + + test("the seed always has core, falling back to the probed version", () => { + expect(versionCheckSeed({}, "2.85.0").versions).toEqual({ + core: { local: "2.85.0", pypi: "2.85.0" }, + "azure-cli": { local: "2.85.0", pypi: "2.85.0" }, + }); + expect(() => versionCheckSeed({})).toThrow(/core/); + }); + + test("the launcher-as-is seed comes from az version -o json", () => { + const azVersion = JSON.stringify({ + "azure-cli": "2.90.0", + "azure-cli-core": "2.90.0", + "azure-cli-telemetry": "1.1.0", + extensions: { "resource-graph": "2.1.1" }, + }); + expect(versionsFromAzVersionJson(azVersion)).toEqual({ + "azure-cli": "2.90.0", + core: "2.90.0", + telemetry: "1.1.0", + }); + }); + + test("the interpreter probe's _get_local_versions() output is flattened", () => { + const text = JSON.stringify({ + "azure-cli": { local: "2.87.0" }, + core: { local: "2.87.0" }, + telemetry: { local: "1.1.0" }, + }); + expect(versionsFromLocalVersionsJson(text)).toEqual(LOCAL_VERSIONS); + }); +}); + +describe("the marker", () => { + test("written only after success, with the endpoint, session and az version", async () => { + await ensureAzureCliConfigured(target, deps(fakeRunner().runner)); + expect(readMarker(configDir)).toMatchObject({ + endpoint: ENDPOINT, + sessionId: "s-1", + azVersion: "2.87.0", + }); + }); + + test("a partial failure leaves no marker, and the error names the step with az's output", async () => { + const { runner } = fakeRunner((argv) => + argv[0] === "login" + ? fail("ERROR: Unable to get endpoints from the cloud.", 1) + : argv[1] === "list" + ? ok("") + : ok() + ); + const error = await ensureAzureCliConfigured(target, deps(runner)).catch((e) => e); + expect(error).toBeInstanceOf(BootstrapError); + expect((error as BootstrapError).step).toBe("login"); + expect((error as BootstrapError).result.stderr).toContain("Unable to get endpoints"); + expect(existsSync(path.join(configDir, MARKER_FILE))).toBe(false); + expect(existsSync(path.join(configDir, LOCK_DIR))).toBe(false); + }); +}); + +describe("locks", () => { + test("two concurrent calls give one bootstrap", async () => { + const { runner, calls } = fakeRunner(async (argv) => { + await new Promise((r) => setTimeout(r, 20)); + return argv[1] === "list" ? ok("") : ok(); + }); + await Promise.all([ + ensureAzureCliConfigured(target, deps(runner)), + ensureAzureCliConfigured(target, deps(runner)), + ]); + expect(calls).toHaveLength(5); + }); + + test("another process's bootstrap (its lock dir) is waited for, and not repeated", async () => { + mkdirSync(path.join(configDir, LOCK_DIR), { recursive: true }); + writeFileSync( + path.join(configDir, LOCK_DIR, "owner.json"), + JSON.stringify({ createdAt: Date.now(), hostname: "other" }) + ); + const { runner, calls } = fakeRunner(); + let slept = 0; + const sleep = async (ms: number) => { + slept++; + if (slept === 3) { + // The other process finishes: it writes the marker and releases its lock. + writeFileSync( + path.join(configDir, MARKER_FILE), + JSON.stringify({ + endpoint: ENDPOINT, + sessionId: "s-1", + azVersion: "2.87.0", + createdAt: "x", + }) + ); + rmSync(path.join(configDir, LOCK_DIR), { recursive: true, force: true }); + } + await new Promise((r) => setTimeout(r, Math.min(ms, 5))); + }; + await ensureAzureCliConfigured(target, deps(runner, { sleep })); + expect(slept).toBeGreaterThanOrEqual(3); + expect(calls).toEqual([]); + }); + + test("a lock dir older than 5 minutes is broken", async () => { + mkdirSync(path.join(configDir, LOCK_DIR), { recursive: true }); + writeFileSync( + path.join(configDir, LOCK_DIR, "owner.json"), + JSON.stringify({ createdAt: Date.now() - STALE_LOCK_MS - 1000, hostname: "crashed" }) + ); + const { runner, calls } = fakeRunner(); + await ensureAzureCliConfigured(target, deps(runner)); + expect(calls).toHaveLength(5); + }); + + test("a fresh lock is respected until it goes stale", async () => { + mkdirSync(path.join(configDir, LOCK_DIR), { recursive: true }); + writeFileSync( + path.join(configDir, LOCK_DIR, "owner.json"), + JSON.stringify({ createdAt: 1_000_000, hostname: "busy" }) + ); + let clock = 1_000_000; + const { runner, calls } = fakeRunner(); + const sleep = async (ms: number) => { + clock += ms * 100; // 25 s per poll: stale after about 12 polls + }; + await ensureAzureCliConfigured(target, deps(runner, { now: () => clock, sleep })); + expect(clock - 1_000_000).toBeGreaterThan(STALE_LOCK_MS); + expect(calls).toHaveLength(5); + }); + + test("a re-bootstrap waits for in-flight commands", async () => { + await ensureAzureCliConfigured(target, deps(fakeRunner().runner)); + let release!: () => void; + const order: string[] = []; + const { runner } = fakeRunner(async (argv) => { + if (argv[0] === "group") { + order.push("command:start"); + await new Promise((r) => (release = r)); + order.push("command:end"); + return ok("[]"); + } + order.push(`bootstrap:${argv[0]} ${argv[1] ?? ""}`.trim()); + return argv[1] === "list" ? ok("") : ok(); + }); + const command = runInProfile(target, runner, ["group", "list"], { timeoutMs: 1000, cwd: root }); + await new Promise((r) => setTimeout(r, 20)); + const rebootstrap = ensureAzureCliConfigured(target, deps(runner), { force: true }); + await new Promise((r) => setTimeout(r, 50)); + expect(order).toEqual(["command:start"]); + release(); + await Promise.all([command, rebootstrap]); + expect(order.slice(0, 3)).toEqual(["command:start", "command:end", "bootstrap:cloud list"]); + }); + + test("the readers-writer lock prefers a waiting writer over new readers", async () => { + const lock = new ReadWriteLock(); + const events: string[] = []; + let releaseFirst!: () => void; + const r1 = lock.read(() => + new Promise((r) => (releaseFirst = r)).then(() => void events.push("r1")) + ); + await new Promise((r) => setTimeout(r, 5)); + const w = lock.write(async () => void events.push("w")); + const r2 = lock.read(async () => void events.push("r2")); + await new Promise((r) => setTimeout(r, 5)); + expect(lock.state).toEqual({ readers: 1, writer: false, waitingWriters: 1 }); + releaseFirst(); + await Promise.all([r1, w, r2]); + expect(events).toEqual(["r1", "w", "r2"]); + }); +}); + +describe("a shared config dir", () => { + test("a marker for another endpoint with that process's fresh lease fails with a clear error", async () => { + mkdirSync(path.join(configDir, LEASE_DIR), { recursive: true }); + const other = "https://azure.localhost.localstack.cloud:4666"; + writeFileSync( + path.join(configDir, MARKER_FILE), + JSON.stringify({ endpoint: other, createdAt: "x" }) + ); + writeFileSync( + path.join(configDir, LEASE_DIR, "other-process.json"), + JSON.stringify({ endpoint: other, hostname: "h", updatedAt: Date.now() }) + ); + const { runner, calls } = fakeRunner(); + await expect(ensureAzureCliConfigured(target, deps(runner))).rejects.toBeInstanceOf( + ConfigDirInUseError + ); + expect(calls).toEqual([]); + }); + + test("a stale lease of the other endpoint does not block a re-point", async () => { + mkdirSync(path.join(configDir, LEASE_DIR), { recursive: true }); + const other = "https://azure.localhost.localstack.cloud:4666"; + writeFileSync( + path.join(configDir, MARKER_FILE), + JSON.stringify({ endpoint: other, createdAt: "x" }) + ); + writeFileSync( + path.join(configDir, LEASE_DIR, "other-process.json"), + JSON.stringify({ endpoint: other, hostname: "h", updatedAt: Date.now() - 16 * 60_000 }) + ); + const { runner, calls } = fakeRunner(); + await ensureAzureCliConfigured(target, deps(runner)); + expect(calls).toHaveLength(5); + }); + + test("this process's own lease never counts as another's", async () => { + await ensureAzureCliConfigured(target, deps(fakeRunner().runner)); + expect(freshLeaseFor(configDir, ENDPOINT, Date.now())).toBe(false); + }); +}); + +describe("self-heal", () => { + test.each([ + "ERROR: Please run 'az login' to setup account.", + 'WARNING: something first\nERROR: Please run "az login" to setup account.', + "ERROR: No subscription found for the tenant.", + "ERROR: Cloud 'LocalStack' is not registered.", + ])("%j triggers exactly one re-bootstrap and one retry", async (stderr) => { + await ensureAzureCliConfigured(target, deps(fakeRunner().runner)); + let commandRuns = 0; + const { runner, calls } = fakeRunner((argv) => { + if (argv[0] === "group") return ++commandRuns === 1 ? fail(stderr) : ok('[{"name":"rg1"}]'); + return argv[1] === "list" ? ok("LocalStack") : ok(); + }); + const result = await runWithSelfHeal(target, deps(runner), ["group", "list"], { + timeoutMs: 1000, + cwd: root, + }); + expect(result.stdout).toContain("rg1"); + expect(commandRuns).toBe(2); + expect(calls.filter((c) => c[0] !== "group")).toHaveLength(5); + }); + + test("a second login failure is returned as it is: no loop", async () => { + await ensureAzureCliConfigured(target, deps(fakeRunner().runner)); + const { runner, calls } = fakeRunner((argv) => + argv[0] === "group" + ? fail("ERROR: Please run 'az login' to setup account.") + : argv[1] === "list" + ? ok("") + : ok() + ); + const result = await runWithSelfHeal(target, deps(runner), ["group", "list"], { + timeoutMs: 1000, + cwd: root, + }); + expect(result.exitCode).toBe(1); + expect(calls.filter((c) => c[0] === "group")).toHaveLength(2); + }); + + test("a genuine error (resource not found) never triggers it", async () => { + await ensureAzureCliConfigured(target, deps(fakeRunner().runner)); + const { runner, calls } = fakeRunner(() => + fail("ERROR: (ResourceNotFound) Resource group 'x' could not be found.", 3) + ); + const result = await runWithSelfHeal(target, deps(runner), ["group", "show", "-n", "x"], { + timeoutMs: 1000, + cwd: root, + }); + expect(result.exitCode).toBe(3); + expect(calls).toEqual([["group", "show", "-n", "x"]]); + }); + + test("needsRebootstrap ignores successes, timeouts and cancellations", () => { + const login = "ERROR: Please run 'az login' to setup account."; + expect(needsRebootstrap(fail(login))).toBe(true); + expect(needsRebootstrap({ ...ok(), stderr: login })).toBe(false); + expect(needsRebootstrap({ ...fail(login), timedOut: true })).toBe(false); + expect(needsRebootstrap({ ...fail(login), aborted: true })).toBe(false); + }); +}); diff --git a/src/lib/azure/bootstrap.ts b/src/lib/azure/bootstrap.ts new file mode 100644 index 0000000..b0a9479 --- /dev/null +++ b/src/lib/azure/bootstrap.ts @@ -0,0 +1,576 @@ +import { randomUUID } from "crypto"; +import { + mkdirSync, + readdirSync, + readFileSync, + renameSync, + rmSync, + statSync, + unlinkSync, + writeFileSync, +} from "fs"; +import os from "os"; +import path from "path"; +import type { AzRunner, AzRunOptions, AzRunResult } from "./types"; + +/** + * The isolated CLI profile. Exactly five `az` calls + * point the config dir at the emulator: `cloud list`, `cloud register|update`, + * `cloud set`, `config set` and a dummy `login`. The bootstrap is the only caller of + * those commands; the policy refuses them for the agent. + */ + +export const LOCALSTACK_CLOUD = "LocalStack"; + +/** The CLI config values, set in one call. */ +export const CLI_CONFIG = [ + "core.instance_discovery=false", + "core.collect_telemetry=false", + "output.show_survey_link=no", + "extension.use_dynamic_install=no", + // az 2.85 sends failed command names, with their parameter names, to + // app.aladdin.microsoft.com. + "core.error_recommendation=off", + "core.output=json", + // Removes `vm create`'s region cost notice. + "core.display_region_identified=false", + // Bicep from the child's PATH, and no aka.ms version lookups. + "bicep.use_binary_from_path=true", + "bicep.check_version=false", + // Never spawn `az upgrade` (read from the source). + "auto-upgrade.enable=no", +]; + +export const DUMMY_LOGIN = [ + "login", + "--service-principal", + "-u", + "any-app", + "-p", + "any-pass", + "--tenant", + "anytenant", + "--only-show-errors", +]; + +export function cloudConfigJson(endpoint: string): string { + return JSON.stringify({ + endpoints: { + activeDirectory: endpoint, + activeDirectoryResourceId: endpoint, + activeDirectoryGraphResourceId: endpoint, + management: `${endpoint}/`, + microsoftGraphResourceId: `${endpoint}/`, + resourceManager: `${endpoint}/`, + logAnalyticsResourceId: endpoint, + }, + }); +} + +export const MARKER_FILE = "localstack-mcp-bootstrap.json"; +export const LOCK_DIR = "localstack-mcp-bootstrap.lock"; +export const LEASE_DIR = "localstack-mcp-leases"; +export const VERSION_CHECK_FILE = "versionCheck.json"; +/** A server killed mid-bootstrap leaves its lock behind; one this old is broken. */ +export const STALE_LOCK_MS = 5 * 60_000; +/** Another server's lease younger than this means it is using the config dir. */ +export const FRESH_LEASE_MS = 15 * 60_000; +const LEASE_REFRESH_MS = 60_000; +const LOCK_POLL_MS = 250; + +/** + * Failures that mean the profile is not (or no longer) pointed at the emulator: one + * re-bootstrap and one retry. A + * genuine error, such as a missing resource, never matches. + */ +export const SELF_HEAL_PATTERNS = [ + /^ERROR: Please run ['"]?az login['"]? to setup account\./im, + /please run ['"]?az login/i, + /no subscriptions? found for/i, + /cloud ['"]?[^'"\n]*['"]? (is )?not (registered|found)/i, +]; + +export function needsRebootstrap(result: AzRunResult): boolean { + if (result.exitCode === 0 || result.timedOut || result.aborted) return false; + return SELF_HEAL_PATTERNS.some((p) => p.test(result.stderr)); +} + +export interface BootstrapMarker { + endpoint: string; + azVersion?: string; + sessionId?: string; + createdAt: string; +} + +export interface BootstrapTarget { + configDir: string; + endpoint: string; + sessionId?: string; + azVersion?: string; + /** The runner's cwd: always the workdir. */ + cwd: string; + timeoutMs: number; +} + +export interface BootstrapDeps { + /** Runs the bootstrap's own steps (always a subprocess: each writes CLI state). */ + runner: AzRunner; + /** Runs the agent's command; the warm worker with LOCALSTACK_AZ_RUNNER=worker. Default: runner. */ + commandRunner?: AzRunner; + /** + * The local versions for the update-check seed: one interpreter call of + * `_get_local_versions()`, or `az version -o json` for a launcher spawned as-is. + * Keys as az uses them: `azure-cli`, `core`, `telemetry`. + */ + readLocalVersions(): Promise>; + now?: () => number; + sleep?: (ms: number) => Promise; + hostname?: string; +} + +/** A bootstrap step failed; `result` carries az's output for the error hints. */ +export class BootstrapError extends Error { + constructor( + readonly step: string, + readonly result: AzRunResult + ) { + super(`The Azure CLI profile bootstrap failed at \`az ${step}\` (exit ${result.exitCode}).`); + this.name = "BootstrapError"; + } +} + +/** Another server is using this config dir with another endpoint. */ +export class ConfigDirInUseError extends Error { + constructor(configDir: string, endpoint: string) { + super( + `The Azure CLI config dir ${configDir} is in use by another LocalStack MCP server for ${endpoint}. ` + + `Re-pointing it would move that server's commands to this emulator. Give each server its own ` + + `LOCALSTACK_AZ_CONFIG_DIR (the default is one per port: ~/.localstack/azure/mcp-config-).` + ); + this.name = "ConfigDirInUseError"; + } +} + +// --------------------------------------------------------------------------- +// Locks: commands are readers, the bootstrap is the writer +// --------------------------------------------------------------------------- + +/** A writer-preferring readers-writer lock, so a re-bootstrap never starves. */ +export class ReadWriteLock { + private readers = 0; + private writer = false; + private waitingWriters = 0; + private readonly waiters: Array<() => void> = []; + + private pump() { + const pending = this.waiters.splice(0); + for (const w of pending) w(); + } + + private async acquireRead() { + while (this.writer || this.waitingWriters > 0) { + await new Promise((r) => this.waiters.push(r)); + } + this.readers++; + } + + private async acquireWrite() { + this.waitingWriters++; + try { + while (this.writer || this.readers > 0) { + await new Promise((r) => this.waiters.push(r)); + } + } finally { + this.waitingWriters--; + } + this.writer = true; + } + + async read(fn: () => Promise): Promise { + await this.acquireRead(); + try { + return await fn(); + } finally { + this.readers--; + this.pump(); + } + } + + async write(fn: () => Promise): Promise { + await this.acquireWrite(); + try { + return await fn(); + } finally { + this.writer = false; + this.pump(); + } + } + + get state() { + return { readers: this.readers, writer: this.writer, waitingWriters: this.waitingWriters }; + } +} + +const locks = new Map(); + +export function profileLock(configDir: string): ReadWriteLock { + const key = path.resolve(configDir).toLowerCase(); + let lock = locks.get(key); + if (!lock) { + lock = new ReadWriteLock(); + locks.set(key, lock); + } + return lock; +} + +/** Forget the in-process locks and this process's lease id (tests). */ +export function resetBootstrapState(): void { + locks.clear(); + leaseWrittenAt.clear(); + processLeaseId = randomUUID(); +} + +// --------------------------------------------------------------------------- +// Files: the marker, the seed, the lock dir and the leases +// --------------------------------------------------------------------------- + +function writeAtomic(file: string, text: string) { + const temp = `${file}.${process.pid}.${randomUUID()}.tmp`; + writeFileSync(temp, text, { mode: 0o600 }); + try { + renameSync(temp, file); + } catch { + // Windows refuses to replace a file another process has open (EPERM). Readers + // of these files tolerate a torn read (the marker re-bootstraps under the lock). + rmSync(temp, { force: true }); + writeFileSync(file, text, { mode: 0o600 }); + } +} + +export function readMarker(configDir: string): BootstrapMarker | undefined { + try { + const marker = JSON.parse(readFileSync(path.join(configDir, MARKER_FILE), "utf8")); + return typeof marker?.endpoint === "string" ? marker : undefined; + } catch { + return undefined; + } +} + +export function markerMatches( + marker: BootstrapMarker | undefined, + target: BootstrapTarget +): boolean { + return ( + Boolean(marker) && + marker!.endpoint === target.endpoint && + marker!.sessionId === target.sessionId && + marker!.azVersion === target.azVersion + ); +} + +/** + * The update-check seed. az reads `versions.core.local` on every + * command and logs any failure as a WARNING, `versions['azure-cli']` for `az + * version`, and parses both times with `%Y-%m-%d %H:%M:%S.%f`, adding 1 or 7 days + * (so the year 9999 would overflow). With these values az never checks for + * updates, which would reach azcliprod.blob.core.windows.net. + */ +export function versionCheckSeed(localVersions: Record, azVersion?: string) { + const versions: Record = { ...localVersions }; + if (azVersion) { + versions.core ??= azVersion; + versions["azure-cli"] ??= azVersion; + } + if (!versions.core) throw new Error("the update-check seed needs the azure-cli-core version"); + return { + versions: Object.fromEntries( + Object.entries(versions).map(([k, v]) => [k, { local: v, pypi: v }]) + ), + update_time: "2999-01-01 00:00:00.000000", + check_time: "2999-01-01 00:00:00.000000", + }; +} + +/** `az version -o json` → the seed's keys (for a launcher spawned as-is). */ +export function versionsFromAzVersionJson(text: string): Record { + const parsed = JSON.parse(text) as Record; + const out: Record = {}; + const map: Record = { + "azure-cli": "azure-cli", + "azure-cli-core": "core", + "azure-cli-telemetry": "telemetry", + }; + for (const [from, to] of Object.entries(map)) { + if (typeof parsed[from] === "string") out[to] = parsed[from] as string; + } + return out; +} + +/** `_get_local_versions()` output ({"core": {"local": "2.87.0"}, ...}) → flat versions. */ +export function versionsFromLocalVersionsJson(text: string): Record { + const parsed = JSON.parse(text) as Record; + const out: Record = {}; + for (const [k, v] of Object.entries(parsed)) { + if (typeof v?.local === "string") out[k] = v.local; + } + return out; +} + +interface LockOwner { + createdAt: number; + hostname: string; +} + +/** The lock's creation time: its owner file, else (not yet written, or a crash) the dir's age. */ +function lockCreatedAt(lockDir: string): number | undefined { + try { + return (JSON.parse(readFileSync(path.join(lockDir, "owner.json"), "utf8")) as LockOwner) + .createdAt; + } catch { + try { + return statSync(lockDir).mtimeMs; + } catch { + return undefined; + } + } +} + +/** + * The cross-process lock: a directory created with mkdir, holding {createdAt, + * hostname}. A lock older than 5 minutes is broken; a lock is never judged by + * checking or killing a PID. + */ +async function withLockDir( + configDir: string, + deps: Required>, + fn: () => Promise +): Promise { + const lockDir = path.join(configDir, LOCK_DIR); + for (;;) { + try { + mkdirSync(lockDir); + writeFileSync( + path.join(lockDir, "owner.json"), + JSON.stringify({ createdAt: deps.now(), hostname: deps.hostname } satisfies LockOwner) + ); + break; + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "EEXIST") throw error; + const createdAt = lockCreatedAt(lockDir); + if (createdAt === undefined) continue; // released meanwhile + if (deps.now() - createdAt > STALE_LOCK_MS) { + // Break it by renaming: of two waiters that judged the same lock stale, only + // one rename succeeds, and a lock re-taken meanwhile is seen by the re-read. + try { + if (lockCreatedAt(lockDir) === createdAt) { + const broken = `${lockDir}.stale-${randomUUID()}`; + renameSync(lockDir, broken); + rmSync(broken, { recursive: true, force: true }); + } + } catch { + // another waiter broke it first + } + continue; + } + await deps.sleep(LOCK_POLL_MS); + } + } + try { + return await fn(); + } finally { + rmSync(lockDir, { recursive: true, force: true }); + } +} + +let processLeaseId = randomUUID(); +const leaseWrittenAt = new Map(); +let leaseExitHook = false; + +interface Lease { + endpoint: string; + hostname: string; + updatedAt: number; +} + +/** Record that this process uses the config dir for its endpoint (refreshed each minute). */ +export function touchLease( + configDir: string, + endpoint: string, + now: number, + hostname: string +): void { + const key = `${configDir}\n${endpoint}`; + if (now - (leaseWrittenAt.get(key) ?? -Infinity) < LEASE_REFRESH_MS) return; + const dir = path.join(configDir, LEASE_DIR); + mkdirSync(dir, { recursive: true }); + const file = path.join(dir, `${processLeaseId}.json`); + writeAtomic(file, JSON.stringify({ endpoint, hostname, updatedAt: now } satisfies Lease)); + leaseWrittenAt.set(key, now); + if (!leaseExitHook) { + leaseExitHook = true; + process.on("exit", () => { + for (const k of leaseWrittenAt.keys()) { + try { + unlinkSync(path.join(k.split("\n")[0], LEASE_DIR, `${processLeaseId}.json`)); + } catch { + // already gone + } + } + }); + } +} + +/** Another process's fresh lease for `endpoint`. */ +export function freshLeaseFor(configDir: string, endpoint: string, now: number): boolean { + const dir = path.join(configDir, LEASE_DIR); + let names: string[]; + try { + names = readdirSync(dir); + } catch { + return false; + } + return names.some((name) => { + if (name === `${processLeaseId}.json` || !name.endsWith(".json")) return false; + try { + const lease = JSON.parse(readFileSync(path.join(dir, name), "utf8")) as Lease; + return lease.endpoint === endpoint && now - lease.updatedAt < FRESH_LEASE_MS; + } catch { + return false; + } + }); +} + +// --------------------------------------------------------------------------- +// The bootstrap +// --------------------------------------------------------------------------- + +function withDefaults(deps: BootstrapDeps) { + return { + ...deps, + now: deps.now ?? (() => Date.now()), + sleep: deps.sleep ?? ((ms: number) => new Promise((r) => setTimeout(r, ms))), + hostname: deps.hostname ?? os.hostname(), + }; +} + +async function mustSucceed(step: string, pending: Promise): Promise { + const result = await pending; + if (result.exitCode !== 0) throw new BootstrapError(step, result); + return result; +} + +async function runBootstrap( + target: BootstrapTarget, + deps: ReturnType +): Promise { + const opts: AzRunOptions = { timeoutMs: target.timeoutMs, cwd: target.cwd }; + // A marker is written last, so a partial bootstrap leaves none (the next call retries). + rmSync(path.join(target.configDir, MARKER_FILE), { force: true }); + // The seed comes BEFORE the first az call: a fresh config dir checks for updates + // on its very first command. + const seed = versionCheckSeed(await deps.readLocalVersions(), target.azVersion); + writeAtomic(path.join(target.configDir, VERSION_CHECK_FILE), JSON.stringify(seed)); + + const listed = await deps.runner.run( + ["cloud", "list", "--query", `[?name=='${LOCALSTACK_CLOUD}'].name`, "-o", "tsv"], + opts + ); + // The stdout, not the result object, says whether the cloud exists. + const exists = listed.exitCode === 0 && listed.stdout.trim() === LOCALSTACK_CLOUD; + const verb = exists ? "update" : "register"; + await mustSucceed( + `cloud ${verb}`, + deps.runner.run( + [ + "cloud", + verb, + "--name", + LOCALSTACK_CLOUD, + "--cloud-config", + cloudConfigJson(target.endpoint), + "--only-show-errors", + ], + opts + ) + ); + await mustSucceed( + "cloud set", + deps.runner.run(["cloud", "set", "--name", LOCALSTACK_CLOUD, "--only-show-errors"], opts) + ); + await mustSucceed( + "config set", + deps.runner.run(["config", "set", ...CLI_CONFIG, "--only-show-errors"], opts) + ); + await mustSucceed("login", deps.runner.run(DUMMY_LOGIN, opts)); + + const marker: BootstrapMarker = { + endpoint: target.endpoint, + azVersion: target.azVersion, + sessionId: target.sessionId, + createdAt: new Date(deps.now()).toISOString(), + }; + writeAtomic(path.join(target.configDir, MARKER_FILE), JSON.stringify(marker)); +} + +/** + * Make sure the config dir is bootstrapped for this endpoint, emulator session and + * az version. Re-bootstraps when any of them changed; `force` re-bootstraps anyway + * (the self-heal). Waits for in-flight commands first (the write lock), and for a + * bootstrap in another server process (the lock dir). + */ +export async function ensureAzureCliConfigured( + target: BootstrapTarget, + rawDeps: BootstrapDeps, + opts: { force?: boolean } = {} +): Promise { + const deps = withDefaults(rawDeps); + mkdirSync(target.configDir, { recursive: true, mode: 0o700 }); + touchLease(target.configDir, target.endpoint, deps.now(), deps.hostname); + if (!opts.force && markerMatches(readMarker(target.configDir), target)) return; + + const stale = opts.force ? readMarker(target.configDir) : undefined; + await profileLock(target.configDir).write(() => + withLockDir(target.configDir, deps, async () => { + const marker = readMarker(target.configDir); + // Another caller finished the job while this one waited. + if (markerMatches(marker, target) && (!opts.force || marker?.createdAt !== stale?.createdAt)) + return; + if ( + marker && + marker.endpoint !== target.endpoint && + freshLeaseFor(target.configDir, marker.endpoint, deps.now()) + ) { + throw new ConfigDirInUseError(target.configDir, marker.endpoint); + } + await runBootstrap(target, deps); + }) + ); +} + +/** Run one command under the profile's read lock. */ +export function runInProfile( + target: BootstrapTarget, + runner: AzRunner, + argv: string[], + opts: AzRunOptions +): Promise { + return profileLock(target.configDir).read(() => runner.run(argv, opts)); +} + +/** + * Run a command; if it fails because the profile is not logged in to the emulator, + * re-bootstrap once and retry once. A second failure is returned + * as it is, and the output layer gives the row-1 hint. + */ +export async function runWithSelfHeal( + target: BootstrapTarget, + deps: BootstrapDeps, + argv: string[], + opts: AzRunOptions +): Promise { + const runner = deps.commandRunner ?? deps.runner; + const first = await runInProfile(target, runner, argv, opts); + if (!needsRebootstrap(first) || opts.signal?.aborted) return first; + await ensureAzureCliConfigured(target, deps, { force: true }); + return runInProfile(target, runner, argv, opts); +} diff --git a/src/lib/azure/child-env.test.ts b/src/lib/azure/child-env.test.ts new file mode 100644 index 0000000..fb49197 --- /dev/null +++ b/src/lib/azure/child-env.test.ts @@ -0,0 +1,301 @@ +import { accessSync, constants, mkdtempSync, rmSync, statSync } from "fs"; +import os from "os"; +import path from "path"; +import { buildAzChildEnv, ensurePrivateDirs, privateDirs, type ChildEnvOptions } from "./child-env"; + +const winConfig = { + configDir: "C:\\Users\\me\\.localstack\\azure\\mcp-config-4566", + homeDir: "C:\\Users\\me\\.localstack\\azure\\mcp-config-4566\\home", + tmpDir: "C:\\Users\\me\\.localstack\\azure\\mcp-config-4566\\tmp", + extensionDir: "C:\\Users\\me\\.localstack\\azure\\mcp-extensions", + egressGuard: true, + inDocker: false, +}; +const posixConfig = { + configDir: "/home/me/.localstack/azure/mcp-config-4566", + homeDir: "/home/me/.localstack/azure/mcp-config-4566/home", + tmpDir: "/home/me/.localstack/azure/mcp-config-4566/tmp", + extensionDir: "/home/me/.localstack/azure/mcp-extensions", + egressGuard: true, + inDocker: false, +}; +const win = (over: Partial = {}): ChildEnvOptions => ({ + platform: "win32", + config: winConfig, + az: { installer: "msi", azInstaller: "MSI" }, + ...over, +}); +const posix = (over: Partial = {}): ChildEnvOptions => ({ + platform: "linux", + config: posixConfig, + az: { installer: "deb", azInstaller: "DEB" }, + ...over, +}); + +// The parent a user's shell could plausibly have: Azure logins, proxies, CA bundles +// and tools that change what az or Bicep does. +const POLLUTED: NodeJS.ProcessEnv = { + Path: "C:\\Windows\\system32;C:\\Program Files\\Microsoft SDKs\\Azure\\CLI2\\wbin", + SystemRoot: "C:\\Windows", + SystemDrive: "C:", + windir: "C:\\Windows", + ComSpec: "C:\\Windows\\system32\\cmd.exe", + PATHEXT: ".COM;.EXE;.BAT;.CMD", + USERNAME: "me", + COMPUTERNAME: "BOX", + NUMBER_OF_PROCESSORS: "8", + PROCESSOR_ARCHITECTURE: "AMD64", + USERPROFILE: "C:\\Users\\me", + HOME: "C:\\Users\\me", + APPDATA: "C:\\Users\\me\\AppData\\Roaming", + LOCALAPPDATA: "C:\\Users\\me\\AppData\\Local", + TEMP: "C:\\Users\\me\\AppData\\Local\\Temp", + TMP: "C:\\Users\\me\\AppData\\Local\\Temp", + AZURE_CONFIG_DIR: "C:\\Users\\x\\.azure", + AZURE_EXTENSION_DIR: "C:\\Users\\x\\.azure\\cliextensions", + AZURE_CLIENT_ID: "real-client", + AZURE_CLIENT_SECRET: "real-secret", + AZURE_TENANT_ID: "real-tenant", + AZURE_SUBSCRIPTION_ID: "real-sub", + ARM_CLIENT_ID: "tf-client", + ARM_CLIENT_SECRET: "tf-secret", + MSI_ENDPOINT: "http://169.254.169.254/metadata/identity", + IDENTITY_ENDPOINT: "http://localhost:42356/msi/token", + IDENTITY_HEADER: "secret-header", + HTTPS_PROXY: "http://corp-proxy:8080", + HTTP_PROXY: "http://corp-proxy:8080", + NO_PROXY: "*", + REQUESTS_CA_BUNDLE: "C:\\certs\\corp.pem", + SSL_CERT_FILE: "C:\\certs\\corp.pem", + PYTHONPATH: "C:\\evil", + PYTHONSTARTUP: "C:\\evil\\startup.py", + LOCALSTACK_AUTH_TOKEN: "ls-secret-token", +}; + +describe("buildAzChildEnv: the allow-list", () => { + test("a polluted parent yields a child with none of the polluting variables", () => { + const env = buildAzChildEnv(POLLUTED, win()); + for (const name of [ + "AZURE_CLIENT_ID", + "AZURE_CLIENT_SECRET", + "AZURE_TENANT_ID", + "AZURE_SUBSCRIPTION_ID", + "ARM_CLIENT_ID", + "ARM_CLIENT_SECRET", + "MSI_ENDPOINT", + "IDENTITY_ENDPOINT", + "IDENTITY_HEADER", + "HTTPS_PROXY", + "HTTP_PROXY", + "NO_PROXY", + "REQUESTS_CA_BUNDLE", + "SSL_CERT_FILE", + "PYTHONPATH", + "PYTHONSTARTUP", + "LOCALSTACK_AUTH_TOKEN", + ]) { + expect(env).not.toHaveProperty(name); + } + expect(Object.values(env).join("\n")).not.toMatch( + /real-secret|tf-secret|ls-secret-token|corp-proxy/ + ); + }); + + test("the required variables are set, and AZURE_CONFIG_DIR is always the isolated dir", () => { + const env = buildAzChildEnv(POLLUTED, win()); + expect(env).toMatchObject({ + AZURE_CONFIG_DIR: winConfig.configDir, + AZURE_EXTENSION_DIR: winConfig.extensionDir, + AZURE_CORE_COLLECT_TELEMETRY: "no", + AZURE_CORE_NO_COLOR: "1", + AZ_INSTALLER: "MSI", + SYSTEMROOT: "C:\\Windows", + SYSTEMDRIVE: "C:", + WINDIR: "C:\\Windows", + COMSPEC: "C:\\Windows\\system32\\cmd.exe", + PATHEXT: ".COM;.EXE;.BAT;.CMD", + USERNAME: "me", + COMPUTERNAME: "BOX", + NUMBER_OF_PROCESSORS: "8", + PROCESSOR_ARCHITECTURE: "AMD64", + }); + expect(env.PATH).toBe(POLLUTED.Path); + expect(env.AZURE_CONFIG_DIR).not.toBe(POLLUTED.AZURE_CONFIG_DIR); + }); + + test("with LOCALSTACK_AZ_EXTENSION_DIR set, the child's AZURE_EXTENSION_DIR equals it", () => { + const env = buildAzChildEnv( + POLLUTED, + win({ config: { ...winConfig, extensionDir: "D:\\az-ext" } }) + ); + expect(env.AZURE_EXTENSION_DIR).toBe("D:\\az-ext"); + }); + + test.each([ + "KUBECONFIG", + "DOCKER_COMMAND", + "GITHUB_ACTIONS", + "TF_BUILD", + "AZURE_EXTENSION_SYS_DIR", + "BICEP_TRACING_ENABLED", + "BICEP_TRUSTED_REGISTRIES", + "AZURE_BICEP_USE_BINARY_FROM_PATH", + "AZURE_BICEP_CHECK_VERSION", + "BROWSER", + ])("%s in the parent is absent from the child", (name) => { + expect(buildAzChildEnv({ ...POLLUTED, [name]: "x" }, win())).not.toHaveProperty(name); + expect(buildAzChildEnv({ PATH: "/usr/bin", [name]: "x" }, posix())).not.toHaveProperty(name); + }); + + test("no proxy variables are set by the env builder (the runner adds the guard's per call)", () => { + const env = buildAzChildEnv(POLLUTED, win({ config: { ...winConfig, egressGuard: false } })); + expect(Object.keys(env).filter((k) => /proxy/i.test(k))).toEqual([]); + }); + + test("LANG and LC_* are inherited on POSIX", () => { + const env = buildAzChildEnv( + { PATH: "/usr/bin", LANG: "de_DE.UTF-8", LC_TIME: "C", LC_ALL: "C.UTF-8" }, + posix() + ); + expect(env).toMatchObject({ LANG: "de_DE.UTF-8", LC_TIME: "C", LC_ALL: "C.UTF-8" }); + }); +}); + +describe("buildAzChildEnv: the private home and temp", () => { + test("Windows: HOME, USERPROFILE, APPDATA, LOCALAPPDATA, TEMP and TMP are the private ones", () => { + const env = buildAzChildEnv(POLLUTED, win()); + expect(env).toMatchObject({ + HOME: winConfig.homeDir, + USERPROFILE: winConfig.homeDir, + APPDATA: `${winConfig.homeDir}\\AppData\\Roaming`, + LOCALAPPDATA: `${winConfig.homeDir}\\AppData\\Local`, + TEMP: winConfig.tmpDir, + TMP: winConfig.tmpDir, + }); + for (const name of ["HOME", "USERPROFILE", "APPDATA", "LOCALAPPDATA", "TEMP", "TMP"]) { + expect(env[name]).not.toBe(POLLUTED[name]); + } + }); + + test("Windows: HOMEDRIVE and HOMEPATH are split from the private home", () => { + const env = buildAzChildEnv(POLLUTED, win()); + expect(env.HOMEDRIVE).toBe("C:"); + expect(env.HOMEPATH).toBe("\\Users\\me\\.localstack\\azure\\mcp-config-4566\\home"); + expect(env.HOMEDRIVE + env.HOMEPATH).toBe(winConfig.homeDir); + }); + + test("Windows: a UNC home splits at the share", () => { + const home = "\\\\server\\share\\me\\cfg\\home"; + const env = buildAzChildEnv(POLLUTED, win({ config: { ...winConfig, homeDir: home } })); + expect(env.HOMEDRIVE).toBe("\\\\server\\share"); + expect(env.HOMEPATH).toBe("\\me\\cfg\\home"); + }); + + test("POSIX: HOME and TMPDIR are the private ones, and no Windows-only variables are set", () => { + const env = buildAzChildEnv({ PATH: "/usr/bin", HOME: "/home/me", TMPDIR: "/tmp" }, posix()); + expect(env).toMatchObject({ + HOME: posixConfig.homeDir, + TMPDIR: posixConfig.tmpDir, + TEMP: posixConfig.tmpDir, + }); + expect(env).not.toHaveProperty("APPDATA"); + expect(env).not.toHaveProperty("HOMEDRIVE"); + expect(env).not.toHaveProperty("PATHEXT"); + }); + + test("ensurePrivateDirs creates the dirs, writable (real filesystem)", () => { + const root = mkdtempSync(path.join(os.tmpdir(), "lsaz-env-")); + try { + const configDir = path.join(root, "mcp-config-4566"); + const opts = { + platform: process.platform, + config: { + ...posixConfig, + configDir, + homeDir: path.join(configDir, "home"), + tmpDir: path.join(configDir, "tmp"), + }, + }; + ensurePrivateDirs(opts); + const dirs = privateDirs(opts); + const expected = [ + dirs.home, + dirs.tmp, + ...(process.platform === "win32" ? [dirs.appData, dirs.localAppData] : []), + ]; + for (const dir of expected) { + expect(statSync(dir).isDirectory()).toBe(true); + accessSync(dir, constants.W_OK); + } + if (process.platform !== "win32") expect(statSync(dirs.home).mode & 0o077).toBe(0); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); +}); + +describe("buildAzChildEnv: Bicep, Docker and launchers spawned as-is", () => { + test("the Bicep binary's directory comes first on PATH", () => { + const env = buildAzChildEnv( + POLLUTED, + win({ bicepDir: "C:\\Users\\me\\.localstack\\azure\\bin" }) + ); + expect(env.PATH.split(";")[0]).toBe("C:\\Users\\me\\.localstack\\azure\\bin"); + const posixEnv = buildAzChildEnv({ PATH: "/usr/bin" }, posix({ bicepDir: "/usr/local/bin" })); + expect(posixEnv.PATH).toBe("/usr/local/bin:/usr/bin"); + }); + + test("DOTNET_SYSTEM_GLOBALIZATION_INVARIANT=1 is set in Docker only", () => { + expect(buildAzChildEnv({ PATH: "/usr/bin" }, posix())).not.toHaveProperty( + "DOTNET_SYSTEM_GLOBALIZATION_INVARIANT" + ); + expect( + buildAzChildEnv({ PATH: "/usr/bin" }, posix({ config: { ...posixConfig, inDocker: true } })) + .DOTNET_SYSTEM_GLOBALIZATION_INVARIANT + ).toBe("1"); + }); + + test("a launcher spawned as-is gets LC_ALL=C.UTF-8 when no UTF-8 locale is inherited", () => { + const asIs = posix({ az: { installer: "launcher-as-is" } }); + expect(buildAzChildEnv({ PATH: "/usr/bin", LANG: "C" }, asIs)).toMatchObject({ + LC_ALL: "C.UTF-8", + PYTHONIOENCODING: "utf-8", + }); + expect(buildAzChildEnv({ PATH: "/usr/bin", LANG: "en_US.UTF-8" }, asIs).LC_ALL).toBeUndefined(); + expect(buildAzChildEnv({ PATH: "/usr/bin", LANG: "C" }, posix())).not.toHaveProperty( + "PYTHONIOENCODING" + ); + }); +}); + +describe("buildAzChildEnv: the variables Bicep may read", () => { + // A .bicepparam reads `readEnvironmentVariable('X')` from az's environment. It is built from an + // allow-list, so only the names listed in LOCALSTACK_AZ_BICEP_ENV may pass (config.bicepEnv). + const withBicepEnv = (names: string[]) => + win({ config: { ...winConfig, bicepEnv: names } as ChildEnvOptions["config"] }); + + test("a listed variable passes through from the server's environment", () => { + const env = buildAzChildEnv( + { ...POLLUTED, BACKEND_SECRET: "s3cret" }, + withBicepEnv(["BACKEND_SECRET"]) + ); + expect(env.BACKEND_SECRET).toBe("s3cret"); + }); + + test("an unlisted variable does not, and a listed one that is unset is simply absent", () => { + const env = buildAzChildEnv( + { ...POLLUTED, BACKEND_SECRET: "s3cret", OTHER_SECRET: "nope" }, + withBicepEnv(["BACKEND_SECRET", "MISSING_ONE"]) + ); + expect(env).not.toHaveProperty("OTHER_SECRET"); + expect(env).not.toHaveProperty("MISSING_ONE"); + }); + + test("the tool's own settings still win over a listed name", () => { + const env = buildAzChildEnv( + { ...POLLUTED, AZURE_CONFIG_DIR: "C:\\Users\\x\\.azure" }, + withBicepEnv(["AZURE_CONFIG_DIR"]) + ); + expect(env.AZURE_CONFIG_DIR).toBe(winConfig.configDir); + }); +}); diff --git a/src/lib/azure/child-env.ts b/src/lib/azure/child-env.ts new file mode 100644 index 0000000..42a1524 --- /dev/null +++ b/src/lib/azure/child-env.ts @@ -0,0 +1,151 @@ +import { mkdirSync } from "fs"; +import path from "path"; +import type { AzExecutable, AzureConfig } from "./types"; + +/** + * The environment `az` runs with. It is built + * from an allow-list, never from a copy of the server's environment: + * - `AZURE_*`, `ARM_*`, `MSI_ENDPOINT`, `IDENTITY_*` and the user's proxy and CA + * variables would point `az` at real Azure or at another profile; + * - `KUBECONFIG`, `DOCKER_COMMAND`, `GITHUB_ACTIONS`, `TF_BUILD`, `BICEP_*`, + * `AZURE_EXTENSION_SYS_DIR` and `BROWSER` each change what `az` or Bicep does. + * The home, AppData and temp directories are private ones inside the tool's config + * dir, so commands that default to `~` (`--generate-ssh-keys`, `aks get-credentials`, + * `ad ... --create-cert`) write there instead of into the user's home. + */ + +/** Inherited on every platform. */ +const INHERITED = ["PATH", "SYSTEMROOT", "SYSTEMDRIVE", "WINDIR", "COMSPEC", "LANG"]; +/** + * Inherited on Windows only. az finds Bicep with `shutil.which("bicep")`, which reads + * PATHEXT on older Pythons; the others describe the machine, and Windows programs expect them. + */ +const INHERITED_WINDOWS = [ + "PATHEXT", + "USERNAME", + "COMPUTERNAME", + "NUMBER_OF_PROCESSORS", + "PROCESSOR_ARCHITECTURE", +]; + +export interface ChildEnvOptions { + platform: NodeJS.Platform; + config: Pick< + AzureConfig, + "configDir" | "homeDir" | "tmpDir" | "extensionDir" | "egressGuard" | "inDocker" + > & + Partial>; + az: Pick; + /** Prepended to PATH, so `bicep.use_binary_from_path=true` finds the resolved binary. */ + bicepDir?: string; +} + +function lookup(env: NodeJS.ProcessEnv, name: string, platform: NodeJS.Platform) { + if (platform !== "win32") return env[name]; + const key = Object.keys(env).find((k) => k.toUpperCase() === name); + return key === undefined ? undefined : env[key]; +} + +/** The private directories inside the config dir (`/home`, `/tmp`). */ +export function privateDirs(opts: Pick) { + const pathApi = opts.platform === "win32" ? path.win32 : path.posix; + const { homeDir, tmpDir } = opts.config; + return { + home: homeDir, + appData: pathApi.join(homeDir, "AppData", "Roaming"), + localAppData: pathApi.join(homeDir, "AppData", "Local"), + tmp: tmpDir, + }; +} + +/** + * Create the config dir and its private home and temp, owner-only. Bicep extracts + * 13.4 MB of native libraries into the temp dir (Windows) or `$HOME/.net` (Linux), + * so both must be writable. + */ +export function ensurePrivateDirs(opts: Pick): void { + const dirs = privateDirs(opts); + mkdirSync(opts.config.configDir, { recursive: true, mode: 0o700 }); + for (const dir of [dirs.home, dirs.tmp]) mkdirSync(dir, { recursive: true, mode: 0o700 }); + if (opts.platform === "win32") { + mkdirSync(dirs.appData, { recursive: true }); + mkdirSync(dirs.localAppData, { recursive: true }); + } +} + +function hasUtf8Locale(env: Record) { + return ["LC_ALL", "LC_CTYPE", "LANG"].some((k) => /utf-?8/i.test(env[k] ?? "")); +} + +/** + * Build the child environment. The egress proxy variables are added per call by the + * runner; `NO_PROXY` is never set, since bypassing the guard would skip + * its name mapping and its per-call records. + */ +export function buildAzChildEnv( + parent: NodeJS.ProcessEnv, + opts: ChildEnvOptions +): Record { + const { platform, config } = opts; + const pathApi = platform === "win32" ? path.win32 : path.posix; + const env: Record = {}; + + const inherited = platform === "win32" ? [...INHERITED, ...INHERITED_WINDOWS] : INHERITED; + for (const name of inherited) { + const value = lookup(parent, name, platform); + if (value !== undefined) env[name] = value; + } + for (const [name, value] of Object.entries(parent)) { + if (/^LC_[A-Z]+$/.test(name) && value !== undefined) env[name] = value; + } + if (opts.bicepDir) { + env.PATH = [opts.bicepDir, env.PATH].filter(Boolean).join(pathApi.delimiter); + } + // The variables a `.bicepparam` may read with readEnvironmentVariable() (LOCALSTACK_AZ_BICEP_ENV): + // only those the user listed; config already refused the token and anything that + // steers az. Set before the tool's own settings below, so those always win. + const sameName = (a: string, b: string) => + platform === "win32" ? a.toUpperCase() === b.toUpperCase() : a === b; + for (const name of config.bicepEnv ?? []) { + if (Object.keys(env).some((key) => sameName(key, name))) continue; // never replace PATH & co. + const value = + platform === "win32" ? lookup(parent, name.toUpperCase(), platform) : parent[name]; + if (value !== undefined) env[name] = value; + } + + // These must always be set here. On Windows, libuv's uv_spawn copies HOMEDRIVE, + // HOMEPATH, TEMP, USERPROFILE and the rest of its required set from the PARENT + // whenever the env block lacks them (verified in runner.test.ts), which would + // silently undo the private home. + const dirs = privateDirs(opts); + env.HOME = dirs.home; + env.USERPROFILE = dirs.home; + env.TEMP = dirs.tmp; + env.TMP = dirs.tmp; + if (platform === "win32") { + const root = path.win32.parse(dirs.home).root; // "C:\" or "\\server\share\" + env.HOMEDRIVE = root.replace(/[\\/]$/, ""); + env.HOMEPATH = dirs.home.slice(env.HOMEDRIVE.length) || "\\"; + env.APPDATA = dirs.appData; + env.LOCALAPPDATA = dirs.localAppData; + } else { + env.TMPDIR = dirs.tmp; + } + + env.AZURE_CONFIG_DIR = config.configDir; + env.AZURE_EXTENSION_DIR = config.extensionDir; + env.AZURE_CORE_COLLECT_TELEMETRY = "no"; + env.AZURE_CORE_NO_COLOR = "1"; + if (opts.az.azInstaller) env.AZ_INSTALLER = opts.az.azInstaller; + if (config.inDocker) { + // A bundled or mounted Linux Bicep runs without ICU, with identical output. + env.DOTNET_SYSTEM_GLOBALIZATION_INVARIANT = "1"; + } + if (opts.az.installer === "launcher-as-is") { + // No interpreter to give `-X utf8`: the launcher's own flags decide, so + // ask for UTF-8 through the locale and, for launchers without -I, PYTHONIOENCODING. + if (!hasUtf8Locale(env)) env.LC_ALL = "C.UTF-8"; + env.PYTHONIOENCODING = "utf-8"; + } + return env; +} diff --git a/src/lib/azure/description.test.ts b/src/lib/azure/description.test.ts new file mode 100644 index 0000000..8837ba6 --- /dev/null +++ b/src/lib/azure/description.test.ts @@ -0,0 +1,35 @@ +import { + AZURE_COMMAND_DESCRIPTION, + AZURE_SUBSCRIPTION_ID, + buildAzureClientDescription, +} from "./description"; + +// The unit part of the catalogue budget. The direct test measures the real tools/list +// entry; this one keeps the description itself inside its share of the budget. + +describe("buildAzureClientDescription", () => { + const workdir200 = "C:\\" + "w".repeat(197); + + test("fills the workdir and the output cap, and names the constant subscription", () => { + const text = buildAzureClientDescription({ workdir: "/work/project", maxOutputChars: 30000 }); + expect(text).toContain("Files must be inside /work/project."); + expect(text).toContain("truncated past 30,000 characters"); + expect(text).toContain(`Subscription: ${AZURE_SUBSCRIPTION_ID}.`); + expect(text).toContain("--no-wait"); + expect(text.startsWith("Run an Azure CLI (az) command")).toBe(true); + }); + + test("is deterministic (cache-friendly)", () => { + const ctx = { workdir: workdir200, maxOutputChars: 30000 }; + expect(buildAzureClientDescription(ctx)).toBe(buildAzureClientDescription(ctx)); + }); + + test("with a 200-character workdir, JSON-escaped, it leaves room in the 3,200-byte entry", () => { + const text = buildAzureClientDescription({ workdir: workdir200, maxOutputChars: 30000 }); + const escaped = Buffer.byteLength(JSON.stringify(text)); + // Measured: the input schema is ~241 bytes and xmcp/the SDK add ~137 per tool, plus + // the name, title and annotations (~250). 2,400 keeps the whole entry under 3,200. + expect(escaped).toBeLessThanOrEqual(2400); + expect(Buffer.byteLength(JSON.stringify(AZURE_COMMAND_DESCRIPTION))).toBeLessThan(120); + }); +}); diff --git a/src/lib/azure/description.ts b/src/lib/azure/description.ts new file mode 100644 index 0000000..aa8c656 --- /dev/null +++ b/src/lib/azure/description.ts @@ -0,0 +1,29 @@ +/** + * The `localstack-azure-client` tool description. It is filled once at server start and + * then static, so clients can cache the tool list. `tools/list` is served before any + * bootstrap, so the subscription is a constant. + */ + +export const AZURE_SUBSCRIPTION_ID = "00000000-0000-0000-0000-000000000000"; + +export const AZURE_COMMAND_DESCRIPTION = + "The Azure CLI command to run, without the leading 'az' (e.g. 'group list')."; + +export function buildAzureClientDescription(ctx: { + workdir: string; + maxOutputChars: number; +}): string { + const limit = ctx.maxOutputChars.toLocaleString("en-US"); + return [ + "Run an Azure CLI (az) command against the local LocalStack for Azure emulator and return its output.", + "", + `- Runs against the local emulator only, never real Azure: the CLI is pre-configured with a "LocalStack" cloud and a dummy login, so all data and secrets are local test data. Subscription: ${AZURE_SUBSCRIPTION_ID}. There is no default location: pass --location (for example westeurope) wherever a command needs one.`, + "- Give ONE command without the leading `az`. No pipes, redirects, chaining, $(...), backticks or newlines outside quotes. Quote values with spaces, JSON or JMESPath: --query \"[?name=='a'].id | [0]\". Filter with --query and use -o tsv for single values.", + "- Creates wait until the resource is ready; don't pass --no-wait unless asked. If your client stops waiting after about a minute, use --no-wait and then poll with show.", + `- No dedicated command? Use \`rest\` with a RELATIVE URL, e.g. rest --method get --url "/subscriptions/${AZURE_SUBSCRIPTION_ID}/resourceGroups/rg1?api-version=2022-09-01".`, + "- Unsure of a command or its parameters? Run it with --help first.", + `- Files must be inside ${ctx.workdir}. Not allowed: login/logout, cloud and config changes, extension installs, upgrade, interactive, and commands that open a browser, shell or tunnel.`, + `- Output: the CLI's JSON, truncated past ${limit} characters. Errors come back verbatim, with a hint when the emulator does not implement an operation.`, + "Examples: group create --name rg1 --location westeurope | storage account create --name st1abc --resource-group rg1 --sku Standard_LRS | keyvault secret set --vault-name kv1abc --name pw --value x", + ].join("\n"); +} diff --git a/src/lib/azure/egress-proxy.test.ts b/src/lib/azure/egress-proxy.test.ts new file mode 100644 index 0000000..e6f9d2e --- /dev/null +++ b/src/lib/azure/egress-proxy.test.ts @@ -0,0 +1,953 @@ +/** + * The egress guard. + * + * Every server here listens on a loopback port the OS picks. Refused hosts are refused + * before any connection is attempted, and injected connect factories only ever connect + * to 127.0.0.1, so nothing leaves the machine. + */ +import dns from "node:dns"; +import { once } from "node:events"; +import http from "node:http"; +import net from "node:net"; +import type { Duplex } from "node:stream"; +import tls from "node:tls"; +import { + HOUSEKEEPING_HOSTS, + isAllowedEgressHost, + startEgressProxy, + type EgressProxyOptions, +} from "./egress-proxy"; +import type { EgressEvent, EgressProxy, EgressRecords } from "./types"; + +const LOOPBACK = "127.0.0.1"; +const NO_RECORDS: EgressRecords = { refused: [], upstream: [], housekeeping: [], allowed: 0 }; + +// TLS with a pre-shared key needs no certificate, so the test carries no key material. +const PSK = Buffer.alloc(32, 7); +const PSK_IDENTITY = "u9-client"; +const PSK_OPTIONS = { ciphers: "PSK-AES128-GCM-SHA256", maxVersion: "TLSv1.2" as const }; + +interface Upstream { + port: number; + /** Connections accepted so far, in order. */ + accepted: net.Socket[]; + close(): Promise; +} + +interface TlsUpstream extends Upstream { + /** The SNI of every handshake. */ + servernames: string[]; +} + +interface WebUpstream extends Upstream { + /** Requests received so far. */ + requests: Array<{ method?: string; url?: string }>; +} + +interface Reply { + status: number; + headers: Record; + body: string; + /** Bytes after the answer: tunnel data after a 200 to CONNECT. */ + rest: Buffer; + socket: net.Socket; +} + +/** Everything a test opens is closed after it, so Jest sees no open handles. */ +const openSockets = new Set(); +const closers: Array<() => Promise> = []; +let nextCall = 0; + +let echo: Upstream; +let tlsEcho: TlsUpstream; +let web: WebUpstream; +let guard: EgressProxy; + +beforeAll(async () => { + echo = await startEcho(); + tlsEcho = await startTlsEcho(); + web = await startWeb(); + guard = await startEgressProxy(); +}); + +afterEach(async () => { + for (const socket of openSockets) socket.destroy(); + openSockets.clear(); + for (const close of closers.splice(0).reverse()) await close(); +}); + +afterAll(async () => { + await guard.close(); + await Promise.all([echo.close(), tlsEcho.close(), web.close()]); +}); + +function newCallId(label: string): string { + nextCall += 1; + return `u9-${label}-${nextCall}`; +} + +async function newGuard(options: EgressProxyOptions = {}): Promise { + const created = await startEgressProxy(options); + closers.push(() => created.close()); + return created; +} + +function listen(server: net.Server, host = LOOPBACK): Promise { + return new Promise((resolve, reject) => { + server.once("error", reject); + server.listen(0, host, () => { + server.off("error", reject); + resolve((server.address() as net.AddressInfo).port); + }); + }); +} + +function closeServer(server: net.Server, sockets: Iterable): Promise { + for (const socket of sockets) socket.destroy(); + return new Promise((resolve) => server.close(() => resolve())); +} + +async function startEcho(host = LOOPBACK): Promise { + const accepted: net.Socket[] = []; + const server = net.createServer((socket) => { + accepted.push(socket); + socket.on("error", () => undefined); + socket.pipe(socket); + }); + const port = await listen(server, host); + return { port, accepted, close: () => closeServer(server, accepted) }; +} + +async function startTlsEcho(): Promise { + const accepted: net.Socket[] = []; + const servernames: string[] = []; + const server = tls.createServer( + { + ...PSK_OPTIONS, + pskCallback: (_socket, identity) => (identity === PSK_IDENTITY ? PSK : null), + }, + (socket) => { + servernames.push(String(socket.servername)); + socket.on("error", () => undefined); + socket.pipe(socket); + } + ); + server.on("connection", (socket: net.Socket) => { + accepted.push(socket); + socket.on("error", () => undefined); + }); + server.on("tlsClientError", () => undefined); + const port = await listen(server); + return { port, accepted, servernames, close: () => closeServer(server, accepted) }; +} + +/** Answers every request with a JSON description of what it received. */ +async function startWeb(): Promise { + const accepted: net.Socket[] = []; + const requests: Array<{ method?: string; url?: string }> = []; + const server = http.createServer((req, res) => { + requests.push({ method: req.method, url: req.url }); + let body = ""; + req.on("data", (chunk: Buffer) => (body += chunk.toString())); + req.on("end", () => { + const answer = JSON.stringify({ + method: req.method, + url: req.url, + host: req.headers.host, + proxyAuthorization: req.headers["proxy-authorization"] ?? null, + body, + }); + res.writeHead(200, { + "Content-Type": "application/json", + "Content-Length": Buffer.byteLength(answer), + "X-Upstream": "u9", + }); + res.end(answer); + }); + }); + server.on("connection", (socket) => accepted.push(socket)); + const port = await listen(server); + return { + port, + accepted, + requests, + close: () => { + server.closeAllConnections(); + return closeServer(server, accepted); + }, + }; +} + +/** A loopback port with nothing listening: the OS just handed it out and it was closed. */ +async function closedPort(): Promise { + const server = net.createServer(); + const port = await listen(server); + await new Promise((resolve) => server.close(() => resolve())); + return port; +} + +function basic(tag: string): string { + return `Basic ${Buffer.from(`${tag}:x`).toString("base64")}`; +} + +function connectRequest(authority: string, tag?: string, extraHeaders: string[] = []): string { + const lines = [`CONNECT ${authority} HTTP/1.1`, `Host: ${authority}`]; + if (tag !== undefined) lines.push(`Proxy-Authorization: ${basic(tag)}`); + return [...lines, ...extraHeaders, "", ""].join("\r\n"); +} + +/** A plain-HTTP proxy request: the absolute URI in the request line. */ +function httpRequest(uri: string, tag?: string, method = "GET", body = ""): string { + const lines = [`${method} ${uri} HTTP/1.1`, `Host: ${new URL(uri).host}`, "Connection: close"]; + if (tag !== undefined) lines.push(`Proxy-Authorization: ${basic(tag)}`); + if (body) lines.push(`Content-Length: ${Buffer.byteLength(body)}`); + return [...lines, "", body].join("\r\n"); +} + +/** Send raw bytes to `port` and read one answer: its head, and a Content-Length body. */ +function exchange(port: number, bytes: string): Promise { + return new Promise((resolve, reject) => { + const socket = net.connect(port, LOOPBACK); + openSockets.add(socket); + let received = Buffer.alloc(0); + let settled = false; + const fail = (error: Error) => { + if (settled) return; + settled = true; + reject(error); + }; + socket.on("error", fail); + socket.on("close", () => + fail(new Error(`closed before an answer: ${JSON.stringify(received.toString("latin1"))}`)) + ); + const onData = (chunk: Buffer) => { + received = Buffer.concat([received, chunk]); + const end = received.indexOf("\r\n\r\n"); + if (end === -1) return; + const [statusLine, ...lines] = received.subarray(0, end).toString("latin1").split("\r\n"); + const headers: Record = {}; + for (const line of lines) { + const colon = line.indexOf(":"); + headers[line.slice(0, colon).trim().toLowerCase()] = line.slice(colon + 1).trim(); + } + const length = Number(headers["content-length"] ?? 0); + const after = received.subarray(end + 4); + if (after.length < length) return; + socket.off("data", onData); + socket.pause(); + settled = true; + resolve({ + status: Number(statusLine.split(" ")[1]), + headers, + body: after.subarray(0, length).toString("utf8"), + rest: after.subarray(length), + socket, + }); + }; + socket.on("data", onData); + socket.write(bytes); + }); +} + +/** Open a CONNECT tunnel through `port` and check that it was established. */ +async function openTunnel(port: number, authority: string, tag: string): Promise { + const reply = await exchange(port, connectRequest(authority, tag)); + expect(reply.status).toBe(200); + return reply.socket; +} + +function readBytes( + socket: Duplex, + length: number, + initial: Buffer = Buffer.alloc(0) +): Promise { + return new Promise((resolve, reject) => { + let got: Buffer = initial; + if (got.length >= length) { + resolve(got.toString()); + return; + } + const onData = (chunk: Buffer) => { + got = Buffer.concat([got, chunk]); + if (got.length < length) return; + socket.off("data", onData); + socket.pause(); + resolve(got.toString()); + }; + socket.on("data", onData); + socket.once("error", reject); + socket.resume(); + }); +} + +/** Write `text` into a tunnel that ends at an echo server, and read it back. */ +function roundTrip(socket: Duplex, text: string): Promise { + const echoed = readBytes(socket, Buffer.byteLength(text)); + socket.write(text); + return echoed; +} + +/** Resolves when `socket` has closed, however it ended. */ +function closed(socket: net.Socket): Promise { + return new Promise((resolve) => socket.once("close", () => resolve())); +} + +function connectOnce(port: number): Promise { + return new Promise((resolve, reject) => { + const socket = net.connect(port, LOOPBACK); + openSockets.add(socket); + socket.once("connect", () => { + socket.destroy(); + resolve(); + }); + socket.once("error", reject); + }); +} + +async function waitFor(condition: () => boolean, what: string, timeoutMs = 3000): Promise { + const deadline = Date.now() + timeoutMs; + while (!condition()) { + if (Date.now() > deadline) throw new Error(`timed out waiting for ${what}`); + await new Promise((resolve) => setTimeout(resolve, 10)); + } +} + +/** A connect factory that records its arguments and connects to that port on 127.0.0.1. */ +function recordingConnect(calls: Array<[number, string]>): EgressProxyOptions["connect"] { + return (port, host) => { + calls.push([port, host]); + return net.connect(port, LOOPBACK); + }; +} + +/** + * Replace the resolver entry points with ones that answer `address`. The tests assert + * that none of them runs. + */ +function spyOnResolvers(address: string): jest.SpyInstance[] { + const answer = + (...result: unknown[]) => + (...args: unknown[]) => { + const callback = args[args.length - 1]; + if (typeof callback === "function") process.nextTick(() => callback(null, ...result)); + }; + const lookup = (_host: string, ...args: unknown[]) => { + const callback = args[args.length - 1]; + const all = args.length > 1 && (args[0] as { all?: boolean } | null)?.all === true; + if (typeof callback !== "function") return; + process.nextTick(() => + all ? callback(null, [{ address, family: 4 }]) : callback(null, address, 4) + ); + }; + return [ + jest.spyOn(dns, "lookup").mockImplementation(lookup as unknown as typeof dns.lookup), + jest + .spyOn(dns, "resolve") + .mockImplementation(answer([address]) as unknown as typeof dns.resolve), + jest + .spyOn(dns, "resolve4") + .mockImplementation(answer([address]) as unknown as typeof dns.resolve4), + jest.spyOn(dns, "resolve6").mockImplementation(answer([]) as unknown as typeof dns.resolve6), + jest.spyOn(dns.promises, "lookup").mockResolvedValue({ address, family: 4 }), + jest.spyOn(dns.promises, "resolve4").mockResolvedValue([address]), + ]; +} + +describe("isAllowedEgressHost", () => { + test.each([ + "localhost.localstack.cloud", + "azure.localhost.localstack.cloud", + "acct.blob.core.azure.localhost.localstack.cloud", + "AZURE.LocalHost.LocalStack.Cloud.", + "localhost", + "127.0.0.1", + "::1", + "[::1]", + "0:0:0:0:0:0:0:1", + ])("allows %s", (host) => { + expect(isAllowedEgressHost(host)).toBe(true); + }); + + test.each([ + "evillocalhost.localstack.cloud", + "localhost.localstack.cloud.evil.example", + "localstack.cloud", + "management.azure.com", + "127.0.0.2", + "::2", + "::ffff:127.0.0.1", + "[localhost]", + "host.docker.internal", + "", + ])("refuses %j", (host) => { + expect(isAllowedEgressHost(host)).toBe(false); + }); +}); + +describe("startEgressProxy, envFor and takeRecords", () => { + test("the guard listens before it resolves, and envFor tags both proxy variables", async () => { + await connectOnce(guard.port); + const id = newCallId("env"); + const url = `http://${id}:x@127.0.0.1:${guard.port}`; + expect(guard.envFor(id)).toEqual({ HTTPS_PROXY: url, HTTP_PROXY: url }); + // The user part is what requests sends as the Basic tag. + expect(new URL(url).username).toBe(id); + expect(guard.envFor(id)).toEqual({ HTTPS_PROXY: url, HTTP_PROXY: url }); + }); + + test("envFor refuses call ids that do not fit into a URL's userinfo", () => { + for (const bad of ["", "a b", "a:b", "a@b", "a/b", "x".repeat(129)]) { + expect(() => guard.envFor(bad)).toThrow(/call id/); + } + }); + + test("takeRecords returns a call's records once, and empty records for unknown calls", async () => { + const id = newCallId("take"); + guard.envFor(id); + await exchange(guard.port, connectRequest("example.com:443", id)); + expect(guard.takeRecords(id)).toEqual({ ...NO_RECORDS, refused: ["example.com"] }); + expect(guard.takeRecords(id)).toEqual(NO_RECORDS); + expect(guard.takeRecords("u9-never-registered")).toEqual(NO_RECORDS); + }); + + test("extra allowed hosts must be names", async () => { + await expect(startEgressProxy({ extraAllowedHosts: ["10.0.0.1"] })).rejects.toThrow(/names/); + await expect(startEgressProxy({ extraAllowedHosts: ["not a host"] })).rejects.toThrow(/names/); + }); +}); + +describe("CONNECT", () => { + test("an allowed CONNECT relays bytes both ways: a TLS handshake and data, SNI intact", async () => { + const id = newCallId("tls"); + guard.envFor(id); + const tunnel = await openTunnel( + guard.port, + `azure.localhost.localstack.cloud:${tlsEcho.port}`, + id + ); + const secure = tls.connect({ + socket: tunnel, + servername: "azure.localhost.localstack.cloud", + ...PSK_OPTIONS, + checkServerIdentity: () => undefined, + pskCallback: () => ({ psk: PSK, identity: PSK_IDENTITY }), + }); + openSockets.add(secure); + await once(secure, "secureConnect"); + expect(await roundTrip(secure, "hello through the tunnel")).toBe("hello through the tunnel"); + expect(tlsEcho.servernames).toContain("azure.localhost.localstack.cloud"); + expect(guard.takeRecords(id)).toEqual({ ...NO_RECORDS, allowed: 1 }); + }); + + test("bytes that arrive together with the CONNECT request are relayed first", async () => { + const id = newCallId("head"); + guard.envFor(id); + const reply = await exchange( + guard.port, + `${connectRequest(`localhost:${echo.port}`, id)}early` + ); + expect(reply.status).toBe(200); + expect(await readBytes(reply.socket, "early".length, reply.rest)).toBe("early"); + }); + + test("the apex, the names under it and the other local names are relayed", async () => { + const id = newCallId("names"); + guard.envFor(id); + const names = [ + "localhost.localstack.cloud", // the LRO polling host + "azure.localhost.localstack.cloud", + "acct.blob.core.azure.localhost.localstack.cloud", + "AZURE.LocalHost.LocalStack.Cloud.", + "localhost", + "127.0.0.1", + ]; + for (const name of names) { + const tunnel = await openTunnel(guard.port, `${name}:${echo.port}`, id); + expect(await roundTrip(tunnel, name)).toBe(name); + tunnel.destroy(); + } + expect(guard.takeRecords(id)).toEqual({ ...NO_RECORDS, allowed: names.length }); + }); + + test("names are mapped to 127.0.0.1 by the guard: no resolver runs, even one answering 203.0.113.5", async () => { + const connects: Array<[number, string]> = []; + const local = await newGuard({ connect: recordingConnect(connects) }); + // Only now: Node's listen() itself runs dns.lookup, even for the literal 127.0.0.1 + // (answered locally). What must never resolve is the name in a CONNECT. + const resolvers = spyOnResolvers("203.0.113.5"); + try { + const id = newCallId("no-dns"); + local.envFor(id); + for (const name of [ + "azure.localhost.localstack.cloud", + "localhost.localstack.cloud", + "localhost", + ]) { + const tunnel = await openTunnel(local.port, `${name}:${echo.port}`, id); + expect(await roundTrip(tunnel, name)).toBe(name); + tunnel.destroy(); + } + expect(connects).toEqual([ + [echo.port, "127.0.0.1"], + [echo.port, "127.0.0.1"], + [echo.port, "127.0.0.1"], + ]); + for (const resolver of resolvers) expect(resolver).not.toHaveBeenCalled(); + } finally { + for (const resolver of resolvers) resolver.mockRestore(); + } + }); + + test("the default connection is made to the address, so it needs no lookup either", async () => { + // Loopback answers, so that not even a regression could reach the network. + const resolvers = spyOnResolvers(LOOPBACK); + try { + const id = newCallId("no-dns-default"); + guard.envFor(id); + const tunnel = await openTunnel( + guard.port, + `azure.localhost.localstack.cloud:${echo.port}`, + id + ); + expect(await roundTrip(tunnel, "direct")).toBe("direct"); + const plain = await exchange( + guard.port, + httpRequest(`http://localhost.localstack.cloud:${web.port}/`, id) + ); + expect(plain.status).toBe(200); + for (const resolver of resolvers) expect(resolver).not.toHaveBeenCalled(); + } finally { + for (const resolver of resolvers) resolver.mockRestore(); + } + }); + + test("a [::1]: authority is parsed and sent to ::1", async () => { + const connects: Array<[number, string]> = []; + const local = await newGuard({ connect: recordingConnect(connects) }); + const id = newCallId("v6"); + local.envFor(id); + // Bracketed, spelled out, and unbracketed as older Pythons send it. + const authorities = [ + `[::1]:${echo.port}`, + `[0:0:0:0:0:0:0:1]:${echo.port}`, + `::1:${echo.port}`, + ]; + for (const authority of authorities) { + const tunnel = await openTunnel(local.port, authority, id); + expect(await roundTrip(tunnel, authority)).toBe(authority); + tunnel.destroy(); + } + const plain = await exchange(local.port, httpRequest(`http://[::1]:${web.port}/v6`, id)); + expect(plain.status).toBe(200); + expect(connects).toEqual([...authorities.map(() => [echo.port, "::1"]), [web.port, "::1"]]); + expect(local.takeRecords(id)).toEqual({ ...NO_RECORDS, allowed: 4 }); + }); + + test("a literal ::1 is relayed to ::1 for real where the machine has IPv6 loopback", async () => { + let echo6: Upstream; + try { + echo6 = await startEcho("::1"); + } catch { + // No IPv6 loopback (some CI containers): the injected-connect test covers the mapping. + return; + } + closers.push(() => echo6.close()); + const id = newCallId("v6-real"); + guard.envFor(id); + const tunnel = await openTunnel(guard.port, `[::1]:${echo6.port}`, id); + expect(await roundTrip(tunnel, "over IPv6")).toBe("over IPv6"); + expect(echo6.accepted).toHaveLength(1); + }); + + test("a refused host gets 403 and is recorded under the call id from Proxy-Authorization", async () => { + const a = newCallId("refused-a"); + const b = newCallId("refused-b"); + guard.envFor(a); + guard.envFor(b); + const reply = await exchange(guard.port, connectRequest("management.azure.com:443", a)); + expect(reply.status).toBe(403); + expect(reply.body).toBe("blocked by egress guard: management.azure.com"); + expect(reply.headers).toMatchObject({ connection: "close", "content-type": "text/plain" }); + // Near misses and a repeat; records hold canonical names, each once. + const more = [ + "Management.Azure.com:443", + "evillocalhost.localstack.cloud:443", + "localhost.localstack.cloud.evil.example:443", + "localstack.cloud:443", + `127.0.0.2:${echo.port}`, + "[::2]:443", + ]; + for (const authority of more) { + expect((await exchange(guard.port, connectRequest(authority, a))).status).toBe(403); + } + expect(guard.takeRecords(a)).toEqual({ + ...NO_RECORDS, + refused: [ + "management.azure.com", + "evillocalhost.localstack.cloud", + "localhost.localstack.cloud.evil.example", + "localstack.cloud", + "127.0.0.2", + "::2", + ], + }); + expect(guard.takeRecords(b)).toEqual(NO_RECORDS); + }); + + test("housekeeping hosts are recorded as housekeeping, not refused", async () => { + expect(HOUSEKEEPING_HOSTS).toEqual([ + "azcliprod.blob.core.windows.net", + "app.aladdin.microsoft.com", + "aka.ms", + "raw.githubusercontent.com", + "appinsights.azureedge.net", + ]); + const id = newCallId("housekeeping"); + guard.envFor(id); + for (const host of HOUSEKEEPING_HOSTS) { + const reply = await exchange(guard.port, connectRequest(`${host}:443`, id)); + expect(reply.status).toBe(403); + expect(reply.body).toBe(`blocked by egress guard: ${host}`); + } + expect(guard.takeRecords(id)).toEqual({ ...NO_RECORDS, housekeeping: [...HOUSEKEEPING_HOSTS] }); + }); + + test("the housekeeping list can be replaced, and extra names go to 127.0.0.1", async () => { + const connects: Array<[number, string]> = []; + const local = await newGuard({ + housekeepingHosts: ["Example.org"], + extraAllowedHosts: ["emulator.test"], + connect: recordingConnect(connects), + }); + const id = newCallId("options"); + local.envFor(id); + expect((await exchange(local.port, connectRequest("example.org:443", id))).status).toBe(403); + expect((await exchange(local.port, connectRequest("aka.ms:443", id))).status).toBe(403); + const tunnel = await openTunnel(local.port, `Emulator.Test:${echo.port}`, id); + expect(await roundTrip(tunnel, "extra")).toBe("extra"); + expect(connects).toEqual([[echo.port, "127.0.0.1"]]); + expect(local.takeRecords(id)).toEqual({ + refused: ["aka.ms"], + upstream: [], + housekeeping: ["example.org"], + allowed: 1, + }); + }); + + test("concurrent calls get separate records", async () => { + const a = newCallId("parallel-a"); + const b = newCallId("parallel-b"); + guard.envFor(a); + guard.envFor(b); + const replies = await Promise.all([ + exchange(guard.port, connectRequest("a.example.com:443", a)), + exchange(guard.port, connectRequest("b.example.com:443", b)), + exchange(guard.port, connectRequest(`azure.localhost.localstack.cloud:${echo.port}`, a)), + exchange(guard.port, connectRequest(`localhost.localstack.cloud:${echo.port}`, b)), + exchange(guard.port, connectRequest("azcliprod.blob.core.windows.net:443", b)), + exchange(guard.port, httpRequest(`http://localhost:${web.port}/parallel`, a)), + ]); + expect(replies.map((reply) => reply.status)).toEqual([403, 403, 200, 200, 403, 200]); + expect(guard.takeRecords(a)).toEqual({ ...NO_RECORDS, refused: ["a.example.com"], allowed: 2 }); + expect(guard.takeRecords(b)).toEqual({ + refused: ["b.example.com"], + upstream: [], + housekeeping: ["azcliprod.blob.core.windows.net"], + allowed: 1, + }); + }); + + test("a missing or unknown call tag gets 407 and is not relayed; the retry with the tag passes", async () => { + const connects: Array<[number, string]> = []; + const local = await newGuard({ connect: recordingConnect(connects) }); + const id = newCallId("challenge"); + local.envFor(id); + const taken = newCallId("taken"); + local.envFor(taken); + local.takeRecords(taken); + const target = `azure.localhost.localstack.cloud:${echo.port}`; + const acceptedBefore = echo.accepted.length; + const untagged = [ + connectRequest(target), // what .NET clients such as Bicep send first + connectRequest(target, "u9-never-registered"), + connectRequest(target, taken), // a straggler of a call that has resolved + connectRequest(target, undefined, ["Proxy-Authorization: Bearer abc"]), + connectRequest(target, undefined, ["Proxy-Authorization: Basic !!!"]), + httpRequest(`http://localhost:${web.port}/untagged`), + ]; + for (const request of untagged) { + const reply = await exchange(local.port, request); + expect(reply.status).toBe(407); + expect(reply.headers["proxy-authenticate"]).toBe('Basic realm="localstack-azure-mcp"'); + } + expect(connects).toEqual([]); + expect(echo.accepted).toHaveLength(acceptedBefore); + // The .NET pattern: the same CONNECT again, now with the tag. + const tunnel = await openTunnel(local.port, target, id); + expect(await roundTrip(tunnel, "after the challenge")).toBe("after the challenge"); + expect(connects).toEqual([[echo.port, "127.0.0.1"]]); + expect(local.takeRecords(id)).toEqual({ ...NO_RECORDS, allowed: 1 }); + }); + + test("malformed requests get 400 and leave no record", async () => { + const id = newCallId("malformed"); + guard.envFor(id); + const malformed = [ + connectRequest("azure.localhost.localstack.cloud", id), // no port + connectRequest("localhost:0", id), + connectRequest("localhost:70000", id), + connectRequest(`[localhost]:${echo.port}`, id), + connectRequest(`localhost:${echo.port}:1`, id), + `GET /relative HTTP/1.1\r\nHost: localhost\r\nProxy-Authorization: ${basic(id)}\r\n\r\n`, + httpRequest(`https://localhost:${web.port}/`, id), // https goes through CONNECT only + "\u0000\u0001 not http at all\r\n\r\n", + ]; + for (const request of malformed) { + expect((await exchange(guard.port, request)).status).toBe(400); + } + expect(guard.takeRecords(id)).toEqual(NO_RECORDS); + }); + + test("an upstream port that refuses gets 502 and an upstream record", async () => { + const dead = await closedPort(); + const id = newCallId("upstream"); + guard.envFor(id); + const tunnel = await exchange( + guard.port, + connectRequest(`localhost.localstack.cloud:${dead}`, id) + ); + expect(tunnel.status).toBe(502); + expect(tunnel.body).toBe("upstream error: ECONNREFUSED"); + const plain = await exchange(guard.port, httpRequest(`http://localhost:${dead}/`, id)); + expect(plain.status).toBe(502); + expect(plain.body).toBe("upstream error: ECONNREFUSED"); + expect(guard.takeRecords(id)).toEqual({ + ...NO_RECORDS, + upstream: ["localhost.localstack.cloud", "localhost"], + }); + }, 20000); + + test("a connect factory that throws gets 500 and leaves the guard up", async () => { + const logs: string[] = []; + const local = await newGuard({ + connect: () => { + throw new Error("factory exploded"); + }, + log: (line) => logs.push(line), + }); + const id = newCallId("throwing"); + local.envFor(id); + expect((await exchange(local.port, connectRequest(`localhost:${echo.port}`, id))).status).toBe( + 500 + ); + expect( + (await exchange(local.port, httpRequest(`http://localhost:${web.port}/`, id))).status + ).toBe(500); + expect((await exchange(local.port, connectRequest("example.com:443", id))).status).toBe(403); + expect(logs.join("\n")).toMatch(/factory exploded/); + expect(local.takeRecords(id)).toEqual({ ...NO_RECORDS, refused: ["example.com"] }); + }); + + test("a client that resets right after the 403 does not crash the process", async () => { + const logs: string[] = []; + const local = await newGuard({ log: (line) => logs.push(line) }); + const id = newCallId("reset"); + local.envFor(id); + const reply = await exchange(local.port, connectRequest("example.com:443", id)); + expect(reply.status).toBe(403); + reply.socket.resetAndDestroy(); + // The guard's end saw the reset, so this ran the path that could crash the process ... + await waitFor( + () => logs.some((line) => /client socket error: (ECONNRESET|EPIPE)/.test(line)), + "the reset to reach the guard" + ); + // ... and the guard still serves. + expect((await exchange(local.port, connectRequest(`localhost:${echo.port}`, id))).status).toBe( + 200 + ); + expect(local.takeRecords(id)).toEqual({ ...NO_RECORDS, refused: ["example.com"], allowed: 1 }); + }); + + test("a reset before the answer is read, or inside a tunnel, is survived too", async () => { + const local = await newGuard(); + const id = newCallId("reset-more"); + local.envFor(id); + const early = net.connect(local.port, LOOPBACK); + openSockets.add(early); + early.on("error", () => undefined); + await once(early, "connect"); + const earlyClosed = closed(early); + early.write(connectRequest("example.com:443", id), () => early.resetAndDestroy()); + await earlyClosed; + + const acceptedBefore = echo.accepted.length; + const tunnel = await openTunnel(local.port, `localhost:${echo.port}`, id); + await waitFor(() => echo.accepted.length > acceptedBefore, "the relayed connection"); + const relayedClosed = closed(echo.accepted[acceptedBefore]); + tunnel.resetAndDestroy(); + // The guard tears down the emulator side as well. + await relayedClosed; + expect((await exchange(local.port, connectRequest("example.com:443", id))).status).toBe(403); + }); + + test("a client that hangs up cleanly ends the relayed connection, and the tunnel closes", async () => { + const id = newCallId("hang-up"); + guard.envFor(id); + const acceptedBefore = echo.accepted.length; + const tunnel = await openTunnel(guard.port, `localhost:${echo.port}`, id); + await waitFor(() => echo.accepted.length > acceptedBefore, "the relayed connection"); + const relayedClosed = closed(echo.accepted[acceptedBefore]); + const tunnelClosed = closed(tunnel); + expect(await roundTrip(tunnel, "last words")).toBe("last words"); + tunnel.resume(); + tunnel.end(); + await Promise.all([relayedClosed, tunnelClosed]); + }); +}); + +describe("plain HTTP", () => { + test("an absolute URI to an allowed host is relayed with its Host, without the call tag", async () => { + const id = newCallId("http"); + guard.envFor(id); + const before = web.requests.length; + const get = await exchange( + guard.port, + httpRequest(`http://LocalHost.LocalStack.Cloud:${web.port}/echo?x=1`, id) + ); + expect(get.status).toBe(200); + expect(get.headers["x-upstream"]).toBe("u9"); + expect(JSON.parse(get.body)).toEqual({ + method: "GET", + url: "/echo?x=1", + host: `localhost.localstack.cloud:${web.port}`, + proxyAuthorization: null, + body: "", + }); + const post = await exchange( + guard.port, + httpRequest(`http://127.0.0.1:${web.port}/upload`, id, "POST", "payload") + ); + expect(JSON.parse(post.body)).toMatchObject({ + method: "POST", + url: "/upload", + body: "payload", + }); + expect(web.requests).toHaveLength(before + 2); + expect(guard.takeRecords(id)).toEqual({ ...NO_RECORDS, allowed: 2 }); + }); + + test("an absolute URI to any other host gets 403 and a record, and is not relayed", async () => { + const id = newCallId("http-refused"); + guard.envFor(id); + const before = web.requests.length; + const refused = await exchange(guard.port, httpRequest("http://example.com/x", id)); + expect(refused.status).toBe(403); + expect(refused.body).toBe("blocked by egress guard: example.com"); + expect(refused.headers.connection).toBe("close"); + const housekeeping = await exchange( + guard.port, + httpRequest("http://raw.githubusercontent.com/Azure/x", id) + ); + expect(housekeeping.status).toBe(403); + expect(web.requests).toHaveLength(before); + expect(guard.takeRecords(id)).toEqual({ + ...NO_RECORDS, + refused: ["example.com"], + housekeeping: ["raw.githubusercontent.com"], + }); + }); +}); + +describe("events", () => { + test("each outcome emits one event with its call id, until unsubscribed", async () => { + const a = newCallId("events-a"); + const b = newCallId("events-b"); + guard.envFor(a); + guard.envFor(b); + const events: Array<[string, EgressEvent]> = []; + const off = guard.onEvent((callId, event) => { + if (callId === a || callId === b) events.push([callId, { ...event }]); + }); + const dead = await closedPort(); + await exchange(guard.port, connectRequest("example.com:443", a)); + await exchange(guard.port, connectRequest("aka.ms:443", b)); + await exchange(guard.port, connectRequest(`localhost.localstack.cloud:${dead}`, a)); + await openTunnel(guard.port, `azure.localhost.localstack.cloud:${echo.port}`, b); + await exchange(guard.port, httpRequest(`http://localhost:${web.port}/events`, a)); + off(); + await exchange(guard.port, connectRequest("example.com:443", a)); + expect(events).toEqual([ + [a, { kind: "refused", host: "example.com" }], + [b, { kind: "housekeeping", host: "aka.ms" }], + [a, { kind: "upstream", host: "localhost.localstack.cloud" }], + [b, { kind: "allowed", host: "azure.localhost.localstack.cloud" }], + [a, { kind: "allowed", host: "localhost" }], + ]); + }, 20000); + + test("a listener that throws or rejects changes nothing for the guard or other listeners", async () => { + const logs: string[] = []; + const local = await newGuard({ log: (line) => logs.push(line) }); + local.onEvent(() => { + throw new Error("listener threw"); + }); + local.onEvent(async () => { + throw new Error("listener rejected"); + }); + const seen: string[] = []; + local.onEvent((_callId, event) => seen.push(event.host)); + const id = newCallId("listeners"); + local.envFor(id); + expect((await exchange(local.port, connectRequest("example.com:443", id))).status).toBe(403); + // The rejection is handled (and logged), so it never becomes an unhandled one. + await waitFor(() => logs.some((line) => /listener rejected/.test(line)), "the rejection"); + expect(logs.join("\n")).toMatch(/listener threw/); + expect(seen).toEqual(["example.com"]); + expect(local.takeRecords(id)).toEqual({ ...NO_RECORDS, refused: ["example.com"] }); + }); +}); + +describe("close", () => { + test("close() destroys open tunnels, stops listening and can be called twice", async () => { + const local = await newGuard(); + const id = newCallId("close"); + local.envFor(id); + const acceptedBefore = echo.accepted.length; + const tunnel = await openTunnel(local.port, `localhost:${echo.port}`, id); + await waitFor(() => echo.accepted.length > acceptedBefore, "the relayed connection"); + const relayedClosed = closed(echo.accepted[acceptedBefore]); + const tunnelClosed = closed(tunnel); + tunnel.resume(); + await local.close(); + await Promise.all([relayedClosed, tunnelClosed]); + await expect(connectOnce(local.port)).rejects.toMatchObject({ code: "ECONNREFUSED" }); + await expect(local.close()).resolves.toBeUndefined(); + // Records outlive the guard, so a call that resolves after shutdown still gets them. + expect(local.takeRecords(id)).toEqual({ ...NO_RECORDS, allowed: 1 }); + }, 20000); +}); + +describe("allowedPorts: only the emulator's ports", () => { + test("a local host on another port gets 403 and a host:port record; the allowed ports relay", async () => { + const other = await closedPort(); + const g = await newGuard({ allowedPorts: new Set([echo.port, web.port]) }); + const id = newCallId("ports"); + g.envFor(id); + const tunnelRefused = await exchange(g.port, connectRequest(`localhost:${other}`, id)); + expect(tunnelRefused.status).toBe(403); + expect(tunnelRefused.body).toBe(`blocked by egress guard: localhost:${other}`); + const httpRefused = await exchange(g.port, httpRequest(`http://127.0.0.1:${other}/admin`, id)); + expect(httpRefused.status).toBe(403); + // The emulator's ports still relay, over CONNECT and plain HTTP. + const tunnel = await openTunnel(g.port, `localhost.localstack.cloud:${echo.port}`, id); + expect(await roundTrip(tunnel, "ping")).toBe("ping"); + tunnel.destroy(); + const get = await exchange(g.port, httpRequest(`http://localhost:${web.port}/echo`, id)); + expect(get.status).toBe(200); + expect(g.takeRecords(id)).toEqual({ + ...NO_RECORDS, + refused: [`localhost:${other}`, `127.0.0.1:${other}`], + allowed: 2, + }); + }); +}); diff --git a/src/lib/azure/egress-proxy.ts b/src/lib/azure/egress-proxy.ts new file mode 100644 index 0000000..151ebb3 --- /dev/null +++ b/src/lib/azure/egress-proxy.ts @@ -0,0 +1,676 @@ +/** + * The egress guard: the loopback proxy that every `az` child reaches the network through, + * via HTTPS_PROXY/HTTP_PROXY. + * + * - It relays only to the emulator: `localhost.localstack.cloud` and every name under it + * (the apex too, because the emulator's LRO `Location` headers point there), plus + * `localhost`, `127.0.0.1` and `::1`. + * - It maps those names to 127.0.0.1 itself (::1 for a literal ::1), which is what + * LocalStack's public DNS answers anyway. No DNS lookup ever happens, so a hijacked or + * blocked answer changes nothing. + * - Every other host gets 403 and is recorded under the call that asked for it, found by + * the tag in the proxy URL's userinfo. The runner reads those records to fail fast and + * to name the blocked host, which az's own error does not do on SDK paths. + * - A request without a known tag gets 407. Python's requests sends the tag up front, but + * .NET children such as Bicep send it only after this challenge. It also keeps + * other local processes from using the guard as a relay. + * + * The guard runs inside the MCP server's process, so no socket error, throw or rejection + * may escape into the event loop: a client that resets right after a 403 must not crash it. + */ + +import { + STATUS_CODES, + createServer, + request, + type ClientRequest, + type IncomingHttpHeaders, + type IncomingMessage, + type OutgoingHttpHeaders, + type Server, + type ServerResponse, +} from "node:http"; +import { connect, isIP, isIPv6, type AddressInfo, type Socket } from "node:net"; +import type { Duplex } from "node:stream"; +import type { EgressEvent, EgressProxy, EgressRecords } from "./types"; + +/** LocalStack's public zone: every name in it resolves to 127.0.0.1. */ +const EMULATOR_ZONE = "localhost.localstack.cloud"; + +/** + * Hosts that az and Bicep call on their own. They are refused like any other host, but + * they are expected blocks, never the command's failure and never a fail-fast trigger. + */ +export const HOUSEKEEPING_HOSTS: readonly string[] = [ + "azcliprod.blob.core.windows.net", // az's update check + "app.aladdin.microsoft.com", // az 2.85's command recommender, until bootstrap turns it off + "aka.ms", // Bicep's public-module index, fetched on every build + "raw.githubusercontent.com", // the `vm create --image ` list; az falls back to its copy + // `functionapp create` without --workspace downloads Application Insights' region map + // (appservice/_create_util.py get_region_mapping). az catches the failure, warns, and still + // creates the function app; as a fail-fast trigger it killed a successful create (L4, L2). + "appinsights.azureedge.net", +]; + +const PROXY_AUTHENTICATE = 'Basic realm="localstack-azure-mcp"'; + +/** How long a closing connection may wait for its peer to hang up or to take the last bytes. */ +const LINGER_MS = 2000; + +/** Tags travel in a URL's userinfo, so they are limited to unreserved URL characters. */ +const CALL_ID = /^[A-Za-z0-9._~-]{1,128}$/; + +/** A host name as it may appear in a CONNECT authority (IP literals are checked apart). */ +const REG_NAME = /^[A-Za-z0-9._-]+$/; + +/** Hop-by-hop headers (RFC 9110, section 7.6.1) are never forwarded on the plain-HTTP path. */ +const HOP_BY_HOP = new Set([ + "connection", + "keep-alive", + "proxy-authenticate", + "proxy-authorization", + "proxy-connection", + "te", + "trailer", + "transfer-encoding", + "upgrade", +]); + +export interface EgressProxyOptions { + /** + * Opens the TCP connection to an allowed target and returns the socket while it is + * still connecting, as `net.connect()` does. `host` is always 127.0.0.1 or ::1, never a + * name, which is how tests prove that no DNS is involved. + */ + connect?: (port: number, host: string) => Socket; + /** More names to relay, matched exactly (in any case) and sent to 127.0.0.1 without DNS. */ + extraAllowedHosts?: readonly string[]; + /** + * The only ports relayed to (the emulator's: its gateway, 443 and its service range), so + * that az cannot reach other local services, such as Docker's API on localhost:2375. + * Unset: any port. + */ + allowedPorts?: ReadonlySet; + /** Replaces HOUSEKEEPING_HOSTS. */ + housekeepingHosts?: readonly string[]; + /** + * Diagnostics, one line each: 400, 407 and 500 answers, upstream failures and socket + * errors. Never wire it to stdout, where the MCP server speaks JSON-RPC. + */ + log?: (line: string) => void; +} + +interface Target { + /** Canonical: lower case, without brackets or a trailing dot. */ + host: string; + port: number; +} + +interface HttpTarget extends Target { + /** `host[:port]` of the absolute URI, which becomes the forwarded Host header. */ + authority: string; + /** Path and query, as sent upstream. */ + path: string; +} + +interface CallState { + refused: Set; + upstream: Set; + housekeeping: Set; + allowed: number; +} + +type Listener = (callId: string, event: EgressEvent) => void; + +/** + * The canonical spelling of a host from a CONNECT authority or an absolute URI: lower + * case, without brackets or a trailing dot, with IP literals in standard form, so that + * `127.1` or `[0:0:0:0:0:0:0:1]` are recognised for what they are. Undefined for anything + * that is not a host. + */ +function canonicalHost(raw: string): string | undefined { + try { + const bracketed = raw.startsWith("[") && raw.endsWith("]"); + const inner = bracketed ? raw.slice(1, -1) : raw; + // Older Pythons send an IPv6 CONNECT target without brackets. + if (isIPv6(inner)) return new URL(`http://[${inner}]/`).hostname.slice(1, -1); + if (bracketed || !REG_NAME.test(raw)) return undefined; + return new URL(`http://${raw}/`).hostname.replace(/\.$/, "") || undefined; + } catch { + return undefined; + } +} + +/** The loopback address the built-in allow-list maps a canonical host to. */ +function builtInTarget(host: string): "127.0.0.1" | "::1" | undefined { + if (host === "::1") return "::1"; + if ( + host === "127.0.0.1" || + host === "localhost" || + host === EMULATOR_ZONE || + host.endsWith(`.${EMULATOR_ZONE}`) + ) { + return "127.0.0.1"; + } + return undefined; +} + +/** + * Whether the guard relays `host` (a name or an IP literal, bracketed or not): the + * built-in list of `localhost.localstack.cloud` and every name under it, `localhost`, + * `127.0.0.1` and `::1`. Case and a trailing dot do not matter. The policy's URL rule and + * the endpoint check can share it. + */ +export function isAllowedEgressHost(host: string): boolean { + const canonical = canonicalHost(host); + return canonical !== undefined && builtInTarget(canonical) !== undefined; +} + +/** A CONNECT target, `host:port` or `[v6]:port`. */ +function parseAuthority(authority: string): Target | undefined { + const colon = authority.lastIndexOf(":"); + const portText = authority.slice(colon + 1); + if (colon <= 0 || !/^\d{1,5}$/.test(portText)) return undefined; + const port = Number(portText); + const host = canonicalHost(authority.slice(0, colon)); + return host !== undefined && port >= 1 && port <= 65535 ? { host, port } : undefined; +} + +/** The target of a plain-HTTP proxy request, which must be an absolute `http://` URI. */ +function parseAbsoluteUri(raw: string): HttpTarget | undefined { + if (!/^http:\/\//i.test(raw)) return undefined; + let url: URL; + try { + url = new URL(raw); + } catch { + return undefined; + } + const host = canonicalHost(url.hostname); + const port = url.port ? Number(url.port) : 80; + if (host === undefined || port < 1) return undefined; + return { host, port, authority: url.host, path: `${url.pathname}${url.search}` }; +} + +/** The call tag of `Proxy-Authorization: Basic base64(:x)`, when there is one. */ +function callTagOf(header: string | undefined): string | undefined { + const match = /^basic +([A-Za-z0-9+/]+={0,2}) *$/i.exec(header ?? ""); + if (!match) return undefined; + const credentials = Buffer.from(match[1], "base64").toString("utf8"); + const colon = credentials.indexOf(":"); + return colon === -1 ? credentials : credentials.slice(0, colon); +} + +function withoutHopByHop(headers: IncomingHttpHeaders): OutgoingHttpHeaders { + // Headers named in `Connection` are hop-by-hop too. + const named = (headers.connection ?? "").toLowerCase().split(","); + const kept: OutgoingHttpHeaders = {}; + for (const [name, value] of Object.entries(headers)) { + if (value !== undefined && !HOP_BY_HOP.has(name) && !named.some((n) => n.trim() === name)) { + kept[name] = value; + } + } + return kept; +} + +function errorCode(error: unknown): string { + const code = (error as { code?: unknown } | null)?.code; + return typeof code === "string" ? code : errorMessage(error); +} + +function errorMessage(error: unknown): string { + return error instanceof Error ? error.message : String(error); +} + +/** Request targets come from any local process: keep them short and on one line. */ +function printable(value: string | undefined): string { + return JSON.stringify((value ?? "").slice(0, 200)); +} + +function isThenable(value: unknown): value is PromiseLike { + return typeof (value as { then?: unknown } | null)?.then === "function"; +} + +/** + * Close one end of a tunnel whose other end is gone: flush what is buffered, then + * destroy. The timer bounds a peer that stops reading. + */ +function endThenDestroy(socket: Duplex): void { + if (socket.destroyed) return; + if (socket.writableFinished) { + socket.destroy(); + return; + } + const timer = setTimeout(() => socket.destroy(), LINGER_MS); + timer.unref(); + socket.once("close", () => clearTimeout(timer)); + socket.once("finish", () => socket.destroy()); + if (!socket.writableEnded) socket.end(); +} + +const defaultConnect = (port: number, host: string): Socket => + // Half-open, so a client that shuts down its sending side still gets the whole answer. + connect({ port, host, allowHalfOpen: true, noDelay: true }); + +class EgressGuard implements EgressProxy { + private readonly server: Server; + private readonly calls = new Map(); + // Wrapped, so the same function can subscribe twice and unsubscribe once. + private readonly listeners = new Set<{ listener: Listener }>(); + private readonly sockets = new Set(); + private readonly connectUpstream: (port: number, host: string) => Socket; + private readonly extraHosts = new Set(); + private readonly housekeeping: ReadonlySet; + private readonly allowedPorts?: ReadonlySet; + private readonly writeLog?: (line: string) => void; + private listeningPort = 0; + private closing?: Promise; + + constructor(options: EgressProxyOptions) { + this.connectUpstream = options.connect ?? defaultConnect; + this.allowedPorts = options.allowedPorts; + this.writeLog = options.log; + for (const name of options.extraAllowedHosts ?? []) { + const host = canonicalHost(name); + if (host === undefined || isIP(host) !== 0) { + throw new TypeError( + `egress guard: extra allowed hosts must be names, got ${printable(name)}` + ); + } + this.extraHosts.add(host); + } + this.housekeeping = new Set( + (options.housekeepingHosts ?? HOUSEKEEPING_HOSTS).map((name) => canonicalHost(name) ?? name) + ); + + this.server = createServer(); + this.server.on("connection", (socket) => this.track(socket)); + this.server.on("connect", (req, socket, head) => this.onConnect(req, socket, head)); + this.server.on("request", (req, res) => this.onRequest(req, res)); + this.server.on("clientError", (error, socket) => this.onClientError(error, socket)); + // Accept failures (EMFILE, ...) arrive here; without a listener they would end the process. + this.server.on("error", (error) => this.log(`server error: ${errorCode(error)}`)); + } + + get port(): number { + return this.listeningPort; + } + + listen(): Promise { + return new Promise((resolve, reject) => { + const onError = (error: Error) => { + this.server.off("listening", onListening); + reject(error); + }; + const onListening = () => { + this.server.off("error", onError); + this.listeningPort = (this.server.address() as AddressInfo).port; + resolve(); + }; + this.server.once("error", onError); + this.server.once("listening", onListening); + this.server.listen(0, "127.0.0.1"); + }); + } + + envFor(callId: string): { HTTPS_PROXY: string; HTTP_PROXY: string } { + if (!CALL_ID.test(callId)) { + throw new TypeError( + `egress guard: a call id must match ${CALL_ID}, got ${printable(callId)}` + ); + } + if (!this.calls.has(callId)) { + this.calls.set(callId, { + refused: new Set(), + upstream: new Set(), + housekeeping: new Set(), + allowed: 0, + }); + } + const url = `http://${callId}:x@127.0.0.1:${this.listeningPort}`; + return { HTTPS_PROXY: url, HTTP_PROXY: url }; + } + + takeRecords(callId: string): EgressRecords { + const call = this.calls.get(callId); + // Forgetting the tag means a straggler of a finished call gets 407, not a relay. + this.calls.delete(callId); + return { + refused: [...(call?.refused ?? [])], + upstream: [...(call?.upstream ?? [])], + housekeeping: [...(call?.housekeeping ?? [])], + allowed: call?.allowed ?? 0, + }; + } + + onEvent(listener: Listener): () => void { + const entry = { listener }; + this.listeners.add(entry); + return () => { + this.listeners.delete(entry); + }; + } + + close(): Promise { + this.closing ??= new Promise((resolve) => { + this.listeners.clear(); + this.server.close(() => resolve()); + for (const socket of this.sockets) socket.destroy(); + }); + return this.closing; + } + + private onConnect(req: IncomingMessage, socket: Duplex, head: Buffer): void { + // First, before anything can throw: Node hands the raw socket over without an 'error' + // listener, and a client that resets right after our answer (curl, requests) would + // take the whole process down with it. + socket.on("error", (error) => this.log(`client socket error: ${errorCode(error)}`)); + try { + const target = parseAuthority(req.url ?? ""); + if (target === undefined) { + this.log(`400 CONNECT ${printable(req.url)}: not host:port`); + this.reply(socket, 400); + return; + } + const callId = this.callOf(req, `CONNECT ${target.host}:${target.port}`); + if (callId === undefined) { + this.reply(socket, 407, "", [`Proxy-Authenticate: ${PROXY_AUTHENTICATE}`]); + return; + } + const address = this.targetFor(target.host); + const refused = this.refusedTarget(target, address); + if (address === undefined || refused !== undefined) { + this.refuse(callId, refused ?? target.host); + this.reply(socket, 403, `blocked by egress guard: ${refused ?? target.host}`); + return; + } + this.tunnel(socket, head, callId, target, address); + } catch (error) { + this.log(`500 CONNECT ${printable(req.url)}: ${errorMessage(error)}`); + this.reply(socket, 500); + } + } + + private onRequest(req: IncomingMessage, res: ServerResponse): void { + req.on("error", (error) => this.log(`request error: ${errorCode(error)}`)); + res.on("error", (error) => this.log(`response error: ${errorCode(error)}`)); + try { + const target = parseAbsoluteUri(req.url ?? ""); + if (target === undefined) { + this.log(`400 ${req.method} ${printable(req.url)}: not an absolute http:// URI`); + this.respond(res, 400, "the egress guard only relays absolute http:// URIs"); + return; + } + const callId = this.callOf(req, `${req.method} ${target.host}:${target.port}`); + if (callId === undefined) { + this.respond(res, 407, "", { "Proxy-Authenticate": PROXY_AUTHENTICATE }); + return; + } + const address = this.targetFor(target.host); + const refused = this.refusedTarget(target, address); + if (address === undefined || refused !== undefined) { + this.refuse(callId, refused ?? target.host); + this.respond(res, 403, `blocked by egress guard: ${refused ?? target.host}`); + return; + } + this.forward(req, res, callId, target, address); + } catch (error) { + this.log(`500 ${req.method} ${printable(req.url)}: ${errorMessage(error)}`); + this.respond(res, 500, ""); + } + } + + private onClientError(error: Error, socket: Duplex): void { + try { + const code = errorCode(error); + this.log(`client error: ${code}`); + // A parse error gets an answer; after a reset or a timeout nobody is left to read one. + if (code.startsWith("HPE_") && socket.writable) this.reply(socket, 400); + else socket.destroy(); + } catch { + socket.destroy(); + } + } + + private tunnel( + client: Duplex, + head: Buffer, + callId: string, + target: Target, + address: string + ): void { + const upstream = this.connectUpstream(target.port, address); + this.track(upstream); + let established = false; + + upstream.on("error", (error) => { + try { + if (established || client.destroyed) { + this.log(`tunnel to ${target.host}:${target.port} ended: ${errorCode(error)}`); + return; + } + // Only a failure while the client still waits is the emulator's: a client that + // left first (the runner killed its call) must not be reported as "emulator down". + this.log(`502 CONNECT ${target.host}:${target.port}: ${errorCode(error)}`); + this.record(callId, "upstream", target.host); + this.reply(client, 502, `upstream error: ${errorCode(error)}`); + } catch (failure) { + this.log(`tunnel error handling failed: ${errorMessage(failure)}`); + client.destroy(); + } + }); + upstream.once("connect", () => { + try { + if (client.destroyed) { + upstream.destroy(); + return; + } + established = true; + client.write("HTTP/1.1 200 Connection Established\r\n\r\n"); + if (head.length > 0) upstream.write(head); + client.pipe(upstream); + upstream.pipe(client); + this.record(callId, "allowed", target.host); + } catch (error) { + this.log(`tunnel to ${target.host}:${target.port} failed: ${errorMessage(error)}`); + client.destroy(); + upstream.destroy(); + } + }); + // Half-closes travel through the pipes; these handle an end that is gone for good. + client.once("close", () => (established ? endThenDestroy(upstream) : upstream.destroy())); + upstream.once("close", () => { + if (established) endThenDestroy(client); + }); + } + + private forward( + req: IncomingMessage, + res: ServerResponse, + callId: string, + target: HttpTarget, + address: string + ): void { + // Opened here, not by http.request, so that a throwing factory lands in the caller's + // catch (a 500) instead of being reported as an upstream failure. + const socket = this.connectUpstream(target.port, address); + this.track(socket); + // The request reports socket errors, but attaches its listener a tick later; this + // covers a factory whose socket fails before that. + socket.on("error", () => undefined); + let answered = false; + let clientGone = false; + + // RFC 9112, section 3.2.2: a proxy sets Host from the absolute URI. + const headers = { ...withoutHopByHop(req.headers), host: target.authority }; + let upstream: ClientRequest; + try { + upstream = request( + { method: req.method, path: target.path, headers, createConnection: () => socket }, + (response) => { + try { + answered = true; + response.on("error", (error) => + this.log(`upstream response error: ${errorCode(error)}`) + ); + this.record(callId, "allowed", target.host); + const status = response.statusCode ?? 502; + const kept = withoutHopByHop(response.headers); + if (response.statusMessage) res.writeHead(status, response.statusMessage, kept); + else res.writeHead(status, kept); + response.pipe(res); + } catch (error) { + this.log(`relaying ${target.host}:${target.port} failed: ${errorMessage(error)}`); + res.destroy(); + } + } + ); + } catch (error) { + socket.destroy(); + throw error; + } + upstream.on("error", (error) => { + try { + if (clientGone) return; + if (answered) { + this.log( + `upstream ${target.host}:${target.port} failed mid-response: ${errorCode(error)}` + ); + res.destroy(); + return; + } + this.log(`502 ${req.method} ${target.host}:${target.port}: ${errorCode(error)}`); + this.record(callId, "upstream", target.host); + this.respond(res, 502, `upstream error: ${errorCode(error)}`); + } catch (failure) { + this.log(`upstream error handling failed: ${errorMessage(failure)}`); + res.destroy(); + } + }); + res.once("close", () => { + if (res.writableFinished) return; + clientGone = true; + upstream.destroy(); + }); + req.pipe(upstream); + } + + /** The registered call behind a request, or undefined (and logged) without a known tag. */ + private callOf(req: IncomingMessage, what: string): string | undefined { + const tag = callTagOf(req.headers["proxy-authorization"]); + if (tag !== undefined && this.calls.has(tag)) return tag; + this.log(`407 ${what}: ${tag === undefined ? "no call tag" : "unknown call tag"}`); + return undefined; + } + + private targetFor(host: string): string | undefined { + return builtInTarget(host) ?? (this.extraHosts.has(host) ? "127.0.0.1" : undefined); + } + + /** `host:port` when an allowed host is asked for a port outside the emulator's, else undefined. */ + private refusedTarget(target: Target, address: string | undefined): string | undefined { + if (address === undefined || !this.allowedPorts || this.allowedPorts.has(target.port)) + return undefined; + return `${target.host}:${target.port}`; + } + + private refuse(callId: string, host: string): void { + this.record(callId, this.housekeeping.has(host) ? "housekeeping" : "refused", host); + } + + private record(callId: string, kind: EgressEvent["kind"], host: string): void { + const call = this.calls.get(callId); + // Records already taken mean the call has resolved: a late connection counts for nobody. + if (call === undefined) return; + if (kind === "allowed") call.allowed += 1; + else call[kind].add(host); + const event: EgressEvent = Object.freeze({ kind, host }); + for (const { listener } of [...this.listeners]) { + try { + const result: unknown = listener(callId, event); + if (isThenable(result)) { + result.then(undefined, (error) => + this.log(`event listener failed: ${errorMessage(error)}`) + ); + } + } catch (error) { + this.log(`event listener failed: ${errorMessage(error)}`); + } + } + } + + /** + * Answer a raw socket and close it. It keeps reading (and discarding) until the client + * hangs up, for at most LINGER_MS, so the answer is not lost to the reset that closing + * with unread input sends. + */ + private reply(socket: Duplex, status: number, body = "", headers: string[] = []): void { + try { + if (!socket.writable) { + socket.destroy(); + return; + } + const lines = [`HTTP/1.1 ${status} ${STATUS_CODES[status] ?? ""}`, ...headers]; + if (body) lines.push("Content-Type: text/plain"); + lines.push(`Content-Length: ${Buffer.byteLength(body)}`, "Connection: close"); + socket.end(`${lines.join("\r\n")}\r\n\r\n${body}`); + socket.resume(); + const timer = setTimeout(() => socket.destroy(), LINGER_MS); + timer.unref(); + socket.once("close", () => clearTimeout(timer)); + } catch (error) { + this.log(`answering ${status} failed: ${errorMessage(error)}`); + socket.destroy(); + } + } + + private respond( + res: ServerResponse, + status: number, + body: string, + headers: OutgoingHttpHeaders = {} + ): void { + try { + if (res.headersSent) { + res.destroy(); + return; + } + res.writeHead(status, { + ...headers, + "Content-Type": "text/plain", + "Content-Length": Buffer.byteLength(body), + Connection: "close", + }); + res.end(body); + } catch (error) { + this.log(`answering ${status} failed: ${errorMessage(error)}`); + res.destroy(); + } + } + + private track(socket: Duplex): void { + if (this.closing) { + socket.destroy(); + return; + } + this.sockets.add(socket); + socket.once("close", () => this.sockets.delete(socket)); + } + + private log(line: string): void { + try { + this.writeLog?.(`egress guard: ${line}`); + } catch { + // A failing logger must not turn a diagnostic into a crash. + } + } +} + +/** + * Start the guard on 127.0.0.1 and a port the OS picks. It resolves only once the guard + * listens: a child started earlier would spend 14 s per call on a proxy that is not there. + */ +export async function startEgressProxy(options: EgressProxyOptions = {}): Promise { + const guard = new EgressGuard(options); + await guard.listen(); + return guard; +} diff --git a/src/lib/azure/emulator.test.ts b/src/lib/azure/emulator.test.ts new file mode 100644 index 0000000..ae60345 --- /dev/null +++ b/src/lib/azure/emulator.test.ts @@ -0,0 +1,293 @@ +import https from "https"; +import net from "net"; +import type { TLSSocket } from "tls"; +import { makeTestCertificate } from "./testing/tls"; +import { + defaultEmulatorDeps, + getAzureEmulatorStatus, + isLoopbackAddress, + resetEmulatorStatusCache, + type EmulatorDeps, +} from "./emulator"; + +// The emulator status. The network is mocked, apart from one real +// HTTPS readiness case on a local TLS server. + +const CONFIG = { + port: 4566, + healthBaseUrl: "http://127.0.0.1:4566", + endpoint: "https://azure.localhost.localstack.cloud:4566", + endpointHost: "azure.localhost.localstack.cloud", + egressGuard: true, +}; + +const AZURE_HEALTH = { edition: "azure-alpha", license: true }; +const INFO = { version: "2026.9.0.dev336:1ee0fa539", edition: "azure-alpha", session_id: "s-1" }; + +function deps(over: Partial & { health?: unknown; info?: unknown } = {}) { + let clock = 0; + // `in`, not a destructuring default: an explicit `health: undefined` means "unreachable". + const { health: _h, info: _i, ...rest } = over; + const health = "health" in over ? over.health : AZURE_HEALTH; + const info = "info" in over ? over.info : INFO; + const d: EmulatorDeps = { + getJson: jest.fn(async (url: string) => (url.endsWith("/health") ? health : info)), + httpsReady: jest.fn(async () => true), + // Closed by default: no health answer then means nothing listens (not running). + tcpOpen: jest.fn(async () => false), + lookup: jest.fn(async () => { + throw new Error("DNS must not be used"); + }), + sleep: jest.fn(async (ms: number) => { + clock += ms; + }), + now: () => clock, + ...rest, + }; + return d; +} + +beforeEach(() => resetEmulatorStatusCache()); + +describe("getAzureEmulatorStatus", () => { + test("a healthy Azure emulator", async () => { + const d = deps(); + expect(await getAzureEmulatorStatus(CONFIG, d)).toEqual({ + ok: true, + edition: "azure-alpha", + license: true, + sessionId: "s-1", + version: "2026.9.0.dev336:1ee0fa539", + }); + expect(d.getJson).toHaveBeenCalledWith( + "http://127.0.0.1:4566/_localstack/health", + expect.any(Number) + ); + }); + + test("unreachable: the start advice and the port variable", async () => { + const status = await getAzureEmulatorStatus(CONFIG, deps({ health: undefined })); + expect(status).toMatchObject({ ok: false, problem: "not-running" }); + expect(status.message).toMatch(/not running at http:\/\/127\.0\.0\.1:4566/); + expect(status.message).toMatch(/service: azure/); + // The documented setting first; the Azure one only for the side-by-side layout. + expect(status.message).toMatch( + /set LOCALSTACK_PORT \(LOCALSTACK_AZURE_PORT only when it runs beside an AWS emulator\)/ + ); + }); + + test("unreachable, with LOCALSTACK_HOSTNAME naming another host: says it does not apply", async () => { + process.env.LOCALSTACK_HOSTNAME = "remote-box"; + try { + const onHost = await getAzureEmulatorStatus(CONFIG, deps({ health: undefined })); + expect(onHost.message).toMatch(/LOCALSTACK_HOSTNAME \(remote-box\) does not apply/); + // In Docker the loopback forwarder tries it, so there is nothing to say. + const inDocker = await getAzureEmulatorStatus( + { ...CONFIG, inDocker: true }, + deps({ health: undefined }) + ); + expect(inDocker.message).not.toMatch(/LOCALSTACK_HOSTNAME/); + process.env.LOCALSTACK_HOSTNAME = "localhost"; + const local = await getAzureEmulatorStatus(CONFIG, deps({ health: undefined })); + expect(local.message).not.toMatch(/LOCALSTACK_HOSTNAME/); + } finally { + delete process.env.LOCALSTACK_HOSTNAME; + } + }); + + test("busy: an open port that answers no health check is not-responding, not not-running", async () => { + const d = deps({ health: undefined, tcpOpen: jest.fn(async () => true) }); + const status = await getAzureEmulatorStatus(CONFIG, d); + expect(status).toMatchObject({ ok: false, problem: "not-responding" }); + expect(status.message).toMatch(/accepts connections but did not answer its health check/); + expect(status.message).toMatch(/Retry in a few seconds/); + expect(status.message).not.toMatch(/Start it with/); + // The short probe, then the longer one, then the port. + expect(d.getJson).toHaveBeenCalledTimes(2); + expect((d.getJson as jest.Mock).mock.calls[1][1]).toBeGreaterThan( + (d.getJson as jest.Mock).mock.calls[0][1] + ); + expect(d.tcpOpen).toHaveBeenCalledWith("127.0.0.1", 4566, expect.any(Number)); + }); + + test("a slow first health probe is rescued by the longer second one", async () => { + let calls = 0; + const d = deps({ + getJson: jest.fn(async (url: string) => + url.endsWith("/health") ? (++calls === 1 ? undefined : AZURE_HEALTH) : INFO + ), + }); + const status = await getAzureEmulatorStatus(CONFIG, d); + expect(status.ok).toBe(true); + expect(d.tcpOpen).not.toHaveBeenCalled(); + }); + + test.each(["pro", "bigdata-pro", "snowflake", undefined])( + "wrong edition: %s", + async (edition) => { + const status = await getAzureEmulatorStatus( + CONFIG, + deps({ health: { edition, license: true } }) + ); + expect(status).toMatchObject({ ok: false, problem: "wrong-edition" }); + expect(status.message).toContain(`edition: ${edition ?? "unknown"}`); + } + ); + + test("license: false", async () => { + const status = await getAzureEmulatorStatus( + CONFIG, + deps({ health: { edition: "azure-alpha", license: false } }) + ); + expect(status).toMatchObject({ ok: false, problem: "license" }); + }); + + test("HTTPS not ready yet: retries for up to 10 s, then fails with the readiness message", async () => { + const httpsReady = jest.fn(async () => false); + const d = deps({ httpsReady }); + const status = await getAzureEmulatorStatus(CONFIG, d); + expect(status).toMatchObject({ ok: false, problem: "https-not-ready", sessionId: "s-1" }); + expect(status.message).toMatch(/did not become ready within 10 s/); + expect(httpsReady.mock.calls.length).toBeGreaterThanOrEqual(20); + expect(httpsReady).toHaveBeenCalledWith( + "127.0.0.1", + 4566, + "azure.localhost.localstack.cloud", + expect.any(Number) + ); + }); + + test("HTTPS that comes up after a few retries passes", async () => { + let calls = 0; + const status = await getAzureEmulatorStatus( + CONFIG, + deps({ httpsReady: async () => ++calls >= 3 }) + ); + expect(status.ok).toBe(true); + expect(calls).toBe(3); + }); + + test("the HTTPS check needs no DNS: a resolver that throws is never called", async () => { + const d = deps(); + await getAzureEmulatorStatus(CONFIG, d); + expect(d.lookup).not.toHaveBeenCalled(); + }); + + test("a session that passed the HTTPS check once is not probed again; a new session is", async () => { + const d = deps(); + await getAzureEmulatorStatus(CONFIG, d); + await getAzureEmulatorStatus(CONFIG, d); + expect(d.httpsReady).toHaveBeenCalledTimes(1); + const restarted = deps({ info: { ...INFO, session_id: "s-2" } }); + const status = await getAzureEmulatorStatus(CONFIG, restarted); + expect(status.sessionId).toBe("s-2"); + expect(restarted.httpsReady).toHaveBeenCalledTimes(1); + }); + + test("an IPv6 endpoint connects to ::1", async () => { + const d = deps(); + await getAzureEmulatorStatus( + { ...CONFIG, endpoint: "https://[::1]:4566", endpointHost: "[::1]" }, + d + ); + expect(d.httpsReady).toHaveBeenCalledWith("::1", 4566, "::1", expect.any(Number)); + }); + + describe("with the guard off, DNS must answer loopback", () => { + const off = { ...CONFIG, egressGuard: false }; + + test("loopback answers pass", async () => { + const status = await getAzureEmulatorStatus(off, deps({ lookup: async () => ["127.0.0.1"] })); + expect(status.ok).toBe(true); + }); + + test("a non-loopback answer is refused", async () => { + const status = await getAzureEmulatorStatus( + off, + deps({ lookup: async () => ["203.0.113.5"] }) + ); + expect(status).toMatchObject({ ok: false, problem: "dns-not-loopback" }); + expect(status.message).toContain("203.0.113.5"); + }); + + test("a failed lookup gives the DNS-rebinding hint", async () => { + const status = await getAzureEmulatorStatus( + off, + deps({ + lookup: async () => { + throw new Error("ENOTFOUND"); + }, + }) + ); + expect(status).toMatchObject({ ok: false, problem: "dns" }); + expect(status.message).toMatch(/DNS-rebinding protection/); + }); + }); + + test("isLoopbackAddress", () => { + expect(isLoopbackAddress("127.0.0.1")).toBe(true); + expect(isLoopbackAddress("127.1.2.3")).toBe(true); + expect(isLoopbackAddress("::1")).toBe(true); + expect(isLoopbackAddress("::ffff:127.0.0.1")).toBe(true); + expect(isLoopbackAddress("10.0.0.1")).toBe(false); + }); +}); + +describe("the default deps against real local servers", () => { + test("httpsReady answers through SNI on 127.0.0.1", async () => { + const pems = makeTestCertificate(); + if (!pems) return; // no openssl here: the mocked cases cover the logic + let sni: string | undefined; + const server = https.createServer(pems, (req, res) => { + sni = (req.socket as TLSSocket).servername || undefined; + res.end("{}"); + }); + await new Promise((r) => server.listen(0, "127.0.0.1", r)); + const port = (server.address() as { port: number }).port; + try { + expect( + await defaultEmulatorDeps.httpsReady( + "127.0.0.1", + port, + "azure.localhost.localstack.cloud", + 3000 + ) + ).toBe(true); + expect(sni).toBe("azure.localhost.localstack.cloud"); + } finally { + await new Promise((r) => server.close(r)); + } + }); + + test("getJson returns undefined for a closed port", async () => { + const server = https.createServer(); + await new Promise((r) => server.listen(0, "127.0.0.1", r)); + const port = (server.address() as { port: number }).port; + await new Promise((r) => server.close(r)); + expect( + await defaultEmulatorDeps.getJson(`http://127.0.0.1:${port}/_localstack/health`, 2000) + ).toBeUndefined(); + expect( + await defaultEmulatorDeps.httpsReady("127.0.0.1", port, "x.localhost.localstack.cloud", 2000) + ).toBe(false); + expect(await defaultEmulatorDeps.tcpOpen("127.0.0.1", port, 2000)).toBe(false); + }); + + test("tcpOpen: a listening port that never answers HTTP is open (the busy case)", async () => { + // Accepts the connection and says nothing, like a stalled emulator. Its sockets are + // destroyed at the end: a socket nobody reads never sees the client leave. + const sockets = new Set(); + const server = net.createServer((socket) => void sockets.add(socket)); + await new Promise((r) => server.listen(0, "127.0.0.1", r)); + const port = (server.address() as { port: number }).port; + try { + expect(await defaultEmulatorDeps.tcpOpen("127.0.0.1", port, 2000)).toBe(true); + expect( + await defaultEmulatorDeps.getJson(`http://127.0.0.1:${port}/_localstack/health`, 500) + ).toBeUndefined(); + } finally { + for (const socket of sockets) socket.destroy(); + await new Promise((r) => server.close(r)); + } + }); +}); diff --git a/src/lib/azure/emulator.ts b/src/lib/azure/emulator.ts new file mode 100644 index 0000000..910dce0 --- /dev/null +++ b/src/lib/azure/emulator.ts @@ -0,0 +1,286 @@ +import { promises as dns } from "dns"; +import http from "http"; +import https from "https"; +import net from "net"; +import type { AzureConfig } from "./types"; + +/** + * Is the LocalStack Azure emulator usable? Health and info are read + * over plain HTTP on 127.0.0.1, never `localhost`: on Windows `localhost` tries ::1 + * first and costs about 2 s per call when the emulator listens on IPv4 only. The ARM + * endpoint's HTTPS readiness is checked by connecting to the loopback address with + * the ARM host as SNI, so no DNS is needed (the egress guard maps those names too). + * The node `http` modules are used instead of fetch so that no proxy setting of the + * server's environment can reroute these checks. + */ + +export type EmulatorProblem = + | "not-running" + | "not-responding" + | "wrong-edition" + | "license" + | "https-not-ready" + | "dns" + | "dns-not-loopback"; + +export interface AzureEmulatorStatus { + ok: boolean; + problem?: EmulatorProblem; + /** The user-facing explanation when `ok` is false. */ + message?: string; + edition?: string; + license?: boolean; + version?: string; + /** Changes whenever the emulator restarts; the bootstrap re-runs on a change. */ + sessionId?: string; +} + +export interface EmulatorDeps { + /** GET a JSON document over plain HTTP; undefined when nothing answers. */ + getJson(url: string, timeoutMs: number): Promise; + /** True when an HTTPS server answers at ip:port for `servername`. */ + httpsReady(ip: string, port: number, servername: string, timeoutMs: number): Promise; + /** True when something accepts a TCP connection at host:port. */ + tcpOpen(host: string, port: number, timeoutMs: number): Promise; + lookup(host: string): Promise; + sleep(ms: number): Promise; + now(): number; +} + +type StatusConfig = Pick< + AzureConfig, + "port" | "healthBaseUrl" | "endpoint" | "endpointHost" | "egressGuard" +> & + Partial>; + +/** LOCALSTACK_PORT, as for every tool; LOCALSTACK_AZURE_PORT only beside an AWS emulator. */ +const PORT_ADVICE = + "If it runs on another port, set LOCALSTACK_PORT (LOCALSTACK_AZURE_PORT only when it runs beside an AWS emulator)."; + +/** + * On the host the Azure tool reaches the emulator on this machine only (the egress guard + * relays nothing else), so a LOCALSTACK_HOSTNAME naming another host does not apply: say so. + */ +function hostnameNote(config: StatusConfig): string { + const hostname = process.env.LOCALSTACK_HOSTNAME?.trim(); + if (!hostname || config.inDocker) return ""; + if (/^(localhost|127\.0\.0\.1|::1|\[::1\]|(.+\.)?localhost\.localstack\.cloud)$/i.test(hostname)) + return ""; + return ` LOCALSTACK_HOSTNAME (${hostname}) does not apply to the Azure tool: on the host it reaches the emulator on this machine only.`; +} + +const HEALTH_TIMEOUT_MS = 3000; +/** The second, longer health probe: a busy emulator can miss the first. */ +const BUSY_HEALTH_TIMEOUT_MS = 10_000; +const HTTPS_READY_WAIT_MS = 10_000; +const HTTPS_RETRY_MS = 500; + +const START_ADVICE = + "Start it with `localstack-management` (action: start, service: azure), or `lstk start --type azure`, then retry."; + +function getJson(url: string, timeoutMs: number): Promise { + return new Promise((resolve) => { + const request = http.get(url, { agent: false, timeout: timeoutMs }, (response) => { + const chunks: Buffer[] = []; + response.on("data", (c: Buffer) => chunks.push(c)); + response.on("end", () => { + try { + resolve(JSON.parse(Buffer.concat(chunks).toString("utf8"))); + } catch { + resolve({}); + } + }); + response.on("error", () => resolve(undefined)); + }); + request.on("timeout", () => request.destroy()); + request.on("error", () => resolve(undefined)); + }); +} + +function httpsReady( + ip: string, + port: number, + servername: string, + timeoutMs: number +): Promise { + return new Promise((resolve) => { + const request = https.request( + { + host: ip, + port, + servername, + path: "/_localstack/health", + method: "GET", + headers: { Host: `${servername}:${port}` }, + agent: false, + timeout: timeoutMs, + // Only readiness is checked here and nothing is sent; az verifies the + // certificate itself on every real call. + rejectUnauthorized: false, + }, + (response) => { + response.resume(); + resolve(true); + } + ); + request.on("timeout", () => request.destroy()); + request.on("error", () => resolve(false)); + request.end(); + }); +} + +function tcpOpen(host: string, port: number, timeoutMs: number): Promise { + return new Promise((resolve) => { + const socket = net.connect({ host, port }); + const done = (open: boolean) => { + socket.destroy(); + resolve(open); + }; + socket.setTimeout(timeoutMs, () => done(false)); + socket.once("connect", () => done(true)); + socket.once("error", () => done(false)); + }); +} + +export const defaultEmulatorDeps: EmulatorDeps = { + getJson, + httpsReady, + tcpOpen, + lookup: async (host) => (await dns.lookup(host, { all: true })).map((a) => a.address), + sleep: (ms) => new Promise((r) => setTimeout(r, ms)), + now: () => Date.now(), +}; + +export function isLoopbackAddress(address: string): boolean { + return /^127\./.test(address) || address === "::1" || /^::ffff:127\./i.test(address); +} + +/** The endpoint's port and the loopback address its host maps to (as the guard does). */ +function endpointTarget(config: StatusConfig) { + const url = new URL(config.endpoint); + const port = url.port ? Number(url.port) : 443; + const ip = /^\[?::1\]?$/.test(url.hostname) ? "::1" : "127.0.0.1"; + return { ip, port, servername: config.endpointHost.replace(/^\[|\]$/g, "") }; +} + +let verifiedSession: string | undefined; + +/** Forget which emulator session passed the HTTPS check (tests). */ +export function resetEmulatorStatusCache(): void { + verifiedSession = undefined; +} + +export async function getAzureEmulatorStatus( + config: StatusConfig, + deps: EmulatorDeps = defaultEmulatorDeps, + /** `localstack-management` polls on its own, so it asks for a single HTTPS probe. */ + opts: { httpsWaitMs?: number } = {} +): Promise { + const httpsWaitMs = opts.httpsWaitMs ?? HTTPS_READY_WAIT_MS; + const healthUrl = `${config.healthBaseUrl}/_localstack/health`; + type Health = { edition?: unknown; license?: unknown } | undefined; + let health = (await deps.getJson(healthUrl, HEALTH_TIMEOUT_MS)) as Health; + if (!health) { + // A busy emulator (starting side-cars under load) can miss the short probe: once more, + // longer. Then an open port that does not answer is busy, not stopped: "not + // running, start it" sent agents to `start`, which answers "already running". + health = (await deps.getJson(healthUrl, BUSY_HEALTH_TIMEOUT_MS)) as Health; + } + if (!health) { + const url = new URL(config.healthBaseUrl); + if (await deps.tcpOpen(url.hostname, Number(url.port || 80), HEALTH_TIMEOUT_MS)) { + return { + ok: false, + problem: "not-responding", + message: `The LocalStack Azure emulator at ${config.healthBaseUrl} accepts connections but did not answer its health check within ${Math.round((HEALTH_TIMEOUT_MS + BUSY_HEALTH_TIMEOUT_MS) / 1000)} s: it is probably busy (for example starting side-car containers) or still starting. Retry in a few seconds. If it stays like this, check its logs (\`localstack-logs-analysis\`, analysisType: logs) or restart it (\`localstack-management\`, action: restart, service: azure).`, + }; + } + return { + ok: false, + problem: "not-running", + message: `The LocalStack Azure emulator is not running at ${config.healthBaseUrl}. ${START_ADVICE} ${PORT_ADVICE}${hostnameNote(config)}`, + }; + } + const edition = typeof health.edition === "string" ? health.edition : undefined; + const license = typeof health.license === "boolean" ? health.license : undefined; + if (!edition || !/azure/i.test(edition)) { + return { + ok: false, + problem: "wrong-edition", + edition, + message: `The emulator at ${config.healthBaseUrl} is not the LocalStack Azure emulator (edition: ${edition ?? "unknown"}). ${START_ADVICE} ${PORT_ADVICE}${hostnameNote(config)}`, + }; + } + if (license === false) { + return { + ok: false, + problem: "license", + edition, + license, + message: `The LocalStack Azure emulator at ${config.healthBaseUrl} reports no active license (health: license false). Start it with a LOCALSTACK_AUTH_TOKEN whose license includes LocalStack for Azure.`, + }; + } + + const info = (await deps.getJson( + `${config.healthBaseUrl}/_localstack/info`, + HEALTH_TIMEOUT_MS + )) as { session_id?: unknown; version?: unknown } | undefined; + const sessionId = typeof info?.session_id === "string" ? info.session_id : undefined; + const version = typeof info?.version === "string" ? info.version : undefined; + + // HTTPS comes up seconds after plain HTTP, so a fresh emulator gets + // up to 10 s. A session that passed once is not probed again. + if (!sessionId || sessionId !== verifiedSession) { + const target = endpointTarget(config); + const deadline = deps.now() + httpsWaitMs; + let ready = await deps.httpsReady(target.ip, target.port, target.servername, HEALTH_TIMEOUT_MS); + while (!ready && deps.now() < deadline) { + await deps.sleep(HTTPS_RETRY_MS); + ready = await deps.httpsReady(target.ip, target.port, target.servername, HEALTH_TIMEOUT_MS); + } + if (!ready) { + return { + ok: false, + problem: "https-not-ready", + edition, + license, + sessionId, + version, + message: `The LocalStack Azure emulator answers on ${config.healthBaseUrl}, but its HTTPS endpoint ${config.endpoint} did not become ready within ${Math.round(httpsWaitMs / 1000)} s. It may still be starting; retry in a few seconds.`, + }; + } + if (sessionId) verifiedSession = sessionId; + } + + // With the guard on, the guard maps the names itself; only with it off does az + // depend on public DNS answering 127.0.0.1. + if (!config.egressGuard) { + let addresses: string[]; + try { + addresses = await deps.lookup(config.endpointHost); + } catch { + return { + ok: false, + problem: "dns", + edition, + license, + sessionId, + version, + message: `${config.endpointHost} did not resolve. It is a public DNS name for 127.0.0.1, and some routers and corporate resolvers block such answers (DNS-rebinding protection). Turn the egress guard back on (unset LOCALSTACK_AZ_EGRESS_GUARD; it needs no DNS), allow localhost.localstack.cloud, or add \`127.0.0.1 ${config.endpointHost}\` to your hosts file.`, + }; + } + if (!addresses.length || !addresses.every(isLoopbackAddress)) { + return { + ok: false, + problem: "dns-not-loopback", + edition, + license, + sessionId, + version, + message: `${config.endpointHost} resolved to ${addresses.join(", ") || "nothing"}, not to 127.0.0.1. With the egress guard off, az would send its requests there. Turn the guard back on (unset LOCALSTACK_AZ_EGRESS_GUARD), or fix the name in your hosts file.`, + }; + } + } + + return { ok: true, edition, license, sessionId, version }; +} diff --git a/src/lib/azure/extension-install.test.ts b/src/lib/azure/extension-install.test.ts new file mode 100644 index 0000000..4f4f8e4 --- /dev/null +++ b/src/lib/azure/extension-install.test.ts @@ -0,0 +1,178 @@ +import * as fs from "fs"; +import * as os from "os"; +import * as path from "path"; +import { + assertNotInProfile, + defaultExtensionDir, + extensionAddArgs, + ExtensionInstallError, + installExtensions, + installerEnv, + type RunAz, +} from "./extension-install"; +import { AZURE_EXTENSION_PINS } from "./extension-pins"; + +// The shared extension installer. Nothing here runs az: the runner is +// injected, and every directory is a temporary one. + +describe("extensionAddArgs", () => { + test("pinned, fail-fast, converging on the pin; --allow-preview only for preview builds", () => { + expect(extensionAddArgs({ name: "fleet", version: "1.11.1", preview: false })).toEqual([ + "extension", + "add", + "--name", + "fleet", + "--version", + "1.11.1", + "--upgrade", + "--yes", + "--only-show-errors", + ]); + expect(extensionAddArgs({ name: "cdn", version: "1.0.0b3", preview: true })).toContain( + "--allow-preview" + ); + }); +}); + +describe("the target directory", () => { + test("defaults to ~/.localstack/azure/mcp-extensions, or LOCALSTACK_AZ_EXTENSION_DIR", () => { + const home = path.join(os.tmpdir(), "home-x"); + expect(defaultExtensionDir({}, home)).toBe( + path.resolve(home, ".localstack", "azure", "mcp-extensions") + ); + expect(defaultExtensionDir({ LOCALSTACK_AZ_EXTENSION_DIR: path.join(home, "ext") }, home)).toBe( + path.resolve(home, "ext") + ); + }); + + test("never inside ~/.azure or the user's AZURE_CONFIG_DIR", () => { + const home = path.join(os.tmpdir(), "home-y"); + const ctx = { homedir: home, env: { AZURE_CONFIG_DIR: path.join(home, "other-profile") } }; + expect(() => assertNotInProfile(path.join(home, ".azure", "cliextensions"), ctx)).toThrow( + /refusing to install into/ + ); + expect(() => assertNotInProfile(path.join(home, ".azure"), ctx)).toThrow(/refusing/); + expect(() => assertNotInProfile(path.join(home, "other-profile", "ext"), ctx)).toThrow( + /refusing/ + ); + expect(() => + assertNotInProfile(path.join(home, ".localstack", "azure", "mcp-extensions"), ctx) + ).not.toThrow(); + }); + + test("a symlink into the profile is refused too (WSL's ~/.azure can link to Windows)", () => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), "lsmcp-ext-link-")); + try { + const home = path.join(root, "home"); + const profile = path.join(home, ".azure"); + fs.mkdirSync(profile, { recursive: true }); + const link = path.join(root, "innocent-looking"); + fs.symlinkSync(profile, link, process.platform === "win32" ? "junction" : "dir"); + expect(() => + assertNotInProfile(path.join(link, "cliextensions"), { homedir: home, env: {} }) + ).toThrow(/refusing/); + } finally { + fs.rmSync(root, { recursive: true, force: true }); + } + }); +}); + +describe("installerEnv", () => { + test("forces the extension dir and a throwaway config dir; the parent's are dropped", () => { + const env = installerEnv( + { PATH: "/bin", AZURE_CONFIG_DIR: "/real/profile", azure_extension_dir: "/real/ext" }, + "/target", + "/tmp/cfg" + ); + expect(env).toMatchObject({ + PATH: "/bin", + AZURE_EXTENSION_DIR: "/target", + AZURE_CONFIG_DIR: "/tmp/cfg", + AZURE_CORE_COLLECT_TELEMETRY: "no", + }); + expect(Object.values(env)).not.toContain("/real/profile"); + expect(Object.values(env)).not.toContain("/real/ext"); + }); +}); + +describe("installExtensions", () => { + let root: string; + beforeEach(() => { + root = fs.mkdtempSync(path.join(os.tmpdir(), "lsmcp-ext-install-")); + }); + afterEach(() => fs.rmSync(root, { recursive: true, force: true })); + + function recorder(failOn?: string) { + const calls: Array<{ + file: string; + args: string[]; + env: NodeJS.ProcessEnv; + configExisted: boolean; + }> = []; + const run: RunAz = async (file, args, env) => { + calls.push({ file, args, env, configExisted: fs.existsSync(env.AZURE_CONFIG_DIR!) }); + return failOn && args.includes(failOn) + ? { status: 1, output: "line 1\nERROR: the index could not be read" } + : { status: 0 }; + }; + return { calls, run }; + } + + test("every pin in order, through the given az, with a fresh config dir removed afterwards", async () => { + const { calls, run } = recorder(); + const dir = path.join(root, "ext"); + const result = await installExtensions({ + pins: AZURE_EXTENSION_PINS, + dir, + az: { file: "/opt/az/bin/python3", prefix: ["-X", "utf8", "-IBm", "azure.cli"] }, + run, + env: { PATH: "/bin", AZURE_CONFIG_DIR: "/real/profile" }, + tmpdir: root, + }); + expect(result).toEqual({ installed: 26, dir }); + expect(fs.existsSync(dir)).toBe(true); + expect(calls.map((c) => c.args[c.args.indexOf("--name") + 1])).toEqual( + AZURE_EXTENSION_PINS.map((p) => p.name) + ); + expect(calls.every((c) => c.file === "/opt/az/bin/python3")).toBe(true); + expect(calls[0].args.slice(0, 4)).toEqual(["-X", "utf8", "-IBm", "azure.cli"]); + const configDir = calls[0].env.AZURE_CONFIG_DIR!; + expect(configDir).not.toBe("/real/profile"); + expect(calls.every((c) => c.env.AZURE_CONFIG_DIR === configDir && c.configExisted)).toBe(true); + expect(calls.every((c) => c.env.AZURE_EXTENSION_DIR === dir)).toBe(true); + expect(fs.existsSync(configDir)).toBe(false); + }); + + test("the first failure stops the run, names the extension, and still removes the config dir", async () => { + const { calls, run } = recorder("dns-resolver"); + const error = await installExtensions({ + pins: AZURE_EXTENSION_PINS, + dir: path.join(root, "ext"), + az: { file: "az", prefix: [] }, + run, + tmpdir: root, + }).catch((e: unknown) => e); + expect(error).toBeInstanceOf(ExtensionInstallError); + expect((error as ExtensionInstallError).pin.name).toBe("dns-resolver"); + expect((error as Error).message).toMatch( + /dns-resolver 1\.2\.0 \(exit 1\):\n[\s\S]*index could not/ + ); + expect(calls).toHaveLength( + AZURE_EXTENSION_PINS.findIndex((p) => p.name === "dns-resolver") + 1 + ); + expect(fs.existsSync(calls[0].env.AZURE_CONFIG_DIR!)).toBe(false); + }); + + test("a spawn error is reported as such", async () => { + const run: RunAz = async () => ({ status: null, error: new Error("spawn az ENOENT") }); + await expect( + installExtensions({ + pins: AZURE_EXTENSION_PINS.slice(0, 1), + dir: path.join(root, "ext"), + az: { file: "az", prefix: [] }, + run, + tmpdir: root, + }) + ).rejects.toThrow(/could not run az: spawn az ENOENT/); + }); +}); diff --git a/src/lib/azure/extension-install.ts b/src/lib/azure/extension-install.ts new file mode 100644 index 0000000..0b3353f --- /dev/null +++ b/src/lib/azure/extension-install.ts @@ -0,0 +1,204 @@ +/** + * Installs the curated Azure CLI extensions. One installer for + * `scripts/install-azure-extensions.mjs` (CI, developers) and the `install-azure-addons` command. + * + * Every pin runs `az extension add --name --version [--allow-preview true] --upgrade + * --yes --only-show-errors` with AZURE_EXTENSION_DIR set to the target dir and AZURE_CONFIG_DIR + * set to a fresh temporary dir, so the user's own ~/.azure is never read or written. `--upgrade` + * makes a re-run converge on the pinned version: without it az keeps whatever version is + * installed and, under --only-show-errors, says nothing. The first failure stops the run. + * + * The script loads this file unbundled, through Node's built-in type stripping (Node 22.18+): + * keep it free of imports of other local modules and of TypeScript syntax that needs a + * transform (enums, namespaces, parameter properties). + */ +import { spawn } from "child_process"; +import * as fs from "fs"; +import * as os from "os"; +import * as path from "path"; + +export interface ExtensionPin { + name: string; + version: string; + preview: boolean; +} + +/** How to run az: a file and the arguments before az's own (the CLI's Python and `-m azure.cli`). */ +export interface AzInvocation { + file: string; + prefix: string[]; +} + +export interface RunResult { + status: number | null; + signal?: string | null; + error?: Error; + /** stdout and stderr, when the runner captured them. */ + output?: string; +} + +export type RunAz = (file: string, args: string[], env: NodeJS.ProcessEnv) => Promise; + +/** The tool's extension dir: LOCALSTACK_AZ_EXTENSION_DIR, else ~/.localstack/azure/mcp-extensions. */ +export function defaultExtensionDir(env: NodeJS.ProcessEnv, homedir: string): string { + return path.resolve( + env.LOCALSTACK_AZ_EXTENSION_DIR || path.join(homedir, ".localstack", "azure", "mcp-extensions") + ); +} + +function foldCase(p: string, platform: NodeJS.Platform): string { + return platform === "win32" || platform === "darwin" ? p.toLowerCase() : p; +} + +export function isInsideDir( + child: string, + parent: string, + platform: NodeJS.Platform = process.platform +): boolean { + const rel = path.relative( + foldCase(path.resolve(parent), platform), + foldCase(path.resolve(child), platform) + ); + return rel === "" || (!rel.startsWith("..") && !path.isAbsolute(rel)); +} + +/** + * The path with symlinks resolved as far as it exists: the nearest existing ancestor is + * resolved and the rest re-appended. (In WSL, ~/.azure can be a link to the Windows profile.) + */ +function realish(p: string): string { + let head = path.resolve(p); + const rest: string[] = []; + for (;;) { + try { + return path.join(fs.realpathSync(head), ...rest); + } catch { + const parent = path.dirname(head); + if (parent === head) return path.resolve(p); + rest.unshift(path.basename(head)); + head = parent; + } + } +} + +/** + * Throws when `dir` lies inside the user's own CLI profile (~/.azure, or their + * AZURE_CONFIG_DIR): installing there would change their real `az`. Both the paths as + * written and their symlink-resolved forms are compared. + */ +export function assertNotInProfile( + dir: string, + ctx: { homedir: string; env: NodeJS.ProcessEnv; platform?: NodeJS.Platform } +): void { + const platform = ctx.platform ?? process.platform; + const profiles = [path.join(ctx.homedir, ".azure"), ctx.env.AZURE_CONFIG_DIR].filter( + (p): p is string => Boolean(p) + ); + for (const profile of profiles) { + const inside = + isInsideDir(dir, profile, platform) || isInsideDir(realish(dir), realish(profile), platform); + if (inside) throw new Error(`refusing to install into ${dir}: it is inside ${profile}`); + } +} + +export function extensionAddArgs(pin: ExtensionPin): string[] { + return [ + "extension", + "add", + "--name", + pin.name, + "--version", + pin.version, + ...(pin.preview ? ["--allow-preview", "true"] : []), + "--upgrade", + "--yes", + "--only-show-errors", + ]; +} + +/** The parent environment with the extension dir and a throwaway config dir forced in. */ +export function installerEnv( + parent: NodeJS.ProcessEnv, + extensionDir: string, + configDir: string +): NodeJS.ProcessEnv { + const env: NodeJS.ProcessEnv = {}; + for (const [key, value] of Object.entries(parent)) { + const upper = key.toUpperCase(); + if (upper === "AZURE_CONFIG_DIR" || upper === "AZURE_EXTENSION_DIR") continue; + env[key] = value; + } + return { + ...env, + AZURE_EXTENSION_DIR: extensionDir, + AZURE_CONFIG_DIR: configDir, + AZURE_CORE_COLLECT_TELEMETRY: "no", + AZURE_CORE_NO_COLOR: "1", + }; +} + +/** Spawns az without a shell; `inherit` shows its output, `pipe` keeps it for an error message. */ +export function spawnAz(stdio: "inherit" | "pipe"): RunAz { + return (file, args, env) => + new Promise((resolve) => { + let output = ""; + const child = spawn(file, args, { env, shell: false, stdio, windowsHide: true }); + child.stdout?.on("data", (d: Buffer) => (output += d.toString())); + child.stderr?.on("data", (d: Buffer) => (output += d.toString())); + child.on("error", (error) => resolve({ status: null, error, output })); + child.on("close", (status, signal) => resolve({ status, signal, output })); + }); +} + +export class ExtensionInstallError extends Error { + pin: ExtensionPin; + exitCode: number; + constructor(message: string, pin: ExtensionPin, exitCode: number) { + super(message); + this.pin = pin; + this.exitCode = exitCode; + } +} + +/** Installs every pin into `dir`, in order; the first failure throws ExtensionInstallError. */ +export async function installExtensions(opts: { + pins: readonly ExtensionPin[]; + dir: string; + az: AzInvocation; + run: RunAz; + env?: NodeJS.ProcessEnv; + tmpdir?: string; + onProgress?: (index: number, total: number, pin: ExtensionPin) => void; +}): Promise<{ installed: number; dir: string }> { + fs.mkdirSync(opts.dir, { recursive: true }); + const configDir = fs.mkdtempSync( + path.join(opts.tmpdir ?? os.tmpdir(), "localstack-az-ext-config-") + ); + try { + const env = installerEnv(opts.env ?? process.env, opts.dir, configDir); + for (const [index, pin] of opts.pins.entries()) { + opts.onProgress?.(index, opts.pins.length, pin); + const run = await opts.run(opts.az.file, [...opts.az.prefix, ...extensionAddArgs(pin)], env); + const label = `${pin.name} ${pin.version}${pin.preview ? " (preview)" : ""}`; + if (run.error) { + throw new ExtensionInstallError( + `could not run ${opts.az.file}: ${run.error.message}`, + pin, + 1 + ); + } + if (run.status !== 0) { + const how = run.status === null ? `signal ${run.signal}` : `exit ${run.status}`; + const tail = run.output?.trim().split(/\r?\n/).slice(-5).join("\n"); + throw new ExtensionInstallError( + `az extension add failed for ${label} (${how})${tail ? `:\n${tail}` : ""}`, + pin, + run.status || 1 + ); + } + } + } finally { + fs.rmSync(configDir, { recursive: true, force: true }); + } + return { installed: opts.pins.length, dir: opts.dir }; +} diff --git a/src/lib/azure/extension-map.test.ts b/src/lib/azure/extension-map.test.ts new file mode 100644 index 0000000..e64ee95 --- /dev/null +++ b/src/lib/azure/extension-map.test.ts @@ -0,0 +1,303 @@ +import { spawnSync } from "child_process"; +import * as fs from "fs"; +import * as os from "os"; +import * as path from "path"; +import { + EXTENSION_COMMANDS, + commandWords, + extensionFor, + listInstalledExtensions, + missingCuratedExtensions, + missingExtensionHint, + parsePinList, +} from "./extension-map"; + +const REPO = path.join(__dirname, "../../.."); +const PIN_LIST = path.join(REPO, "docker", "azure-extensions.txt"); +const SCRIPT = path.join(REPO, "scripts", "install-azure-extensions.mjs"); + +// The 26 curated extensions, the six preview-only ones among them. +const CURATED = [ + "acrcssc", + "acrquery", + "acrtransfer", + "application-insights", + "azure-firewall", + "bastion", + "cdn", + "dns-resolver", + "documentdb", + "edge-action", + "eventgrid", + "express-route-cross-connection", + "fleet", + "front-door", + "ip-group", + "k8s-configuration", + "k8s-extension", + "monitor-control-service", + "nsp", + "resource-graph", + "scheduled-query", + "staticwebapp", + "virtual-network-manager", + "virtual-network-tap", + "virtual-wan", + "webapp", +]; +const PREVIEW_ONLY = [ + "acrcssc", + "cdn", + "edge-action", + "eventgrid", + "scheduled-query", + "virtual-network-tap", +]; + +describe("the pin list (docker/azure-extensions.txt)", () => { + const pins = parsePinList(fs.readFileSync(PIN_LIST, "utf8")); + + it("pins exactly the 26 curated extensions", () => { + expect(pins.map((p) => p.name).sort()).toEqual([...CURATED].sort()); + }); + + it("marks the six preview-only builds, and pins application-insights to the stable 1.2.3", () => { + expect( + pins + .filter((p) => p.preview) + .map((p) => p.name) + .sort() + ).toEqual([...PREVIEW_ONLY].sort()); + expect(pins.find((p) => p.name === "application-insights")).toEqual({ + name: "application-insights", + version: "1.2.3", + preview: false, + }); + // A preview flag goes with a pre-release version and nothing else. + for (const p of pins) expect(p.preview).toBe(/[a-z]/.test(p.version)); + }); + + it("pins the versions checked with azure-cli 2.90.0 (2026-09-27)", () => { + const versions = Object.fromEntries(pins.map((p) => [p.name, p.version])); + expect(versions).toMatchObject({ + acrcssc: "1.0.0b8", + cdn: "1.0.0b3", + fleet: "1.11.1", + "k8s-extension": "1.9.1", + "resource-graph": "2.1.1", + "virtual-network-manager": "3.0.2", + webapp: "0.4.0", + }); + }); +}); + +describe("parsePinList", () => { + it("reads names, versions and the preview flag, skipping comments and blank lines", () => { + const text = + "\uFEFF# header\r\n\r\n fleet 1.11.1 \r\ncdn 1.0.0b3 preview\r\n # indented comment\n"; + expect(parsePinList(text)).toEqual([ + { name: "fleet", version: "1.11.1", preview: false }, + { name: "cdn", version: "1.0.0b3", preview: true }, + ]); + expect(parsePinList("")).toEqual([]); + }); + + it.each([ + ["fleet", /line 1: expected "name version \[preview\]"/], + ["fleet 1.0 preview extra", /line 1: expected/], + ["fleet 1.0 # pinned", /line 1: expected/], + ["cdn 1.0.0b3 beta", /the third field must be "preview"/], + ["fleet latest", /invalid version "latest"/], + ["-x 1.0", /invalid extension name/], + ["fleet 1.0\nfleet 1.1", /line 2: fleet is listed twice/], + ])("refuses %j", (text, message) => { + expect(() => parsePinList(text)).toThrow(message); + }); +}); + +describe("EXTENSION_COMMANDS", () => { + it("maps the same 26 extensions as the pin list", () => { + const pinned = parsePinList(fs.readFileSync(PIN_LIST, "utf8")) + .map((p) => p.name) + .sort(); + expect(Object.keys(EXTENSION_COMMANDS).sort()).toEqual(pinned); + }); + + it("claims no core group", () => { + const groups = Object.values(EXTENSION_COMMANDS).flat(); + // Core parents (and containerapp, core in 2.85-2.90) are never keys of their own. + for (const core of [ + "acr", + "monitor", + "network", + "webapp", + "staticwebapp", + "eventgrid", + "containerapp", + "network vnet", + "network nic", + ]) { + expect(groups).not.toContain(core); + } + // Every one-word key is a group that no core version has (cdn and afd from 2.90 on). + const topLevel = groups.filter((g) => !g.includes(" ")).sort(); + expect(topLevel).toEqual( + [ + "afd", + "cdn", + "dns-resolver", + "documentdb", + "edge-action", + "fleet", + "graph", + "k8s-configuration", + "k8s-extension", + ].sort() + ); + expect(new Set(groups).size).toBe(groups.length); + for (const g of groups) expect(g).toMatch(/^[a-z0-9-]+( [a-z0-9-]+)*$/); + }); +}); + +describe("extensionFor", () => { + it("names the extension of the path az did not recognise", () => { + expect(extensionFor(["graph"])).toBe("resource-graph"); + expect(extensionFor(["monitor", "app-insights"])).toBe("application-insights"); + expect(extensionFor(["k8s-extension"])).toBe("k8s-extension"); + expect(extensionFor(["network", "front-door"])).toBe("front-door"); + expect(extensionFor(["network", "vnet", "tap"])).toBe("virtual-network-tap"); + expect(extensionFor(["afd", "rule", "action", "show"])).toBe("cdn"); + }); + + it("matches the whole path only, so a typo under a known group is not a missing extension", () => { + expect(extensionFor(["monitor"])).toBeUndefined(); + expect(extensionFor(["afd", "bogus"])).toBeUndefined(); + expect(extensionFor(["graph", "query"])).toBeUndefined(); + expect(extensionFor(["containerapp"])).toBeUndefined(); + expect(extensionFor([])).toBeUndefined(); + }); + + it("stops at the first flag", () => { + expect(commandWords(["graph", "--query", "x", "y"])).toEqual(["graph"]); + expect(extensionFor(["graph", "-q", "Resources"])).toBe("resource-graph"); + }); + + it("is silent when the extension is installed", () => { + expect(extensionFor(["graph"], new Set(["resource-graph"]))).toBeUndefined(); + expect(extensionFor(["graph"], new Set(["fleet"]))).toBe("resource-graph"); + }); +}); + +describe("missingExtensionHint", () => { + it("has the two extension hints: the add-ons command on a host, the image's set in Docker", () => { + expect(missingExtensionHint("graph", "resource-graph", false)).toBe( + "`az graph` comes from the `resource-graph` Azure CLI extension, which is not installed for " + + "this tool (automatic installs are disabled). Install the tool's extensions once with " + + "`npx -y @localstack/localstack-mcp-server install-azure-addons`; meanwhile `rest` with a " + + "relative URL usually works." + ); + expect(missingExtensionHint("graph", "resource-graph", false)).not.toMatch(/wizard|\binit\b/); + expect(missingExtensionHint("graph", "resource-graph", true)).toBe( + "`resource-graph` is not in this image's curated set; use `rest` with a relative URL." + ); + }); +}); + +describe("listInstalledExtensions", () => { + let dir: string; + beforeEach(() => { + dir = fs.mkdtempSync(path.join(os.tmpdir(), "lsmcp-ext-")); + }); + afterEach(() => { + fs.rmSync(dir, { recursive: true, force: true }); + }); + + it("counts a directory with package metadata, like az does", () => { + fs.mkdirSync(path.join(dir, "resource-graph", "resource_graph-2.1.1.dist-info"), { + recursive: true, + }); + fs.mkdirSync(path.join(dir, "fleet", "azext_fleet"), { recursive: true }); + fs.writeFileSync(path.join(dir, ".install-cdn.log"), ""); + const installed = listInstalledExtensions(dir); + expect([...installed]).toEqual(["resource-graph"]); + expect(missingCuratedExtensions(installed)).toHaveLength(25); + expect(missingCuratedExtensions(installed)).not.toContain("resource-graph"); + }); + + it("returns nothing for a missing directory", () => { + expect(listInstalledExtensions(path.join(dir, "absent")).size).toBe(0); + }); +}); + +// The installer imports parsePinList from this module through Node's type stripping (Node 22.18+). +const typeStripping = Boolean((process.features as unknown as Record).typescript); + +(typeStripping ? describe : describe.skip)("scripts/install-azure-extensions.mjs --dry-run", () => { + let dir: string; + beforeEach(() => { + dir = fs.mkdtempSync(path.join(os.tmpdir(), "lsmcp-ext-dry-")); + }); + afterEach(() => { + fs.rmSync(dir, { recursive: true, force: true }); + }); + + const script = (args: string[], env: NodeJS.ProcessEnv = process.env) => + spawnSync(process.execPath, [SCRIPT, ...args], { encoding: "utf8", env, timeout: 30_000 }); + + it("prints one fail-fast install per pin and changes nothing", () => { + const target = path.join(dir, "extensions"); + // Not `python.exe` off Windows: under WSL the script rightly refuses a Windows az. + const python = path.join(dir, "fake", process.platform === "win32" ? "python.exe" : "python3"); + const result = script(["--dry-run", "--dir", target, "--az-python", python]); + expect(result.status).toBe(0); + const lines = result.stdout.trim().split(/\r?\n/); + expect(lines).toContain(`# AZURE_EXTENSION_DIR=${path.resolve(target)}`); + const commands = lines.filter((line) => !line.startsWith("#")); + const pins = parsePinList(fs.readFileSync(PIN_LIST, "utf8")); + expect(commands).toHaveLength(pins.length); + pins.forEach((pin, i) => { + const flags = pin.preview ? " --allow-preview true" : ""; + const argv = + `-X utf8 -IBm azure.cli extension add --name ${pin.name} --version ${pin.version}` + + `${flags} --upgrade --yes --only-show-errors`; + expect(commands[i].endsWith(argv)).toBe(true); + expect(commands[i].startsWith(python.includes(" ") ? `"${python}"` : python)).toBe(true); + }); + expect(fs.existsSync(target)).toBe(false); + }); + + it("defaults to LOCALSTACK_AZ_EXTENSION_DIR", () => { + const target = path.join(dir, "from-env"); + const result = script(["--dry-run", "--az", "az"], { + ...process.env, + LOCALSTACK_AZ_EXTENSION_DIR: target, + }); + expect(result.status).toBe(0); + expect(result.stdout).toContain(`# AZURE_EXTENSION_DIR=${path.resolve(target)}`); + }); + + it("refuses a pin list it cannot parse, and a target inside ~/.azure", () => { + const list = path.join(dir, "pins.txt"); + fs.writeFileSync(list, "fleet 1.11.1 # trailing comment\n"); + const bad = script(["--dry-run", "--list", list, "--az", "az"]); + expect(bad.status).toBe(2); + expect(bad.stderr).toContain("line 1: expected"); + + // Only compares paths: nothing under ~/.azure is read or written. + const inside = script([ + "--dry-run", + "--dir", + path.join(os.homedir(), ".azure", "cliextensions"), + "--az", + "az", + ]); + expect(inside.status).toBe(2); + expect(inside.stderr).toContain("refusing to install into"); + }); + + it("rejects unknown arguments", () => { + const result = script(["--dry-run", "--bogus"]); + expect(result.status).toBe(2); + expect(result.stderr).toContain("unknown argument --bogus"); + }); +}); diff --git a/src/lib/azure/extension-map.ts b/src/lib/azure/extension-map.ts new file mode 100644 index 0000000..dc179a8 --- /dev/null +++ b/src/lib/azure/extension-map.ts @@ -0,0 +1,202 @@ +/** + * The curated Azure CLI extensions and the command groups they add, for the + * missing-extension hint. + * + * The tool runs `az` with `extension.use_dynamic_install=no`, so a command from an extension that + * is not installed fails like a typo: `ERROR: 'graph' is misspelled or not recognized by the + * system.`. az never names the extension, so the hint needs this map. + * + * This file is also loaded, unbundled, by `scripts/install-azure-extensions.mjs` through Node's + * built-in TypeScript type stripping. Keep it free of runtime imports of other local modules and of + * TypeScript syntax that needs a transform (enums, namespaces, parameter properties). + */ +import * as fs from "fs"; +import * as path from "path"; + +export interface PinnedExtension { + name: string; + version: string; + /** A preview-only build: `az extension add` needs `--allow-preview true`. */ + preview: boolean; +} + +/** + * Parses `docker/azure-extensions.txt`: one `name version [preview]` per line, `#` comment lines + * and blank lines ignored. The Docker build reads the same file with `while read -r name version + * flag`, so a trailing comment would land in `flag` there; this parser refuses it instead. + */ +export function parsePinList(text: string): PinnedExtension[] { + const pins: PinnedExtension[] = []; + const seen = new Set(); + const lines = text.replace(/^/, "").split(/\r?\n/); + lines.forEach((raw, index) => { + const line = raw.trim(); + if (!line || line.startsWith("#")) return; + const where = `line ${index + 1}`; + const fields = line.split(/\s+/); + if (fields.length < 2 || fields.length > 3) { + throw new Error(`${where}: expected "name version [preview]", got "${line}"`); + } + const [name, version, flag] = fields; + if (!/^[A-Za-z0-9][A-Za-z0-9._-]*$/.test(name)) { + throw new Error(`${where}: invalid extension name "${name}"`); + } + if (!/^\d+(\.\d+)*([a-z]+\d*)?$/.test(version)) { + throw new Error(`${where}: invalid version "${version}" for ${name}`); + } + if (flag !== undefined && flag !== "preview") { + throw new Error(`${where}: the third field must be "preview", got "${flag}"`); + } + if (seen.has(name)) { + throw new Error(`${where}: ${name} is listed twice`); + } + seen.add(name); + pins.push({ name, version, preview: flag === "preview" }); + }); + return pins; +} + +/** + * The command groups (or commands) each curated extension adds, keyed by extension name. + * + * Each entry is the shortest command path that core `az` does not know: the token az reports as + * "misspelled or not recognized" when the extension is missing. Derived by comparing the + * command table of azure-cli 2.90.0 with the 26 extensions installed (every command whose + * source is `ext:`) with the core-only table, then checked against the core tables of + * az 2.85.0 (MSI) and 2.87.0 (pip), dumped offline on + * 2026-09-27. The keys agree across the three versions except for `cdn`, see below. + * + * `containerapp` is core in 2.85-2.90 (its extension overrides core commands and is not curated), + * so it is not here. At start the handler builds the set of installed extensions + * (`listInstalledExtensions`) and passes it to `extensionFor`. + */ +export const EXTENSION_COMMANDS: Readonly> = { + acrcssc: ["acr supply-chain"], + acrquery: ["acr query"], + acrtransfer: ["acr export-pipeline", "acr import-pipeline", "acr pipeline-run"], + "application-insights": ["monitor app-insights"], + "azure-firewall": ["network firewall"], + bastion: ["network bastion"], + // `afd` and `cdn` are core (deprecated) in 2.85 and 2.87 and extension-only from 2.90. On the + // older versions az recognises both groups itself, so only the six leaf commands their core + // lacks can be reported as missing there; `afd` and `cdn` match only on 2.90 and later. + cdn: [ + "afd", + "cdn", + "afd rule action show", + "afd rule action update", + "afd rule action wait", + "afd rule condition show", + "afd rule condition update", + "afd rule condition wait", + ], + "dns-resolver": ["dns-resolver"], + documentdb: ["documentdb"], + "edge-action": ["edge-action"], + eventgrid: ["eventgrid namespace"], + "express-route-cross-connection": ["network cross-connection"], + fleet: ["fleet"], + "front-door": ["network front-door"], + "ip-group": ["network ip-group"], + "k8s-configuration": ["k8s-configuration"], + "k8s-extension": ["k8s-extension"], + "monitor-control-service": ["monitor data-collection"], + nsp: ["network perimeter"], + "resource-graph": ["graph"], + "scheduled-query": ["monitor scheduled-query"], + staticwebapp: ["staticwebapp dbconnection"], + "virtual-network-manager": ["network manager"], + "virtual-network-tap": ["network nic vtap-config", "network vnet tap"], + "virtual-wan": [ + "network p2s-vpn-gateway", + "network vhub", + "network vpn-gateway", + "network vpn-server-config", + "network vpn-site", + "network vwan", + ], + webapp: ["webapp scan"], +}; + +const GROUP_TO_EXTENSION: ReadonlyMap = new Map( + Object.entries(EXTENSION_COMMANDS).flatMap(([extension, groups]) => + groups.map((group): [string, string] => [group, extension]) + ) +); + +/** The command words of an argv: the leading tokens before the first flag. */ +export function commandWords(tokens: readonly string[]): string[] { + const words: string[] = []; + for (const token of tokens) { + if (token.startsWith("-")) break; + words.push(token); + } + return words; +} + +/** + * The curated extension that provides exactly this command path, when it is not installed. + * + * `tokens` is the command path up to and including the token az did not recognise, for example + * `["monitor", "app-insights"]` → `application-insights`. The match is exact on purpose: for + * `afd bogus` on az 2.85 the unknown token is `bogus` under the core `afd` group, which is a typo, + * not a missing extension. Returns undefined when the extension is in `installed`; without + * `installed`, every curated extension counts as missing. + */ +export function extensionFor( + tokens: readonly string[], + installed?: ReadonlySet +): string | undefined { + const words = commandWords(tokens); + if (words.length === 0) return undefined; + const extension = GROUP_TO_EXTENSION.get(words.join(" ")); + if (!extension || installed?.has(extension)) return undefined; + return extension; +} + +/** + * The `extension` hint. On a host (npx) the extensions are installed once, by the user, with the + * `install-azure-addons` command (spelled out here: this file has no runtime imports); in the + * image all curated extensions are installed, so a miss there means the extension is not part of + * the image. + */ +export function missingExtensionHint(group: string, ext: string, inDocker: boolean): string { + if (inDocker) { + return `\`${ext}\` is not in this image's curated set; use \`rest\` with a relative URL.`; + } + return ( + `\`az ${group}\` comes from the \`${ext}\` Azure CLI extension, which is not installed ` + + `for this tool (automatic installs are disabled). Install the tool's extensions once with ` + + "`npx -y @localstack/localstack-mcp-server install-azure-addons`; meanwhile `rest` with a " + + "relative URL usually works." + ); +} + +/** + * The extensions installed in an `AZURE_EXTENSION_DIR`. Like az itself, a subdirectory counts when + * it holds package metadata (`*.dist-info` or `*.egg-info`); a missing dir means none. + */ +export function listInstalledExtensions(dir: string): Set { + const installed = new Set(); + let entries: fs.Dirent[]; + try { + entries = fs.readdirSync(dir, { withFileTypes: true }); + } catch { + return installed; + } + for (const entry of entries) { + if (!entry.isDirectory()) continue; + try { + const inner = fs.readdirSync(path.join(dir, entry.name)); + if (inner.some((name) => /\..+-info$/.test(name))) installed.add(entry.name); + } catch { + // unreadable: treat as not installed + } + } + return installed; +} + +/** The curated extensions that are not installed, for a start-up warning. */ +export function missingCuratedExtensions(installed: ReadonlySet): string[] { + return Object.keys(EXTENSION_COMMANDS).filter((name) => !installed.has(name)); +} diff --git a/src/lib/azure/extension-pins.test.ts b/src/lib/azure/extension-pins.test.ts new file mode 100644 index 0000000..6cbe39f --- /dev/null +++ b/src/lib/azure/extension-pins.test.ts @@ -0,0 +1,11 @@ +import * as fs from "fs"; +import * as path from "path"; +import { parsePinList } from "./extension-map"; +import { AZURE_EXTENSION_PINS } from "./extension-pins"; + +// The wizard's embedded copy of docker/azure-extensions.txt must equal the file, which the +// Docker build and the install script read. Update extension-pins.ts with it. +test("the embedded pin list equals docker/azure-extensions.txt", () => { + const file = path.join(__dirname, "../../../docker/azure-extensions.txt"); + expect(AZURE_EXTENSION_PINS).toEqual(parsePinList(fs.readFileSync(file, "utf8"))); +}); diff --git a/src/lib/azure/extension-pins.ts b/src/lib/azure/extension-pins.ts new file mode 100644 index 0000000..5f059dc --- /dev/null +++ b/src/lib/azure/extension-pins.ts @@ -0,0 +1,35 @@ +import type { PinnedExtension } from "./extension-map"; + +/** + * `docker/azure-extensions.txt`, embedded for the `install-azure-addons` command: the npm package + * ships only `dist/`, so the command cannot read the file at run time. The Docker build and the install + * script read the file itself; `extension-pins.test.ts` keeps the two equal. + */ +export const AZURE_EXTENSION_PINS: readonly PinnedExtension[] = [ + { name: "acrcssc", version: "1.0.0b8", preview: true }, + { name: "acrquery", version: "1.0.1", preview: false }, + { name: "acrtransfer", version: "2.0.0", preview: false }, + { name: "application-insights", version: "1.2.3", preview: false }, + { name: "azure-firewall", version: "2.2.1", preview: false }, + { name: "bastion", version: "1.4.3", preview: false }, + { name: "cdn", version: "1.0.0b3", preview: true }, + { name: "dns-resolver", version: "1.2.0", preview: false }, + { name: "documentdb", version: "1.0.0", preview: false }, + { name: "edge-action", version: "1.0.0b4", preview: true }, + { name: "eventgrid", version: "1.0.0b2", preview: true }, + { name: "express-route-cross-connection", version: "1.0.0", preview: false }, + { name: "fleet", version: "1.11.1", preview: false }, + { name: "front-door", version: "2.3.0", preview: false }, + { name: "ip-group", version: "1.0.1", preview: false }, + { name: "k8s-configuration", version: "2.3.0", preview: false }, + { name: "k8s-extension", version: "1.9.1", preview: false }, + { name: "monitor-control-service", version: "1.2.0", preview: false }, + { name: "nsp", version: "1.1.0", preview: false }, + { name: "resource-graph", version: "2.1.1", preview: false }, + { name: "scheduled-query", version: "1.0.0b2", preview: true }, + { name: "staticwebapp", version: "1.0.1", preview: false }, + { name: "virtual-network-manager", version: "3.0.2", preview: false }, + { name: "virtual-network-tap", version: "1.0.0b2", preview: true }, + { name: "virtual-wan", version: "1.0.1", preview: false }, + { name: "webapp", version: "0.4.0", preview: false }, +]; diff --git a/src/lib/azure/local-hosts.ts b/src/lib/azure/local-hosts.ts new file mode 100644 index 0000000..90cc193 --- /dev/null +++ b/src/lib/azure/local-hosts.ts @@ -0,0 +1,39 @@ +import { isAllowedEgressHost } from "./egress-proxy"; + +/** + * The host names `az` may reach. The + * endpoint override, the policy's URL rule and the egress guard share one check, the + * guard's own, so they always agree (it also canonicalises case, a trailing dot, + * brackets and IP spellings such as `127.1`). LocalStack's public DNS answers + * 127.0.0.1 for the `localhost.localstack.cloud` names, and the guard maps them itself + * without DNS. The apex matters: the emulator's long-running-operation `Location` + * headers use it. + */ +export function isLocalHost(host: string): boolean { + return isAllowedEgressHost(host); +} + +/** + * The ports `az` may reach on those hosts: the emulator's gateway port, 443 (its other default + * listener, which port-less endpoint URLs use) and its external service range, which Service + * Bus and Event Hubs endpoints, Cosmos DB's document endpoint and the AKS API server use. The + * range is EXTERNAL_SERVICE_PORTS_START..END when set, else 4510-4560 as this server publishes + * it. The policy's URL rule and the egress guard share it. + */ +export function emulatorPorts( + gatewayPorts: readonly number[], + env: NodeJS.ProcessEnv = process.env +): ReadonlySet { + const read = (...names: string[]) => Number(names.map((n) => env[n]?.trim()).find(Boolean)); + const start = + read("EXTERNAL_SERVICE_PORTS_START", "LOCALSTACK_EXTERNAL_SERVICE_PORTS_START") || 4510; + const end = read("EXTERNAL_SERVICE_PORTS_END", "LOCALSTACK_EXTERNAL_SERVICE_PORTS_END"); + const last = Math.min(end >= start ? end : start + 50, start + 1000); + const ports = new Set([...gatewayPorts, 443]); + for (let port = start; port <= last; port++) ports.add(port); + return ports; +} + +/** The same list as a pattern, for messages and documentation. */ +export const LOCAL_HOST = + /(^|\.)localhost\.localstack\.cloud$|^localhost$|^127\.0\.0\.1$|^\[?::1\]?$/i; diff --git a/src/lib/azure/loopback-forwarder.test.ts b/src/lib/azure/loopback-forwarder.test.ts new file mode 100644 index 0000000..52203a7 --- /dev/null +++ b/src/lib/azure/loopback-forwarder.test.ts @@ -0,0 +1,353 @@ +import net from "net"; +import { + DEFAULT_SERVICE_PORTS, + ensureLoopbackForwarder, + planMirrors, + resetLoopbackForwarder, + startForwarder, + totalForwarderConnections, + type Forwarder, + type ForwarderDeps, +} from "./loopback-forwarder"; + +// Ephemeral ports only: binding 127.0.0.1:4566 or 4510-4560 on a developer +// machine would collide with a running emulator's published ports. + +async function echoServer(): Promise<{ port: number; close: () => Promise }> { + const sockets = new Set(); + const server = net.createServer((socket) => { + sockets.add(socket); + socket.on("close", () => sockets.delete(socket)); + socket.on("data", (chunk: Buffer) => + socket.write(Buffer.concat([Buffer.from("echo:"), chunk])) + ); + }); + await new Promise((r) => server.listen(0, "127.0.0.1", r)); + return { + port: (server.address() as net.AddressInfo).port, + close: async () => { + for (const s of sockets) s.destroy(); + await new Promise((r) => server.close(r)); + }, + }; +} + +async function freePort(): Promise { + const probe = net.createServer(); + await new Promise((r) => probe.listen(0, "127.0.0.1", r)); + const port = (probe.address() as net.AddressInfo).port; + await new Promise((r) => probe.close(r)); + return port; +} + +function roundTrip(port: number, payload: string): Promise { + return new Promise((resolve, reject) => { + const socket = net.connect(port, "127.0.0.1"); + let received = ""; + socket.on("data", (d) => { + received += d.toString(); + if (received.length >= payload.length + 5) { + socket.end(); + resolve(received); + } + }); + socket.on("error", reject); + socket.write(payload); + }); +} + +describe("startForwarder", () => { + test("pipes bytes both ways, for several concurrent connections, and counts them", async () => { + const upstream = await echoServer(); + const listen = await freePort(); + const forwarder = await startForwarder("127.0.0.1", [{ listen, target: upstream.port }]); + try { + const answers = await Promise.all( + ["a", "bb", "ccc", "dddd", "✓✓"].map((p) => roundTrip(listen, p)) + ); + expect(answers).toEqual(["echo:a", "echo:bb", "echo:ccc", "echo:dddd", "echo:✓✓"]); + expect(forwarder.connections()).toBe(5); + expect(forwarder.failures()).toBe(0); + } finally { + await forwarder.close(); + await upstream.close(); + } + }); + + test("closes cleanly: the port is free again and new connections are refused", async () => { + const upstream = await echoServer(); + const listen = await freePort(); + const forwarder = await startForwarder("127.0.0.1", [{ listen, target: upstream.port }]); + await roundTrip(listen, "x"); + await forwarder.close(); + await expect(roundTrip(listen, "x")).rejects.toThrow(); + await upstream.close(); + }); + + test("an unreachable target counts a failure and closes the client", async () => { + const dead = await freePort(); + const listen = await freePort(); + const forwarder = await startForwarder("127.0.0.1", [{ listen, target: dead }]); + try { + await new Promise((resolve) => { + const socket = net.connect(listen, "127.0.0.1"); + socket.on("close", () => resolve()); + socket.on("error", () => undefined); + }); + expect(forwarder.failures()).toBe(1); + } finally { + await forwarder.close(); + } + }); + + test("a busy service port is skipped; a busy gateway port is an error", async () => { + const upstream = await echoServer(); + const gateway = await freePort(); + // The upstream's own port is taken on 127.0.0.1: listening there must fail. + const forwarder = await startForwarder("127.0.0.1", [ + { listen: gateway, target: upstream.port }, + { listen: upstream.port, target: upstream.port }, + ]); + expect(forwarder.ports).toEqual([{ listen: gateway, target: upstream.port }]); + await forwarder.close(); + await expect( + startForwarder("127.0.0.1", [{ listen: upstream.port, target: 1 }]) + ).rejects.toThrow(/could not listen/); + await upstream.close(); + }); +}); + +describe("planMirrors", () => { + const lstk = { + ipAddresses: ["172.18.0.5"], + bindings: [ + { containerPort: 4566, hostPort: 4566 }, + { containerPort: 4510, hostPort: 4510 }, + { containerPort: 4559, hostPort: 4559 }, + ], + }; + + test("the port list comes from PortBindings", () => { + expect(planMirrors(4566, lstk, "host")).toEqual([ + { listen: 4566, target: 4566 }, + { listen: 4510, target: 4510 }, + { listen: 4559, target: 4559 }, + ]); + }); + + test("a port-shifted emulator: host targets use host ports, the container IP its container ports", () => { + const shifted = { + ipAddresses: ["172.18.0.9"], + bindings: [ + { containerPort: 4566, hostPort: 4666 }, + { containerPort: 4510, hostPort: 4610 }, + ], + }; + expect(planMirrors(4666, shifted, "host")).toEqual([ + { listen: 4666, target: 4666 }, + { listen: 4610, target: 4610 }, + ]); + expect(planMirrors(4666, shifted, "container")).toEqual([ + { listen: 4666, target: 4566 }, + { listen: 4610, target: 4510 }, + ]); + }); + + test("without a container, the default range 4510-4560", () => { + const mirrors = planMirrors(4566, undefined, "host"); + expect(mirrors[0]).toEqual({ listen: 4566, target: 4566 }); + expect(mirrors.slice(1).map((m) => m.listen)).toEqual(DEFAULT_SERVICE_PORTS); + expect(DEFAULT_SERVICE_PORTS[0]).toBe(4510); + expect(DEFAULT_SERVICE_PORTS[DEFAULT_SERVICE_PORTS.length - 1]).toBe(4560); + }); +}); + +describe("ensureLoopbackForwarder (mocked Docker socket, no real listener)", () => { + afterEach(() => resetLoopbackForwarder()); + + const fakeForwarder = (target: string): Forwarder => ({ + target, + ports: [], + connections: () => 0, + failures: () => 0, + close: async () => undefined, + }); + + function deps(over: Partial = {}) { + const start = jest.fn(async (target: string) => fakeForwarder(target)); + const d: ForwarderDeps = { + inDocker: true, + port: 4566, + canConnect: jest.fn(async () => false), + findEmulator: jest.fn(async () => ({ + ipAddresses: ["172.18.0.5"], + bindings: [{ containerPort: 4566, hostPort: 4566 }], + })), + log: jest.fn(), + start: start as unknown as ForwarderDeps["start"], + ...over, + }; + return { d, start }; + } + + test("never listens outside Docker", async () => { + const { d, start } = deps({ inDocker: false }); + expect(await ensureLoopbackForwarder(d)).toBeUndefined(); + expect(start).not.toHaveBeenCalled(); + expect(d.canConnect).not.toHaveBeenCalled(); + }); + + test("is skipped when 127.0.0.1: already answers (--network host)", async () => { + const { d, start } = deps({ + canConnect: jest.fn(async (host: string) => host === "127.0.0.1"), + }); + expect(await ensureLoopbackForwarder(d)).toBeUndefined(); + expect(start).not.toHaveBeenCalled(); + }); + + test("LOCALSTACK_AZURE_FORWARD_TARGET comes first, and is used without a probe", async () => { + const { d, start } = deps({ forwardTarget: "10.1.2.3" }); + const forwarder = await ensureLoopbackForwarder(d); + expect(forwarder?.target).toBe("10.1.2.3"); + expect(start).toHaveBeenCalledTimes(1); + }); + + test("then LOCALSTACK_HOSTNAME, when it names the emulator's host and answers", async () => { + const { d } = deps({ + hostname: "localstack-main", + canConnect: jest.fn( + async (host: string) => host === "localstack-main" || host === "host.docker.internal" + ), + }); + expect((await ensureLoopbackForwarder(d))?.target).toBe("localstack-main"); + }); + + test("an unreachable LOCALSTACK_HOSTNAME falls through to the other targets", async () => { + const { d } = deps({ + hostname: "nowhere", + canConnect: jest.fn(async (host: string) => host === "172.18.0.5"), + }); + expect((await ensureLoopbackForwarder(d))?.target).toBe("172.18.0.5"); + }); + + test("then host.docker.internal when it answers", async () => { + const { d } = deps({ + canConnect: jest.fn(async (host: string) => host === "host.docker.internal"), + }); + expect((await ensureLoopbackForwarder(d))?.target).toBe("host.docker.internal"); + }); + + test("then the emulator container's IP, with its container ports", async () => { + const { d, start } = deps({ + canConnect: jest.fn(async (host: string) => host === "172.18.0.5"), + }); + expect((await ensureLoopbackForwarder(d))?.target).toBe("172.18.0.5"); + expect(start.mock.calls[0][0]).toBe("172.18.0.5"); + }); + + test("no target answering: nothing listens, and the next call tries again", async () => { + const { d, start } = deps(); + expect(await ensureLoopbackForwarder(d)).toBeUndefined(); + expect(start).not.toHaveBeenCalled(); + (d.canConnect as jest.Mock).mockImplementation( + async (host: string) => host === "host.docker.internal" + ); + expect((await ensureLoopbackForwarder(d))?.target).toBe("host.docker.internal"); + }); + + test("a started forwarder is reused", async () => { + const { d, start } = deps({ forwardTarget: "10.1.2.3" }); + await ensureLoopbackForwarder(d); + await ensureLoopbackForwarder(d); + expect(start).toHaveBeenCalledTimes(1); + }); + + // A forwarder with a real gateway mirror, so the per-call target check has a port to probe. + function trackedDeps(ip: { current: string }) { + const closed: string[] = []; + let connections = 0; + const start = jest.fn( + async (target: string, mirrors: Array<{ listen: number; target: number }>) => + ({ + target, + ports: mirrors.slice(0, 1), + connections: () => (connections += 3), + failures: () => 0, + close: async () => { + closed.push(target); + }, + }) as Forwarder + ); + // An empty IP means the emulator is stopped: nothing answers and no container is found. + const canConnect = jest.fn(async (host: string) => ip.current !== "" && host === ip.current); + const d: ForwarderDeps = { + inDocker: true, + port: 4566, + canConnect, + findEmulator: jest.fn(async () => ({ + ipAddresses: ip.current ? [ip.current] : [], + bindings: [{ containerPort: 4566, hostPort: 4566 }], + })), + log: jest.fn(), + start: start as unknown as ForwarderDeps["start"], + }; + return { d, start, closed, canConnect }; + } + + test("a restarted emulator on a new container IP: the old target is closed and the new one used", async () => { + const ip = { current: "172.18.0.5" }; + const { d, start, closed } = trackedDeps(ip); + expect((await ensureLoopbackForwarder(d))?.target).toBe("172.18.0.5"); + ip.current = "172.18.0.9"; // the restart moved the container + expect((await ensureLoopbackForwarder(d))?.target).toBe("172.18.0.9"); + expect(closed).toEqual(["172.18.0.5"]); + expect(start).toHaveBeenCalledTimes(2); + expect((d.log as jest.Mock).mock.calls.flat().join("\n")).toMatch(/stopped answering/); + }); + + test("a live forwarder is not mistaken for an emulator on 127.0.0.1 (it answers there itself)", async () => { + const ip = { current: "172.18.0.5" }; + const { d, start, canConnect } = trackedDeps(ip); + await ensureLoopbackForwarder(d); + canConnect.mockClear(); + canConnect.mockImplementation( + async (host: string) => host === "127.0.0.1" || host === ip.current + ); + expect((await ensureLoopbackForwarder(d))?.target).toBe("172.18.0.5"); + expect(start).toHaveBeenCalledTimes(1); + expect(canConnect.mock.calls.map((c) => c[0])).toEqual(["172.18.0.5"]); + }); + + test("the emulator stopped: nothing listens until it answers again", async () => { + const ip = { current: "172.18.0.5" }; + const { d, start, closed } = trackedDeps(ip); + await ensureLoopbackForwarder(d); + ip.current = ""; + expect(await ensureLoopbackForwarder(d)).toBeUndefined(); + expect(closed).toEqual(["172.18.0.5"]); + ip.current = "172.18.0.5"; + expect((await ensureLoopbackForwarder(d))?.target).toBe("172.18.0.5"); + expect(start).toHaveBeenCalledTimes(2); + }); + + test("concurrent calls start one forwarder", async () => { + const ip = { current: "172.18.0.5" }; + const { d, start } = trackedDeps(ip); + await Promise.all([ + ensureLoopbackForwarder(d), + ensureLoopbackForwarder(d), + ensureLoopbackForwarder(d), + ]); + expect(start).toHaveBeenCalledTimes(1); + }); + + test("the connection total keeps the counts of replaced forwarders", async () => { + const ip = { current: "172.18.0.5" }; + const { d } = trackedDeps(ip); + await ensureLoopbackForwarder(d); + ip.current = "172.18.0.9"; + await ensureLoopbackForwarder(d); // retires the first (its counter reads 3) + // the retired 3, plus the live one's next reading (6: the fake counts up by 3 per read) + expect(await totalForwarderConnections()).toBe(9); + }); +}); diff --git a/src/lib/azure/loopback-forwarder.ts b/src/lib/azure/loopback-forwarder.ts new file mode 100644 index 0000000..cf40f15 --- /dev/null +++ b/src/lib/azure/loopback-forwarder.ts @@ -0,0 +1,306 @@ +import net from "net"; +import { DockerApiClient } from "../docker/docker.client"; + +/** + * The loopback forwarder. Inside the Docker + * image the emulator is not on the server's 127.0.0.1, yet everything else assumes it + * is: the health checks, the egress guard (which maps the emulator's names to + * 127.0.0.1 without DNS) and az's ARM endpoint. So, in Docker only, the server listens + * on 127.0.0.1 on the emulator's ports and pipes each connection to the real emulator. + * TLS stays end to end: bytes are relayed, never terminated. + */ + +/** The emulator's external service ports when its bindings cannot be read (the server's own spec). */ +export const DEFAULT_SERVICE_PORTS = Array.from({ length: 51 }, (_, i) => 4510 + i); + +export interface PortMirror { + /** The port on the server's 127.0.0.1. */ + listen: number; + /** The port on the target. */ + target: number; +} + +export interface Forwarder { + readonly target: string; + readonly ports: PortMirror[]; + /** Connections relayed so far; L5 reads it to prove traffic went through. */ + connections(): number; + failures(): number; + close(): Promise; +} + +/** Relay every connection on 127.0.0.1: to :. */ +export async function startForwarder( + target: string, + mirrors: PortMirror[], + opts: { listenHost?: string; log?: (line: string) => void } = {} +): Promise { + const listenHost = opts.listenHost ?? "127.0.0.1"; + let connections = 0; + let failures = 0; + const sockets = new Set(); + const servers: net.Server[] = []; + const listening: PortMirror[] = []; + + for (const mirror of mirrors) { + const server = net.createServer((client) => { + connections++; + sockets.add(client); + const upstream = net.connect(mirror.target, target); + sockets.add(upstream); + const done = () => { + client.destroy(); + upstream.destroy(); + sockets.delete(client); + sockets.delete(upstream); + }; + client.on("error", done); + client.on("close", done); + upstream.on("error", () => { + failures++; + done(); + }); + upstream.on("close", done); + client.pipe(upstream); + upstream.pipe(client); + }); + server.on("error", (error) => + opts.log?.(`forwarder: ${listenHost}:${mirror.listen}: ${error.message}`) + ); + const ok = await new Promise((resolve) => { + server.once("error", () => resolve(false)); + server.listen(mirror.listen, listenHost, () => resolve(true)); + }); + if (ok) { + servers.push(server); + listening.push(mirror); + } else if (mirror === mirrors[0]) { + // The gateway port is the one that matters; a busy service port is only skipped. + await Promise.all(servers.map((s) => new Promise((r) => s.close(r)))); + throw new Error(`the loopback forwarder could not listen on ${listenHost}:${mirror.listen}`); + } + } + + return { + target, + ports: listening, + connections: () => connections, + failures: () => failures, + async close() { + for (const socket of sockets) socket.destroy(); + await Promise.all(servers.map((s) => new Promise((r) => s.close(r)))); + }, + }; +} + +/** What the forwarder needs to know about the emulator container, from the Docker socket. */ +export interface EmulatorContainer { + /** The container's IPs (a shared network makes one reachable). */ + ipAddresses: string[]; + /** Container port → host port, for the gateway and the service ports. */ + bindings: Array<{ containerPort: number; hostPort: number }>; +} + +export interface ForwarderDeps { + inDocker: boolean; + /** LOCALSTACK_AZURE_PORT: the port az's endpoint names. */ + port: number; + /** LOCALSTACK_AZURE_FORWARD_TARGET. */ + forwardTarget?: string; + /** LOCALSTACK_HOSTNAME: where the documented configuration says the emulator lives. */ + hostname?: string; + canConnect(host: string, port: number, timeoutMs: number): Promise; + findEmulator(): Promise; + log(line: string): void; + /** Tests: replaces the real listener, so no fixed port is ever bound. */ + start?: typeof startForwarder; +} + +export function canConnect(host: string, port: number, timeoutMs: number): Promise { + return new Promise((resolve) => { + const socket = net.connect(port, host); + const finish = (ok: boolean) => { + socket.destroy(); + resolve(ok); + }; + socket.setTimeout(timeoutMs, () => finish(false)); + socket.once("connect", () => finish(true)); + socket.once("error", () => finish(false)); + }); +} + +/** The emulator container through the mounted Docker socket, when there is one. */ +async function findEmulatorViaDocker(): Promise { + try { + const docker = new DockerApiClient(); + const metadata = await docker.inspectContainer( + await docker.findLocalStackContainer({ stack: "azure" }) + ); + const bindings: EmulatorContainer["bindings"] = []; + for (const [key, hosts] of Object.entries(metadata.portBindings ?? {})) { + const containerPort = parseInt(key, 10); + for (const host of hosts ?? []) { + const hostPort = parseInt(host.HostPort ?? "", 10); + if (containerPort && hostPort) bindings.push({ containerPort, hostPort }); + } + } + return { ipAddresses: metadata.ipAddresses ?? [], bindings }; + } catch { + return undefined; + } +} + +export function defaultForwarderDeps(config: { + inDocker: boolean; + port: number; + forwardTarget?: string; +}): ForwarderDeps { + return { + inDocker: config.inDocker, + port: config.port, + forwardTarget: config.forwardTarget, + hostname: process.env.LOCALSTACK_HOSTNAME?.trim() || undefined, + canConnect, + findEmulator: findEmulatorViaDocker, + log: (line) => process.stderr.write(`[localstack-azure-client] ${line}\n`), + }; +} + +/** + * The mirrors for one target. For `host.docker.internal` (or an explicit target) the + * host ports are the targets; for the container's own IP the container ports are. + */ +export function planMirrors( + port: number, + emulator: EmulatorContainer | undefined, + kind: "host" | "container" +): PortMirror[] { + const gateway = emulator?.bindings.find((b) => b.hostPort === port); + const mirrors: PortMirror[] = [ + { listen: port, target: kind === "container" ? (gateway?.containerPort ?? 4566) : port }, + ]; + const services = emulator?.bindings.filter( + (b) => b.containerPort >= 4510 && b.containerPort <= 4560 + ); + const servicePairs = services?.length + ? services.map((b) => ({ + listen: b.hostPort, + target: kind === "container" ? b.containerPort : b.hostPort, + })) + : DEFAULT_SERVICE_PORTS.map((p) => ({ listen: p, target: p })); + for (const pair of servicePairs) { + if (!mirrors.some((m) => m.listen === pair.listen)) mirrors.push(pair); + } + return mirrors; +} + +let active: Promise | undefined; +/** Connections of forwarders already replaced, so the exit line counts every one. */ +let retiredConnections = 0; +/** The last forwarder started, read by the synchronous exit hook. */ +let latestForwarder: Forwarder | undefined; +let exitHook = false; + +/** The running forwarder, if any (tests and the test envelope). */ +export async function activeForwarder(): Promise { + return active ? active.catch(() => undefined) : undefined; +} + +/** Connections relayed by every forwarder this process started (the exit line). */ +export async function totalForwarderConnections(): Promise { + return retiredConnections + ((await activeForwarder())?.connections() ?? 0); +} + +export async function resetLoopbackForwarder(): Promise { + const forwarder = await activeForwarder(); + await forwarder?.close(); + active = undefined; + latestForwarder = undefined; + retiredConnections = 0; +} + +/** + * Start the forwarder in Docker only, and only when 127.0.0.1: does not already + * answer (for example under `--network host`). Target order: + * LOCALSTACK_AZURE_FORWARD_TARGET, then host.docker.internal, then the emulator + * container's IP through the Docker socket (this needs a shared network). When no + * target answers, nothing listens and the health check reports the emulator as down. + * + * Every call checks that a running forwarder's target still answers: a restarted + * emulator container can come back on another IP, and the old target would then fail + * every call. A target that stopped answering is closed and looked up again. + */ +export function ensureLoopbackForwarder(deps?: ForwarderDeps): Promise { + if (!deps) { + // Imported lazily so that a server that never runs an Azure command reads no config. + // eslint-disable-next-line @typescript-eslint/no-require-imports + const { azureConfig } = require("./services") as typeof import("./services"); + deps = defaultForwarderDeps(azureConfig()); + } + const d = deps; + if (!d.inDocker) return Promise.resolve(undefined); + // Chained, so concurrent calls check and replace the forwarder one at a time. + const previous = active; + active = (async () => { + const current = previous ? await previous.catch(() => undefined) : undefined; + if (current) { + const gateway = current.ports[0]; + if (!gateway || (await d.canConnect(current.target, gateway.target, 1000))) return current; + d.log( + `forwarder: ${current.target}:${gateway.target} stopped answering; looking for the emulator again` + ); + retiredConnections += current.connections(); + if (latestForwarder === current) latestForwarder = undefined; + await current.close(); + } else if (await d.canConnect("127.0.0.1", d.port, 500)) { + // Checked only without a forwarder of our own, which would answer here itself. + return undefined; + } + // Undefined when no target answers yet (the emulator may not be up): the next call tries again. + return startFirstReachable(d); + })(); + return active; +} + +async function startFirstReachable(d: ForwarderDeps): Promise { + const emulator = await d.findEmulator(); + const candidates: Array<{ host: string; kind: "host" | "container"; explicit?: boolean }> = []; + if (d.forwardTarget) candidates.push({ host: d.forwardTarget, kind: "host", explicit: true }); + // LOCALSTACK_HOSTNAME names where the emulator lives (a container on a shared network): + // checked like the others, so host.docker.internal keeps its fallback to the container IP. + if ( + d.hostname && + !/^(localhost|127\.0\.0\.1|::1|\[::1\]|host\.docker\.internal)$/i.test(d.hostname) + ) { + candidates.push({ host: d.hostname, kind: "host" }); + } + candidates.push({ host: "host.docker.internal", kind: "host" }); + for (const ip of emulator?.ipAddresses ?? []) candidates.push({ host: ip, kind: "container" }); + for (const candidate of candidates) { + const mirrors = planMirrors(d.port, emulator, candidate.kind); + if (!candidate.explicit && !(await d.canConnect(candidate.host, mirrors[0].target, 2000))) + continue; + const forwarder = await (d.start ?? startForwarder)(candidate.host, mirrors, { log: d.log }); + d.log( + `forwarder: 127.0.0.1:${d.port} -> ${candidate.host}:${mirrors[0].target} (${forwarder.ports.length} ports)` + ); + if (!exitHook) { + exitHook = true; + process.on("exit", () => { + try { + // Synchronous here: the active forwarder, if any, is the one just started or a replacement. + const live = latestForwarder?.connections() ?? 0; + process.stderr.write(`forwarder connections=${retiredConnections + live}\n`); + } catch { + // stderr already gone + } + }); + } + latestForwarder = forwarder; + return forwarder; + } + d.log( + `forwarder: no target answered on port ${d.port} (tried ${candidates.map((c) => c.host).join(", ")})` + ); + return undefined; +} diff --git a/src/lib/azure/output.test.ts b/src/lib/azure/output.test.ts new file mode 100644 index 0000000..5a95062 --- /dev/null +++ b/src/lib/azure/output.test.ts @@ -0,0 +1,1182 @@ +import * as crypto from "crypto"; +import * as fs from "fs"; +import * as path from "path"; +import * as zlib from "zlib"; +import { + GUARD_OFF_NOTE, + STDERR_FAILURE_CHARS, + bicepMissing, + bicepRegistryUnsupported, + classifyFailure, + commandTooLong, + cutHelpPage, + formatAzResult, + headCut, + prepareStderr, + redactProxyUrls, + reflowHelp, + tailCut, + type FormatOptions, + type ResultEnvelope, +} from "./output"; +import { extensionFor } from "./extension-map"; +import type { AzFailureClass, AzRunResult, EgressRecords } from "./types"; + +// Byte copies of raw az streams, stored base64 so that git cannot rewrite their CRLFs. +interface Stream { + bytes: number; + sha256: string; + encoding: "base64" | "gzip+base64"; + data: string; +} + +interface Fixture { + id: string; + classId: AzFailureClass | null; + guardOn: boolean; + argv: string[]; + exitCode: number; + synthetic: boolean; + basis?: string; + isHelp?: boolean; + program?: string; + egress?: EgressRecords; + stdout?: Stream; + stderr?: Stream; + reflowed?: { chars: number; sha256: string }; +} + +const INDEX = path.join(__dirname, "../../../tests/fixtures/azure/stderr/index.json"); +const FIXTURES: Fixture[] = JSON.parse(fs.readFileSync(INDEX, "utf8")).fixtures; + +function bytesOf(stream: Stream): Buffer { + const raw = Buffer.from(stream.data, "base64"); + return stream.encoding === "gzip+base64" ? zlib.gunzipSync(raw) : raw; +} + +// The runner decodes both streams with StringDecoder("utf8"); these are the same strings. +const text = (stream?: Stream) => (stream ? bytesOf(stream).toString("utf8") : ""); +const sha256 = (data: Buffer | string) => crypto.createHash("sha256").update(data).digest("hex"); + +function fixture(id: string): Fixture { + const found = FIXTURES.find((f) => f.id === id); + if (!found) throw new Error(`no fixture ${id}`); + return found; +} + +const NO_EGRESS: EgressRecords = { refused: [], upstream: [], housekeeping: [], allowed: 0 }; + +function run(overrides: Partial = {}): AzRunResult { + return { + exitCode: 1, + stdout: "", + stderr: "", + timedOut: false, + aborted: false, + truncated: false, + durationMs: 1_000, + egress: NO_EGRESS, + ...overrides, + }; +} + +function runOf(f: Fixture, overrides: Partial = {}): AzRunResult { + return run({ + exitCode: f.exitCode, + stdout: text(f.stdout), + stderr: text(f.stderr), + egress: f.egress ?? NO_EGRESS, + ...overrides, + }); +} + +function options(overrides: Partial = {}): FormatOptions { + return { + argv: [], + notes: [], + isHelp: false, + guardOn: true, + port: 4566, + timeoutSeconds: 300, + maxOutputChars: 30_000, + maxHelpChars: 30_000, + envelope: false, + inDocker: false, + ...overrides, + }; +} + +function format(f: Fixture, opts: Partial = {}, result?: Partial) { + const response = formatAzResult( + runOf(f, result), + options({ argv: f.argv, guardOn: f.guardOn, ...opts }) + ); + return response.content[0].text; +} + +const firstLineOf = (s: string) => s.split("\n")[0]; +const FIRST_LINE = /^❌ \*\*Command Failed\*\* \(exit (\d+|none), ([a-z-]+)\)$/; + +/** True when a string holds a lone surrogate, i.e. a cut split a code point. */ +function hasLoneSurrogate(s: string): boolean { + for (let i = 0; i < s.length; i++) { + const c = s.charCodeAt(i); + if (c >= 0xd800 && c <= 0xdbff) { + const next = s.charCodeAt(i + 1); + if (!(next >= 0xdc00 && next <= 0xdfff)) return true; + i++; + } else if (c >= 0xdc00 && c <= 0xdfff) { + return true; + } + } + return false; +} + +describe("fixtures (tests/fixtures/azure/stderr/index.json)", () => { + it.each(FIXTURES.map((f) => [f.id, f] as const))("%s decodes to its recorded bytes", (_id, f) => { + const streams = [f.stdout, f.stderr].filter((s): s is Stream => s !== undefined); + expect(streams.length).toBeGreaterThan(0); + for (const stream of streams) { + const bytes = bytesOf(stream); + expect(bytes.length).toBe(stream.bytes); + expect(sha256(bytes)).toBe(stream.sha256); + // Valid UTF-8, as az prints it with -X utf8. + expect(() => new TextDecoder("utf-8", { fatal: true }).decode(bytes)).not.toThrow(); + } + }); + + it("keeps Windows line endings byte for byte", () => { + const raw = bytesOf(fixture("not-implemented-native").stderr!).toString("latin1"); + expect(raw).toContain("\r\n"); + expect(raw.replace(/\r\n/g, "")).not.toContain("\n"); + }); + + it("covers every class az's output can show (too-long and timeout are flags, tested below)", () => { + const classes = new Set( + FIXTURES.map((f) => f.classId).filter((c) => c !== null && c !== "other") + ); + expect([...classes].sort()).toEqual( + [ + "login", + "conn-refused", + "dns", + "discovery", + "not-implemented", + "provider", + "no-route", + "extension", + "unknown-command", + "argument", + "not-found", + "cli-error", + "emulator-error", + "needs-yes", + "egress-refused", + "guard-down", + "bicep-missing", + "bicep-registry", + "azcopy", + ].sort() + ); + }); + + it("marks reconstructed streams as synthetic and says what they are based on", () => { + const synthetic = FIXTURES.filter((f) => f.synthetic).map((f) => f.id); + expect(synthetic.sort()).toEqual( + [ + "ok-update-check-warning", + "emulator-error", + "needs-yes", + "egress-refused-rest", + "egress-refused-sdk", + "guard-down", + ].sort() + ); + for (const f of FIXTURES.filter((x) => x.synthetic)) + expect(f.basis!.length).toBeGreaterThan(60); + }); +}); + +describe("the first line of every failure", () => { + const failures = FIXTURES.filter((f) => f.classId !== null); + + it.each(failures.map((f) => [f.id, f] as const))( + "%s: starts with ❌ and carries the exit code and class id", + (_id, f) => { + const out = format(f); + expect(out.startsWith("❌")).toBe(true); + const line = firstLineOf(out); + expect(line).toMatch(FIRST_LINE); + expect(line).toBe(`❌ **Command Failed** (exit ${f.exitCode}, ${f.classId})`); + expect(out).not.toContain("\r"); + expect(out).not.toContain("Traceback (most recent call last)"); + } + ); + + it("never names a host or a path, only the code and the class id", () => { + const sdk = format(fixture("egress-refused-sdk")); + expect(firstLineOf(sdk)).toBe("❌ **Command Failed** (exit 1, egress-refused)"); + expect(sdk.indexOf("mcpzzzz.blob.core.windows.net")).toBeGreaterThan(firstLineOf(sdk).length); + }); + + it("is `exit none` when the tool stopped az or never started it", () => { + const n1 = fixture("not-implemented-native"); + const cases: Array<[Partial, string]> = [ + [{ timedOut: true }, "timeout"], + [{ aborted: true }, "cancelled"], + [{ exitCode: null }, "not-implemented"], + [{ failFast: "refused", egress: { ...NO_EGRESS, refused: ["x.example"] } }, "egress-refused"], + [{ exitCode: null, tooLong: true, stderr: "" }, "too-long"], + [{ exitCode: null, spawnError: "spawn python ENOENT", stderr: "" }, "spawn-error"], + ]; + for (const [overrides, classId] of cases) { + expect(firstLineOf(format(n1, {}, overrides))).toBe( + `❌ **Command Failed** (exit none, ${classId})` + ); + } + }); + + it("is the same contract for the answers given before spawning", () => { + expect(firstLineOf(commandTooLong().content[0].text)).toBe( + "❌ **Command Failed** (exit none, too-long)" + ); + const bicep = bicepMissing({ inDocker: false }).content[0].text; + expect(firstLineOf(bicep)).toBe("❌ **Command Failed** (exit none, bicep-missing)"); + const registry = bicepRegistryUnsupported("br/public:avm/res/storage/storage-account:0.9.1"); + expect(firstLineOf(registry.content[0].text)).toBe( + "❌ **Command Failed** (exit none, bicep-registry)" + ); + }); +}); + +describe("the failure classes", () => { + it("login: not logged in, after the self-heal retry", () => { + const out = format(fixture("login-group-list")); + expect(out).toBe( + "❌ **Command Failed** (exit 1, login)\n\n" + + "ERROR: Please run 'az login' to setup account.\n\n" + + "The tool's Azure CLI profile could not log in to the LocalStack emulator. Check that the " + + "emulator is running and healthy (`localstack-management` action `status`, service `azure`)." + ); + }); + + it("conn-refused with the guard off: host and port come from the stderr", () => { + const out = format(fixture("conn-native-group-list")); + expect(firstLineOf(out)).toBe("❌ **Command Failed** (exit 1, conn-refused)"); + expect(out).toContain( + "Could not connect to the LocalStack Azure emulator at `azure.localhost.localstack.cloud:4599`: " + + "nothing is listening. Start it with `localstack-management` (`action: start`, `service: azure`), " + + "or `lstk start --type azure`, then retry. If it runs on another port, set `LOCALSTACK_PORT` " + + "(`LOCALSTACK_AZURE_PORT` only when it runs beside an AWS emulator)." + ); + expect(out).toContain(GUARD_OFF_NOTE); + }); + + it("conn-refused: the `az rest` traceback is stripped to its ERROR lines", () => { + const f = fixture("conn-rest-traceback"); + expect(f.stderr!.bytes).toBe(5_123); + const out = format(f); + const stderr = prepareStderr(text(f.stderr)); + expect(stderr.split("\n")).toEqual([ + "ERROR: The command failed with an unexpected error. Here is the traceback:", + expect.stringMatching( + /^ERROR: HTTPSConnectionPool\(host='azure\.localhost\.localstack\.cloud', port=4599\)/ + ), + "(Python traceback omitted)", + ]); + expect(out).toContain(stderr); + expect(out).not.toContain('File "D:\\a\\_work'); + expect(out).not.toContain("To check existing issues"); + expect(out).toContain("`azure.localhost.localstack.cloud:4599`"); + }); + + it("conn-refused: a failed `az login` connect exits 2", () => { + expect(firstLineOf(format(fixture("conn-login")))).toBe( + "❌ **Command Failed** (exit 2, conn-refused)" + ); + }); + + it("conn-refused with the guard on: only the guard's upstream record counts", () => { + // The guard answers 502 and records the host; its stderr regex is for the guard-off case. + const guardOn = format(fixture("conn-native-group-list"), { guardOn: true }); + expect(firstLineOf(guardOn)).not.toContain("conn-refused"); + + const out = formatAzResult( + run({ + exitCode: 1, + failFast: "upstream", + egress: { ...NO_EGRESS, upstream: ["azure.localhost.localstack.cloud"] }, + }), + options({ argv: ["group", "list"] }) + ).content[0].text; + expect(firstLineOf(out)).toBe("❌ **Command Failed** (exit none, conn-refused)"); + expect(out).toContain("at `azure.localhost.localstack.cloud:4566`: nothing is listening"); + expect(out).not.toContain(GUARD_OFF_NOTE); + }); + + it("dns: a name outside LocalStack gets the containment answer", () => { + for (const id of ["dns-rest-traceback", "dns-native"]) { + const f = fixture(id); + const c = classifyFailure(runOf(f), { guardOn: false, argv: f.argv }); + expect(c.classId).toBe("dns"); + expect(c.details.host).toBe("mcp-nxdomain.invalid"); + expect(c.hint).toBe("`az` can only reach the LocalStack emulator from this tool."); + } + const out = format(fixture("dns-rest-traceback")); + // The warning before the error stays; the 4.6 KB traceback goes. + expect(out).toContain("WARNING: Can't derive appropriate Azure AD resource from --url"); + expect(out.length).toBeLessThan(1_500); + }); + + it("dns: a LocalStack name that did not resolve gets the resolver advice", () => { + // C8d-group-list-nxdomain with the emulator's own name in place of the .invalid host. + const stderr = text(fixture("dns-native").stderr).replace( + /mcp-nxdomain\.invalid/g, + "azure.localhost.localstack.cloud" + ); + const off = classifyFailure(run({ stderr }), { guardOn: false }); + expect(off.hint).toBe( + "`azure.localhost.localstack.cloud` did not resolve. It is a public DNS name for 127.0.0.1, " + + "and some routers and corporate resolvers block such answers (DNS-rebinding protection). " + + "Turn the egress guard back on (it needs no DNS), allow `localhost.localstack.cloud`, or " + + "add `127.0.0.1 azure.localhost.localstack.cloud` to your hosts file." + ); + const on = classifyFailure(run({ stderr }), { guardOn: true }); + expect(on.classId).toBe("dns"); + expect(on.hint).not.toContain("Turn the egress guard back on"); + }); + + it("discovery: endpoint discovery maps through conn-refused with its Error detail", () => { + const f = fixture("discovery-register"); + const on = classifyFailure(runOf(f), { guardOn: true, argv: f.argv }); + expect(on.classId).toBe("discovery"); + expect(on.hint).toContain("`azure.localhost.localstack.cloud:4599`: nothing is listening"); + // With the guard off, the conn-refused regex comes first. + expect(classifyFailure(runOf(f), { guardOn: false }).classId).toBe("conn-refused"); + // A detail that is neither conn-refused nor dns still gets an answer. + const other = classifyFailure( + run({ + stderr: + "ERROR: Unable to get endpoints from the cloud.\r\nPlease ensure you have network connection. Error detail: SSLError\r\n", + }), + { guardOn: true } + ); + expect(other.classId).toBe("discovery"); + expect(other.hint).toContain("`/metadata/endpoints`"); + }); + + it("not-implemented: the unimplemented operation, the coverage list and the port", () => { + const out = format(fixture("not-implemented-native"), { port: 4666 }); + expect(out).toContain( + "The LocalStack Azure emulator does not implement `GET /subscriptions/00000000-0000-0000-0000-000000000000/providers/Microsoft.Authorization/locks` yet. " + + "This is an emulator limitation, so retrying will not help. See the implemented operations at " + + "http://127.0.0.1:4666/_localstack/coverage and https://docs.localstack.cloud/azure/." + ); + expect(out).not.toContain("register a provider"); + const rest = classifyFailure(runOf(fixture("not-implemented-rest")), { guardOn: true }); + expect(rest.details).toEqual({ + method: "GET", + path: "/subscriptions/00000000-0000-0000-0000-000000000000/providers/Microsoft.Authorization/locks", + }); + }); + + it("not-implemented: a provider registration the CLI made on its own", () => { + const out = format(fixture("not-implemented-register")); + expect(out).toContain( + "The path ends in `/providers/Microsoft.Management/register`: the CLI tried to register a " + + "provider the emulator does not emulate." + ); + }); + + it("provider: the namespace, native (exit 3) and `az rest`", () => { + const native = fixture("provider-native"); + expect(firstLineOf(format(native))).toBe("❌ **Command Failed** (exit 3, provider)"); + expect(format(native)).toContain( + "The LocalStack Azure emulator does not emulate the `Microsoft.Cache` resource provider. " + + "Supported providers: http://127.0.0.1:4566/_localstack/coverage." + ); + expect(classifyFailure(runOf(fixture("provider-rest")), { guardOn: true }).details).toEqual({ + namespace: "Microsoft.McpFake", + }); + }); + + it("no-route: no route says the operation may not be emulated", () => { + for (const id of [ + "no-route-bogus-type", + "no-route-wrong-method", + "no-route-unimplemented-post", + ]) { + expect(format(fixture(id))).toContain( + "The emulator has no route for this request. Check the URL path, HTTP method and " + + "api-version. If they are right, the operation may not be emulated: see " + + "http://127.0.0.1:4566/_localstack/coverage." + ); + } + }); + + it("extension: a missing curated extension, named by the command-to-extension map", () => { + const cases: Array<[string, string, string]> = [ + ["extension-graph", "graph", "resource-graph"], + ["extension-app-insights", "monitor app-insights", "application-insights"], + ["extension-k8s-extension", "k8s-extension", "k8s-extension"], + ]; + for (const [id, group, extension] of cases) { + const f = fixture(id); + const c = classifyFailure(runOf(f), { guardOn: true, argv: f.argv }); + expect(c.classId).toBe("extension"); + expect(c.details).toMatchObject({ group, extension }); + expect(c.hint).toBe( + `\`az ${group}\` comes from the \`${extension}\` Azure CLI extension, which is not ` + + "installed for this tool (automatic installs are disabled). Install the tool's " + + "extensions once with `npx -y @localstack/localstack-mcp-server install-azure-addons`; " + + "meanwhile `rest` with a relative URL usually works." + ); + } + const f = fixture("extension-graph"); + const image = classifyFailure(runOf(f), { guardOn: true, argv: f.argv, inDocker: true }); + expect(image.hint).toBe( + "`resource-graph` is not in this image's curated set; use `rest` with a relative URL." + ); + }); + + it("extension vs unknown-command: an installed extension or a typo under a core group is not a missing extension", () => { + const f = fixture("extension-graph"); + const installed = new Set(["resource-graph"]); + const c = classifyFailure(runOf(f), { + guardOn: true, + argv: f.argv, + extensionFor: (tokens) => extensionFor(tokens, installed), + }); + expect(c.classId).toBe("unknown-command"); + // `afd` and `containerapp` are core in 2.85 and 2.87: a bad verb under them is a typo. + const typo = (argv: string[], token: string) => + classifyFailure( + run({ + exitCode: 2, + stderr: `ERROR: '${token}' is misspelled or not recognized by the system.\r\n`, + }), + { guardOn: true, argv } + ).classId; + expect(typo(["afd", "profle", "list"], "profle")).toBe("unknown-command"); + expect(typo(["containerapp", "lst"], "lst")).toBe("unknown-command"); + // On 2.90 the whole `afd` group is the cdn extension's. + expect(typo(["afd", "profile", "list", "-g", "x"], "afd")).toBe("extension"); + }); + + it("unknown-command: an unknown command keeps the CLI's own suggestion and adds no hint", () => { + const f = fixture("unknown-command-typo"); + const c = classifyFailure(runOf(f), { guardOn: true, argv: f.argv }); + expect(c).toEqual({ + classId: "unknown-command", + hint: undefined, + details: { token: "lst", suggestion: "list" }, + }); + expect(format(f)).toBe( + `❌ **Command Failed** (exit 2, unknown-command)\n\n${prepareStderr(text(f.stderr))}` + ); + expect( + classifyFailure(runOf(fixture("unknown-command-subcommand")), { guardOn: true }).classId + ).toBe("unknown-command"); + }); + + it("argument: argument errors point at --help and keep the CLI's examples", () => { + for (const id of [ + "argument-required", + "argument-unrecognized", + "argument-required-location", + "argument-choice", + "argument-jmespath", + ]) { + const out = format(fixture(id)); + expect(firstLineOf(out)).toBe("❌ **Command Failed** (exit 2, argument)"); + expect(out.endsWith("Run the command with `--help` to see its arguments.")).toBe(true); + } + expect(format(fixture("argument-required"))).toContain("Examples from AI knowledge base:"); + }); + + it("not-found: not found adds nothing, and never says resource group not found", () => { + for (const id of ["not-found-group", "not-found-storage", "not-found-rest"]) { + const f = fixture(id); + const out = format(f); + expect(out).toBe( + `❌ **Command Failed** (exit ${f.exitCode}, not-found)\n\n${prepareStderr(text(f.stderr))}` + ); + expect(out).not.toMatch(/ResourceGroupNotFound|resource group not found/i); + } + }); + + it("cli-error: an unexpected CLI error is matched on after its traceback is stripped", () => { + const f = fixture("cli-error-traceback"); + // The stripped traceback holds a row-7 text; it must not decide the class. + expect(text(f.stderr)).toContain("The requested URL was not found on the server"); + const out = format(f); + expect(firstLineOf(out)).toBe("❌ **Command Failed** (exit 1, cli-error)"); + expect(out).toBe( + "❌ **Command Failed** (exit 1, cli-error)\n\n" + + "ERROR: The command failed with an unexpected error. Here is the traceback:\n" + + "ERROR: 'error'\n(Python traceback omitted)" + ); + }); + + it("cli-error lets a later class answer", () => { + const stderr = + "ERROR: The command failed with an unexpected error. Here is the traceback:\r\n" + + "ERROR: Unable to prompt for confirmation as no tty available. Use --yes.\r\n" + + 'Traceback (most recent call last):\r\n File "x.py", line 1, in \r\nEOFError\r\n'; + expect(classifyFailure(run({ stderr }), { guardOn: true }).classId).toBe("needs-yes"); + }); + + it("emulator-error (synthetic): an emulator internal error points at the emulator logs", () => { + const out = format(fixture("emulator-error")); + expect(out).toContain( + "The LocalStack Azure emulator hit an internal error while handling `GET /subscriptions/00000000-0000-0000-0000-000000000000/providers/Microsoft.Storage/storageAccounts`. " + + "This is an emulator bug; the details are in the emulator logs (`localstack-logs-analysis`, analysisType `logs`)." + ); + }); + + it("needs-yes (synthetic): a prompt without --yes", () => { + expect(format(fixture("needs-yes"))).toContain( + "This command asks for confirmation. Re-run it with `--yes`." + ); + }); + + it("egress-refused: a refused egress host replaces az's text with the tool's line", () => { + const sdk = format(fixture("egress-refused-sdk")); + expect(sdk).toBe( + "❌ **Command Failed** (exit 1, egress-refused)\n\n" + + "Blocked a connection to `mcpzzzz.blob.core.windows.net`: this tool only lets `az` " + + "talk to the local emulator. Use a relative URL with `rest`, or a command that stays on the emulator." + ); + const rest = format(fixture("egress-refused-rest")); + expect(rest).toContain("Blocked a connection to `management.azure.com`"); + expect(rest).not.toContain("Unable to connect to proxy"); + expect(rest).not.toContain("WARNING"); + }); + + it("egress-refused: the backup regex, which never reports a housekeeping host", () => { + const sdkText = text(fixture("egress-refused-sdk").stderr); + const lost = classifyFailure(run({ stderr: sdkText }), { guardOn: true }); + expect(lost.classId).toBe("egress-refused"); + expect(format(fixture("egress-refused-sdk"), {}, { egress: NO_EGRESS })).toContain( + "Blocked a connection to a host outside the emulator" + ); + const housekeeping = classifyFailure( + run({ stderr: sdkText, egress: { ...NO_EGRESS, housekeeping: ["aka.ms"] } }), + { guardOn: true } + ); + expect(housekeeping.classId).not.toBe("egress-refused"); + // With the guard off a 403 came from someone else's proxy. + expect(classifyFailure(run({ stderr: sdkText }), { guardOn: false }).classId).toBe("other"); + }); + + it("too-long: too long for Windows, before spawning or from the spawn error", () => { + const hint = + "The command is too long for Windows (limit 32,767 characters). Put large values, such as " + + "JSON bodies or templates, in a file inside the working directory and pass `@`."; + expect(commandTooLong().content[0].text).toBe( + `❌ **Command Failed** (exit none, too-long)\n\n${hint}` + ); + expect(classifyFailure(run({ exitCode: null, tooLong: true }), { guardOn: true })).toEqual({ + classId: "too-long", + hint, + details: {}, + }); + expect( + classifyFailure(run({ exitCode: null, spawnError: "spawn ENAMETOOLONG" }), { guardOn: true }) + .classId + ).toBe("too-long"); + }); + + it("timeout: a timeout whose stderr holds a not-implemented message is answered as a timeout", () => { + const f = fixture("not-implemented-native"); + const out = format(f, {}, { timedOut: true }); + expect(firstLineOf(out)).toBe("❌ **Command Failed** (exit none, timeout)"); + expect(out).toContain( + "`az` did not finish within 300 s and was stopped. For long-running creates, run the " + + "command with `--no-wait`, then poll with `show`." + ); + expect(out).not.toContain("does not implement"); + expect(out).not.toContain("Claude Desktop"); + }); + + it("timeout: the hint names Claude Desktop's ~60 s limit", () => { + const f = fixture("not-implemented-native"); + const desktop = format( + f, + { client: { name: "claude-ai", version: "0.1.0" } }, + { timedOut: true } + ); + expect(desktop).toContain("Claude Desktop also stops waiting for a tool call after about 60 s"); + const code = format( + f, + { client: { name: "claude-code", version: "2.1.0" } }, + { timedOut: true } + ); + expect(code).not.toContain("Claude Desktop"); + }); + + it("guard-down: the guard-down text is not answered as conn-refused", () => { + const f = fixture("guard-down"); + const on = format(f); + expect(firstLineOf(on)).toBe("❌ **Command Failed** (exit 1, guard-down)"); + expect(on).toContain( + "Internal error: the tool's egress guard was not running. Restart the MCP server; if it happens again, report it." + ); + expect(on).not.toContain("Could not connect to the LocalStack Azure emulator"); + // With the guard off the child has no proxy variables: a proxy error is the system proxy's. + const off = classifyFailure(runOf(f), { guardOn: false }); + expect(off.classId).not.toBe("guard-down"); + expect(off.classId).not.toBe("conn-refused"); + }); + + it("bicep-missing: the three texts az prints", () => { + const hint = + "Bicep templates need the Bicep CLI, which the LocalStack Azure tool could not find. " + + "Install it with `npx -y @localstack/localstack-mcp-server install-azure-addons`, or put Bicep on your PATH " + + "(https://learn.microsoft.com/azure/azure-resource-manager/bicep/install), or set `LOCALSTACK_AZ_BICEP_PATH` " + + "to a bicep binary, then retry. " + + "Or deploy compiled ARM JSON: `--template-file main.json`. (`az bicep install` and `az config set` are not available through this tool.)"; + for (const id of [ + "bicep-missing-path-mode", + "bicep-missing-version", + "bicep-missing-latest-lookup", + ]) { + const out = format(fixture(id)); + expect(firstLineOf(out)).toBe("❌ **Command Failed** (exit 1, bicep-missing)"); + expect(out).toContain(hint); + } + // The aka.ms lookup failed through the guard: a housekeeping block, not egress-refused. + expect(format(fixture("bicep-missing-latest-lookup"))).toContain( + "Note: the egress guard also blocked `aka.ms`" + ); + }); + + it("bicep-missing: the host and image variants of the hint, and the pre-spawn answer", () => { + const host = bicepMissing({ inDocker: false }).content[0].text; + expect(host).toContain( + "Install it with `npx -y @localstack/localstack-mcp-server install-azure-addons`, or put " + + "Bicep on your PATH (https://learn.microsoft.com/azure/azure-resource-manager/bicep/install), " + + "or set `LOCALSTACK_AZ_BICEP_PATH` to a bicep binary, then retry." + ); + expect(host).not.toMatch(/wizard|\binit\b/); + expect(bicepMissing({ inDocker: true }).content[0].text).toBe( + "❌ **Command Failed** (exit none, bicep-missing)\n\n" + + "This image's Bicep CLI is missing. Set `LOCALSTACK_AZ_BICEP_PATH` to a mounted Linux bicep " + + "binary, or deploy compiled ARM JSON (`--template-file main.json`)." + ); + // Pre-spawn checks come first in the evaluation order. + const c = classifyFailure(run({ timedOut: true }), { guardOn: true, bicepFound: false }); + expect(c.classId).toBe("bicep-missing"); + }); + + it("bicep-registry: BCP192 prints the proxy URL, which is redacted with its tag", () => { + const f = fixture("bicep-registry-bcp192"); + expect(text(f.stderr)).toContain("http://call-0025:x@127.0.0.1:56739/"); + const out = format(f); + expect(firstLineOf(out)).toBe("❌ **Command Failed** (exit 1, bicep-registry)"); + expect(out).not.toContain("call-0025"); + expect(out).toContain("http://[redacted]@127.0.0.1:56739/"); + expect(out).toContain( + "Bicep registry modules (`br:`, `br/public:`) cannot be restored: the tool only reaches the " + + "local emulator. Copy the module into the workdir, and reference it by relative path." + ); + expect(classifyFailure(runOf(f), { guardOn: true }).details).toEqual({ + reference: "br:mcr.microsoft.com/bicep/avm/res/storage/storage-account:0.9.1", + }); + const direct = format(fixture("bicep-registry-bcp192-direct")); + expect(direct).not.toContain("bicep-1790513910081"); + expect(direct).toContain("http://[redacted]@127.0.0.1:52336/"); + }); + + it("bicep-registry: BCP446 and the pre-spawn answer", () => { + const f = fixture("bicep-registry-bcp446"); + expect(classifyFailure(runOf(f), { guardOn: true })).toMatchObject({ + classId: "bicep-registry", + details: { reference: "mcp.invalid" }, + }); + const ref = "br/public:avm/res/storage/storage-account:0.9.1"; + expect(bicepRegistryUnsupported(ref).content[0].text).toBe( + "❌ **Command Failed** (exit none, bicep-registry)\n\n" + + `The template uses the registry module \`${ref}\`.\n\n` + + "Bicep registry modules (`br:`, `br/public:`) cannot be restored: the tool only reaches the " + + "local emulator. Copy the module into the workdir, and reference it by relative path." + ); + }); + + it("azcopy: azcopy download", () => { + const out = format(fixture("azcopy")); + expect(firstLineOf(out)).toBe("❌ **Command Failed** (exit 1, azcopy)"); + expect(out).toContain( + "This command needs azcopy, which `az` would download from the internet. Use " + + "`storage blob upload-batch`, `download-batch` or `delete-batch` instead." + ); + }); + + it("no match: other, with the trimmed stderr and no hint", () => { + const f = fixture("prep-image-alias-warning"); + const out = format(f); + expect(firstLineOf(out)).toBe("❌ **Command Failed** (exit 1, other)"); + expect(out).toContain("ERROR: Proximity Placement Group 'mcp-no-such-ppg' does not exist."); + expect(out.trim().endsWith("does not exist.")).toBe(true); + }); + + it("says so when az failed without a message", () => { + const out = formatAzResult(run({ exitCode: 1 }), options()).content[0].text; + expect(out).toBe( + "❌ **Command Failed** (exit 1, other)\n\n`az` exited without printing an error message." + ); + }); + + it("cancelled and spawn errors", () => { + const cancelled = formatAzResult(run({ aborted: true, timedOut: true }), options()).content[0] + .text; + expect(firstLineOf(cancelled)).toBe("❌ **Command Failed** (exit none, cancelled)"); + expect(cancelled).toContain("The client cancelled the call, and `az` was stopped."); + const spawn = formatAzResult( + run({ exitCode: null, spawnError: "spawn C:\\az\\python.exe ENOENT" }), + options() + ); + expect(spawn.content[0].text).toBe( + "❌ **Command Failed** (exit none, spawn-error)\n\n" + + "The tool could not start the Azure CLI: spawn C:\\az\\python.exe ENOENT\n\n" + + "Check the `az` installation, or set `LOCALSTACK_AZ_PATH` to the `az` launcher or its Python." + ); + }); + + it("evaluates the runner flags and the guard's records before the stderr rows", () => { + const n1 = runOf(fixture("not-implemented-native")); + const refused = { + ...NO_EGRESS, + refused: ["x.example"], + upstream: ["azure.localhost.localstack.cloud"], + }; + expect( + classifyFailure({ ...n1, aborted: true, timedOut: true }, { guardOn: true }).classId + ).toBe("cancelled"); + expect( + classifyFailure({ ...n1, timedOut: true, egress: refused }, { guardOn: true }).classId + ).toBe("timeout"); + expect(classifyFailure({ ...n1, egress: refused }, { guardOn: true }).classId).toBe( + "egress-refused" + ); + expect( + classifyFailure( + { ...n1, egress: { ...NO_EGRESS, upstream: ["localhost.localstack.cloud"] } }, + { guardOn: true } + ).classId + ).toBe("conn-refused"); + }); +}); + +describe("stderr preparation", () => { + it("normalises CRLF in every fixture", () => { + for (const f of FIXTURES) expect(prepareStderr(text(f.stderr))).not.toContain("\r"); + }); + + it("keeps ERROR lines that follow a traceback", () => { + const raw = + 'ERROR: first\r\nTraceback (most recent call last):\r\n File "a"\r\nValueError\r\nERROR: second\r\nplain\r\n'; + expect(prepareStderr(raw)).toBe("ERROR: first\n(Python traceback omitted)\nERROR: second"); + }); + + it("drops the image-alias warning and keeps the others", () => { + const f = fixture("prep-image-alias-warning"); + expect(text(f.stderr)).toContain("WARNING: Failed to retrieve image alias doc"); + const prepared = prepareStderr(text(f.stderr)); + expect(prepared).not.toContain("image alias"); + expect(prepared).toContain("WARNING: The default value of '--size' will be changed"); + expect(prepared).toContain("WARNING: SSH key files"); + // A different error name is not the refused-host fallback, so it stays. + const other = + "WARNING: Failed to retrieve image alias doc 'x'. Error: 'JSONDecodeError'. Use local copy instead."; + expect(prepareStderr(other)).toBe(other); + }); + + it("drops the update-check warning", () => { + const f = fixture("ok-update-check-warning"); + expect(prepareStderr(text(f.stderr))).toBe(""); + expect(format(f)).not.toContain("WARNING"); + }); + + it("caps at about 4,000 characters, keeping the head", () => { + const raw = `ERROR: head\n${"x".repeat(80)}\n`.repeat(200); + const prepared = prepareStderr(raw); + expect(prepared.length).toBeLessThanOrEqual(STDERR_FAILURE_CHARS); + expect(prepared.length).toBeGreaterThan(STDERR_FAILURE_CHARS - 200); + expect(prepared.startsWith("ERROR: head\n")).toBe(true); + expect(prepared).toMatch(/\n\[\.\.\. [\d,]+ more characters cut\]$/); + expect(prepareStderr(raw, { maxChars: 500 }).length).toBeLessThanOrEqual(500); + }); + + it("never splits a surrogate pair at the cap", () => { + for (let n = 3_990; n < 4_000; n++) { + const raw = `${"a".repeat(n)}${"😀".repeat(40)}`; + expect(hasLoneSurrogate(prepareStderr(raw))).toBe(false); + } + }); +}); + +describe("redactProxyUrls", () => { + it("removes the call tag from loopback proxy URLs", () => { + expect(redactProxyUrls("proxy 'http://call-0025:x@127.0.0.1:56739/' failed")).toBe( + "proxy 'http://[redacted]@127.0.0.1:56739/' failed" + ); + expect(redactProxyUrls("http://2f1c1a0e-8e4b-4c7e-9a7b-8b0c1d2e3f40:x@127.0.0.1:61234")).toBe( + "http://[redacted]@127.0.0.1:61234" + ); + expect(redactProxyUrls("HTTP://tag:x@LOCALHOST:1 and http://tag:x@[::1]:2")).toBe( + "HTTP://[redacted]@LOCALHOST:1 and http://[redacted]@[::1]:2" + ); + }); + + it("leaves other URLs alone", () => { + const s = + "https://azure.localhost.localstack.cloud:4566/x and http://127.0.0.1:4566/_localstack/health"; + expect(redactProxyUrls(s)).toBe(s); + expect(redactProxyUrls("https://user@example.com/")).toBe("https://user@example.com/"); + }); +}); + +describe("success output", () => { + it("is az's stdout with LF line endings", () => { + const f = fixture("ok-group-list"); + const out = format(f); + expect(out).toBe(text(f.stdout).replace(/\r\n/g, "\n").replace(/\n+$/, "")); + expect(out.startsWith("❌")).toBe(false); + }); + + it("puts notes first, then the output, then the warnings", () => { + const f = fixture("ok-warning-storage-create"); + const note = "Rewrote the management.azure.com URL to a relative one."; + const out = format(f, { notes: [note] }); + expect(out).toBe( + `${note}\n\nSucceeded\n\n` + + "WARNING: The --min-tls-version argument values TLS1_0 and TLS1_1 have been retired on 2026/02/03 and will be removed on 2026/03/03." + ); + }); + + it("caps the warnings at 2,000 characters", () => { + const stderr = `WARNING: ${"w".repeat(3_000)}\r\n`; + const out = formatAzResult(run({ exitCode: 0, stdout: "ok\r\n", stderr }), options()).content[0] + .text; + const warnings = out.slice(out.indexOf("WARNING")); + expect(warnings.length).toBeLessThanOrEqual(2_000); + expect(warnings).toMatch(/more characters cut\]$/); + }); + + it("says so when az printed nothing", () => { + const out = formatAzResult(run({ exitCode: 0 }), options()).content[0].text; + expect(out).toBe("The command succeeded and printed no output."); + }); + + it("carries the guard-off note on successes and failures, and only then", () => { + const ok = formatAzResult(run({ exitCode: 0, stdout: "[]\r\n" }), options({ guardOn: false })) + .content[0].text; + expect(ok).toBe(`${GUARD_OFF_NOTE}\n\n[]`); + const failed = format(fixture("not-implemented-native"), { guardOn: false }); + expect(failed.endsWith(GUARD_OFF_NOTE)).toBe(true); + expect(format(fixture("not-implemented-native"))).not.toContain("egress guard"); + }); + + it("notes a refused host even when the command still succeeded", () => { + const out = formatAzResult( + run({ exitCode: 0, stdout: "{}\r\n", egress: { ...NO_EGRESS, refused: ["example.com"] } }), + options() + ).content[0].text; + expect(out).toBe( + "Note: the egress guard blocked a connection to `example.com`; the command still succeeded.\n\n{}" + ); + }); + + describe("functionapp create without --workspace (the App Insights region map)", () => { + // az's own warning when its App Insights setup fails (appservice/custom.py), after the + // function app itself was created. + const warning = + "WARNING: Error while trying to create and configure an Application Insights for the Function App. " + + "Please use the Azure Portal to create and configure the Application Insights, if needed.\r\n"; + const created = (egress: EgressRecords) => + formatAzResult( + run({ exitCode: 0, stdout: '{"name": "fa"}\r\n', stderr: warning, egress }), + options() + ).content[0].text; + + it("is a success, with a note on getting Application Insights, when the refused host was the region map", () => { + const out = created({ ...NO_EGRESS, housekeeping: ["appinsights.azureedge.net"] }); + expect(out.startsWith("❌")).toBe(false); + expect(out).toContain("the function app was created without Application Insights"); + expect(out).toContain("--workspace "); + expect(out).toContain("--disable-app-insights true"); + expect(out).toContain('{"name": "fa"}'); + }); + + it("adds no note when App Insights failed for another reason", () => { + expect(created(NO_EGRESS)).not.toContain("appinsights.azureedge.net"); + }); + + it("the region-map host is a housekeeping host: named in stderr, it never counts as a refusal", () => { + // requests' text for the guard's 403, with no guard records: only the static list decides. + const stderr = + "ERROR: HTTPSConnectionPool(host='appinsights.azureedge.net', port=443): Max retries exceeded " + + "with url: /portal/regionMapping.json (Caused by ProxyError('Unable to connect to proxy', " + + "OSError('Tunnel connection failed: 403 blocked by egress guard: appinsights.azureedge.net')))\r\n"; + const failed = classifyFailure(run({ exitCode: 1, stderr }), { guardOn: true }); + expect(failed.classId).not.toBe("egress-refused"); + }); + }); + + it("lists housekeeping blocks as expected when the command failed for another reason", () => { + const out = format( + fixture("not-implemented-native"), + {}, + { + egress: { ...NO_EGRESS, housekeeping: ["azcliprod.blob.core.windows.net"] }, + } + ); + expect(out.split("\n\n").pop()).toBe( + "Note: the egress guard also blocked `azcliprod.blob.core.windows.net`, which the tool always " + + "refuses (housekeeping calls such as update checks); these blocks are expected." + ); + }); + + it("puts the failure's parts in order: first line, stderr, output, hint, notes", () => { + const f = fixture("not-implemented-native"); + const out = format(f, { notes: ["policy note"] }, { stdout: "partial\r\n" }); + const parts = out.split("\n\n"); + expect(parts[0]).toBe("❌ **Command Failed** (exit 1, not-implemented)"); + expect(parts[1]).toMatch(/^ERROR: \(NotImplemented\)/); + expect(parts[2]).toBe("Output:\npartial"); + expect(parts[3]).toMatch(/^The LocalStack Azure emulator does not implement/); + expect(parts[4]).toBe("policy note"); + }); +}); + +describe("truncation", () => { + it("caps `account list-locations` (31 KB, just over the cap) and keeps its UTF-8", () => { + const f = fixture("ok-list-locations-utf8"); + const all = text(f.stdout).replace(/\r\n/g, "\n").replace(/\n+$/, ""); + expect(all).toContain("Querétaro"); + expect(all.length).toBe(30_158); + const out = format(f); + const [shown, note] = out.split("\n\n"); + expect(shown.length).toBeLessThanOrEqual(30_000); + expect(all.startsWith(shown)).toBe(true); + expect(note).toBe( + `[Output truncated: showing the first ${shown.length.toLocaleString("en-US")} of 30,158 characters. ` + + 'Narrow it with --query and -o tsv, for example --query "[].name" -o tsv.]' + ); + }); + + it("keeps output of exactly the cap, and cuts one character more", () => { + const at = formatAzResult( + run({ exitCode: 0, stdout: "x".repeat(1_000) }), + options({ maxOutputChars: 1_000 }) + ); + expect(at.content[0].text).toBe("x".repeat(1_000)); + const over = formatAzResult( + run({ exitCode: 0, stdout: "x".repeat(1_001) }), + options({ maxOutputChars: 1_000 }) + ); + expect(over.content[0].text).toMatch( + /^x{1000}\n\n\[Output truncated: showing the first 1,000 of 1,001 characters\./ + ); + }); + + it("handles 4.5 MB of `provider list` output", () => { + // The size of a recorded `provider list -o json`: 4,521,778 bytes with CRLF line endings. + const block = + ' {\r\n "id": "/subscriptions/00000000-0000-0000-0000-000000000000/providers/Microsoft.Example",\r\n' + + ' "namespace": "Microsoft.Example",\r\n "registrationState": "Registered",\r\n "resourceTypes": []\r\n },\r\n'; + let stdout = `[\r\n${block.repeat(Math.ceil(4_521_778 / block.length))}]\r\n`; + stdout = stdout.slice(0, 4_521_775) + "]\r\n"; + expect(stdout.length).toBeGreaterThanOrEqual(4_500_000); + const started = Date.now(); + const response = formatAzResult(run({ exitCode: 0, stdout }), options({ envelope: true })); + // ~150 ms alone; a loaded full-suite run on WSL took over 2 s. The bound only has to + // catch a quadratic regression. + expect(Date.now() - started).toBeLessThan(10_000); + const out = response.content[0].text; + const total = stdout.replace(/\r\n/g, "\n").replace(/\n+$/, "").length; + expect(out.length).toBeLessThan(30_300); + expect(out).toContain(`of ${total.toLocaleString("en-US")} characters`); + // The envelope still carries every byte. + const envelope: ResultEnvelope = JSON.parse(response.content[1].text); + expect(envelope.stdout).toBe(stdout); + }); + + it("never splits a surrogate pair in the output cap", () => { + for (let n = 990; n < 1_000; n++) { + const stdout = `${"a".repeat(n)}${"😀".repeat(20)}`; + const out = formatAzResult(run({ exitCode: 0, stdout }), options({ maxOutputChars: 1_000 })) + .content[0].text; + expect(hasLoneSurrogate(out)).toBe(false); + } + expect(headCut("ab😀", 3)).toBe("ab"); + expect(tailCut("😀ab", 3)).toBe("ab"); + expect(headCut("ab😀", 4)).toBe("ab😀"); + }); +}); + +describe("help pages", () => { + const reflowOf = (f: Fixture) => reflowHelp(text(f.stdout)); + + it("re-flows each recorded help page to its expected length and hash", () => { + for (const id of ["help-az", "help-vm-create", "help-aks-create"]) { + const f = fixture(id); + const reflowed = reflowOf(f); + expect(reflowed.length).toBe(f.reflowed!.chars); + expect(sha256(reflowed)).toBe(f.reflowed!.sha256); + } + }); + + // The recorded sizes count the blank line that starts every page and the two newlines that end it; + // the response drops them. + const page = (f: Fixture) => reflowOf(f).replace(/^\n+/, "").replace(/\n+$/, ""); + + it("vm create is 28.7 KB after re-flow and is not cut", () => { + const f = fixture("help-vm-create"); + expect(f.stdout!.bytes).toBe(116_394); + expect(reflowOf(f).length).toBe(28_742); + const out = format(f, { isHelp: true }); + expect(out).toBe(page(f)); + expect(out.length).toBe(28_739); + expect(out).not.toContain("characters of this help page cut"); + }); + + it("aks create is 43 KB after re-flow and is cut in the middle, keeping head and Examples", () => { + const f = fixture("help-aks-create"); + expect(reflowOf(f).length).toBe(43_240); + const reflowed = page(f); + const out = format(f, { isHelp: true }); + expect(out.length).toBeLessThanOrEqual(30_000); + expect(out.length).toBeGreaterThan(29_000); + const examples = reflowed.slice(reflowed.lastIndexOf("\nExamples\n") + 1); + expect(examples.length).toBe(10_028); + expect(out.endsWith(examples)).toBe(true); + expect(out.startsWith(reflowed.slice(0, 2_000))).toBe(true); + expect(out).toMatch( + /\n\n\[\.\.\. [\d,]+ characters of this help page cut here; the head and the Examples are kept \.\.\.\]\n\n/ + ); + const head = out.slice(0, out.indexOf("\n\n[...")); + expect(reflowed.startsWith(head)).toBe(true); + }); + + it("appends az's warnings after the page", () => { + const f = fixture("help-az"); + const out = format(f, { isHelp: true }); + expect(out.startsWith(`${page(f)}\n\n`)).toBe(true); + expect( + out.endsWith( + "\n\nWARNING: You have 2 update(s) available. Consider updating your CLI installation with 'az upgrade'\nWARNING:" + ) + ).toBe(true); + }); + + it("joins lines indented 20 or more spaces and collapses runs of spaces", () => { + const page = [ + "Arguments", + " --name -n [Required] : The storage account name. It must", + " be unique.", + " not joined: 19 spaces", + "", + " after a blank line", + " a b c d e", + ].join("\r\n"); + expect(reflowHelp(page).split("\n")).toEqual([ + "Arguments", + " --name -n [Required] : The storage account name. It must be unique.", + " not joined: 19 spaces", + "", + " after a blank line", + " a b c d e", + ]); + }); + + it("falls back to the last part of the page when there is no small Examples section", () => { + const page = Array.from({ length: 400 }, (_, i) => `line ${i} ${"z".repeat(60)}`).join("\n"); + const cut = cutHelpPage(page, 5_000); + expect(cut.length).toBeLessThanOrEqual(5_000); + expect(cut.startsWith("line 0 ")).toBe(true); + expect(cut.endsWith(`line 399 ${"z".repeat(60)}`)).toBe(true); + expect(hasLoneSurrogate(cutHelpPage("😀".repeat(3_000), 1_001))).toBe(false); + }); +}); + +describe("test envelope", () => { + it("is present only with envelope: true", () => { + const f = fixture("ok-warning-storage-create"); + expect(formatAzResult(runOf(f), options()).content).toHaveLength(1); + const withEnvelope = formatAzResult(runOf(f), options({ envelope: true })); + expect(withEnvelope.content).toHaveLength(2); + expect(withEnvelope.content[0].text).toBe(formatAzResult(runOf(f), options()).content[0].text); + }); + + it("carries az's stdout byte for byte, warnings excluded", () => { + // A success with a warning round-trips through $(…). + for (const id of ["ok-warning-storage-create", "ok-list-locations-utf8", "ok-group-list"]) { + const f = fixture(id); + const response = formatAzResult(runOf(f), options({ envelope: true, notes: ["n1"] })); + const envelope: ResultEnvelope = JSON.parse(response.content[1].text); + expect(Buffer.from(envelope.stdout, "utf8").equals(bytesOf(f.stdout!))).toBe(true); + expect(envelope).toMatchObject({ + exitCode: 0, + classId: null, + truncated: false, + notes: ["n1"], + }); + } + }); + + it("carries the class id, the exit code, redacted stderr and the runner's truncation", () => { + const f = fixture("bicep-registry-bcp192"); + const response = formatAzResult( + runOf(f, { truncated: true }), + options({ envelope: true, argv: f.argv }) + ); + const envelope: ResultEnvelope = JSON.parse(response.content[1].text); + expect(envelope.classId).toBe("bicep-registry"); + expect(envelope.exitCode).toBe(1); + expect(envelope.truncated).toBe(true); + expect(envelope.stderr).toContain("\r\n"); + expect(envelope.stderr).not.toContain("call-0025"); + expect(envelope.stderr).toBe(redactProxyUrls(text(f.stderr))); + expect(envelope.notes).toEqual([expect.stringContaining("more than the tool captures")]); + }); +}); + +describe("classifyFailure: az's own not-found exit code", () => { + // az exits 3 for ResourceNotFoundError. The emulator words some not-found answers with codes + // the text pattern does not know (Event Hubs, Service Bus, Cosmos, Key Vault, subnets, ...), so + // a `show` of a missing child came back as the vague "other". The exit code is az's own verdict. + const unknownNotFound = + "ERROR: (NotFound) The resource 'hub1' was not found.\nCode: NotFound\nMessage: The resource 'hub1' was not found."; + + test("exit code 3 with an unrecognised not-found text is classified not-found", () => { + const c = classifyFailure(run({ exitCode: 3, stderr: unknownNotFound }), { guardOn: false }); + expect(c.classId).toBe("not-found"); + }); + + test("the same text with a generic exit code 1 is still other (the exit code decides)", () => { + const c = classifyFailure(run({ exitCode: 1, stderr: unknownNotFound }), { guardOn: false }); + expect(c.classId).toBe("other"); + }); + + test("exit code 3 does not override a more specific class", () => { + const c = classifyFailure( + run({ exitCode: 3, stderr: "ERROR: unrecognized arguments: --nope" }), + { guardOn: false } + ); + expect(c.classId).toBe("argument"); + }); +}); + +describe("classifyFailure: Bicep readEnvironmentVariable", () => { + test("BCP427 gets its own class, naming the variable and how to pass it", () => { + const stderr = + 'ERROR: C:\\work\\main.bicepparam(11,47) : Error BCP427: Environment variable "BACKEND_SECRET" does not exist and there\'s no default value set. [https://aka.ms/bicep/core-diagnostics#BCP427]'; + const c = classifyFailure(run({ exitCode: 1, stderr }), { guardOn: false }); + expect(c.classId).toBe("bicep-env"); + expect(c.hint).toContain("BACKEND_SECRET"); + expect(c.hint).toContain("LOCALSTACK_AZ_BICEP_ENV"); + expect(c.details).toEqual({ variable: "BACKEND_SECRET" }); + }); +}); diff --git a/src/lib/azure/output.ts b/src/lib/azure/output.ts new file mode 100644 index 0000000..c5db35d --- /dev/null +++ b/src/lib/azure/output.ts @@ -0,0 +1,794 @@ +/** + * Output formatting and error hints of the Azure client. + * + * Every failure starts with `❌ **Command Failed** (exit N, )`. The repo's analytics + * record the first line of an ❌ response as `error_message`, so that line carries only the exit + * code and the class id, never a value from the command. N is az's exit code, or + * `none` when az did not exit by itself: it never started, the handler answered before spawning, + * or the tool stopped it (timeout, client cancel, egress fail-fast), where the code only reflects + * the kill and differs between Windows and POSIX. + * + * Caps count UTF-16 code units (JS string length), and every cut keeps surrogate pairs whole. + */ +import { ResponseBuilder } from "../../core/response-builder"; +import { HOUSEKEEPING_HOSTS as GUARD_HOUSEKEEPING_HOSTS } from "./egress-proxy"; +import { + commandWords, + extensionFor as curatedExtensionFor, + missingExtensionHint, +} from "./extension-map"; +import type { AzFailureClass, AzRunResult, ToolTextResponse } from "./types"; + +export interface ClientInfo { + name?: string; + version?: string; +} + +export interface ClassifyContext { + guardOn: boolean; + /** + * Pass only for commands that need Bicep (`policy.needsBicep`): false means no binary was + * found, which is answered before spawning with `bicep-missing`. + */ + bicepFound?: boolean; + /** The command-to-extension lookup bound to the installed set (`extension-map.ts`). */ + extensionFor?: (tokens: string[]) => string | undefined; + /** The argv that ran, without the leading `az`. */ + argv?: string[]; + /** The rest are only for the hint texts. */ + port?: number; + timeoutSeconds?: number; + client?: ClientInfo; + inDocker?: boolean; +} + +export interface FailureClassification { + classId: AzFailureClass; + hint?: string; + /** + * What the hint was built from (hosts, paths, names). These can hold user values: never pass + * them to analytics. + */ + details: Record; +} + +export interface FormatOptions { + argv: string[]; + /** Policy notes (for example a rewritten management.azure.com URL). */ + notes: string[]; + isHelp: boolean; + guardOn: boolean; + port: number; + timeoutSeconds: number; + maxOutputChars: number; + maxHelpChars: number; + client?: ClientInfo; + /** Tests only (LOCALSTACK_AZ_TEST_ENVELOPE=1): append the JSON result envelope. */ + envelope: boolean; + inDocker: boolean; + extensionFor?: (tokens: string[]) => string | undefined; +} + +/** The second content item with `envelope: true`. */ +export interface ResultEnvelope { + exitCode: number | null; + /** az's stdout exactly as the runner decoded it: CRLF kept, never capped. */ + stdout: string; + /** az's stderr with proxy URLs redacted, otherwise as decoded. */ + stderr: string; + /** Policy and tool notes; display-only notes (truncation) are left out. */ + notes: string[]; + classId: AzFailureClass | null; + /** The runner hit its capture cap, so stdout or stderr is incomplete. */ + truncated: boolean; + /** + * The tool ended az (timeout, cancel, a refused host, the output cap) or never started it: + * exitCode is then not az's own (null on POSIX, 1 after taskkill, even 0). Always set by + * this server; optional for readers of envelopes from older builds. + */ + stoppedByTool?: boolean; +} + +export const STDERR_FAILURE_CHARS = 4_000; +export const STDERR_SUCCESS_CHARS = 2_000; + +/** Every response of a call with LOCALSTACK_AZ_EGRESS_GUARD=0 carries this note. */ +export const GUARD_OFF_NOTE = + "Note: the egress guard (containment layer 4) is off (`LOCALSTACK_AZ_EGRESS_GUARD=0`), so " + + "`az` can reach hosts other than the emulator. Use this for debugging only."; + +// The docs landing page of LocalStack for Azure; the emulator's own list is the coverage endpoint. +const AZURE_DOCS_URL = "https://docs.localstack.cloud/azure/"; + +// Refused on every run, never a failure: the guard's own list, for the backup regex +// of `egress-refused`, which must not report these. +const HOUSEKEEPING_HOSTS = new Set(GUARD_HOUSEKEEPING_HOSTS); + +// `functionapp create` catches a failed App Insights setup and warns with this line +// (appservice/custom.py). Without --workspace, the failure is its refused region-map download. +const APP_INSIGHTS_SKIPPED = + /^WARNING: Error while trying to create and configure an Application Insights for the Function App\./m; +const APP_INSIGHTS_REGION_HOST = "appinsights.azureedge.net"; +const APP_INSIGHTS_NOTE = + "Note: the function app was created without Application Insights. Without `--workspace`, " + + "az first downloads Application Insights' region map from appinsights.azureedge.net, which " + + "this tool never contacts. To get an Application Insights component, create a Log Analytics " + + "workspace (`monitor log-analytics workspace create`) and pass `--workspace `; " + + "`--disable-app-insights true` skips it without a warning."; + +// Warnings caused only by refused housekeeping hosts, which prepareStderr drops. +const HOUSEKEEPING_WARNINGS = [ + /^WARNING: Unable to check if your CLI is up-to-date\. Check your internet connection\.\s*$/, + /^WARNING: Failed to retrieve image alias doc '[^']*'\. Error: 'ConnectionError'\. Use local copy instead\.\s*$/, +]; + +const TRACEBACK_START = /^Traceback \(most recent call last\):\s*$/; +const TRACEBACK_OMITTED = "(Python traceback omitted)"; + +// The guard's per-call URL `http://:x@127.0.0.1:`; Bicep's BCP192 prints it. +const PROXY_URL = /\b(https?:\/\/)[^\s'"<>@/]+@(127\.0\.0\.1|localhost|\[::1\])(:\d+)?/gi; + +// The stderr detectors, all with the m flag: stderr often starts with a WARNING line. +const RE = { + login: /^ERROR: Please run 'az login' to setup account\./m, + connRefused: + /Failed to establish a new connection: .*(actively refused|Connection refused|WinError 10061|Errno 111)/m, + connHostPort: /HTTPS?Connection\(host='([^']+)', port=(\d+)\)/, + viaProxy: /Unable to connect to proxy/, + // Alternations report the leftmost match, so the values come from separate expressions. + dns: /Failed to resolve '|NameResolutionError/m, + dnsHost: /Failed to resolve '([^']+)'/, + hostInText: /host='([^']+)'/, + discovery: /^ERROR: Unable to get endpoints from the cloud\./m, + notImplemented: /The API operation '([A-Z]+) ([^']+)' is not yet implemented in LocalStack\./m, + notImplementedCode: /\(NotImplemented\)|"code": "NotImplemented"/m, + provider: /The resource namespace '[^']+' is invalid\.|InvalidResourceNamespace/m, + providerNamespace: /The resource namespace '([^'/]+)(\/[^']*)?' is invalid\./, + noRoute: /The requested URL was not found on the server/m, + notRecognized: /'([\w-]+)' is misspelled or not recognized by the system\./m, + didYouMean: /Did you mean '([\w-]+)' \?/m, + argument: + /^ERROR: (the following arguments are required: |unrecognized arguments: |argument [^:]+: invalid )|is not a valid value for '/m, + notFound: + /\((ResourceNotFound|ResourceGroupNotFound|ParentResourceNotFound)\)|"code": "Resource\w*NotFound"/m, + cliError: /The command failed with an unexpected error\. Here is the traceback:/m, + emulatorError: /An unexpected error occurred while processing '([A-Z]+) ([^']+)'/m, + needsYes: /Unable to prompt for confirmation as no tty available\. Use --yes\./m, + bicepMissing: + /Could not find the "bicep" executable on PATH|Bicep CLI not found\. Install it now by running "az bicep install"\.|Error while attempting to retrieve the latest Bicep version:/m, + bicepRegistry: + /Error BCP192: Unable to restore the artifact with reference "br:|Error BCP446: Restore from registry "([^"]+)" is blocked/m, + bicepReference: /reference "(br:[^"]+)"/, + bicepEnv: /Error BCP427: Environment variable "([^"]+)" does not exist/m, + azcopy: /Error while attempting to download azcopy\./m, + guardDown: /Unable to connect to proxy', NewConnectionError\(/m, + guardDownErrno: /10061|Connection refused|Errno 111/m, + tunnel403: /Tunnel connection failed: 403 Forbidden/m, + register: /\/providers\/([^/]+)\/register$/i, +}; + +// Clients that stop waiting for a tool call long before LOCALSTACK_AZ_TIMEOUT_SECONDS. +// Claude Desktop (and claude.ai) identify themselves as `claude-ai`. +const SHORT_LIMIT_CLIENTS: Array<{ match: RegExp; label: string; seconds: number }> = [ + { match: /^claude-ai$|claude[\s-]?desktop/i, label: "Claude Desktop", seconds: 60 }, +]; + +// --------------------------------------------------------------------------------------------- +// Text helpers + +const isHighSurrogate = (code: number) => code >= 0xd800 && code <= 0xdbff; +const isLowSurrogate = (code: number) => code >= 0xdc00 && code <= 0xdfff; +const fmt = (n: number) => n.toLocaleString("en-US"); + +export function normalizeNewlines(text: string): string { + return text.replace(/\r\n/g, "\n"); +} + +/** The first `max` code units, never ending inside a surrogate pair. */ +export function headCut(text: string, max: number): string { + if (text.length <= max) return text; + let end = Math.max(0, Math.floor(max)); + if (end > 0 && isHighSurrogate(text.charCodeAt(end - 1))) end -= 1; + return text.slice(0, end); +} + +/** The last `max` code units, never starting inside a surrogate pair. */ +export function tailCut(text: string, max: number): string { + if (text.length <= max) return text; + let start = text.length - Math.max(0, Math.floor(max)); + if (start < text.length && isLowSurrogate(text.charCodeAt(start))) start += 1; + return text.slice(start); +} + +/** Like headCut, but back to the last line break when one is within `slack` of the cut. */ +function headCutAtLine(text: string, max: number, slack: number): string { + const head = headCut(text, max); + if (head.length === text.length) return head; + const newline = head.lastIndexOf("\n"); + return newline > 0 && newline >= head.length - slack ? head.slice(0, newline) : head; +} + +/** Cuts to at most `max` characters, the marker included; the marker gets the cut count. */ +function capWithMarker(text: string, max: number, marker: (cut: number) => string): string { + if (text.length <= max) return text; + const room = Math.max(0, max - marker(text.length).length); + const head = headCutAtLine(text, room, Math.min(200, Math.floor(room / 10))); + return head + marker(text.length - head.length); +} + +function trimBlankLines(text: string): string { + return text.replace(/^(?:[ \t]*\n)+/, "").replace(/\s+$/, ""); +} + +const unique = (items: readonly string[]) => [...new Set(items.filter(Boolean))]; +/** Markdown inline code; the value comes from az or the user, so no backtick may close it. */ +const inlineCode = (value: string) => `\`${value.replace(/`/g, "'").replace(/\s+/g, " ")}\``; + +/** Replaces the userinfo of loopback proxy URLs (the guard's per-call tag) with `[redacted]`. */ +export function redactProxyUrls(text: string): string { + return text.replace(PROXY_URL, (_match, scheme: string, host: string, port?: string) => { + return `${scheme}[redacted]@${host}${port ?? ""}`; + }); +} + +function prepareLines(raw: string): string { + const kept: string[] = []; + let inTraceback = false; + for (const line of normalizeNewlines(raw).split("\n")) { + if (TRACEBACK_START.test(line)) { + if (!inTraceback) kept.push(TRACEBACK_OMITTED); + inTraceback = true; + continue; + } + // After a traceback only az's own ERROR: lines are worth keeping. + if (inTraceback && !line.startsWith("ERROR:")) continue; + if (HOUSEKEEPING_WARNINGS.some((re) => re.test(line))) continue; + kept.push(line); + } + return redactProxyUrls(trimBlankLines(kept.join("\n"))); +} + +/** + * The stderr preparation: CRLF to LF, Python tracebacks stripped (the ERROR: lines + * stay), the two housekeeping warnings dropped, proxy URLs redacted, capped keeping the head. + */ +export function prepareStderr(raw: string, opts: { maxChars?: number } = {}): string { + const max = opts.maxChars ?? STDERR_FAILURE_CHARS; + return capWithMarker(prepareLines(raw), max, (cut) => `\n[... ${fmt(cut)} more characters cut]`); +} + +/** + * Re-flows an az help page: a line indented 20 or more spaces is a wrapped part of + * the description column and joins the previous line, and runs of spaces inside a line collapse + * to two. Pages shrink to 25-78 % and stay readable. + */ +export function reflowHelp(text: string): string { + // No lookbehind (the compile target is ES2016); the lookahead leaves the next word unconsumed. + const collapse = (line: string) => line.replace(/(\S) {3,}(?=\S)/g, "$1 "); + const out: string[] = []; + for (const raw of normalizeNewlines(text).split("\n")) { + const line = raw.replace(/[ \t]+$/, ""); + const content = line.trim(); + const indent = line.length - line.replace(/^ +/, "").length; + const previous = out.length > 0 ? out[out.length - 1] : undefined; + if (content && indent >= 20 && previous !== undefined && previous.trim() !== "") { + out[out.length - 1] = `${previous} ${collapse(content)}`; + } else { + out.push(collapse(line)); + } + } + return out.join("\n"); +} + +function lastMatchIndex(text: string, re: RegExp): number { + let index = -1; + for (let m = re.exec(text); m; m = re.exec(text)) index = m.index; + return index; +} + +/** + * Cuts a re-flowed help page to `max` characters, keeping the head (the command and its + * required arguments) and the tail (the Examples, which az prints last). The Examples section is + * kept whole when it fits in half the room, else the last 40 % of the room is kept. + */ +export function cutHelpPage(text: string, max: number): string { + if (text.length <= max) return text; + const marker = (cut: number) => + `\n\n[... ${fmt(cut)} characters of this help page cut here; ` + + `the head and the Examples are kept ...]\n\n`; + const room = Math.max(0, max - marker(text.length).length); + const examples = lastMatchIndex(text, /^Examples[ \t]*$/gm); + let tail = examples >= 0 && text.length - examples <= room / 2 ? text.slice(examples) : ""; + if (!tail) { + const rough = tailCut(text, Math.floor(room * 0.4)); + const lineStart = rough.indexOf("\n"); + tail = lineStart >= 0 && lineStart < rough.length - 1 ? rough.slice(lineStart + 1) : rough; + } + const headRoom = room - tail.length; + const head = headCutAtLine(text, headRoom, Math.min(2_000, Math.floor(headRoom / 5))); + return head.replace(/\n+$/, "") + marker(text.length - head.length - tail.length) + tail; +} + +// --------------------------------------------------------------------------------------------- +// Hints. They say "may not be implemented" unless the emulator itself said so, and +// never assume Azure's error codes: the emulator reports a missing group as ResourceNotFound. + +const coverageUrl = (port: number) => `http://127.0.0.1:${port}/_localstack/coverage`; + +const LOGIN_HINT = + "The tool's Azure CLI profile could not log in to the LocalStack emulator. Check that the " + + "emulator is running and healthy (`localstack-management` action `status`, service `azure`)."; + +function connRefusedHint(hostPort: string): string { + return ( + `Could not connect to the LocalStack Azure emulator at ${inlineCode(hostPort)}: nothing is ` + + "listening. Start it with `localstack-management` (`action: start`, `service: azure`), or " + + "`lstk start --type azure`, then retry. If it runs on another port, set `LOCALSTACK_PORT` " + + "(`LOCALSTACK_AZURE_PORT` only when it runs beside an AWS emulator)." + ); +} + +function dnsHint(host: string | undefined, guardOn: boolean): string { + if (host && /(^|\.)localhost\.localstack\.cloud$/i.test(host)) { + const fixes = guardOn + ? "Allow `localhost.localstack.cloud`" + : "Turn the egress guard back on (it needs no DNS), allow `localhost.localstack.cloud`"; + return ( + `${inlineCode(host)} did not resolve. It is a public DNS name for 127.0.0.1, and some routers ` + + "and corporate resolvers block such answers (DNS-rebinding protection). " + + `${fixes}, or add ${inlineCode(`127.0.0.1 ${host}`)} to your hosts file.` + ); + } + return "`az` can only reach the LocalStack emulator from this tool."; +} + +function notImplementedHint(port: number, method?: string, opPath?: string): string { + const what = method && opPath ? inlineCode(`${method} ${opPath}`) : "this operation"; + let hint = + `The LocalStack Azure emulator does not implement ${what} yet. This is an emulator ` + + "limitation, so retrying will not help. See the implemented operations at " + + `${coverageUrl(port)} and ${AZURE_DOCS_URL}.`; + const register = opPath ? RE.register.exec(opPath) : null; + if (register) { + hint += + ` The path ends in ${inlineCode(`/providers/${register[1]}/register`)}: the CLI tried to ` + + "register a provider the emulator does not emulate."; + } + return hint; +} + +function providerHint(port: number, namespace?: string): string { + const which = namespace + ? `the ${inlineCode(namespace)} resource provider` + : "this resource provider"; + return ( + `The LocalStack Azure emulator does not emulate ${which}. Supported providers: ` + + `${coverageUrl(port)}.` + ); +} + +function noRouteHint(port: number): string { + return ( + "The emulator has no route for this request. Check the URL path, HTTP method and " + + "api-version. If they are right, the operation may not be emulated: see " + + `${coverageUrl(port)}.` + ); +} + +const ARGUMENT_HINT = "Run the command with `--help` to see its arguments."; + +function emulatorErrorHint(method?: string, opPath?: string): string { + const what = method && opPath ? inlineCode(`${method} ${opPath}`) : "this request"; + return ( + `The LocalStack Azure emulator hit an internal error while handling ${what}. This is an ` + + "emulator bug; the details are in the emulator logs (`localstack-logs-analysis`, " + + "analysisType `logs`)." + ); +} + +const NEEDS_YES_HINT = "This command asks for confirmation. Re-run it with `--yes`."; + +function egressRefusedMessage(hosts: string[]): string { + const target = + hosts.length === 0 + ? "a connection to a host outside the emulator" + : `a connection to ${hosts.map(inlineCode).join(", ")}`; + return ( + `Blocked ${target}: this tool only lets \`az\` talk to the local emulator. Use a relative ` + + "URL with `rest`, or a command that stays on the emulator." + ); +} + +const TOO_LONG_HINT = + "The command is too long for Windows (limit 32,767 characters). Put large values, such as " + + "JSON bodies or templates, in a file inside the working directory and pass `@`."; + +function timeoutHint(timeoutSeconds?: number, client?: ClientInfo): string { + const within = timeoutSeconds ? `within ${fmt(timeoutSeconds)} s` : "in time"; + let hint = + `\`az\` did not finish ${within} and was stopped. For long-running creates, run the ` + + "command with `--no-wait`, then poll with `show`."; + const limited = SHORT_LIMIT_CLIENTS.find((c) => client?.name && c.match.test(client.name)); + if (limited) { + hint += + ` ${limited.label} also stops waiting for a tool call after about ${limited.seconds} s, ` + + "so use `--no-wait` for anything that can take longer."; + } + return hint; +} + +const CANCELLED_HINT = + "The client cancelled the call, and `az` was stopped. The operation may have partly run on " + + "the emulator: check its state with `show` or `list` before retrying."; + +const GUARD_DOWN_HINT = + "Internal error: the tool's egress guard was not running. Restart the MCP server; if it " + + "happens again, report it."; + +const SPAWN_ERROR_HINT = + "Check the `az` installation, or set `LOCALSTACK_AZ_PATH` to the `az` launcher or its Python."; + +const BICEP_INSTALL_URL = "https://learn.microsoft.com/azure/azure-resource-manager/bicep/install"; + +function bicepMissingHint(inDocker: boolean): string { + if (inDocker) { + return ( + "This image's Bicep CLI is missing. Set `LOCALSTACK_AZ_BICEP_PATH` to a mounted Linux " + + "bicep binary, or deploy compiled ARM JSON (`--template-file main.json`)." + ); + } + const install = + "Install it with `npx -y @localstack/localstack-mcp-server install-azure-addons`, or put " + + `Bicep on your PATH (${BICEP_INSTALL_URL}), or set \`LOCALSTACK_AZ_BICEP_PATH\` to a bicep ` + + "binary, then retry."; + return ( + "Bicep templates need the Bicep CLI, which the LocalStack Azure tool could not find. " + + `${install} Or deploy compiled ARM JSON: \`--template-file main.json\`. ` + + "(`az bicep install` and `az config set` are not available through this tool.)" + ); +} + +const BICEP_REGISTRY_HINT = + "Bicep registry modules (`br:`, `br/public:`) cannot be restored: the tool only reaches the " + + "local emulator. Copy the module into the workdir, and reference it by relative path."; + +/** BCP427: a `.bicepparam` reads a variable that az's clean environment does not have. */ +function bicepEnvHint(variable: string): string { + return ( + `The parameter file reads \`${variable}\` with readEnvironmentVariable(), but az (and so ` + + "Bicep) runs with a clean environment: the tool passes only the variables listed in " + + `LOCALSTACK_AZ_BICEP_ENV. Add \`${variable}\` to LOCALSTACK_AZ_BICEP_ENV in this MCP server's ` + + "configuration and set it there too, or pass the value as a parameter instead " + + "(`--parameters =`)." + ); +} + +const AZCOPY_HINT = + "This command needs azcopy, which `az` would download from the internet. Use " + + "`storage blob upload-batch`, `download-batch` or `delete-batch` instead."; + +/** `discovery` gets the `conn-refused` or `dns` hint, from the `Error detail` az prints after it. */ +function discoveryHint(text: string, guardOn: boolean, port: number): string { + const detail = text.slice(text.search(RE.discovery)); + if (RE.connRefused.test(detail)) { + const m = RE.connHostPort.exec(detail); + return connRefusedHint(m ? `${m[1]}:${m[2]}` : `127.0.0.1:${port}`); + } + if (RE.dns.test(detail)) { + return dnsHint(RE.dnsHost.exec(detail)?.[1] ?? RE.hostInText.exec(detail)?.[1], guardOn); + } + return ( + "The CLI could not read the emulator's endpoints (`/metadata/endpoints`). Check that the " + + "LocalStack Azure emulator is running and healthy (`localstack-management` action " + + "`status`, service `azure`)." + ); +} + +// --------------------------------------------------------------------------------------------- +// Classification + +const failure = ( + classId: AzFailureClass, + hint?: string, + details: Record = {} +): FailureClassification => ({ classId, hint, details }); + +/** + * The failure class of a run; the first match wins. + * 1. pre-spawn: command too long (`too-long`), no Bicep binary (`bicep-missing`, only with + * `bicepFound`), and a failed spawn; + * 2. runner flags: aborted (`cancelled`), then `timedOut` (`timeout`); + * 3. the guard's record: a refused host (`egress-refused`), then an upstream failure + * (`conn-refused`); + * 4. `guard-down`, only with the guard on (with it off, a proxy error is the system proxy's); + * 5. the stderr detectors on the prepared stderr: `login`, `conn-refused` (guard off only), + * `dns`, `discovery`, `not-implemented`, `provider`, `no-route`, `extension` or + * `unknown-command`, `argument`, `not-found`, `emulator-error`, `needs-yes`, + * `bicep-missing`, `bicep-registry`, `bicep-env` and `azcopy`. `cli-error` has no hint of + * its own and matching goes on; it is the class only when nothing else matches. The backup + * regex of `egress-refused` comes last, and never reports a housekeeping host; + * 6. no match: `other`. + * Matching sees the prepared stderr before the display cap, so a long warning cannot hide it. + */ +export function classifyFailure(result: AzRunResult, ctx: ClassifyContext): FailureClassification { + const port = ctx.port ?? 4566; + const inDocker = ctx.inDocker ?? false; + + if (result.tooLong || /ENAMETOOLONG/.test(result.spawnError ?? "")) { + return failure("too-long", TOO_LONG_HINT); + } + if (ctx.bicepFound === false) { + return failure("bicep-missing", bicepMissingHint(inDocker), { + source: "precheck", + }); + } + if (result.spawnError) { + return failure("spawn-error", SPAWN_ERROR_HINT, { error: result.spawnError }); + } + + if (result.aborted) return failure("cancelled", CANCELLED_HINT); + if (result.timedOut) { + return failure("timeout", timeoutHint(ctx.timeoutSeconds, ctx.client), { + timeoutSeconds: String(ctx.timeoutSeconds ?? ""), + client: ctx.client?.name ?? "", + }); + } + + const refused = unique(result.egress.refused); + if (refused.length > 0 || result.failFast === "refused") { + return failure("egress-refused", undefined, { hosts: refused.join(",") }); + } + const upstream = unique(result.egress.upstream); + if (upstream.length > 0 || result.failFast === "upstream") { + const host = upstream[0] ?? "127.0.0.1"; + const hostPort = /:\d+$/.test(host) ? host : `${host}:${port}`; + return failure("conn-refused", connRefusedHint(hostPort), { hostPort, source: "guard" }); + } + + const text = prepareLines(result.stderr); + if (ctx.guardOn && RE.guardDown.test(text) && RE.guardDownErrno.test(text)) { + return failure("guard-down", GUARD_DOWN_HINT); + } + + let m: RegExpExecArray | null; + if (RE.login.test(text)) return failure("login", LOGIN_HINT); + if (!ctx.guardOn && RE.connRefused.test(text) && !RE.viaProxy.test(text)) { + m = RE.connHostPort.exec(text); + const hostPort = m ? `${m[1]}:${m[2]}` : `127.0.0.1:${port}`; + return failure("conn-refused", connRefusedHint(hostPort), { hostPort, source: "stderr" }); + } + if (RE.dns.test(text)) { + const host = RE.dnsHost.exec(text)?.[1] ?? RE.hostInText.exec(text)?.[1]; + return failure("dns", dnsHint(host, ctx.guardOn), host ? { host } : {}); + } + if (RE.discovery.test(text)) { + return failure("discovery", discoveryHint(text, ctx.guardOn, port)); + } + if ((m = RE.notImplemented.exec(text))) { + return failure("not-implemented", notImplementedHint(port, m[1], m[2]), { + method: m[1], + path: m[2], + }); + } + if (RE.notImplementedCode.test(text)) { + return failure("not-implemented", notImplementedHint(port)); + } + if (RE.provider.test(text)) { + const namespace = RE.providerNamespace.exec(text)?.[1]; + return failure("provider", providerHint(port, namespace), namespace ? { namespace } : {}); + } + if (RE.noRoute.test(text)) return failure("no-route", noRouteHint(port)); + if ((m = RE.notRecognized.exec(text))) { + const token = m[1]; + const group = pathUpTo(ctx.argv, token); + const lookup = ctx.extensionFor ?? ((tokens: string[]) => curatedExtensionFor(tokens)); + const extension = lookup(group); + if (extension) { + const joined = group.join(" "); + return failure("extension", missingExtensionHint(joined, extension, inDocker), { + token, + group: joined, + extension, + }); + } + const suggestion = RE.didYouMean.exec(text)?.[1]; + return failure("unknown-command", undefined, suggestion ? { token, suggestion } : { token }); + } + if (RE.argument.test(text)) return failure("argument", ARGUMENT_HINT); + // az exits 3 for its ResourceNotFoundError, whatever the service's error wording: the emulator + // words some not-found answers with codes the text pattern does not know (Event Hubs, Service + // Bus, Cosmos, Key Vault, subnets, ...), so the exit code is the reliable signal. The + // more specific classes above still win. + if (RE.notFound.test(text) || result.exitCode === 3) return failure("not-found"); + const cliError = RE.cliError.test(text); + if ((m = RE.emulatorError.exec(text))) { + return failure("emulator-error", emulatorErrorHint(m[1], m[2]), { method: m[1], path: m[2] }); + } + if (RE.needsYes.test(text)) return failure("needs-yes", NEEDS_YES_HINT); + if (RE.bicepMissing.test(text)) { + return failure("bicep-missing", bicepMissingHint(inDocker), { + source: "stderr", + }); + } + if ((m = RE.bicepRegistry.exec(text))) { + const reference = RE.bicepReference.exec(text)?.[1] ?? m[1]; + return failure("bicep-registry", BICEP_REGISTRY_HINT, reference ? { reference } : {}); + } + if ((m = RE.bicepEnv.exec(text))) { + return failure("bicep-env", bicepEnvHint(m[1]), { variable: m[1] }); + } + if (RE.azcopy.test(text)) return failure("azcopy", AZCOPY_HINT); + if (ctx.guardOn && RE.tunnel403.test(text)) { + const host = RE.hostInText.exec(text)?.[1]; + const housekeeping = new Set([...HOUSEKEEPING_HOSTS, ...result.egress.housekeeping]); + const onlyHousekeeping = host + ? housekeeping.has(host.toLowerCase()) + : result.egress.housekeeping.length > 0; + if (!onlyHousekeeping) { + return failure("egress-refused", undefined, { hosts: host ?? "", source: "stderr" }); + } + } + return cliError ? failure("cli-error") : failure("other"); +} + +/** The command path up to and including `token`, the word az did not recognise. */ +function pathUpTo(argv: string[] | undefined, token: string): string[] { + const words = commandWords(argv ?? []); + const index = words.indexOf(token); + return index >= 0 ? words.slice(0, index + 1) : [token]; +} + +// --------------------------------------------------------------------------------------------- +// Responses + +function firstLine(exit: string, classId: AzFailureClass): string { + return `❌ **Command Failed** (exit ${exit}, ${classId})`; +} + +function stoppedByTool(result: AzRunResult): boolean { + return ( + result.exitCode === null || + result.timedOut || + result.aborted || + result.failFast !== undefined || + result.tooLong === true || + result.spawnError !== undefined + ); +} + +function isSuccess(result: AzRunResult): boolean { + return result.exitCode === 0 && !stoppedByTool(result); +} + +function formatStdout(stdout: string, max: number): { text: string; note?: string } { + const all = normalizeNewlines(stdout).replace(/\n+$/, ""); + if (all.length <= max) return { text: all }; + const shown = headCutAtLine(all, max, Math.min(1_000, Math.floor(max / 10))); + return { + text: shown, + note: + `[Output truncated: showing the first ${fmt(shown.length)} of ${fmt(all.length)} ` + + 'characters. Narrow it with --query and -o tsv, for example --query "[].name" -o tsv.]', + }; +} + +const NO_OUTPUT = "The command succeeded and printed no output."; + +function hostsNote(prefix: string, hosts: string[], suffix: string): string { + return `${prefix} ${hosts.map(inlineCode).join(", ")}${suffix}`; +} + +/** Formats a finished `az` run for the model. */ +export function formatAzResult(result: AzRunResult, opts: FormatOptions): ToolTextResponse { + const notes = [...opts.notes]; + if (!opts.guardOn) notes.push(GUARD_OFF_NOTE); + if (result.truncated) { + notes.push( + "Note: `az` printed more than the tool captures, so it was stopped and its output is " + + "incomplete. Narrow the output with `--query` or `-o tsv`." + ); + } + + const blocks: string[] = []; + let classId: AzFailureClass | null = null; + + if (isSuccess(result)) { + const refused = unique(result.egress.refused); + if (refused.length > 0) { + const prefix = "Note: the egress guard blocked a connection to"; + notes.push(hostsNote(prefix, refused, "; the command still succeeded.")); + } + if ( + APP_INSIGHTS_SKIPPED.test(result.stderr) && + result.egress.housekeeping.includes(APP_INSIGHTS_REGION_HOST) + ) { + notes.push(APP_INSIGHTS_NOTE); + } + if (notes.length > 0) blocks.push(notes.join("\n")); + if (opts.isHelp) { + const page = reflowHelp(result.stdout).replace(/^\n+/, "").replace(/\n+$/, ""); + blocks.push(cutHelpPage(page, opts.maxHelpChars)); + } else { + const out = formatStdout(result.stdout, opts.maxOutputChars); + blocks.push(out.text || NO_OUTPUT); + if (out.note) blocks.push(out.note); + } + const warnings = prepareStderr(result.stderr, { maxChars: STDERR_SUCCESS_CHARS }); + if (warnings) blocks.push(warnings); + } else { + const c = classifyFailure(result, { + guardOn: opts.guardOn, + extensionFor: opts.extensionFor, + argv: opts.argv, + port: opts.port, + timeoutSeconds: opts.timeoutSeconds, + client: opts.client, + inDocker: opts.inDocker, + }); + classId = c.classId; + blocks.push(firstLine(stoppedByTool(result) ? "none" : String(result.exitCode), c.classId)); + if (c.classId === "egress-refused") { + // az's own text never names the refused host on SDK paths: the tool's line replaces it. + const hosts = c.details.hosts ? c.details.hosts.split(",") : []; + blocks.push(egressRefusedMessage(hosts)); + } else { + const stderr = prepareStderr(result.stderr); + if (stderr) blocks.push(stderr); + if (c.classId === "spawn-error" && result.spawnError) { + blocks.push(`The tool could not start the Azure CLI: ${result.spawnError}`); + } + } + const out = formatStdout(result.stdout, opts.maxOutputChars); + if (out.text) blocks.push(`Output:\n${out.text}${out.note ? `\n${out.note}` : ""}`); + if (blocks.length === 1 && !c.hint) { + blocks.push("`az` exited without printing an error message."); + } + if (c.hint) blocks.push(c.hint); + const housekeeping = unique(result.egress.housekeeping); + if (housekeeping.length > 0 && c.classId !== "egress-refused") { + const suffix = + ", which the tool always refuses (housekeeping calls such as update checks); these " + + "blocks are expected."; + notes.push(hostsNote("Note: the egress guard also blocked", housekeeping, suffix)); + } + if (notes.length > 0) blocks.push(notes.join("\n")); + } + + const response: ToolTextResponse = ResponseBuilder.markdown(blocks.join("\n\n")); + if (opts.envelope) { + const envelope: ResultEnvelope = { + exitCode: result.exitCode, + stdout: result.stdout, + stderr: redactProxyUrls(result.stderr), + notes, + classId, + truncated: result.truncated, + stoppedByTool: stoppedByTool(result), + }; + response.content.push({ type: "text", text: JSON.stringify(envelope) }); + } + return response; +} + +/** `bicep-missing`, answered before spawning when a command needs Bicep and none was found. */ +export function bicepMissing(opts: { inDocker: boolean }): ToolTextResponse { + return ResponseBuilder.markdown( + `${firstLine("none", "bicep-missing")}\n\n${bicepMissingHint(opts.inDocker)}` + ); +} + +/** + * `bicep-registry`, answered before spawning when a `.bicep` input references a registry + * module. On Windows a successful restore would write the real %USERPROFILE%\.bicep. + */ +export function bicepRegistryUnsupported(ref: string): ToolTextResponse { + return ResponseBuilder.markdown( + `${firstLine("none", "bicep-registry")}\n\nThe template uses the registry module ${inlineCode(ref)}.` + + `\n\n${BICEP_REGISTRY_HINT}` + ); +} + +/** `too-long`: the Windows command line would exceed its limit, so az is not spawned. */ +export function commandTooLong(): ToolTextResponse { + return ResponseBuilder.markdown(`${firstLine("none", "too-long")}\n\n${TOO_LONG_HINT}`); +} diff --git a/src/lib/azure/policy.corpus.test.ts b/src/lib/azure/policy.corpus.test.ts new file mode 100644 index 0000000..65cf1a1 --- /dev/null +++ b/src/lib/azure/policy.corpus.test.ts @@ -0,0 +1,109 @@ +/** + * The samples corpus through the shared tokenizer and the policy. + * + * Every `az` command in localstack-azure-samples (pinned commit) must tokenize to exactly the + * bash-generated argv, and get the expected policy verdict: pass, except the seven sample steps + * the policy refuses, recorded with their rule ids. The fixture and the scripts + * that regenerate it live in tests/fixtures/azure/corpus/ (see its README). + */ +import { splitCliArgs } from "../cli/argv"; +import { evaluateAzCommand } from "./policy"; +import type { PolicyOptions } from "./types"; + +interface CorpusCase { + command: string; + argv: string[]; + origins: string[]; + expect: "ok" | { ruleId: string }; +} +interface CorpusFixture { + _metadata: { + source_commit: string; + distinct_count: number; + refused_occurrences: number; + command_count: number; + }; + cases: CorpusCase[]; +} + +// resolveJsonModule is off, so the fixture is loaded with require and typed here. +// eslint-disable-next-line @typescript-eslint/no-var-requires +const fixture: CorpusFixture = require("../../../tests/fixtures/azure/corpus/samples-az-corpus.json"); + +// The three tokenizer options the Azure client uses. +const AZURE_TOKENIZER = { + quotedControlChars: true, + keepEmptyQuoted: true, + bashDoubleQuoteEscapes: true, +} as const; + +// A workdir the samples' relative paths (main.bicep, planner_website.zip, ...) resolve inside, so +// the file rule allows them. POSIX, for host-independent resolution. +const corpusOpts: PolicyOptions = { + workdir: "/work/samples", + homeDir: "/work/.mcp/azure/home", + platform: "linux", +}; + +describe("samples corpus", () => { + test("the fixture has 736 distinct cases and 7 refused occurrences", () => { + expect(fixture.cases.length).toBe(736); + expect(fixture._metadata.distinct_count).toBe(736); + expect(fixture._metadata.command_count).toBe(1985); + expect(fixture._metadata.refused_occurrences).toBe(7); + }); + + test("every case tokenizes to the expected bash argv and gets the expected verdict", () => { + const tokenizerMismatches: string[] = []; + const verdictMismatches: string[] = []; + let refusedOccurrences = 0; + + for (const testCase of fixture.cases) { + const argv = splitCliArgs(testCase.command, AZURE_TOKENIZER); + if (JSON.stringify(argv) !== JSON.stringify(testCase.argv)) { + tokenizerMismatches.push( + `${testCase.origins[0]}: ${testCase.command}\n got ${JSON.stringify(argv)}\n want ${JSON.stringify(testCase.argv)}` + ); + } + + const result = evaluateAzCommand(testCase.command, corpusOpts); + if (testCase.expect === "ok") { + if (!result.ok) { + verdictMismatches.push( + `${testCase.origins[0]}: expected ok, refused ${result.ruleId}: ${testCase.command}` + ); + } + } else { + refusedOccurrences += testCase.origins.length; + if (result.ok) { + verdictMismatches.push( + `${testCase.origins[0]}: expected refusal ${testCase.expect.ruleId}, got ok: ${testCase.command}` + ); + } else if (result.ruleId !== testCase.expect.ruleId) { + verdictMismatches.push( + `${testCase.origins[0]}: expected ${testCase.expect.ruleId}, got ${result.ruleId}: ${testCase.command}` + ); + } + } + } + + expect(tokenizerMismatches.join("\n")).toBe(""); + expect(verdictMismatches.join("\n")).toBe(""); + // Exactly the seven known sample steps are refused, no more, no fewer. + expect(refusedOccurrences).toBe(7); + }); + + test("the seven expected refusals are the acr login and container exec sample steps", () => { + const refused = fixture.cases.filter((c) => c.expect !== "ok"); + expect(refused).toHaveLength(2); + const acrLogin = refused.find((c) => c.command.startsWith("acr login")); + const containerExec = refused.find((c) => c.command.startsWith("container exec")); + expect(acrLogin?.expect).toEqual({ ruleId: "denied:acr-login" }); + expect(acrLogin?.origins).toHaveLength(6); + expect(containerExec?.expect).toEqual({ ruleId: "denied:container-exec" }); + expect(containerExec?.origins).toHaveLength(1); + // And the policy really refuses them. + expect(evaluateAzCommand(acrLogin!.command, corpusOpts).ok).toBe(false); + expect(evaluateAzCommand(containerExec!.command, corpusOpts).ok).toBe(false); + }); +}); diff --git a/src/lib/azure/policy.leak.test.ts b/src/lib/azure/policy.leak.test.ts new file mode 100644 index 0000000..cf3d320 --- /dev/null +++ b/src/lib/azure/policy.leak.test.ts @@ -0,0 +1,171 @@ +/** + * L3, the policy half: the leak commands. + * + * Agents driving `az` against the emulator sent these `az rest` calls with an absolute + * https://management.azure.com URL, so real Azure was their target and only the guard stopped + * them. They are recorded in tests/fixtures/azure/leak-commands.json. Every one must now be + * rewritten to a relative URL before anything is spawned, with nothing else in the argv changed, + * and no argv element may still name the ARM host, except as a `--resource` token audience (the + * leak commands have none). + * The live half (GET and DELETE replayed on a CI emulator) is in tests/azure/egress.live.test.ts. + */ +import { splitCliArgs } from "../cli/argv"; +import { evaluateAzCommand } from "./policy"; +import type { PolicyOptions, PolicyResult } from "./types"; + +interface LeakCommand { + id: string; + method: string; + calls: number; + command: string; +} +interface LeakFixture { + calls: number; + distinct: number; + byMethod: Record; + byMethodDistinct: Record; + redactions: Array<{ id: string; category: string; chars: number }>; + commands: LeakCommand[]; +} + +// resolveJsonModule is off, so the fixture is loaded with require and typed here. +// eslint-disable-next-line @typescript-eslint/no-var-requires +const fixture: LeakFixture = require("../../../tests/fixtures/azure/leak-commands.json"); + +// The three tokenizer options the Azure client uses, as in the policy. +const AZURE_TOKENIZER = { + quotedControlChars: true, + keepEmptyQuoted: true, + bashDoubleQuoteEscapes: true, +} as const; + +// Synthetic paths on both path flavours: the leak commands name no file, so the file rule +// passes them on either, and the URL rule does not depend on the platform. +const PLATFORMS: Array<[string, PolicyOptions]> = [ + ["linux", { workdir: "/work/project", homeDir: "/work/.mcp/azure/home", platform: "linux" }], + [ + "win32", + { + workdir: "C:\\work\\project", + homeDir: "C:\\work\\.mcp\\azure\\home", + protectedDirs: ["C:\\Users\\me\\.azure"], + platform: "win32", + }, + ], +]; + +/** The ARM host itself; `schema.management.azure.com` is another host and does not count. */ +const ARM_HOST = /(? { + if (!result.ok) throw new Error(`${id}: refused ${result.ruleId}: ${result.message}`); + return result; +} + +/** Index of the URL value in an argv (`--url `; the leak commands never use `--url=`). */ +function urlIndex(argv: string[]): number { + const flag = argv.findIndex((token) => URL_FLAGS.has(token)); + return flag === -1 ? -1 : flag + 1; +} + +describe("L3 leak commands (absolute management.azure.com calls that agents sent)", () => { + test("the fixture holds 154 distinct commands from 182 calls", () => { + expect(fixture.calls).toBe(182); + expect(fixture.distinct).toBe(154); + expect(fixture.commands).toHaveLength(154); + expect(new Set(fixture.commands.map((c) => c.command)).size).toBe(154); + expect(fixture.commands.reduce((n, c) => n + c.calls, 0)).toBe(182); + expect(fixture.byMethod).toEqual({ DELETE: 7, GET: 87, PATCH: 13, POST: 49, PUT: 26 }); + for (const leak of fixture.commands) { + expect(leak.command.startsWith("rest ")).toBe(true); + expect(leak.command).toContain("https://management.azure.com"); + } + }); + + test("no private key material is left in the fixture (the certificate bodies are redacted)", () => { + const redacted = fixture.commands.filter((c) => c.command.includes(REDACTION_MARKER)); + expect(redacted.map((c) => c.id)).toEqual(fixture.redactions.map((r) => r.id)); + for (const leak of fixture.commands) { + expect(leak.command).not.toMatch(/[A-Za-z0-9+/]{200,}/); + expect(leak.command).not.toMatch(/-----BEGIN [A-Z ]*PRIVATE KEY-----/); + } + }); + + describe.each(PLATFORMS)("through evaluateAzCommand (%s paths)", (_name, opts) => { + test("every command is rewritten to a relative URL with a note", () => { + const problems: string[] = []; + for (const leak of fixture.commands) { + const result = evaluateAzCommand(leak.command, opts); + if (!result.ok) { + problems.push(`${leak.id}: refused ${result.ruleId}`); + continue; + } + if (result.outcome !== "rewritten") problems.push(`${leak.id}: outcome ${result.outcome}`); + const value = result.argv[urlIndex(result.argv)]; + if (value === undefined || !value.startsWith("/")) + problems.push(`${leak.id}: the URL is not relative: ${value}`); + if (!result.notes.some((n) => n.startsWith("Rewrote the management.azure.com URL"))) + problems.push(`${leak.id}: no rewrite note`); + } + expect(problems.join("\n")).toBe(""); + }); + + test("no argv element names the ARM host, except as a --resource audience", () => { + const problems: string[] = []; + let exempted = 0; + for (const leak of fixture.commands) { + const { argv } = ok(evaluateAzCommand(leak.command, opts), leak.id); + argv.forEach((token, i) => { + if (!ARM_HOST.test(token)) return; + if (argv[i - 1] === "--resource" || token.startsWith("--resource=")) exempted++; + else problems.push(`${leak.id}: argv[${i}] still names management.azure.com`); + }); + } + expect(problems.join("\n")).toBe(""); + // None of the leak commands asks for a token audience, so the exemption never applied. + expect(exempted).toBe(0); + }); + + test("the rewrite keeps the path and query, and changes no other argv element", () => { + const problems: string[] = []; + for (const leak of fixture.commands) { + const original = splitCliArgs(leak.command, AZURE_TOKENIZER); + const { argv } = ok(evaluateAzCommand(leak.command, opts), leak.id); + const at = urlIndex(original); + if (at === -1 || argv.length !== original.length) { + problems.push(`${leak.id}: argv shape changed`); + continue; + } + const expected = original[at].replace(ARM_ORIGIN, "") || "/"; + if (argv[at] !== expected) problems.push(`${leak.id}: ${argv[at]} !== ${expected}`); + original.forEach((token, i) => { + if (i !== at && argv[i] !== token) problems.push(`${leak.id}: argv[${i}] changed`); + }); + } + expect(problems.join("\n")).toBe(""); + }); + }); + + test("the only other management.azure.com names are ARM template $schema URIs in --body", () => { + // The 20 deployment create and validate bodies carry + // "$schema": "https://schema.management.azure.com/...": data sent to the emulator, which + // az never fetches, and a different host from the ARM endpoint. + const withSchemaHost: string[] = []; + const outsideBody: string[] = []; + for (const leak of fixture.commands) { + const { argv } = ok(evaluateAzCommand(leak.command, PLATFORMS[0][1]), leak.id); + argv.forEach((token, i) => { + if (!/management\.azure\.com/i.test(token)) return; + withSchemaHost.push(leak.id); + const isBody = argv[i - 1] === "--body" || argv[i - 1] === "-b"; + if (!isBody || !/"\$schema":\s*"https:\/\/schema\.management\.azure\.com\//.test(token)) + outsideBody.push(`${leak.id}: argv[${i}]`); + }); + } + expect(outsideBody).toEqual([]); + expect(new Set(withSchemaHost).size).toBe(20); + }); +}); diff --git a/src/lib/azure/policy.test.ts b/src/lib/azure/policy.test.ts new file mode 100644 index 0000000..b72de4d --- /dev/null +++ b/src/lib/azure/policy.test.ts @@ -0,0 +1,911 @@ +import fs from "fs"; +import os from "os"; +import path from "path"; +import { + analyticsFields, + DENIED, + DENIED_FLAGS, + evaluateAzCommand, + LOCAL_HOST, + parseDenylistFile, + REQUIRED_FLAGS, + scanBicepModules, +} from "./policy"; +import { emulatorPorts } from "./local-hosts"; +import type { PolicyOptions, PolicyResult } from "./types"; + +// Synthetic win32 paths keep the lexical file rule deterministic on any host: the paths do not +// exist, so the realpath step is skipped and only the pure-lexical logic runs. +const WORKDIR = "C:\\work\\project"; +const HOMEDIR = "C:\\work\\.mcp\\azure\\home"; // the tool's private home +const PROTECTED = ["C:\\Users\\me\\.azure", "C:\\Users\\me\\.ssh", "C:\\work\\.mcp\\azure"]; + +const baseOpts: PolicyOptions = { + workdir: WORKDIR, + homeDir: HOMEDIR, + protectedDirs: PROTECTED, + platform: "win32", +}; + +function evalCmd(command: string, overrides: Partial = {}): PolicyResult { + return evaluateAzCommand(command, { ...baseOpts, ...overrides }); +} + +function expectOk(result: PolicyResult): Extract { + if (!result.ok) throw new Error(`expected ok, got refusal ${result.ruleId}: ${result.message}`); + return result; +} + +function expectRefused(result: PolicyResult): Extract { + if (result.ok) throw new Error(`expected refusal, got ok (${JSON.stringify(result.argv)})`); + return result; +} + +describe("evaluateAzCommand: strip, start rule and tokenizer", () => { + test("strips exactly one leading az", () => { + expect(expectOk(evalCmd("az group list")).argv).toEqual(["group", "list"]); + expect(expectOk(evalCmd("group list")).argv).toEqual(["group", "list"]); + }); + + test("refuses a second az (only one is stripped)", () => { + const r = expectRefused(evalCmd("az az group list")); + expect(r.ruleId).toBe("start"); + expect(r.argv).toEqual(["az", "group", "list"]); + }); + + test("refuses azlocal", () => { + expect(expectRefused(evalCmd("azlocal group list")).ruleId).toBe("start"); + expect(expectRefused(evalCmd("az azlocal group list")).ruleId).toBe("start"); + }); + + test("refuses a leading global flag", () => { + expect(expectRefused(evalCmd("--debug group list")).ruleId).toBe("start"); + }); + + test("a CliSyntaxError becomes a syntax refusal with the tokenizer's message", () => { + const r = expectRefused(evalCmd("group list | tee out")); + expect(r.ruleId).toBe("syntax"); + expect(r.message).toBe("Command contains forbidden shell syntax."); + const q = expectRefused(evalCmd("group list 'unterminated")); + expect(q.ruleId).toBe("syntax"); + expect(q.message).toBe("Command contains an unterminated quote."); + }); + + test("keeps quoted control chars, empty quoted args and bash double-quote escapes", () => { + // Newline inside single quotes is data, not a separator (the 72 corpus commands). + expect(expectOk(evalCmd("monitor diagnostic-settings create --logs '[\n{}]'")).argv).toEqual([ + "monitor", + "diagnostic-settings", + "create", + "--logs", + "[\n{}]", + ]); + // Empty quoted argument kept. + expect(expectOk(evalCmd('tag create --value ""')).argv).toEqual([ + "tag", + "create", + "--value", + "", + ]); + }); +}); + +describe("evaluateAzCommand: denied groups and verbs (one per family)", () => { + const cases: Array<[string, string]> = [ + ["login", "denied:login"], + ["logout", "denied:logout"], + ["account clear", "denied:account-clear"], + ["cloud set --name x", "denied:cloud-set"], + ["cloud register --name x", "denied:cloud-register"], + ["config set x y", "denied:config"], + ["configure --defaults group=rg", "denied:configure"], + ["init", "denied:init"], + ["extension add --name foo", "denied:extension-add"], + ["upgrade", "denied:upgrade"], + ["bicep install", "denied:bicep-install"], + ["bicep restore --file main.bicep", "denied:bicep-restore"], + ["aks install-cli", "denied:aks-install-cli"], + ["find --cli x", "denied:find"], + ["interactive", "denied:interactive"], + ["aks browse --name c --resource-group rg", "denied:aks-browse"], + ["webapp ssh --name w --resource-group rg", "denied:webapp-ssh"], + ["container exec --name c --resource-group rg --exec-command sh", "denied:container-exec"], + ["webapp log tail --name w --resource-group rg", "denied:webapp-log-tail"], + ["network bastion ssh --name b --resource-group rg", "denied:network-bastion-ssh"], + ["acr check-health", "denied:acr-check-health"], + ["storage copy --source a --destination b", "denied:storage-copy"], + ["storage blob sync -c c -s src", "denied:storage-blob-sync"], + ["postgres flexible-server deploy setup", "denied:postgres-flexible-server-deploy"], + ["cognitiveservices agent create --name a --source .", "denied:cognitiveservices-agent-create"], + ["devops project list", "denied:devops"], + ]; + test.each(cases)("refuses `%s`", (command, ruleId) => { + const r = expectRefused(evalCmd(command)); + expect(r.ruleId).toBe(ruleId); + expect(r.title).toBeTruthy(); + expect(r.message).toContain("not allowed"); + }); + + test("cloud show and cloud list keep working (only mutating verbs are denied)", () => { + expect(evalCmd("cloud show").ok).toBe(true); + expect(evalCmd("cloud list").ok).toBe(true); + expect(expectOk(evalCmd("cloud show --query name")).argv).toEqual([ + "cloud", + "show", + "--query", + "name", + ]); + }); + + test("config get is allowed, config set is refused", () => { + expect(evalCmd("config get core.output").ok).toBe(true); + expect(expectRefused(evalCmd("config set core.output=json")).ruleId).toBe("denied:config"); + }); +}); + +describe("evaluateAzCommand: flag rules", () => { + test("refuses --follow anywhere", () => { + expect( + expectRefused(evalCmd("container logs --name c --resource-group rg --follow")).ruleId + ).toBe("denied:flag"); + }); + test("refuses --login-with-github", () => { + expect( + expectRefused(evalCmd("webapp deployment github-actions add --login-with-github")).ruleId + ).toBe("denied:flag"); + }); + test("refuses webapp up --launch-browser", () => { + expect(expectRefused(evalCmd("webapp up --launch-browser --name w")).ruleId).toBe( + "denied:flag" + ); + }); + test("refuses containerapp up --source and containerapp create --repo", () => { + expect(expectRefused(evalCmd("containerapp up --source . --name a")).ruleId).toBe( + "denied:flag" + ); + expect(expectRefused(evalCmd("containerapp create --repo o/r --name a")).ruleId).toBe( + "denied:flag" + ); + }); + test("acr login is refused without --expose-token and allowed with it", () => { + const r = expectRefused(evalCmd('acr login --name "myacr"')); + expect(r.ruleId).toBe("denied:acr-login"); + expect(r.message).toContain("--expose-token"); + expect(evalCmd('acr login --name "myacr" --expose-token').ok).toBe(true); + expect(evalCmd('acr login --name "myacr" -t').ok).toBe(true); + }); + + test("a required flag is not required to read the command's help", () => { + // `az acr login --help` only prints help: it never runs docker login. + for (const cmd of ["acr login --help", "acr login -h", 'acr login --name "myacr" --help']) { + const r = expectOk(evalCmd(cmd)); + expect(r.isHelp).toBe(true); + } + // Help does not lift a denial, though. + expect(expectRefused(evalCmd("login --help")).ruleId).toMatch(/^denied:/); + }); + + test("prefix-aware: abbreviated denied and required flags are handled", () => { + expect(expectRefused(evalCmd("container logs --name c --resource-group rg --fol")).ruleId).toBe( + "denied:flag" + ); + expect(expectRefused(evalCmd("webapp deployment github-actions add --login-with")).ruleId).toBe( + "denied:flag" + ); + expect(expectRefused(evalCmd("webapp up --launch --name w")).ruleId).toBe("denied:flag"); + expect(expectRefused(evalCmd("containerapp up --sour . --name a")).ruleId).toBe("denied:flag"); + // An abbreviation of the required --expose-token satisfies it. + expect(evalCmd('acr login --name "myacr" --expose').ok).toBe(true); + }); +}); + +describe("evaluateAzCommand: URL rule (request targets only)", () => { + test("rewrites a management.azure.com rest --url to a relative path, with a note", () => { + const r = expectOk( + evalCmd("rest --url https://management.azure.com/subscriptions/0/x?api-version=2022-09-01") + ); + expect(r.argv).toEqual(["rest", "--url", "/subscriptions/0/x?api-version=2022-09-01"]); + expect(r.outcome).toBe("rewritten"); + expect(r.notes.join(" ")).toContain("management.azure.com"); + }); + + test("rewrite keeps the leading slash and matches :443 and an upper-case host", () => { + const r = expectOk(evalCmd("rest --uri=https://MANAGEMENT.AZURE.COM:443/providers/foo")); + expect(r.argv).toEqual(["rest", "--uri=/providers/foo"]); + expect(r.outcome).toBe("rewritten"); + }); + + test("allows the local emulator hosts unchanged", () => { + const r = expectOk( + evalCmd("rest --url https://localhost.localstack.cloud:4566/subscriptions/0") + ); + expect(r.argv[2]).toBe("https://localhost.localstack.cloud:4566/subscriptions/0"); + expect(r.outcome).toBe("ok"); + expect(evalCmd("rest --url https://azure.localhost.localstack.cloud:4566/x").ok).toBe(true); + expect(evalCmd("rest --url http://127.0.0.1:4566/x").ok).toBe(true); + }); + + test("with the emulator's ports set, a local host on any other port is refused", () => { + const localPorts = emulatorPorts([4566]); + const refusedHost = (command: string) => { + const r = expectRefused(evalCmd(command, { localPorts })); + expect(r.ruleId).toBe("url:blocked"); + return r.message; + }; + // Docker's unauthenticated API and a local model server: not the emulator. + expect(refusedHost("rest --url http://localhost:2375/containers/json")).toContain( + "localhost:2375" + ); + expect(refusedHost("rest --method post --url http://127.0.0.1:11434/api/generate")).toContain( + "127.0.0.1:11434" + ); + expect(refusedHost("rest --url http://localhost.localstack.cloud/x")).toContain(":80"); + // The gateway, 443 (a port-less https URL) and the service range stay allowed. + for (const url of [ + "https://azure.localhost.localstack.cloud:4566/subscriptions/0", + "https://app--env.aca.azure.localhost.localstack.cloud/", + "https://ns.servicebus.azure.localhost.localstack.cloud:4521/", + "http://127.0.0.1:4560/x", + ]) + expectOk(evalCmd(`rest --url ${url}`, { localPorts })); + // Without the set (the default), any port, as before. + expectOk(evalCmd("rest --url http://localhost:2375/containers/json")); + }); + + test("emulatorPorts: the gateway ports, 443 and the service range from the environment", () => { + const byDefault = emulatorPorts([4566, 4567]); + expect([4566, 4567, 443, 4510, 4560].every((p) => byDefault.has(p))).toBe(true); + expect([2375, 11434, 80, 4561, 4509].some((p) => byDefault.has(p))).toBe(false); + const shifted = emulatorPorts([4666], { EXTERNAL_SERVICE_PORTS_START: "4610" }); + expect(shifted.has(4610) && shifted.has(4660) && !shifted.has(4510)).toBe(true); + }); + + test("refuses other absolute URLs on request-target flags", () => { + expect(expectRefused(evalCmd("rest --url https://graph.microsoft.com/v1.0/me")).ruleId).toBe( + "url:blocked" + ); + expect( + expectRefused( + evalCmd( + "deployment group create -g rg --template-uri https://raw.githubusercontent.com/x/y/main.json" + ) + ).ruleId + ).toBe("url:blocked"); + expect(expectRefused(evalCmd("rest --url https://api.loganalytics.io/v1/x")).ruleId).toBe( + "url:blocked" + ); + }); + + test("acr build refuses a git/github URL source and allows a local dot source", () => { + expect(expectRefused(evalCmd("acr build --registry r https://github.com/x/y.git")).ruleId).toBe( + "url:blocked" + ); + expect(evalCmd("acr build --registry r --image i:v1 .").ok).toBe(true); + }); + + test("prefix-aware: abbreviated URL flags are caught like the full flags", () => { + expect(expectRefused(evalCmd("rest --ur https://evil.example.com/x")).ruleId).toBe( + "url:blocked" + ); + const rewritten = expectOk(evalCmd("rest --ur https://management.azure.com/subscriptions/1")); + expect(rewritten.argv).toEqual(["rest", "--ur", "/subscriptions/1"]); + expect(rewritten.outcome).toBe("rewritten"); + expect( + expectRefused( + evalCmd( + "deployment group create -g rg --template-u https://raw.githubusercontent.com/x/main.json" + ) + ).ruleId + ).toBe("url:blocked"); + }); + + test("IPv6 and odd spellings: the guard's own host check decides", () => { + // A bracketed IPv6 host is parsed as a host, not skipped as "not a URL". + expect(expectRefused(evalCmd("rest --url https://[2001:db8::1]/x")).ruleId).toBe("url:blocked"); + expect(evalCmd("rest --url https://[::1]:4566/subscriptions/0").ok).toBe(true); + // Canonical loopback spellings the guard relays are allowed here too. + expect(evalCmd("rest --url https://127.1:4566/subscriptions/0").ok).toBe(true); + expect(evalCmd("rest --url https://LOCALHOST.localstack.cloud:4566/x").ok).toBe(true); + // Userinfo never makes a host look like the management host or a local one. + expect( + expectRefused(evalCmd("rest --url https://management.azure.com@evil.example/x")).ruleId + ).toBe("url:blocked"); + expect(expectRefused(evalCmd("rest --url https://localhost@evil.example/x")).ruleId).toBe( + "url:blocked" + ); + }); + + test("never rewrites the value of an unrelated flag (--resource is a token audience)", () => { + const r = expectOk( + evalCmd("account get-access-token --resource=https://management.azure.com/") + ); + expect(r.argv).toEqual([ + "account", + "get-access-token", + "--resource=https://management.azure.com/", + ]); + expect(r.outcome).toBe("ok"); + }); +}); + +describe("evaluateAzCommand: file rule", () => { + test("generated table: storage blob download -f is bounded by the workdir", () => { + expect( + expectRefused(evalCmd("storage blob download -c c -n b -f C:\\Windows\\win.ini")).ruleId + ).toBe("file:outside-workdir"); + expect(evalCmd("storage blob download -c c -n b -f hello.txt").ok).toBe(true); + }); + + test("generated table: bicep build --outfile outside the workdir is refused", () => { + expect( + expectRefused(evalCmd("bicep build --file main.bicep --outfile C:\\x.json")).ruleId + ).toBe("file:outside-workdir"); + }); + + test("the outside-workdir refusal says how to move the working directory", () => { + // A GUI client starts the server in its own folder, so the user must learn the setting's name. + const r = expectRefused(evalCmd("storage blob upload -c c -n b -f C:\\Users\\me\\data\\x.txt")); + expect(r.ruleId).toBe("file:outside-workdir"); + expect(r.message).toContain("LOCALSTACK_AZ_WORKDIR"); + // Still value-free: the refused path itself is never echoed. + expect(r.message).not.toContain("x.txt"); + }); + + test("supplement: storage blob download-batch -d ..\\.. is refused", () => { + expect(expectRefused(evalCmd("storage blob download-batch -c c -d ..\\..")).ruleId).toBe( + "file:outside-workdir" + ); + }); + + test("prefix-aware: abbreviated file flags are bounded by the workdir", () => { + expect( + expectRefused(evalCmd("deployment group create -g rg --template-f C:\\Windows\\win.ini")) + .ruleId + ).toBe("file:outside-workdir"); + expect( + expectRefused(evalCmd("storage blob download -c c -n b --fi C:\\Windows\\win.ini")).ruleId + ).toBe("file:outside-workdir"); + expect(evalCmd("storage blob download -c c -n b --fi hello.txt").ok).toBe(true); + }); + + test("aks create --ssh-key-value ~/.ssh/id_rsa.pub resolves into the private home (allowed)", () => { + expect(evalCmd("aks create -g rg -n c --ssh-key-value ~/.ssh/id_rsa.pub").ok).toBe(true); + }); + + test("@~ resolves into the private home; @ the real home is refused", () => { + expect(evalCmd("ad sp create-for-rbac --cert @~/.ssh/id_rsa").ok).toBe(true); + const r = expectRefused(evalCmd("ad sp create-for-rbac --cert @C:\\Users\\me\\.ssh\\id_rsa")); + expect(["file:protected", "file:outside-workdir"]).toContain(r.ruleId); + }); + + test("a `~` path missing from the private home gets a note saying what `~` is", () => { + const r = expectOk( + evalCmd("keyvault secret set --vault-name v --name n --file ~/.azure/azureProfile.json") + ); + expect(r.notes).toHaveLength(1); + expect(r.notes[0]).toContain(`private home (${HOMEDIR}), not your home directory`); + expect(r.notes[0]).toContain("~/.azure/azureProfile.json does not exist there"); + expect(r.notes[0]).toContain("never available through this tool"); + // One note per command, however many `~` arguments it has. + const two = expectOk( + evalCmd("rest --method post --url /x --body @~/a.json --headers @~/b.json") + ); + expect(two.notes).toHaveLength(1); + // No `~`, no note. + expect(expectOk(evalCmd("storage blob upload -c c -n b -f hello.txt")).notes).toEqual([]); + }); + + test("a `~` file that exists in the private home gets no note", () => { + const home = fs.mkdtempSync(path.join(os.tmpdir(), "lsaz-policy-home-")); + fs.mkdirSync(path.join(home, ".ssh")); + fs.writeFileSync(path.join(home, ".ssh", "id_rsa.pub"), "ssh-rsa AAAA uat"); + const r = expectOk( + evaluateAzCommand("aks create -g rg -n c --ssh-key-value ~/.ssh/id_rsa.pub", { + workdir: home, + homeDir: home, + protectedDirs: [], + platform: process.platform, + }) + ); + expect(r.notes).toEqual([]); + }); + + test("refuses absolute paths outside the workdir", () => { + expect(expectRefused(evalCmd("storage blob download -c c -n b -f /etc/passwd")).ok).toBe(false); + expect( + expectRefused(evalCmd("storage blob upload -c c -n b -f C:\\Windows\\win.ini")).ruleId + ).toBe("file:outside-workdir"); + }); + + test("refuses UNC, \\\\?\\ and drive-relative forms", () => { + expect( + expectRefused(evalCmd("storage blob upload -c c -n b -f \\\\host\\share\\x")).ruleId + ).toBe("file:unsupported-path"); + expect(expectRefused(evalCmd("storage blob upload -c c -n b -f \\\\?\\C:\\x")).ruleId).toBe( + "file:unsupported-path" + ); + expect(expectRefused(evalCmd("storage blob upload -c c -n b -f C:relative")).ruleId).toBe( + "file:unsupported-path" + ); + }); + + test("a workdir-relative file and an absolute file inside the workdir are allowed", () => { + expect(evalCmd("storage blob upload -c c -n b -f sub/data.bin").ok).toBe(true); + expect(evalCmd("storage blob upload -c c -n b -f C:\\work\\project\\sub\\data.bin").ok).toBe( + true + ); + }); + + test("a missing main.bicep inside the workdir passes (files need not exist)", () => { + const r = expectOk(evalCmd("deployment group create -g rg --template-file main.bicep")); + expect(r.needsBicep).toBe(true); + }); + + test("ARM ids are not treated as paths", () => { + expect( + evalCmd( + "role assignment create --scope /subscriptions/0/resourceGroups/rg --role Reader --assignee x" + ).ok + ).toBe(true); + }); + + test("--parameters: an assignment is not a path; @file and a .bicepparam are", () => { + expect(evalCmd("deployment group create -g rg --parameters location=westeurope").ok).toBe(true); + expect(evalCmd("deployment group create -g rg --parameters @params.json").ok).toBe(true); + const r = expectOk(evalCmd("deployment group create -g rg --parameters main.bicepparam")); + expect(r.needsBicep).toBe(true); + expect( + expectRefused(evalCmd("deployment group create -g rg --parameters ..\\..\\secret.json")) + .ruleId + ).toBe("file:outside-workdir"); + }); +}); + +describe("evaluateAzCommand: protected directories (workdir is the user's home)", () => { + // The server's cwd is a "fake home"; a relative .ssh/.azure path would pass the workdir check. + const fakeHome = "C:\\fakehome"; + const configDir = "C:\\fakehome\\.mcp\\azure"; + const opts: Partial = { + workdir: fakeHome, + homeDir: `${configDir}\\home`, + protectedDirs: [`${fakeHome}\\.azure`, `${fakeHome}\\.ssh`, configDir], + }; + + test("a relative path into a protected dir is refused", () => { + expect( + expectRefused(evalCmd("storage blob upload -c c -n b -f .ssh/id_rsa", opts)).ruleId + ).toBe("file:protected"); + expect( + expectRefused(evalCmd("storage blob upload -c c -n b -f .azure/config", opts)).ruleId + ).toBe("file:protected"); + }); + + test("an ordinary project file is allowed", () => { + expect(evalCmd("storage blob upload -c c -n b -f project/a.txt", opts).ok).toBe(true); + }); + + test("the private home stays allowed even though the config dir contains it", () => { + expect(evalCmd("aks create -g rg -n c --ssh-key-value ~/.ssh/id_rsa.pub", opts).ok).toBe(true); + }); + + test("a path at the config dir root is refused", () => { + expect( + expectRefused(evalCmd(`storage blob download -c c -n b -f ${configDir}\\clouds.config`, opts)) + .ruleId + ).toBe("file:protected"); + }); +}); + +describe("evaluateAzCommand: arguments az opens a file from by itself", () => { + // Some arguments are not marked as files in az's metadata, yet az opens the path it is given: + // a structured (AAZ) argument such as `--tags`, or a `validate_file_or_dict` argument, loads a + // JSON/YAML file whenever the value names an EXISTING path; and a few plain arguments (such as + // `rest --output-file`, `apim api import --specification-path`) are opened by az's own code. + // The existing-path check needs real files, so these tests use a temp tree. + const root = fs.mkdtempSync(path.join(os.tmpdir(), "policy-d31-")); + const workdir = path.join(root, "work"); + const home = path.join(root, "private-home"); + const protectedDir = path.join(root, "userhome", ".azure"); + const outside = path.join(root, "outside"); + for (const d of [workdir, home, protectedDir, outside]) fs.mkdirSync(d, { recursive: true }); + const inProtected = path.join(protectedDir, "probe.json"); + const inOutside = path.join(outside, "o.json"); + const inWorkdir = path.join(workdir, "tags.json"); + for (const f of [inProtected, inOutside, inWorkdir]) fs.writeFileSync(f, '{"k": "v"}'); + const opts: PolicyOptions = { + workdir, + homeDir: home, + protectedDirs: [protectedDir], + platform: process.platform, + }; + const run = (command: string) => evaluateAzCommand(command, opts); + afterAll(() => fs.rmSync(root, { recursive: true, force: true })); + + test("a structured argument naming an existing file in a protected dir is refused", () => { + expect(expectRefused(run(`network vnet create -g rg -n v --tags ${inProtected}`)).ruleId).toBe( + "file:protected" + ); + }); + + test("a structured argument naming an existing file outside the workdir is refused", () => { + expect(expectRefused(run(`network vnet create -g rg -n v --tags ${inOutside}`)).ruleId).toBe( + "file:outside-workdir" + ); + }); + + test("inline tags, a file in the workdir and a path that does not exist still work", () => { + expect(run("network vnet create -g rg -n v --tags env=dev team=a").ok).toBe(true); + expect(run(`network vnet create -g rg -n v --tags ${inWorkdir}`).ok).toBe(true); + // az reads only a path that exists, so a missing one is plain text to it. + expect(run(`network vnet create -g rg -n v --tags ${path.join(outside, "none.json")}`).ok).toBe( + true + ); + }); + + test("a value naming an existing DIRECTORY is left alone: az reads only files (--probe-path /)", () => { + // A remote path that names a local directory (the filesystem root here) is not a file az could + // read; the L2 matrix's `afd origin-group create --probe-path /` was refused before this. + const probe = + "afd origin-group create --origin-group-name og --profile-name p -g rg --probe-path /"; + expect(run(probe).ok).toBe(true); + expect(run(`network vnet create -g rg -n v --tags ${outside}`).ok).toBe(true); + }); + + test("every value after a structured argument is checked, not only the first", () => { + expect( + expectRefused(run(`network vnet create -g rg -n v --tags env=dev ${inProtected}`)).ruleId + ).toBe("file:protected"); + }); + + test("a validate_file_or_dict argument is covered too", () => { + const cmd = `vm extension set -g rg --vm-name vm -n ext --publisher p --settings ${inProtected}`; + expect(expectRefused(run(cmd)).ruleId).toBe("file:protected"); + }); + + test("rest --output-file is bounded by the workdir, even for a target that does not exist yet", () => { + // az WRITES the response here, so the target need not exist: it must be checked unconditionally. + const url = "--method get --url /subscriptions?api-version=2022-09-01"; + const newTarget = path.join(outside, "does-not-exist-yet.json"); + expect(expectRefused(run(`rest ${url} --output-file ${newTarget}`)).ruleId).toBe( + "file:outside-workdir" + ); + expect(expectRefused(run(`rest ${url} --output-file ${inProtected}`)).ruleId).toBe( + "file:protected" + ); + expect(run(`rest ${url} --output-file out.json`).ok).toBe(true); + }); + + test("apim api import --specification-path is bounded by the workdir", () => { + const cmd = + "apim api import -g rg -n s --path p --specification-format OpenApiJson " + + `--specification-path ${inOutside}`; + expect(expectRefused(run(cmd)).ruleId).toBe("file:outside-workdir"); + }); +}); + +describe("evaluateAzCommand: symlink escape (realpath, I/O)", () => { + test("a symlink out of the workdir is refused via realpath", () => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), "policy-sym-")); + const workdir = path.join(root, "work"); + const outside = path.join(root, "outside"); + fs.mkdirSync(workdir); + fs.mkdirSync(outside); + fs.writeFileSync(path.join(outside, "secret.txt"), "x"); + let linked = false; + try { + fs.symlinkSync(outside, path.join(workdir, "link"), "junction"); + linked = true; + } catch { + // No privilege to create a link on this machine: skip the assertion gracefully. + } + if (linked) { + const r = evaluateAzCommand("storage blob upload -c c -n b -f link/secret.txt", { + workdir, + homeDir: path.join(root, "home"), + platform: process.platform, + }); + expect(r.ok).toBe(false); + if (!r.ok) expect(r.ruleId).toBe("file:outside-workdir"); + } + fs.rmSync(root, { recursive: true, force: true }); + }); + + test("a workdir and a private home reached through a link keep their files allowed", () => { + // macOS's /var links to /private/var, and a Windows temp dir can be an 8.3 short name + // (C:\Users\RUNNER~1): a file inside such a workdir has a real path that differs from its + // lexical one, and must still count as inside. An escape through a link stays refused. + const root = fs.mkdtempSync(path.join(os.tmpdir(), "policy-linkdir-")); + const real = path.join(root, "real"); + const link = path.join(root, "link"); + fs.mkdirSync(path.join(real, "home"), { recursive: true }); + fs.mkdirSync(path.join(root, "outside")); + fs.writeFileSync(path.join(real, "main.json"), "{}"); + fs.writeFileSync(path.join(real, "home", "id.pub"), "ssh-rsa x"); + fs.writeFileSync(path.join(root, "outside", "secret.txt"), "x"); + let linked = false; + try { + fs.symlinkSync(real, link, "junction"); + fs.symlinkSync(path.join(root, "outside"), path.join(real, "out"), "junction"); + linked = true; + } catch { + // No privilege to create a link on this machine: skip the assertions gracefully. + } + if (linked) { + const opts = { workdir: link, homeDir: path.join(link, "home"), platform: process.platform }; + expectOk(evaluateAzCommand("deployment group create -g rg --template-file main.json", opts)); + expectOk(evaluateAzCommand("aks create -g rg -n c --ssh-key-value ~/id.pub", opts)); + const escape = evaluateAzCommand("storage blob upload -c c -n b -f out/secret.txt", opts); + expect(escape.ok).toBe(false); + if (!escape.ok) expect(escape.ruleId).toBe("file:outside-workdir"); + } + fs.rmSync(root, { recursive: true, force: true }); + }); + + describe("roots and `..` through a link (a /home that is /var/home, a linked HOME)", () => { + const roots: string[] = []; + afterAll(() => roots.forEach((r) => fs.rmSync(r, { recursive: true, force: true }))); + const layout = () => { + const root = fs.realpathSync.native(fs.mkdtempSync(path.join(os.tmpdir(), "policy-canon-"))); + roots.push(root); + for (const d of ["real/work", "real/me/.ssh", "real/me/shared/x"]) + fs.mkdirSync(path.join(root, d), { recursive: true }); + fs.writeFileSync(path.join(root, "real/work/main.json"), "{}"); + fs.writeFileSync(path.join(root, "real/me/.ssh/id_rsa"), "key"); + // Directory links: junctions on Windows (no privilege needed), symlinks elsewhere. + fs.symlinkSync(path.join(root, "real"), path.join(root, "link"), "junction"); + fs.symlinkSync( + path.join(root, "real/me/shared"), + path.join(root, "real/work/shared"), + "junction" + ); + return { real: path.join(root, "real"), link: path.join(root, "link") }; + }; + const on = + (workdir: string, homeDir: string, protectedDirs: string[] = []) => + (command: string) => + evaluateAzCommand(command, { workdir, homeDir, protectedDirs, platform: process.platform }); + + test("a workdir spelled through a link keeps its existing files inside", () => { + const { link } = layout(); + const run = on(path.join(link, "work"), path.join(link, "h")); + expectOk(run("deployment group create -g rg --template-file main.json")); + }); + + test("a protected dir spelled through a link still protects reads and new files", () => { + const { real, link } = layout(); + const protectedSsh = path.join(link, "me", ".ssh"); + const run = on(path.join(real, "me"), path.join(link, "me", ".localstack", "home"), [ + protectedSsh, + ]); + for (const command of [ + "ad sp create-for-rbac --cert @.ssh/id_rsa", + "storage blob download -c c -n b -f .ssh/authorized_keys", + ]) { + const refused = expectRefused(run(command)); + expect(refused.ruleId).toBe("file:protected"); + expect(refused.message).toContain(protectedSsh); // as configured, not its real path + } + }); + + (process.platform === "win32" ? test.skip : test)( + "`..` after a link is checked where the OS resolves it (Win32 collapses `..` first)", + () => { + const { real } = layout(); + const run = on(path.join(real, "work"), path.join(real, "h")); + const refused = expectRefused( + run("storage blob upload -c c -n b -f shared/x/../../.ssh/id_rsa") + ); + expect(refused.ruleId).toBe("file:outside-workdir"); + } + ); + }); +}); + +describe("evaluateAzCommand: bicep detection", () => { + test("needsBicep for a .bicep template, a .bicepparam file and the bicep group", () => { + expect( + expectOk(evalCmd("deployment group create -g rg --template-file main.bicep")).needsBicep + ).toBe(true); + expect( + expectOk(evalCmd("deployment group create -g rg --parameters main.bicepparam")).needsBicep + ).toBe(true); + expect(expectOk(evalCmd("bicep build --file main.bicep")).needsBicep).toBe(true); + }); + test("no bicep for an ARM JSON template", () => { + expect( + expectOk(evalCmd("deployment group create -g rg --template-file main.json")).needsBicep + ).toBe(false); + }); +}); + +describe("evaluateAzCommand: help and version", () => { + test("--help/-h anywhere sets isHelp", () => { + expect(expectOk(evalCmd("group create --help")).isHelp).toBe(true); + expect(expectOk(evalCmd("--help")).isHelp).toBe(true); + expect(expectOk(evalCmd("group list -h")).isHelp).toBe(true); + }); + + test("version and bare --version return the local marker", () => { + const v = expectOk(evalCmd("version")); + expect(v.local).toBe("version"); + expect(v.outcome).toBe("local"); + const dash = expectOk(evalCmd("--version")); + expect(dash.local).toBe("version"); + }); + + test("postgres flexible-server create --version 16 still runs", () => { + const r = expectOk(evalCmd("postgres flexible-server create -g rg -n s --version 16")); + expect(r.local).toBeUndefined(); + expect(r.argv).toContain("--version"); + }); +}); + +describe("parseDenylistFile", () => { + test("splits on whitespace and ignores comments and blank lines", () => { + const text = [ + "# a comment", + "", + "keyvault secret # inline comment", + " storage account ", + "\t", + ].join("\n"); + expect(parseDenylistFile(text)).toEqual([ + ["keyvault", "secret"], + ["storage", "account"], + ]); + }); + + test("a parsed denylist prefix is enforced", () => { + const extraDenied = parseDenylistFile("keyvault secret\n# nope\n"); + const r = expectRefused( + evalCmd("keyvault secret show --vault-name v --name n", { extraDenied }) + ); + expect(r.ruleId).toBe("denied:keyvault-secret"); + expect(evalCmd("keyvault key show --vault-name v --name n", { extraDenied }).ok).toBe(true); + }); +}); + +describe("analyticsFields: value-free", () => { + test("command_path and flag_names carry no values; each flag cut at first =", () => { + const command = 'keyvault secret set --vault-name v --name pw --value "s3cr3t!" --tags a=b'; + const policy = evalCmd(command); + const fields = analyticsFields(command, policy); + expect(fields.command_path).toBe("keyvault secret set"); + expect(fields.flag_names).toBe("--vault-name,--name,--value,--tags"); + expect(fields.policy_outcome).toBe("ok"); + }); + + test("a value stuck to a short option is cut off: -pS3cret is -p (az accepts that form)", () => { + const sql = "sql server create -g rg -n srv -u adminuser -pS3cretPass1x"; + expect(analyticsFields(sql, evalCmd(sql)).flag_names).toBe("-g,-n,-u,-p"); + const acr = "acr login -n reg -u user -pRegistryPassw0rd --expose-token"; + expect(analyticsFields(acr, evalCmd(acr)).flag_names).toBe("-n,-u,-p,--expose-token"); + }); + + test("policy_outcome covers ok, rewritten, local, denied and syntax", () => { + expect(analyticsFields("group list", evalCmd("group list")).policy_outcome).toBe("ok"); + expect( + analyticsFields( + "rest --url https://management.azure.com/x", + evalCmd("rest --url https://management.azure.com/x") + ).policy_outcome + ).toBe("rewritten"); + expect(analyticsFields("version", evalCmd("version")).policy_outcome).toBe("local"); + expect(analyticsFields("login", evalCmd("login")).policy_outcome).toBe("denied:login"); + expect( + analyticsFields( + "storage blob download -f C:\\x", + evalCmd("storage blob download -c c -n b -f C:\\x") + ).policy_outcome + ).toBe("denied:file:outside-workdir"); + expect(analyticsFields("group list | x", evalCmd("group list | x")).policy_outcome).toBe( + "syntax" + ); + }); + + test("property: a random secret never appears in any analytics field", () => { + const alphabet = "ABCDEFabcdef0123456789!@#$%^&*();:'\"|<>`$ =/\\-_."; + const randomSecret = (): string => { + const len = 8 + Math.floor(Math.random() * 24); + let s = ""; + for (let i = 0; i < len; i++) s += alphabet[Math.floor(Math.random() * alphabet.length)]; + return s; + }; + for (let i = 0; i < 400; i++) { + const secret = randomSecret(); + const commands = [ + `keyvault secret set --vault-name v --name pw --value ${JSON.stringify(secret)}`, + `sql server create -g rg -n s --admin-user a --admin-password ${JSON.stringify(secret)}`, + `sql server create -g rg -n s -u a -p${JSON.stringify(secret)}`, + `storage account keys list --admin-password=${JSON.stringify(secret)}`, + `storage blob upload --sas-token ${JSON.stringify(secret)} -c c -n n -f f`, + `webapp config appsettings set --settings ${JSON.stringify("CONN=" + secret + ";Key=" + secret)}`, + `deployment group create -g rg --parameters @${JSON.stringify(secret + ".json")}`, + ]; + for (const command of commands) { + const policy = evalCmd(command); + const fields = analyticsFields(command, policy); + for (const value of Object.values(fields)) { + expect(value.includes(secret)).toBe(false); + } + } + } + }); +}); + +describe("scanBicepModules", () => { + test("returns the first registry module reference inside the workdir, else undefined", () => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), "policy-bicep-")); + fs.writeFileSync( + path.join(root, "reg.bicep"), + "module x 'br:myregistry.azurecr.io/bicep/modules/vnet:v1' = {}\n" + ); + fs.writeFileSync(path.join(root, "local.bicep"), "module y './local.bicep' = {}\n"); + expect( + scanBicepModules(["deployment", "group", "create", "--template-file", "reg.bicep"], root) + ).toBe("br:myregistry.azurecr.io/bicep/modules/vnet:v1"); + expect( + scanBicepModules(["deployment", "group", "create", "--template-file", "local.bicep"], root) + ).toBeUndefined(); + // A .bicep argument outside the workdir is not read. + expect( + scanBicepModules(["deployment", "group", "create", "--template-file", "..\\reg.bicep"], root) + ).toBeUndefined(); + fs.rmSync(root, { recursive: true, force: true }); + }); + + test("uses an injected reader when provided (no disk I/O)", () => { + const files: Record = { + [path.resolve("/w", "m.bicep")]: + "module spec 'ts:00000000/rg/spec:1.0' = {\n name: 'x'\n}\n", + }; + expect( + scanBicepModules(["deployment", "group", "create", "-f", "m.bicep"], "/w", (p) => files[p]) + ).toBe("ts:00000000/rg/spec:1.0"); + }); + + const scan = (content: string, file = "t.bicep") => + scanBicepModules(["deployment", "group", "create", "-f", file], "/w", () => content); + + test("property names that contain `ts:` or `br:` are not registry references (L2)", () => { + const vnet = [ + "resource vnet 'Microsoft.Network/virtualNetworks@2023-09-01' = {", + " name: 'v'", + " properties: {", + " addressSpace: { addressPrefixes: ['10.0.0.0/16'] }", + " subnets: [ { name: 'default', properties: { addressPrefix: '10.0.0.0/24' } } ]", + " }", + "}", + "output hosts string = 'ts: a timestamp, not a module'", + "var abr = { cobr: 1, abr: 2 }", + "module local './nested.bicep' = { name: 'n' }", + ].join("\n"); + expect(scan(vnet)).toBeUndefined(); + }); + + test("every registry reference form is caught: module, using, extends, import", () => { + expect(scan("module avm 'br/public:avm/res/network/virtual-network:0.1.0' = {")).toBe( + "br/public:avm/res/network/virtual-network:0.1.0" + ); + expect(scan("using 'br:contoso.azurecr.io/bicep/app:v2'\nparam x = 1", "p.bicepparam")).toBe( + "br:contoso.azurecr.io/bicep/app:v2" + ); + expect(scan("extends 'br:contoso.azurecr.io/params/base:v1'", "p.bicepparam")).toBe( + "br:contoso.azurecr.io/params/base:v1" + ); + expect(scan("import { tags } from 'br:contoso.azurecr.io/bicep/shared:v1'")).toBe( + "br:contoso.azurecr.io/bicep/shared:v1" + ); + }); +}); + +describe("exported policy data", () => { + test("DENIED, DENIED_FLAGS, REQUIRED_FLAGS and LOCAL_HOST are populated", () => { + expect(DENIED.length).toBeGreaterThan(20); + expect(DENIED_FLAGS.some((f) => f.flag === "--follow")).toBe(true); + expect(REQUIRED_FLAGS[0].flag).toBe("--expose-token"); + expect(LOCAL_HOST.test("localhost.localstack.cloud")).toBe(true); + expect(LOCAL_HOST.test("azure.localhost.localstack.cloud")).toBe(true); + expect(LOCAL_HOST.test("management.azure.com")).toBe(false); + }); +}); diff --git a/src/lib/azure/policy.ts b/src/lib/azure/policy.ts new file mode 100644 index 0000000..9a9e626 --- /dev/null +++ b/src/lib/azure/policy.ts @@ -0,0 +1,1129 @@ +/** + * The Azure command policy: a pure decision over one `az` command string. + * + * It never spawns anything and never reaches the network. It decides, before the emulator is + * touched, whether a command may run, must be rewritten (a management.azure.com URL made + * relative), is answered locally (`version`), or is refused with a stable rule id. The handler + * runs this first, so a refusal costs nothing. + * + * The tokenizer is the shared one in src/lib/cli/argv.ts, run with the three options that make + * it accept every `az` command in the Azure samples (policy.corpus.test.ts). + */ + +import fs from "fs"; +import path from "path"; +import { CliSyntaxError, splitCliArgs } from "../cli/argv"; +import { isLocalHost, LOCAL_HOST } from "./local-hosts"; +import type { PolicyOptions, PolicyResult } from "./types"; + +/** The platform-specific path flavour (path.win32 / path.posix), so resolution is host-independent. */ +type PlatformPath = typeof path.win32; + +// The generated file-argument table (scripts/gen-az-file-args.py). tsconfig has +// resolveJsonModule off, so it is loaded with require and typed here. +interface FileArgsEntry { + /** All option strings (long and short) az annotates as a file/directory for this command. */ + flags: string[]; + /** The subset that is nargs '+'/'*', so it consumes several path values. */ + greedy?: string[]; + /** The command takes a positional path (acr build / acr run source). */ + positional?: boolean; +} +interface FileArgsTable { + _metadata: Record; + commands: Record; +} +// eslint-disable-next-line @typescript-eslint/no-var-requires +const GENERATED_TABLE: FileArgsTable = require("./az-file-args.generated.json"); + +/** + * Path arguments az 2.87 does NOT annotate with `file_type` or a file/directory completer, so + * the generated table misses them, yet they read or write a local path outside the workdir. + * Only the LOCAL side of each is listed: for `download-batch` + * `--destination` is local (the source is a remote container); for `upload-batch` `--source` + * is local. Recheck this list when the az pin moves, in case az adds the annotations. + */ +const EXTRA_FILE_ARGS: Record = { + "storage blob download-batch": { flags: ["--destination", "-d"] }, + "storage file download-batch": { flags: ["--destination", "-d"] }, + "storage blob upload-batch": { flags: ["--source", "-s"] }, + "storage file upload-batch": { flags: ["--source", "-s"] }, + "storage fs directory download": { flags: ["--destination-path", "-d"] }, + "storage fs directory upload": { flags: ["--source", "-s"] }, + "webapp deploy": { flags: ["--src-path"] }, + "functionapp deploy": { flags: ["--src-path"] }, + "webapp deployment source config-zip": { flags: ["--src"] }, + "functionapp deployment source config-zip": { flags: ["--src"] }, + "logicapp deployment source config-zip": { flags: ["--src"] }, + "webapp config container set": { flags: ["--multicontainer-config-file"] }, + "webapp create": { flags: ["--multicontainer-config-file"] }, + "container create": { flags: ["--file", "-f"] }, + "container container-group-profile create": { flags: ["--file", "-f"] }, + "aks command invoke": { flags: ["--file", "-f"] }, + "ts export": { flags: ["--output-folder"] }, + "appconfig kv export": { flags: ["--path"] }, + // az opens these itself in custom code, and they always take a path (never inline), so they are + // checked unconditionally like the rest of this list — a write target need not exist yet, so the + // existing-path safety net below would miss it. `rest --output-file` writes the + // response; the `apim` ones read/write a spec or schema file. + rest: { flags: ["--output-file"] }, + "apim api import": { flags: ["--specification-path"] }, + "apim api export": { flags: ["--file-path", "-f"] }, + "apim api schema create": { flags: ["--schema-path"] }, +}; + +// --------------------------------------------------------------------------- +// Policy data. Exported so tests and the cross-check can read the same source. +// --------------------------------------------------------------------------- + +const deny = ( + reason: string, + ...prefixes: string[][] +): Array<{ match: string[]; reason: string }> => prefixes.map((match) => ({ match, reason })); + +/** Denied command groups and verbs. Prefix matches. */ +export const DENIED: Array<{ match: string[]; reason: string }> = [ + ...deny("the CLI is already logged in to the emulator with a dummy account", ["login"]), + ...deny("this would break routing to the emulator", ["logout"], ["account", "clear"]), + ...deny( + "this would re-point the CLI away from the emulator", + ["cloud", "register"], + ["cloud", "unregister"], + ["cloud", "update"], + ["cloud", "set"] + ), + // `config get` is allowed below; every other `config`, plus `configure`/`init`, writes state. + ...deny("the tool manages the CLI configuration", ["config"], ["configure"], ["init"]), + ...deny( + "extensions are pre-installed; adding or updating one downloads from the internet", + ["extension", "add"], + ["extension", "update"], + ["extension", "remove"] + ), + ...deny( + "this downloads or installs software on this machine", + ["upgrade"], + ["bicep", "install"], + ["bicep", "upgrade"], + ["bicep", "uninstall"], + ["bicep", "list-versions"], + ["aks", "install-cli"] // also runs `setx path`, rewriting the real user PATH + ), + ...deny( + "Bicep registry modules cannot be reached from the local emulator", + ["bicep", "restore"], + ["bicep", "publish"] + ), + ...deny("this calls a Microsoft web service", ["find"], ["feedback"], ["survey"]), + ...deny("this needs an interactive session", ["interactive"], ["self-test"]), + ...deny( + "this opens a browser on this machine", + ["aks", "browse"], + ["webapp", "browse"], + ["containerapp", "browse"] + ), + ...deny( + "this opens a shell, tunnel or live stream", + ["webapp", "ssh"], + ["webapp", "create-remote-connection"], + ["container", "exec"], + ["container", "attach"], + ["containerapp", "exec"], + ["webapp", "log", "tail"], + ["appservice", "plan", "managed-instance", "instance", "connect"], + ["network", "bastion", "ssh"], + ["network", "bastion", "rdp"], + ["network", "bastion", "tunnel"] + ), + ...deny( + "this runs Docker or downloads tools on this machine", + ["acr", "check-health"], + ["acr", "helm"], + ["aks", "check-acr"], + ["storage", "copy"], + ["storage", "remove"], + ["storage", "blob", "sync"], // storage copy/remove/sync auto-install azcopy from aka.ms + ["backup", "restore", "files", "mount-rp"], + ["mysql", "flexible-server", "deploy"], + ["postgres", "flexible-server", "deploy"] + ), + // Denied outright until its local-source flag is wired from _params.py. With a + // local source it makes az run `docker build`/`push` on the shared engine. + ...deny("this builds and pushes images on the shared Docker engine", [ + "cognitiveservices", + "agent", + "create", + ]), + ...deny( + "Azure DevOps is not emulated", + ["devops"], + ["boards"], + ["repos"], + ["pipelines"], + ["artifacts"] + ), +]; + +/** Verbs that survive a broader group denial. */ +export const ALLOWED_EXCEPTIONS: string[][] = [["config", "get"]]; + +/** Flags that are refused, globally or for a specific command. */ +export const DENIED_FLAGS: Array<{ command?: string[]; flag: string; reason: string }> = [ + { flag: "--follow", reason: "it streams output until killed" }, + { flag: "--login-with-github", reason: "it opens a browser login" }, + { command: ["webapp", "up"], flag: "--launch-browser", reason: "it opens a browser" }, + { command: ["webapp", "up"], flag: "-b", reason: "it opens a browser" }, + { command: ["webapp", "up"], flag: "--logs", reason: "it streams logs until killed" }, + { + command: ["containerapp", "up"], + flag: "--source", + reason: "it builds an image on the shared Docker engine", + }, + { + command: ["containerapp", "up"], + flag: "--repo", + reason: "it wires up a GitHub Actions build", + }, + { + command: ["containerapp", "create"], + flag: "--source", + reason: "it builds an image on the shared Docker engine", + }, + { + command: ["containerapp", "create"], + flag: "--repo", + reason: "it wires up a GitHub Actions build", + }, +]; + +/** Flags a command must carry, or it is refused. */ +export const REQUIRED_FLAGS: Array<{ + command: string[]; + flag: string; + aliases?: string[]; + reason: string; +}> = [ + { + command: ["acr", "login"], + flag: "--expose-token", + aliases: ["-t"], + reason: + "without `--expose-token` az runs `docker login` on this machine; re-run it with `--expose-token`", + }, +]; + +/** + * Local hosts the URL rule and the egress guard both allow: one + * shared check in local-hosts.ts. Re-exported here for the policy's own tests. + */ +export { LOCAL_HOST }; + +/** The ARM host whose absolute URLs are rewritten to a relative path (matched case-insensitively). */ +const MANAGEMENT_HOST = "management.azure.com"; + +/** URL-target flags whose value az turns into a request or a download. */ +const REST_URL_FLAGS = new Set(["--url", "--uri", "-u"]); // only for the `rest` command +const FETCH_URL_FLAGS = new Set(["--template-uri", "--multicontainer-config-file"]); // any command +const PARAMETERS_FLAGS = new Set(["--parameters", "-p"]); // URL or file, depending on the value + +// --------------------------------------------------------------------------- +// Denylist file parsing. +// --------------------------------------------------------------------------- + +/** + * Parse LOCALSTACK_AZ_DENYLIST_FILE: one command prefix per line, split on whitespace. Blank + * lines and `#` comments are ignored. The result is passed to evaluateAzCommand + * as opts.extraDenied, so the policy itself stays pure. + */ +export function parseDenylistFile(text: string): string[][] { + const prefixes: string[][] = []; + for (const rawLine of text.split(/\r?\n/)) { + const withoutComment = rawLine.replace(/#.*$/, ""); + const words = withoutComment.trim().split(/\s+/).filter(Boolean); + if (words.length > 0) prefixes.push(words); + } + return prefixes; +} + +// --------------------------------------------------------------------------- +// Small helpers. +// --------------------------------------------------------------------------- + +const START_TOKEN = /^[a-z][a-z0-9-]*$/; +const HELP_TOKENS = new Set(["--help", "-h"]); +const ARM_ID = /^\/(subscriptions|providers|tenants)\//i; +const ABSOLUTE_URL = /^[A-Za-z][A-Za-z0-9+.-]*:\/\//; + +function effectivePlatform(opts: PolicyOptions): NodeJS.Platform { + return opts.platform ?? process.platform; +} + +/** True when a command's leading words match `prefix` exactly. */ +function hasPrefix(argv: string[], prefix: string[]): boolean { + if (prefix.length > argv.length) return false; + return prefix.every((word, index) => argv[index] === word); +} + +/** + * Match an option token against a canonical flag, honouring argparse's unambiguous-prefix + * abbreviation for long flags (verified: `cloud list --out tsv` and `--que` both work). A token + * `--

` with p.length >= 2 that is a strict prefix of a long flag counts as that flag; short + * `-x` flags match only exactly. When a prefix could match several path/URL flags we still apply + * the rule (argparse would reject it as ambiguous, so being stricter is safe). + */ +function flagMatches(tokenFlag: string, fullFlag: string): boolean { + if (tokenFlag === fullFlag) return true; + return ( + fullFlag.startsWith("--") && + tokenFlag.startsWith("--") && + tokenFlag.length >= 4 && // "--" plus at least two characters + fullFlag.length > tokenFlag.length && + fullFlag.startsWith(tokenFlag) + ); +} + +/** True when a token's flag matches any flag in the set (exact or an unambiguous prefix). */ +function matchesAnyFlag(tokenFlag: string, flags: Iterable): boolean { + for (const flag of flags) if (flagMatches(tokenFlag, flag)) return true; + return false; +} + +interface ParsedOption { + flag: string; + /** The value packed into the token itself (`--flag=v`, `-fv`, `-f=v`), else undefined. */ + value?: string; + /** How an inline value was attached, so a rewrite can rebuild the token. */ + sep: "=" | ""; +} + +/** Split one option token into its flag and any attached value (no lookup of the next token). */ +function parseOption(token: string): ParsedOption { + if (token.startsWith("--")) { + const eq = token.indexOf("="); + if (eq >= 0) return { flag: token.slice(0, eq), value: token.slice(eq + 1), sep: "=" }; + return { flag: token, sep: "" }; + } + // A single-dash short option, possibly with a stuck or `=`-joined value (`-f/x`, `-f=/x`). + const flag = token.slice(0, 2); + const rest = token.slice(2); + if (rest === "") return { flag, sep: "" }; + if (rest.startsWith("=")) return { flag, value: rest.slice(1), sep: "=" }; + return { flag, value: rest, sep: "" }; +} + +/** The path-taking flags of the command in `argv`, from the generated table plus the supplement. */ +function commandFileFlags(argv: string[]): { + generic: Set; + greedy: Set; + params: Set; + positional: boolean; +} { + const generic = new Set(); + const greedy = new Set(); + const params = new Set(); + let positional = false; + + const apply = (entry: { flags: string[]; greedy?: string[]; positional?: boolean }) => { + const hasParameters = entry.flags.includes("--parameters"); + for (const flag of entry.flags) { + // `--parameters` (and its `-p` alias, only where `--parameters` exists) needs the special + // rule below, not the plain path check, so it is routed to `params`. + if (flag === "--parameters" || (hasParameters && flag === "-p")) { + params.add(flag); + continue; + } + generic.add(flag); + } + for (const flag of entry.greedy ?? []) { + if (flag === "--parameters" || (hasParameters && flag === "-p")) continue; + greedy.add(flag); + } + if (entry.positional) positional = true; + }; + + // The command path is the longest table key whose words are a prefix of argv (so + // "storage blob download-batch" is matched, not "storage blob download"). + for (const source of [GENERATED_TABLE.commands, EXTRA_FILE_ARGS]) { + let bestWords = -1; + let bestEntry: { flags: string[]; greedy?: string[]; positional?: boolean } | undefined; + for (const [key, entry] of Object.entries(source)) { + const words = key.split(" "); + if (words.length > bestWords && hasPrefix(argv, words)) { + bestWords = words.length; + bestEntry = entry; + } + } + if (bestEntry) apply(bestEntry); + } + return { generic, greedy, params, positional }; +} + +// --------------------------------------------------------------------------- +// The file rule: resolve a path value and decide whether it stays inside the workdir. +// --------------------------------------------------------------------------- + +type PathVerdict = + | { kind: "ok" } + | { kind: "arm" } + | { kind: "outside" } + | { kind: "protected"; dir: string } + | { kind: "unsupported"; why: string }; + +function stripAt(value: string): string { + return value.startsWith("@") ? value.slice(1) : value; +} + +/** + * Classify one path value. Lexical first (resolve, normalise, case-insensitive on win32/darwin): + * a protected or unsupported verdict there is final. Then physical, as the OS will open the path: + * the value and every root through realpath (their longest existing prefix), so a symlinked /var + * or /home, an 8.3 name, a junction or a mapped drive neither pushes a workdir file outside nor + * hides a protected dir, even for a write target that does not exist yet. The physical step is + * the one bit of I/O the policy does; a policy for another platform than the host's (as the tests + * simulate) stays lexical, so it stays pure and deterministic. + */ +function classifyPath(rawValue: string, opts: PolicyOptions): PathVerdict { + const value = stripAt(rawValue); + if (value === "" || value === "-") return { kind: "ok" }; // empty or stdin: not a filesystem path + if (ARM_ID.test(value)) return { kind: "arm" }; + + const plat = effectivePlatform(opts); + const P = plat === "win32" ? path.win32 : path.posix; + + if (plat === "win32") { + if (/^[\\/]{2}/.test(value)) { + // \\host\share (also attempts outbound SMB auth), \\?\ and \\.\ device paths. + return { kind: "unsupported", why: "a UNC or device path" }; + } + if (/^[A-Za-z]:([^\\/]|$)/.test(value)) { + // Drive-relative `C:x` or a bare drive `C:`: it resolves against a per-drive cwd, not ours. + return { kind: "unsupported", why: "a drive-relative path" }; + } + } + + // `~` means the tool's private home, as it does for az itself. + const isTilde = value === "~" || value.startsWith("~/") || value.startsWith("~\\"); + const expanded = isTilde ? opts.homeDir + value.slice(1) : value; + const base = isTilde ? opts.homeDir : opts.workdir; + const resolvedLexical = P.resolve(base, expanded); + + // A lexical "outside" is not final: the physical check below decides it. + const lexical = containmentVerdict(resolvedLexical, opts, P); + if (lexical.kind !== "ok" && lexical.kind !== "outside") return lexical; + if (plat !== process.platform) return lexical; + + // Compare like with like: the path and every root through realpath, so a workdir reached + // through a link (macOS /var -> /private/var, Windows C:\Users\RUNNER~1) keeps its files, and a + // home spelled through a link still protects ~/.ssh, for a write target too. + const real = (p: string) => realPrefix(p, P); + const protectedDirs = opts.protectedDirs ?? []; + const realProtected = protectedDirs.map((dir) => dir && real(dir)); + const realOpts: PolicyOptions = { + ...opts, + workdir: real(opts.workdir), + homeDir: opts.homeDir && real(opts.homeDir), + protectedDirs: realProtected, + }; + // POSIX applies `..` after symlinks (`shared/x/../..` leaves the workdir when `shared` is a + // link), so the path goes in unnormalised; Win32 collapses `..` before it opens anything, so + // there the lexical path is what az opens. + const asOpened = + plat === "win32" + ? resolvedLexical + : P.isAbsolute(expanded) + ? expanded + : base + P.sep + expanded; + const physical = containmentVerdict(real(asOpened), realOpts, P); + if (physical.kind === "protected") { + // Name the dir as it is configured, not by its real path. + return { + kind: "protected", + dir: protectedDirs[realProtected.indexOf(physical.dir)] || physical.dir, + }; + } + return physical; +} + +/** `p` with its longest existing prefix through realpath; the rest keeps its spelling. */ +function realPrefix(p: string, P: PlatformPath): string { + let head = p; + const tail: string[] = []; + for (;;) { + try { + return P.join(fs.realpathSync.native(head), ...tail); + } catch { + const parent = P.dirname(head); + if (parent === head) return P.resolve(p); + tail.unshift(P.basename(head)); + head = parent; + } + } +} + +function containmentVerdict(resolved: string, opts: PolicyOptions, P: PlatformPath): PathVerdict { + const plat = effectivePlatform(opts); + const inside = (parent: string): boolean => isInside(resolved, parent, plat, P); + + // The private home is always allowed, even though a protected dir (the config dir) contains it. + if (opts.homeDir && inside(opts.homeDir)) return { kind: "ok" }; + // Protected dirs are checked before the workdir allowance: when the workdir is the user's home + // or an ancestor, a relative `.ssh/id_rsa` would otherwise pass the workdir check. + for (const dir of opts.protectedDirs ?? []) { + if (dir && inside(dir)) return { kind: "protected", dir }; + } + if (inside(opts.workdir)) return { kind: "ok" }; + return { kind: "outside" }; +} + +function isInside(child: string, parent: string, plat: NodeJS.Platform, P: PlatformPath): boolean { + const fold = (p: string) => (plat === "win32" || plat === "darwin" ? p.toLowerCase() : p); + const c = fold(P.resolve(child)); + let pa = fold(P.resolve(parent)); + if (c === pa) return true; + if (!pa.endsWith(P.sep)) pa += P.sep; + return c.startsWith(pa); +} + +// --------------------------------------------------------------------------- +// The URL rule (request targets only). +// --------------------------------------------------------------------------- + +type UrlVerdict = + | { action: "none" } // not an absolute URL: leave it to the file rule / pass through + | { action: "allow" } // a local host + | { action: "rewrite"; rewritten: string } + | { action: "refuse"; host: string }; + +function classifyUrl(value: string, localPorts?: ReadonlySet): UrlVerdict { + // The host is a bracketed IPv6 literal or a name (userinfo stays in it, so + // `https://management.azure.com@evil.example` is not the management host). + const match = value.match( + /^([A-Za-z][A-Za-z0-9+.-]*):\/\/(\[[^\]]*\]|[^/:?#[\]]+)(:\d+)?([/?#].*)?$/ + ); + if (!match) return { action: "none" }; + const scheme = match[1].toLowerCase(); + const host = match[2]; + const port = match[3] ?? ""; + const rest = match[4] ?? ""; + const isHttp = scheme === "http" || scheme === "https"; + + if (isHttp && host.toLowerCase() === MANAGEMENT_HOST && (port === "" || port === ":443")) { + // Rewrite https://management.azure.com/

to /

, keeping the leading slash. + const rewritten = rest.startsWith("/") ? rest : "/"; + return { action: "rewrite", rewritten }; + } + if (isLocalHost(host)) { + // Only the emulator's own ports: localhost:2375 is Docker's API, 11434 a local model server. + const effectivePort = port ? Number(port.slice(1)) : scheme === "https" ? 443 : 80; + if (localPorts && (isHttp || port) && !localPorts.has(effectivePort)) + return { action: "refuse", host: `${host}:${effectivePort}` }; + return { action: "allow" }; + } + return { action: "refuse", host }; +} + +// --------------------------------------------------------------------------- +// Refusal builders. +// --------------------------------------------------------------------------- + +function refuse(title: string, message: string, ruleId: string, argv?: string[]): PolicyResult { + return { ok: false, title, message, ruleId, argv }; +} + +function slug(prefix: string[]): string { + return prefix.join("-"); +} + +// --------------------------------------------------------------------------- +// The pipeline. +// --------------------------------------------------------------------------- + +/** + * Decide what to do with one `az` command string. Pure: no spawn, no network, and no file read + * except the guarded realpath check in classifyPath. See the module comment for the pipeline. + */ +export function evaluateAzCommand(command: string, opts: PolicyOptions): PolicyResult { + // 1. Trim, then strip ONE leading `az`. + let body = command.trim(); + const azStrip = body.match(/^az(\s+|$)/); + if (azStrip) body = body.slice(azStrip[0].length).trim(); + + // 2. Tokenize with the three Azure options; a syntax error stops here. + let argv: string[]; + try { + argv = splitCliArgs(body, { + quotedControlChars: true, + keepEmptyQuoted: true, + bashDoubleQuoteEscapes: true, + }); + } catch (error) { + const message = + error instanceof CliSyntaxError ? error.message : "Command could not be parsed."; + return refuse("Command not understood", message, "syntax"); + } + + if (argv.length === 0) { + return refuse( + "No command given", + "Give one `az` command without the leading `az`, for example `group list`.", + "start" + ); + } + + // 3. A second `az`, or `azlocal`, is refused (one `az` was already stripped). + if (argv[0] === "az" || argv[0] === "azlocal") { + return refuse( + "Extra executable in the command", + "Give just the command, without the `az` or `azlocal` executable, for example `group list`.", + "start", + argv + ); + } + + // 4. Start rule: a command group/verb, or a bare `--help`/`-h`/`--version`. + const first = argv[0]; + const startOk = START_TOKEN.test(first) || HELP_TOKENS.has(first) || first === "--version"; + if (!startOk) { + return refuse( + "Not a valid command", + "Start with a command group or verb (e.g. `group list`). A leading global flag such as " + + "`--debug` is not supported here.", + "start", + argv + ); + } + + // 5. Denied groups and verbs, honouring the `config get` exception. + const denialCheck = checkDenied(argv, opts); + if (denialCheck) return denialCheck; + + // 6. The URL rule (request targets only): rewrite, allow local, or refuse. May change argv. + const notes: string[] = []; + const urlResult = applyUrlRule(argv, notes, opts.localPorts); + if (!urlResult.ok) return urlResult.refusal; + argv = urlResult.argv; + + // 7. The file rule on the (possibly rewritten) argv. + const fileRefusal = applyFileRule(argv, opts, notes); + if (fileRefusal) return fileRefusal; + + // 8. Bicep detection and help detection. + const needsBicep = detectBicep(argv); + const isHelp = argv.some((token) => HELP_TOKENS.has(token)); + + // 9. The `version` marker (answered locally by the handler), last so a real `--version 16` flag + // on `postgres flexible-server create` still runs. + if (argv[0] === "version" || (argv.length === 1 && argv[0] === "--version")) { + return { + ok: true, + local: "version", + argv, + notes, + isHelp: false, + needsBicep: false, + outcome: "local", + }; + } + + return { + ok: true, + argv, + notes, + isHelp, + needsBicep, + outcome: notes.length > 0 ? "rewritten" : "ok", + }; +} + +function checkDenied(argv: string[], opts: PolicyOptions): PolicyResult | null { + const isException = ALLOWED_EXCEPTIONS.some((prefix) => hasPrefix(argv, prefix)); + + for (const entry of DENIED) { + if (!hasPrefix(argv, entry.match)) continue; + // `config get` survives the whole-group `config` denial. + if (isException && entry.match.length === 1 && entry.match[0] === "config") continue; + return refuse( + "Command not allowed", + `\`az ${entry.match.join(" ")}\` is not allowed here: ${entry.reason}.`, + `denied:${slug(entry.match)}`, + argv + ); + } + + for (const prefix of opts.extraDenied ?? []) { + if (prefix.length > 0 && hasPrefix(argv, prefix)) { + return refuse( + "Command not allowed", + `\`az ${prefix.join(" ")}\` is blocked by the configured denylist (LOCALSTACK_AZ_DENYLIST_FILE).`, + `denied:${slug(prefix)}`, + argv + ); + } + } + + // Denied flags, global or per command. Prefix-aware: `--fol` counts as `--follow` (argparse + // abbreviation), so an abbreviation cannot slip a denied flag past the policy. + const tokenFlags = argv + .filter((token) => token.startsWith("-")) + .map((token) => parseOption(token).flag); + for (const rule of DENIED_FLAGS) { + if (rule.command && !hasPrefix(argv, rule.command)) continue; + if (tokenFlags.some((tokenFlag) => flagMatches(tokenFlag, rule.flag))) { + const scope = rule.command ? `\`az ${rule.command.join(" ")}\` ` : ""; + return refuse( + "Flag not allowed", + `${scope}cannot be run with \`${rule.flag}\`: ${rule.reason}.`, + "denied:flag", + argv + ); + } + } + + // Required flags (acr login --expose-token). An abbreviation such as `--expose` satisfies it. + // A help request needs none: `az acr login --help` only prints help. + const helpRequest = argv.some((token) => HELP_TOKENS.has(token)); + for (const rule of REQUIRED_FLAGS) { + if (helpRequest || !hasPrefix(argv, rule.command)) continue; + const satisfied = + tokenFlags.some((tokenFlag) => flagMatches(tokenFlag, rule.flag)) || + (rule.aliases ?? []).some((alias) => tokenFlags.includes(alias)); + if (!satisfied) { + return refuse( + "Missing required flag", + `\`az ${rule.command.join(" ")}\` needs \`${rule.flag}\`: ${rule.reason}.`, + `denied:${slug(rule.command)}`, + argv + ); + } + } + return null; +} + +interface UrlPassResult { + ok: boolean; + argv: string[]; + refusal: PolicyResult; +} + +function applyUrlRule( + argv: string[], + notes: string[], + localPorts?: ReadonlySet +): UrlPassResult { + const out = [...argv]; + const isRest = argv[0] === "rest"; + const isAcrBuildRun = argv[0] === "acr" && (argv[1] === "build" || argv[1] === "run"); + const ok = (): UrlPassResult => ({ ok: true, argv: out, refusal: {} as PolicyResult }); + const bad = (r: PolicyResult): UrlPassResult => ({ ok: false, argv: out, refusal: r }); + + const refuseHost = ( + flagLabel: string, + host: string, + kind: "request" | "download" + ): UrlPassResult => { + // `rest` takes a request target; the other URL flags make az download a file. + const advice = + kind === "request" + ? "For ARM calls, use a relative URL such as `/subscriptions//resourceGroups?api-version=2022-09-01`." + : "Download the file into the working directory yourself and pass it as a local path instead."; + return bad( + refuse( + "Address not allowed", + `${flagLabel} points at \`${host}\`, which the tool cannot reach: it only talks to the ` + + `local emulator. ${advice}`, + "url:blocked", + argv + ) + ); + }; + + const handleUrlValue = ( + flagLabel: string, + value: string, + write: (rewritten: string) => void, + kind: "request" | "download" = "download" + ): UrlPassResult | null => { + const verdict = classifyUrl(value, localPorts); + if (verdict.action === "refuse") return refuseHost(flagLabel, verdict.host, kind); + if (verdict.action === "rewrite") { + write(verdict.rewritten); + notes.push( + `Rewrote the ${MANAGEMENT_HOST} URL of ${flagLabel} to the relative path \`${verdict.rewritten}\`.` + ); + } + return null; + }; + + for (let i = 0; i < out.length; i++) { + const token = out[i]; + if (!token.startsWith("-")) continue; + const parsed = parseOption(token); + // Prefix-aware, so `rest --ur` and `--template-u` are caught like the full flags. + const isRestFlag = isRest && matchesAnyFlag(parsed.flag, REST_URL_FLAGS); + const isFetchFlag = matchesAnyFlag(parsed.flag, FETCH_URL_FLAGS); + const isParams = matchesAnyFlag(parsed.flag, PARAMETERS_FLAGS); + if (!isRestFlag && !isFetchFlag && !isParams) continue; + + const label = `\`${parsed.flag}\``; + const kind = isRestFlag ? "request" : "download"; + if (parsed.value !== undefined) { + // `--parameters` only gets the URL rule when the value is actually a URL. + if (isParams && classifyUrl(parsed.value).action === "none") continue; + const result = handleUrlValue( + label, + parsed.value, + (rewritten) => { + out[i] = parsed.flag + parsed.sep + rewritten; + }, + kind + ); + if (result) return result; + } else if (i + 1 < out.length && !out[i + 1].startsWith("-")) { + const value = out[i + 1]; + if (isParams && classifyUrl(value).action === "none") continue; + const valueIndex = i + 1; + const result = handleUrlValue( + label, + value, + (rewritten) => { + out[valueIndex] = rewritten; + }, + kind + ); + if (result) return result; + } + } + + // acr build / acr run: a URL as the positional source is a request target too. + if (isAcrBuildRun) { + for (const idx of positionalIndexes(out, 2)) { + const value = out[idx]; + const lower = value.toLowerCase(); + const isGitOrOci = + lower.startsWith("git@") || + lower.startsWith("git://") || + lower.startsWith("github.com/") || + lower.startsWith("oci://"); + if (isGitOrOci) return refuseHost("the `acr` source", value, "download"); + if (ABSOLUTE_URL.test(value)) { + const result = handleUrlValue("the `acr` source", value, (rewritten) => { + out[idx] = rewritten; + }); + if (result) return result; + } + } + } + + return ok(); +} + +/** Indexes of positional tokens (a bare token whose predecessor is not an option). */ +function positionalIndexes(argv: string[], startAfter: number): number[] { + const out: number[] = []; + for (let i = startAfter; i < argv.length; i++) { + const token = argv[i]; + if (token.startsWith("-")) continue; + const previous = argv[i - 1]; + if (i === startAfter || previous === undefined || !previous.startsWith("-")) out.push(i); + } + return out; +} + +function applyFileRule(argv: string[], opts: PolicyOptions, notes: string[]): PolicyResult | null { + const { generic, greedy, params, positional } = commandFileFlags(argv); + + // `~` is the tool's private home. When a `~` path does not exist there, say so: a + // user who meant their own ~/.ssh or ~/.azure would otherwise see only az's "No such file", + // naming a path of ours. One note per command. + let tildeNoted = false; + const noteMissingTilde = (rawValue: string, flagLabel: string): void => { + const value = stripAt(rawValue); + const isTilde = value === "~" || value.startsWith("~/") || value.startsWith("~\\"); + if (tildeNoted || !isTilde || !opts.homeDir) return; + const P = effectivePlatform(opts) === "win32" ? path.win32 : path.posix; + let exists = false; + try { + exists = fs.existsSync(P.resolve(opts.homeDir, `.${value.slice(1)}`)); + } catch { + exists = false; + } + if (exists) return; + tildeNoted = true; + notes.push( + `\`~\` in ${flagLabel} is the tool's private home (${opts.homeDir}), not your home directory, ` + + `and ${value} does not exist there. Files in your own home, including ~/.azure, ~/.ssh, ` + + "~/.kube and ~/.docker, are never available through this tool: copy what a command needs " + + "into the working directory." + ); + }; + + const check = (rawValue: string, flagLabel: string): PolicyResult | null => { + const verdict = classifyPath(rawValue, opts); + if (verdict.kind === "ok") noteMissingTilde(rawValue, flagLabel); + if (verdict.kind === "ok" || verdict.kind === "arm") return null; + if (verdict.kind === "protected") { + return refuse( + "Path not allowed", + `${flagLabel} points into a protected directory (${verdict.dir}): the tool keeps az away ` + + "from your own Azure CLI profile, SSH keys, kube and Docker config.", + "file:protected", + argv + ); + } + if (verdict.kind === "unsupported") { + return refuse( + "Path not allowed", + `${flagLabel} is ${verdict.why}, which is not supported: use a path inside the working directory.`, + "file:unsupported-path", + argv + ); + } + return refuse( + "Path not allowed", + `${flagLabel} points outside the working directory (${opts.workdir}): files must be inside it. ` + + "To use files elsewhere, set LOCALSTACK_AZ_WORKDIR in this MCP server's configuration to " + + "the folder that holds them (a GUI client starts the server in its own folder).", + "file:outside-workdir", + argv + ); + }; + + const paramsIsPath = (value: string): boolean => { + // `--parameters` is a path only as `@file`, or a bare value ending .json/.bicepparam. An + // inline assignment such as `location=westeurope` is not a path. + if (value.startsWith("@")) return true; + if (value.includes("=")) return false; + return /\.(json|bicepparam)$/i.test(value); + }; + + for (let i = 0; i < argv.length; i++) { + const token = argv[i]; + + // The `@file` rule applies to any token, whatever its flag (az loads `@file` everywhere). + const atValue = atFileValue(token); + if (atValue !== undefined) { + const result = check(atValue, "an `@file` argument"); + if (result) return result; + } + + if (token.startsWith("-")) { + const parsed = parseOption(token); + const label = `\`${parsed.flag}\``; + // Prefix-aware (argparse abbreviation): `--template-f` counts as `--template-file`, `--fi` + // as `--file`. `--parameters` is nargs '+' in az, so it and its abbreviations are greedy. + const isParams = matchesAnyFlag(parsed.flag, params); + const isGeneric = !isParams && matchesAnyFlag(parsed.flag, generic); + if (!isGeneric && !isParams) continue; + const isGreedy = isParams || matchesAnyFlag(parsed.flag, greedy); + + const values: string[] = []; + if (parsed.value !== undefined) { + values.push(parsed.value); + } else if (isGreedy) { + let j = i + 1; + while (j < argv.length && !argv[j].startsWith("-")) values.push(argv[j++]); + } else if (i + 1 < argv.length && !argv[i + 1].startsWith("-")) { + values.push(argv[i + 1]); + } + + for (const value of values) { + if (value.startsWith("@")) continue; // handled by the @file scan + if (isParams && !paramsIsPath(value)) continue; + const result = check(value, label); + if (result) return result; + } + } + } + + // acr build / acr run: a local positional source must resolve inside the workdir. + if (positional && argv[0] === "acr" && (argv[1] === "build" || argv[1] === "run")) { + for (const idx of positionalIndexes(argv, 2)) { + const value = argv[idx]; + const lower = value.toLowerCase(); + if (ABSOLUTE_URL.test(value) || lower.startsWith("git@") || lower.startsWith("github.com/")) { + continue; // a remote source: handled by the URL rule + } + if (value.toLowerCase() === "/dev/null") continue; // ACR's null context + const result = check(value, "the `acr` source"); + if (result) return result; + } + } + + // Safety net. Some arguments az does not annotate as files are still opened by az when + // their value names an existing path: AAZ structured arguments (`--tags`, `--settings`, ...) read + // JSON/YAML through `os.path.exists`, and so do `validate_file_or_dict` arguments. The generated + // table cannot list them (az gives no file marker), so we mirror az's own gate: any value that + // resolves to an existing file must satisfy the same containment check. Values that are inline + // (`key=value`, `[...]`) or name no existing file are left exactly as az leaves them, so ordinary + // tags and names keep working. Write targets that need not exist are handled by the table above. + for (const token of argv) { + if (token.startsWith("@")) continue; // handled by the @file scan + const value = token.startsWith("-") ? parseOption(token).value : token; + if (value === undefined || value === "" || value.startsWith("@")) continue; + if (!valueNamesExistingPath(value, opts)) continue; + const result = check(value, "a file argument"); + if (result) return result; + } + + return null; +} + +/** + * True when `rawValue`, resolved the way az resolves it (relative to the workdir, with `~` as the + * tool's private home), names an existing FILE. az's own gate is `os.path.exists(expanduser(v))` + * followed by `get_file_json/yaml`, which fails on a directory without reading anything, so only + * a file can leak content. Checking files only keeps remote paths that happen to name a local + * directory out of the net (`afd origin-group create --probe-path /` is the filesystem root: the + * L2 matrix's policy check caught that). ARM ids and the empty/`-` values are never local paths. + */ +function valueNamesExistingPath(rawValue: string, opts: PolicyOptions): boolean { + const value = stripAt(rawValue); + if (value === "" || value === "-" || ARM_ID.test(value)) return false; + const P = effectivePlatform(opts) === "win32" ? path.win32 : path.posix; + const isTilde = value === "~" || value.startsWith("~/") || value.startsWith("~\\"); + const base = isTilde ? (opts.homeDir ?? opts.workdir) : opts.workdir; + const expanded = isTilde && opts.homeDir ? opts.homeDir + value.slice(1) : value; + try { + return fs.statSync(P.resolve(base, expanded)).isFile(); + } catch { + return false; // absent, unreadable or not a path at all: az would not read it either + } +} + +/** The path a token loads with `@` (a leading `@`, or `--flag=@path`), else undefined. */ +function atFileValue(token: string): string | undefined { + if (token.startsWith("@") && token.length > 1) return token.slice(1); + if (token.startsWith("-")) { + const eq = token.indexOf("="); + if (eq >= 0) { + const value = token.slice(eq + 1); + if (value.startsWith("@") && value.length > 1) return value.slice(1); + } + } + return undefined; +} + +/** A `.bicep`/`.bicepparam` argument, or the `bicep` group, needs the Bicep binary. */ +function detectBicep(argv: string[]): boolean { + if (argv[0] === "bicep") return true; + for (const token of argv) { + const value = token.includes("=") ? token.slice(token.indexOf("=") + 1) : token; + const bare = stripAt(value); + if (/\.(bicep|bicepparam)$/i.test(bare)) return true; + } + return false; +} + +// --------------------------------------------------------------------------- +// Analytics: value-free fields derived from the command and the verdict. +// --------------------------------------------------------------------------- + +/** + * Derive the value-free analytics fields. A secret value must never appear in any of + * them: `command_path` is only the leading command words (values come after a flag or are + * quoted), and `flag_names` keeps only each flag's name (`--flag=value` and a value stuck to a + * short option, `-pS3cret`, lose their value). + */ +export function analyticsFields( + command: string, + policy: PolicyResult +): { command_path: string; flag_names: string; policy_outcome: string } { + const argv = policy.argv ?? argvForAnalytics(command); + + // command_path: only the leading command words. Values sit after a flag or a quote, so they + // never reach here (and the strict word shape excludes anything with a symbol or upper case). + const commandWords: string[] = []; + for (const token of argv) { + if (START_TOKEN.test(token)) commandWords.push(token); + else break; + } + + // flag_names: the flag names only: `--flag=value` and a stuck `-pS3cret` (which az accepts) + // lose their value. A value that itself starts with `-` is indistinguishable from a flag, so + // the token right after a space-separated option is always treated as that option's value and + // dropped. This can undercount store-true flags, but it guarantees no secret value is ever + // emitted (a property test in policy.test.ts checks it). + const flagNames: string[] = []; + let expectValue = false; + for (const token of argv) { + if (expectValue) { + expectValue = false; + continue; + } + if (token === "--" || !token.startsWith("-")) continue; + const { flag, value } = parseOption(token); + flagNames.push(flag); + if (value === undefined) expectValue = true; // its value is the next token + } + + return { + command_path: commandWords.join(" "), + flag_names: flagNames.join(","), + policy_outcome: policyOutcome(policy), + }; +} + +/** A tokenizer-free split for the analytics path when the real tokenizer failed (syntax error). */ +function argvForAnalytics(command: string): string[] { + let body = command.trim(); + const azStrip = body.match(/^az(\s+|$)/); + if (azStrip) body = body.slice(azStrip[0].length).trim(); + return body.split(/\s+/).filter(Boolean); +} + +function policyOutcome(policy: PolicyResult): string { + if (policy.ok) { + if (policy.local === "version") return "local"; + return policy.outcome === "rewritten" ? "rewritten" : "ok"; + } + if (policy.ruleId === "syntax") return "syntax"; + return policy.ruleId.startsWith("denied:") ? policy.ruleId : `denied:${policy.ruleId}`; +} + +// --------------------------------------------------------------------------- +// Bicep module scan (handler-side I/O, for `bicep-registry`). +// --------------------------------------------------------------------------- + +// A registry reference is a single-quoted string after `module `, `using` or `extends` +// (.bicepparam), or `import ... from`. A bare `br:`/`ts:` anywhere in the file is not: property +// names such as `subnets:` contain `ts:`, and matching them refused every VNet template (L2). +const REGISTRY_MODULE = + /(?:\bmodule\s+[A-Za-z_]\w*|\busing|\bextends|\bfrom)\s+'((?:br:|br\/|ts:)[^'\r\n]*)'/; + +/** + * Read the `.bicep` inputs named in argv (inside the workdir only) and return the first registry + * module reference (`br:`, `br/`, `ts:`) found, else undefined. Registry modules cannot be + * restored against the emulator, so the handler answers before spawning az. This does I/O; + * the tokenizer-level policy (evaluateAzCommand) does not call it. + */ +export function scanBicepModules( + argv: string[], + workdir: string, + readFile?: (p: string) => string | undefined +): string | undefined { + const read = + readFile ?? + ((p: string): string | undefined => { + try { + return fs.readFileSync(p, "utf8"); + } catch { + return undefined; + } + }); + + for (const token of argv) { + const value = token.includes("=") ? token.slice(token.indexOf("=") + 1) : token; + const bare = stripAt(value); + if (!/\.(bicep|bicepparam)$/i.test(bare)) continue; + + const resolved = path.resolve(workdir, bare); + const rel = path.relative(workdir, resolved); + if (rel.startsWith("..") || path.isAbsolute(rel)) continue; // outside the workdir + + const content = read(resolved); + if (!content) continue; + const match = content.match(REGISTRY_MODULE); + if (match) return match[1]; + } + return undefined; +} diff --git a/src/lib/azure/resolve-az.test.ts b/src/lib/azure/resolve-az.test.ts new file mode 100644 index 0000000..88fa666 --- /dev/null +++ b/src/lib/azure/resolve-az.test.ts @@ -0,0 +1,672 @@ +import path from "path"; +import { + AzResolveError, + BicepPathError, + compareVersions, + listInstalledExtensions, + localVersionText, + locateAz, + locateBicep, + parseBashLauncher, + pathEntries, + probeArgs, + pythonPrefix, + resolveAz, + resolveBicep, + type AzProbeOutcome, + type LocatedAz, + type ResolveAzContext, + type ResolveBicepContext, + type ResolveFs, +} from "./resolve-az"; + +// Resolving az on every install layout, the launcher texts, and Bicep. +// A fake filesystem keeps every case OS-independent. + +interface FakeFile { + text?: string; + exec?: boolean; + /** POSIX symlink target. */ + link?: string; +} + +function fakeFs(files: Record, platform: NodeJS.Platform): ResolveFs { + const key = (p: string) => + platform === "win32" ? path.win32.normalize(p).toLowerCase() : path.posix.normalize(p); + const table = new Map(Object.entries(files).map(([p, f]) => [key(p), f])); + const resolve = (p: string): string | undefined => { + const f = table.get(key(p)); + if (!f) return undefined; + return f.link ? resolve(f.link) : path.posix.normalize(p); + }; + return { + isFile: (p) => table.has(key(p)), + readText: (p) => { + const real = platform === "win32" ? p : (resolve(p) ?? p); + return table.get(key(real))?.text; + }, + realpath: (p) => (platform === "win32" ? (table.has(key(p)) ? p : undefined) : resolve(p)), + isExecutable: (p) => { + const real = platform === "win32" ? p : resolve(p); + const f = real ? table.get(key(real)) : undefined; + return Boolean(f) && (platform === "win32" || f!.exec !== false); + }, + listDir: (dir) => { + const prefix = key(dir) + (platform === "win32" ? "\\" : "/"); + const names = new Set(); + for (const k of table.keys()) { + if (k.startsWith(prefix)) names.add(k.slice(prefix.length).split(/[\\/]/)[0]); + } + return [...names]; + }, + }; +} + +// The launcher texts, verbatim, as the installers write them. +const MSI_CMD = `::\r\n:: Microsoft Azure CLI - Windows Installer - Author file components script\r\n:: Copyright (C) Microsoft Corporation. All Rights Reserved.\r\n::\r\n\r\n@IF EXIST "%~dp0\\..\\python.exe" (\r\n SET AZ_INSTALLER=MSI\r\n "%~dp0\\..\\python.exe" -IBm azure.cli %*\r\n) ELSE (\r\n echo Failed to load python executable.\r\n exit /b 1\r\n)\r\n`; +const MSI_BASH = `#!/usr/bin/env bash\n\nAZ_INSTALLER=MSI "$(dirname "\${BASH_SOURCE[0]}")/../python.exe" -IBm azure.cli "$@"\n`; +const PIP_BAT = `@echo off\r\nsetlocal\r\n\r\nSET PYTHONPATH=%~dp0\\src;%PYTHONPATH%\r\nSET AZ_INSTALLER=PIP\r\n\r\nIF EXIST "%~dp0\\python.exe" (\r\n "%~dp0\\python.exe" -m azure.cli %*\r\n) ELSE (\r\n python -m azure.cli %*\r\n)\r\n`; +const pipScript = (python: string) => + `#!${python}\r\n\nimport sys\nimport os\n\nos.execl(sys.executable, sys.executable, '-m', 'azure.cli', *sys.argv[1:])\n`; +const DEB = + '#!/usr/bin/env bash\nbin_dir=`cd "$(dirname "$BASH_SOURCE[0]")"; pwd`\nAZ_INSTALLER=DEB "$bin_dir"/../../opt/az/bin/python3 -Im azure.cli "$@"\n'; + +const CLI2 = "C:\\Program Files\\Microsoft SDKs\\Azure\\CLI2"; +const PY313 = "C:\\Users\\me\\AppData\\Local\\Programs\\Python\\Python313"; +const HOST_PREFIX = ["-X", "utf8", "-W", "ignore::SyntaxWarning", "-IBm", "azure.cli"]; + +const msiFiles: Record = { + [`${CLI2}\\wbin\\az.cmd`]: { text: MSI_CMD }, + [`${CLI2}\\wbin\\az`]: { text: MSI_BASH }, + [`${CLI2}\\python.exe`]: {}, +}; +const pipFiles: Record = { + [`${PY313}\\Scripts\\az.bat`]: { text: PIP_BAT }, + [`${PY313}\\Scripts\\az`]: { text: pipScript(`${PY313}\\python.exe`) }, + [`${PY313}\\python.exe`]: {}, +}; + +function winCtx(files: Record, env: NodeJS.ProcessEnv = {}): ResolveAzContext { + return { env, platform: "win32", homedir: "C:\\Users\\me", fs: fakeFs(files, "win32") }; +} + +function posixCtx( + files: Record, + env: NodeJS.ProcessEnv = {}, + procVersion?: string +): ResolveAzContext { + return { env, platform: "linux", homedir: "/home/me", fs: fakeFs(files, "linux"), procVersion }; +} + +const locateError = (ctx: ResolveAzContext): AzResolveError => { + try { + locateAz(ctx); + } catch (error) { + return error as AzResolveError; + } + throw new Error("expected locateAz to throw"); +}; + +describe("locateAz: LOCALSTACK_AZ_PATH", () => { + test("the MSI az.cmd maps to CLI2\\python.exe with -X utf8 -IBm and AZ_INSTALLER=MSI", () => { + const exe = locateAz(winCtx(msiFiles, { LOCALSTACK_AZ_PATH: `${CLI2}\\wbin\\az.cmd` })); + expect(exe).toEqual({ + file: `${CLI2}\\python.exe`.replace("wbin\\..\\", ""), + prefixArgs: HOST_PREFIX, + installer: "explicit", + azInstaller: "MSI", + }); + }); + + test("a python.exe is used as it is", () => { + const exe = locateAz(winCtx(msiFiles, { LOCALSTACK_AZ_PATH: `${CLI2}\\python.exe` })); + expect(exe.file).toBe(`${CLI2}\\python.exe`); + expect(exe.prefixArgs).toEqual(HOST_PREFIX); + expect(exe.azInstaller).toBe("MSI"); + }); + + test("a venv's python is used by its own path, not the system Python it links to", () => { + // `python3 -m venv ~/az` links bin/python3 to the system Python, and Python finds the venv + // (pyvenv.cfg) next to the path it was started by: the link's target has no azure-cli. + const files = { + "/home/me/az/bin/python3": { link: "/usr/bin/python3.12" }, + "/usr/bin/python3.12": {}, + }; + const exe = locateAz(posixCtx(files, { LOCALSTACK_AZ_PATH: "/home/me/az/bin/python3" })); + expect(exe.file).toBe("/home/me/az/bin/python3"); + expect(exe.prefixArgs).toEqual(HOST_PREFIX); + expect(exe.installer).toBe("explicit"); + }); + + test.each(["az.cmd", "C:\\nowhere\\az.cmd"])( + "a relative or missing path is a hard error, with no fallback to PATH: %s", + (value) => { + const ctx = winCtx(msiFiles, { LOCALSTACK_AZ_PATH: value, PATH: `${CLI2}\\wbin` }); + expect(locateError(ctx).message).toMatch(/LOCALSTACK_AZ_PATH must be an absolute path/); + } + ); + + test("an explicit unknown wrapper is a hard error, even with a good az on PATH", () => { + const files = { + ...msiFiles, + "C:\\tools\\az.cmd": { text: "@echo off\r\ncall other.cmd %*\r\n" }, + }; + const ctx = winCtx(files, { LOCALSTACK_AZ_PATH: "C:\\tools\\az.cmd", PATH: `${CLI2}\\wbin` }); + expect(locateError(ctx).message).toMatch(/not a known Azure CLI launcher/); + }); +}); + +describe("locateAz: PATH order and entry parsing", () => { + test("wbin before Python313\\Scripts gives the MSI; reversed gives pip", () => { + const files = { ...msiFiles, ...pipFiles }; + const msiFirst = locateAz(winCtx(files, { PATH: `${CLI2}\\wbin;${PY313}\\Scripts` })); + expect(msiFirst).toMatchObject({ file: `${CLI2}\\python.exe`, installer: "msi" }); + const pipFirst = locateAz(winCtx(files, { PATH: `${PY313}\\Scripts;${CLI2}\\wbin` })); + expect(pipFirst).toMatchObject({ + file: `${PY313}\\python.exe`, + installer: "pip", + azInstaller: "PIP", + }); + }); + + test("quoted, empty, trailing-backslash and relative PATH entries are handled", () => { + expect(pathEntries({ Path: `"${CLI2}\\wbin\\";;.\\bin;relative;C:x;C:\\ok` }, "win32")).toEqual( + [`${CLI2}\\wbin\\`, "C:\\ok"] + ); + const exe = locateAz(winCtx(msiFiles, { Path: `;.\\bin;"${CLI2}\\wbin\\"` })); + expect(exe.file).toBe(`${CLI2}\\python.exe`); + }); + + test("extensions match case-insensitively (AZ.CMD)", () => { + const files = { [`${CLI2}\\wbin\\AZ.CMD`]: { text: MSI_CMD }, [`${CLI2}\\python.exe`]: {} }; + expect(locateAz(winCtx(files, { PATH: `${CLI2}\\wbin` })).installer).toBe("msi"); + }); + + test("a directory holding both az and az.cmd maps once, never to the extensionless file", () => { + const exe = locateAz(winCtx(msiFiles, { PATH: `${CLI2}\\wbin` })); + expect(exe.file.toLowerCase().endsWith("python.exe")).toBe(true); + expect(exe.prefixArgs).toEqual(HOST_PREFIX); + }); + + test("the MSI's Git Bash launcher alone also maps to its python.exe", () => { + const files = { [`${CLI2}\\wbin\\az`]: { text: MSI_BASH }, [`${CLI2}\\python.exe`]: {} }; + expect(locateAz(winCtx(files, { PATH: `${CLI2}\\wbin` })).file).toBe(`${CLI2}\\python.exe`); + }); +}); + +describe("locateAz: broken, pip and unknown Windows launchers", () => { + test("an MSI az.cmd without ..\\python.exe is a broken MSI install, with no shell fallback", () => { + const files = { + [`${CLI2}\\wbin\\az.cmd`]: { text: MSI_CMD }, + [`${CLI2}\\wbin\\az`]: { text: MSI_BASH }, + }; + const error = locateError(winCtx(files, { PATH: `${CLI2}\\wbin` })); + expect(error.reasons).toEqual([expect.stringMatching(/broken MSI install/)]); + }); + + test("pip: the venv python.exe comes first", () => { + const venv = "C:\\proj\\.venv\\Scripts"; + const files = { + [`${venv}\\az.bat`]: { text: PIP_BAT }, + [`${venv}\\python.exe`]: {}, + "C:\\proj\\.venv\\python.exe": {}, + }; + expect(locateAz(winCtx(files, { PATH: venv })).file).toBe(`${venv}\\python.exe`); + }); + + test("pip: the base install's python.exe when there is no venv python", () => { + const files = { + [`${PY313}\\Scripts\\az.bat`]: { text: PIP_BAT }, + [`${PY313}\\python.exe`]: {}, + }; + expect(locateAz(winCtx(files, { PATH: `${PY313}\\Scripts` })).file).toBe( + `${PY313}\\python.exe` + ); + }); + + test("pip: the sibling az's shebang (CRLF stripped) for a user-site layout", () => { + const userScripts = "C:\\Users\\me\\AppData\\Roaming\\Python\\Python313\\Scripts"; + const files = { + [`${userScripts}\\az.bat`]: { text: PIP_BAT }, + [`${userScripts}\\az`]: { text: pipScript(`${PY313}\\python.exe`) }, + [`${PY313}\\python.exe`]: {}, + }; + expect(locateAz(winCtx(files, { PATH: userScripts })).file).toBe(`${PY313}\\python.exe`); + }); + + test("a build-machine shebang that does not exist is rejected; python is never taken from PATH", () => { + const scripts = "C:\\odd\\Scripts"; + const files = { + [`${scripts}\\az.bat`]: { text: PIP_BAT }, + [`${scripts}\\az`]: { + text: pipScript("D:\\a\\_work\\1\\s\\build_scripts\\windows\\artifacts\\cli\\python.exe"), + }, + "C:\\Windows\\python.exe": {}, + }; + const error = locateError(winCtx(files, { PATH: `${scripts};C:\\Windows` })); + expect(error.reasons.join("\n")).toMatch(/pip launcher without a usable python\.exe/); + }); + + test.each([ + ["an unknown .cmd wrapper", "C:\\shims\\az.cmd", "@echo off\r\nnode az.js %*\r\n"], + ["an az.exe shim", "C:\\shims\\az.exe", ""], + ])("%s is rejected with the LOCALSTACK_AZ_PATH hint", (_label, file, text) => { + const error = locateError(winCtx({ [file]: { text } }, { PATH: "C:\\shims" })); + expect(error.reasons.join("\n")).toMatch(/LOCALSTACK_AZ_PATH/); + }); + + test("nothing on PATH falls through to the known MSI location", () => { + const exe = locateAz( + winCtx( + { [`${CLI2}\\python.exe`]: {} }, + { ProgramFiles: "C:\\Program Files", PATH: "C:\\empty" } + ) + ); + expect(exe).toMatchObject({ + file: `${CLI2}\\python.exe`, + installer: "msi", + azInstaller: "MSI", + }); + }); + + test("nothing anywhere gives 'az not found' with install advice", () => { + const error = locateError(winCtx({}, { PATH: "C:\\empty" })); + expect(error.message).toMatch(/was not found/); + expect(error.message).toMatch(/install-azure-cli/); + }); +}); + +describe("locateAz: POSIX launchers", () => { + test("a symlink is realpath'd, and a pip/venv script is rewritten to its shebang python", () => { + const files = { + "/home/me/bin/az": { link: "/home/me/tools/azcli/bin/az" }, + "/home/me/tools/azcli/bin/az": { + text: pipScript("/home/me/tools/azcli/bin/python").replace(/\r/g, ""), + }, + "/home/me/tools/azcli/bin/python": {}, + }; + expect(locateAz(posixCtx(files, { PATH: "/home/me/bin:/usr/bin" }))).toEqual({ + file: "/home/me/tools/azcli/bin/python", + prefixArgs: HOST_PREFIX, + installer: "pip", + azInstaller: "PIP", + }); + }); + + test("the deb launcher gives its interpreter /opt/az/bin/python3 with -X utf8 -IBm", () => { + const files = { "/usr/bin/az": { text: DEB }, "/opt/az/bin/python3": {} }; + expect(locateAz(posixCtx(files, { PATH: "/usr/bin" }))).toEqual({ + file: "/opt/az/bin/python3", + prefixArgs: HOST_PREFIX, + installer: "deb", + azInstaller: "DEB", + }); + }); + + test("an unparseable bash launcher is spawned as-is", () => { + const text = + '#!/usr/bin/env bash\nAZ_INSTALLER=RPM PYTHONPATH="$bin_dir/../lib64/az/lib/python3.9/site-packages" python3 -sm azure.cli "$@"\n'; + const exe = locateAz(posixCtx({ "/usr/bin/az": { text } }, { PATH: "/usr/bin" })); + expect(exe).toEqual({ file: "/usr/bin/az", prefixArgs: [], installer: "launcher-as-is" }); + }); + + test("an env-python shebang is refused (it would take python from PATH)", () => { + const text = "#!/usr/bin/env python3\nimport os\nos.execl('-m', 'azure.cli')\n"; + const error = locateError( + posixCtx({ "/usr/local/bin/az": { text } }, { PATH: "/usr/local/bin" }) + ); + expect(error.reasons.join("\n")).toMatch(/through env/); + }); + + test("a non-executable az on PATH is ignored", () => { + const files = { "/usr/bin/az": { text: DEB, exec: false }, "/opt/az/bin/python3": {} }; + const error = locateError(posixCtx(files, { PATH: "/usr/bin" })); + expect(error.message).toMatch(/was not found/); + }); + + test("known locations are tried when PATH has no az", () => { + const files = { "/opt/homebrew/bin/az": { text: DEB.replace("DEB", "HOMEBREW") } }; + const exe = locateAz(posixCtx(files, { PATH: "/nothing" })); + expect(exe.installer).toBe("launcher-as-is"); // its interpreter does not exist here + }); +}); + +describe("locateAz: without az, the answer says how to install it (as the Snowflake tool does for snow)", () => { + test("the docs, the minimum version, an install command per platform, then the add-ons", () => { + const error = locateError(posixCtx({}, { PATH: "/nothing" })); + expect(error.message).toMatch(/^The Azure CLI \(az\) was not found\./); + for (const expected of [ + "https://learn.microsoft.com/cli/azure/install-azure-cli", + "2.85 or newer", + "winget install --exact --id Microsoft.AzureCLI", + "brew install azure-cli", + "curl -sL https://aka.ms/InstallAzureCLIDeb | sudo bash", + "npx -y @localstack/localstack-mcp-server install-azure-addons", + "LOCALSTACK_AZ_PATH", + ]) { + expect(error.message).toContain(expected); + } + }); +}); + +describe("locateAz: WSL", () => { + const wslFiles: Record = { + "/mnt/c/Program Files/Microsoft SDKs/Azure/CLI2/wbin/az": { text: MSI_BASH }, + "/mnt/c/Users/me/AppData/Local/Programs/Python/Python313/Scripts/az": { + text: pipScript("C:\\Users\\me\\AppData\\Local\\Programs\\Python\\Python313\\python.exe"), + }, + }; + const wslPath = + "/mnt/c/Program Files/Microsoft SDKs/Azure/CLI2/wbin:/mnt/c/Users/me/AppData/Local/Programs/Python/Python313/Scripts"; + + test("Windows candidates are skipped with the reason, and the error explains the real-~/.azure trap", () => { + const error = locateError( + posixCtx(wslFiles, { PATH: wslPath, WSL_DISTRO_NAME: "Ubuntu-22.04" }) + ); + expect(error.reasons).toHaveLength(2); + expect(error.reasons.every((r) => /through WSL/.test(r))).toBe(true); + expect(error.message).toMatch(/does not forward AZURE_CONFIG_DIR/); + }); + + test("/proc/version alone detects WSL", () => { + const error = locateError( + posixCtx(wslFiles, { PATH: wslPath }, "Linux version 5.15 (microsoft-standard-WSL2)") + ); + expect(error.message).toMatch(/does not forward AZURE_CONFIG_DIR/); + }); + + test("a Linux az later on PATH still wins, without a login-shell PATH", () => { + const files = { + ...wslFiles, + "/home/me/bin/az": { text: pipScript("/home/me/venv/bin/python").replace(/\r/g, "") }, + "/home/me/venv/bin/python": {}, + }; + const exe = locateAz( + posixCtx(files, { PATH: `${wslPath}:/home/me/bin`, WSL_DISTRO_NAME: "Ubuntu" }) + ); + expect(exe.file).toBe("/home/me/venv/bin/python"); + }); + + test("a Windows az under another automount root (`root = /` mounts C: at /c) is refused", () => { + const cli2 = "/c/Program Files/Microsoft SDKs/Azure/CLI2"; + const error = locateError( + posixCtx( + { [`${cli2}/wbin/az`]: { text: MSI_BASH }, [`${cli2}/python.exe`]: {} }, + { PATH: `${cli2}/wbin`, WSL_DISTRO_NAME: "Ubuntu" } + ) + ); + expect(error.reasons.join("\n")).toMatch(/through WSL/); + }); + + test("a launcher whose shebang runs a Windows python.exe is refused under WSL", () => { + const error = locateError( + posixCtx( + { + "/home/me/bin/az": { text: pipScript("/c/Python313/python.exe").replace(/\r/g, "") }, + "/c/Python313/python.exe": {}, + }, + { PATH: "/home/me/bin", WSL_DISTRO_NAME: "Ubuntu" } + ) + ); + expect(error.reasons.join("\n")).toMatch(/through WSL/); + }); + + test("an .exe candidate under WSL is refused even outside /mnt", () => { + const error = locateError( + posixCtx( + { "/usr/local/bin/az": { link: "/opt/win/az.exe" }, "/opt/win/az.exe": { text: "MZ" } }, + { + PATH: "/usr/local/bin", + WSL_DISTRO_NAME: "Ubuntu", + } + ) + ); + expect(error.reasons.join("\n")).toMatch(/through WSL/); + }); +}); + +describe("parseBashLauncher and the spawn prefix", () => { + test("the deb text", () => { + expect(parseBashLauncher(DEB, "/usr/bin", "linux")).toEqual({ + python: "/opt/az/bin/python3", + installer: "DEB", + }); + }); + + test("the MSI Git Bash text", () => { + expect(parseBashLauncher(MSI_BASH, "/c/cli2/wbin", "linux")).toEqual({ + python: "/c/cli2/python.exe", + installer: "MSI", + }); + }); + + test("a Homebrew-style absolute interpreter", () => { + const text = + '#!/usr/bin/env bash\nAZ_INSTALLER=HOMEBREW /opt/homebrew/Cellar/azure-cli/2.90.0/libexec/bin/python -Im azure.cli "$@"\n'; + expect(parseBashLauncher(text, "/opt/homebrew/bin", "darwin")).toEqual({ + python: "/opt/homebrew/Cellar/azure-cli/2.90.0/libexec/bin/python", + installer: "HOMEBREW", + }); + }); + + test("a PYTHONPATH launcher is not parsed (-I would ignore PYTHONPATH)", () => { + expect( + parseBashLauncher('PYTHONPATH=/x python3 -sm azure.cli "$@"', "/usr/bin", "linux") + ).toBeUndefined(); + }); + + test("the image form: a pycache prefix replaces -B", () => { + expect(pythonPrefix({ pycacheDir: "/tmp/localstack-az-pycache" })).toEqual([ + "-X", + "utf8", + "-W", + "ignore::SyntaxWarning", + "-X", + "pycache_prefix=/tmp/localstack-az-pycache", + "-Im", + "azure.cli", + ]); + expect(pythonPrefix({})).toEqual(HOST_PREFIX); + expect(pythonPrefix({ safePathFallback: true })).toEqual([ + "-X", + "utf8", + "-W", + "ignore::SyntaxWarning", + "-P", + "-Bm", + "azure.cli", + ]); + expect(probeArgs(false).slice(0, 3)).toEqual(["-X", "utf8", "-IB"]); + expect(probeArgs(true).slice(0, 4)).toEqual(["-X", "utf8", "-P", "-B"]); + }); +}); + +describe("resolveAz: the probe", () => { + const ctx = () => winCtx(msiFiles, { PATH: `${CLI2}\\wbin` }); + + test("a successful probe gives the version", async () => { + const probe = jest.fn(async (_exe: LocatedAz, _fallback: boolean): Promise => ({ + ok: true, + python: "3.13.11", + core: "2.85.0", + })); + const exe = await resolveAz(ctx(), probe); + expect(exe.version).toBe("2.85.0"); + expect(probe).toHaveBeenCalledTimes(1); + expect(probe.mock.calls[0][1]).toBe(false); + }); + + test("an import failure under -I retries with -P on Python 3.11+", async () => { + const probe = jest.fn(async (_exe: LocatedAz, fallback: boolean): Promise => + fallback + ? { ok: true, python: "3.12.1", core: "2.87.0" } + : { ok: false, python: "3.12.1", error: "ModuleNotFoundError" } + ); + const exe = await resolveAz(ctx(), probe); + expect(exe.prefixArgs).toEqual([ + "-X", + "utf8", + "-W", + "ignore::SyntaxWarning", + "-P", + "-Bm", + "azure.cli", + ]); + expect(exe.version).toBe("2.87.0"); + }); + + test("no -P retry below Python 3.11", async () => { + const probe = jest.fn(async (): Promise => ({ + ok: false, + python: "3.10.12", + error: "ModuleNotFoundError", + })); + await expect(resolveAz(ctx(), probe)).rejects.toThrow( + /could not be started: ModuleNotFoundError/ + ); + expect(probe).toHaveBeenCalledTimes(1); + }); + + test("a version below the minimum is an error", async () => { + const probe = async (): Promise => ({ + ok: true, + python: "3.11.9", + core: "2.84.1", + }); + await expect(resolveAz(ctx(), probe)).rejects.toThrow(/needs 2\.85\.0 or newer/); + }); + + test("compareVersions", () => { + expect(compareVersions("2.85.0", "2.85.0")).toBe(0); + expect(compareVersions("2.9.0", "2.85.0")).toBeLessThan(0); + expect(compareVersions("2.90.0", "2.85.0")).toBeGreaterThan(0); + expect(compareVersions("0.47.16", "0.14.85")).toBeGreaterThan(0); + }); +}); + +describe("resolveBicep", () => { + const home = "/home/me"; + const bicepCtx = ( + files: Record, + env: NodeJS.ProcessEnv = {} + ): ResolveBicepContext => ({ + env, + platform: "linux", + homedir: home, + fs: fakeFs(files, "linux"), + canonical: (p) => path.posix.resolve(p), + excludeDirs: [`${home}/.azure`], + }); + const all = { + "/opt/bicep/bicep": {}, + [`${home}/.localstack/azure/bin/bicep`]: {}, + "/usr/local/bin/bicep": {}, + [`${home}/.azure/bin/bicep`]: {}, + }; + + test("the order: explicit, then the tool's own, then PATH", () => { + const env = { PATH: `${home}/.azure/bin:/usr/local/bin` }; + expect( + locateBicep(bicepCtx(all, { ...env, LOCALSTACK_AZ_BICEP_PATH: "/opt/bicep/bicep" }))?.source + ).toBe("explicit"); + expect(locateBicep(bicepCtx(all, env))).toEqual({ + path: `${home}/.localstack/azure/bin/bicep`, + dir: `${home}/.localstack/azure/bin`, + source: "tool", + }); + const { [`${home}/.localstack/azure/bin/bicep`]: _own, ...rest } = all; + expect(locateBicep(bicepCtx(rest, env))).toEqual({ + path: "/usr/local/bin/bicep", + dir: "/usr/local/bin", + source: "path", + }); + }); + + test("~/.azure/bin/bicep is never chosen, even when that directory is on PATH", () => { + const files = { [`${home}/.azure/bin/bicep`]: {} }; + expect(locateBicep(bicepCtx(files, { PATH: `${home}/.azure/bin` }))).toBeUndefined(); + }); + + test.each(["/opt/bicep/missing/bicep", "/opt/bicep/bicep-cli", "relative/bicep"])( + "a missing or misnamed LOCALSTACK_AZ_BICEP_PATH is a hard error: %s", + (value) => { + const files = { "/opt/bicep/bicep-cli": {}, "relative/bicep": {} }; + expect(() => locateBicep(bicepCtx(files, { LOCALSTACK_AZ_BICEP_PATH: value }))).toThrow( + BicepPathError + ); + expect(() => locateBicep(bicepCtx(files, { LOCALSTACK_AZ_BICEP_PATH: value }))).toThrow( + /must be an existing file named bicep or bicep.exe/ + ); + } + ); + + test("Windows looks for bicep.exe and matches the explicit name case-insensitively", () => { + const ctx: ResolveBicepContext = { + env: { LOCALSTACK_AZ_BICEP_PATH: "C:\\tools\\Bicep.EXE" }, + platform: "win32", + homedir: "C:\\Users\\me", + fs: fakeFs({ "C:\\tools\\Bicep.EXE": {} }, "win32"), + canonical: (p) => path.win32.resolve(p).toLowerCase(), + excludeDirs: [], + }; + expect(locateBicep(ctx)?.path).toBe("C:\\tools\\Bicep.EXE"); + }); + + test("a version below 0.14.85 is flagged for .bicepparam", async () => { + const files = { "/usr/local/bin/bicep": {} }; + const old = await resolveBicep( + bicepCtx(files, { PATH: "/usr/local/bin" }), + async () => "Bicep CLI version 0.13.1 (e3ac80d678)" + ); + expect(old).toMatchObject({ version: "0.13.1", supportsBicepparam: false }); + const pinned = await resolveBicep( + bicepCtx(files, { PATH: "/usr/local/bin" }), + async () => "Bicep CLI version 0.47.16 (8e2a0e3c1a)" + ); + expect(pinned).toMatchObject({ version: "0.47.16", supportsBicepparam: true }); + }); + + test("no Bicep anywhere resolves to undefined", async () => { + expect( + await resolveBicep(bicepCtx({}, { PATH: "/usr/bin" }), async () => undefined) + ).toBeUndefined(); + }); +}); + +describe("version, answered locally", () => { + test("the installed extensions are read from their METADATA, and the text is JSON plus a note", () => { + const fs = fakeFs( + { + "/ext/resource-graph/resource_graph-2.1.1.dist-info/METADATA": { + text: "Metadata-Version: 2.1\nName: resource-graph\nVersion: 2.1.1\n", + }, + "/ext/front-door/front_door-1.3.0.dist-info/METADATA": { + text: "Name: front-door\nVersion: 1.3.0\n", + }, + "/ext/broken/readme.txt": { text: "x" }, + }, + "linux" + ); + const extensions = listInstalledExtensions("/ext", fs, "linux"); + expect(extensions).toEqual([ + { name: "front-door", version: "1.3.0" }, + { name: "resource-graph", version: "2.1.1" }, + ]); + const text = localVersionText( + { + file: "/opt/az/bin/python3", + prefixArgs: HOST_PREFIX, + installer: "deb", + azInstaller: "DEB", + version: "2.90.0", + }, + extensions + ); + expect(JSON.parse(text.split("\n\n")[0])).toEqual({ + "azure-cli": "2.90.0", + "azure-cli-core": "2.90.0", + extensions: { "front-door": "1.3.0", "resource-graph": "2.1.1" }, + }); + expect(text).toMatch(/without running `az version`/); + }); +}); diff --git a/src/lib/azure/resolve-az.ts b/src/lib/azure/resolve-az.ts new file mode 100644 index 0000000..bc907ce --- /dev/null +++ b/src/lib/azure/resolve-az.ts @@ -0,0 +1,678 @@ +import { accessSync, constants, readdirSync, readFileSync, realpathSync, statSync } from "fs"; +import path from "path"; +import type { AzExecutable } from "./types"; + +/** + * Finding `az` and Bicep. Every launcher is mapped to the CLI's own Python, which is + * then spawned as ` -X utf8 -W ignore::SyntaxWarning -IBm azure.cli`: + * - `-I` keeps a planted `azure/cli/__main__.py` in the working directory from + * running in place of `az` (plain `-m` runs it); + * - `-X utf8` is the only UTF-8 fix that works under `-I` (PYTHON* variables are + * ignored there), and without it non-cp1252 values come back empty with exit 0; + * - `-W ignore::SyntaxWarning` keeps extensions' compile-time warnings out of stderr. + * `.cmd`, `.bat` and extensionless launchers are never spawned, and `python` is never + * taken from PATH. + */ + +export const MIN_AZ_VERSION = "2.85.0"; +/** `.bicepparam` files need this Bicep release or newer. */ +export const MIN_BICEPPARAM_VERSION = "0.14.85"; + +export interface ResolveFs { + isFile(p: string): boolean; + /** The first `maxBytes` of a file as UTF-8; undefined when unreadable. */ + readText(p: string, maxBytes?: number): string | undefined; + realpath(p: string): string | undefined; + /** POSIX: a regular file with an execute bit. win32: a regular file. */ + isExecutable(p: string): boolean; + listDir(p: string): string[]; +} + +export const nodeResolveFs: ResolveFs = { + isFile(p) { + try { + return statSync(p).isFile(); + } catch { + return false; + } + }, + readText(p, maxBytes = 64 * 1024) { + try { + return readFileSync(p).subarray(0, maxBytes).toString("utf8"); + } catch { + return undefined; + } + }, + realpath(p) { + try { + return realpathSync.native(p); + } catch { + return undefined; + } + }, + isExecutable(p) { + try { + if (!statSync(p).isFile()) return false; + if (process.platform !== "win32") accessSync(p, constants.X_OK); + return true; + } catch { + return false; + } + }, + listDir(p) { + try { + return readdirSync(p); + } catch { + return []; + } + }, +}; + +export interface ResolveAzContext { + /** The server's own environment (PATH, ProgramFiles, LOCALSTACK_AZ_PATH, WSL_DISTRO_NAME). */ + env: NodeJS.ProcessEnv; + platform: NodeJS.Platform; + homedir: string; + fs: ResolveFs; + /** POSIX only: the text of /proc/version, for WSL detection. */ + procVersion?: string; + /** LOCALSTACK_AZ_PYCACHE_DIR: write bytecode there instead of `-B`. */ + pycacheDir?: string; +} + +/** `az` could not be resolved; `reasons` lists every candidate that was skipped. */ +export class AzResolveError extends Error { + constructor( + message: string, + public readonly reasons: string[] = [] + ) { + super(message); + this.name = "AzResolveError"; + } +} + +export type LocatedAz = Omit; + +type Mapped = { ok: true; exe: LocatedAz } | { ok: false; reason: string }; + +/** The command that installs the Azure tool's add-ons: its Azure CLI extensions and Bicep. */ +export const AZURE_ADDONS_COMMAND = "npx -y @localstack/localstack-mcp-server install-azure-addons"; + +/** + * How to install the Azure CLI, in the form the Snowflake tool gives for `snow`: the official + * documentation, then one install command per platform. The README lists the same. + */ +export const AZURE_CLI_INSTALL_OPTIONS = + "Install the Azure CLI (2.85 or newer) by following the official documentation:\n" + + "https://learn.microsoft.com/cli/azure/install-azure-cli\n\n" + + "Installation options:\n" + + "- Windows: winget install --exact --id Microsoft.AzureCLI\n" + + "- macOS: brew install azure-cli\n" + + "- Debian or Ubuntu: curl -sL https://aka.ms/InstallAzureCLIDeb | sudo bash"; + +const INSTALL_ADVICE = + `${AZURE_CLI_INSTALL_OPTIONS}\n\n` + + `After installing it, run \`${AZURE_ADDONS_COMMAND}\` for the Azure CLI extensions and Bicep ` + + "this tool uses. If your Azure CLI is somewhere this tool does not look, set LOCALSTACK_AZ_PATH " + + "to its launcher or its Python."; + +function api(platform: NodeJS.Platform) { + return platform === "win32" ? path.win32 : path.posix; +} + +/** Environment lookup that ignores case on Windows, where `Path` and `PATH` are one variable. */ +export function envValue(env: NodeJS.ProcessEnv, name: string, platform: NodeJS.Platform) { + if (platform !== "win32") return env[name]; + const key = Object.keys(env).find((k) => k.toUpperCase() === name.toUpperCase()); + return key === undefined ? undefined : env[key]; +} + +/** PATH entries in order: surrounding quotes stripped, empty and relative entries skipped. */ +export function pathEntries(env: NodeJS.ProcessEnv, platform: NodeJS.Platform): string[] { + const raw = envValue(env, "PATH", platform) ?? ""; + const pathApi = api(platform); + return raw + .split(platform === "win32" ? ";" : ":") + .map((entry) => entry.trim().replace(/^"(.*)"$/, "$1")) + .filter((entry) => entry && pathApi.isAbsolute(entry) && !/^[A-Za-z]:(?![\\/])/.test(entry)); +} + +export function isWsl(ctx: Pick) { + return ( + ctx.platform !== "win32" && + (Boolean(ctx.env.WSL_DISTRO_NAME) || /microsoft/i.test(ctx.procVersion ?? "")) + ); +} + +/** + * The interpreter flags of a spawn prefix, for a `-c` call (the local-versions probe, the + * warm worker): `-X utf8 -W ignore::SyntaxWarning -IBm azure.cli` -> `-X utf8 -W ignore::SyntaxWarning -IB`. + */ +export function interpreterFlags(prefixArgs: string[]): string[] { + const flags = prefixArgs.slice(0, -1); + const last = flags.pop() ?? ""; + const stripped = last.replace(/m$/, ""); + return stripped === "-" || stripped === "" ? flags : [...flags, stripped]; +} + +/** The spawn prefix for a mapped Python. */ +export function pythonPrefix(opts: { pycacheDir?: string; safePathFallback?: boolean }) { + const cache = opts.pycacheDir ? ["-X", `pycache_prefix=${opts.pycacheDir}`] : []; + // With a cache dir the image's stripped .pyc files are rebuilt there once; `-B` + // would forbid that write even with a prefix set (verified on Python 3.13). + const isolation = opts.safePathFallback + ? opts.pycacheDir + ? ["-P", "-m"] + : ["-P", "-Bm"] + : opts.pycacheDir + ? ["-Im"] + : ["-IBm"]; + // Extension code is compiled on every run under -B, and some of it (monitor-control-service) + // has invalid string escapes: without the filter each such call ends with a SyntaxWarning and + // a source line on stderr, which the answer then carries. -W works under -I. + return ["-X", "utf8", "-W", "ignore::SyntaxWarning", ...cache, ...isolation, "azure.cli"]; +} + +function firstLine(text: string) { + return text.split("\n", 1)[0].replace(/\r$/, ""); +} + +/** The interpreter of a `#!` line, when it is an absolute path to a Python. */ +function shebangPython(text: string, platform: NodeJS.Platform): string | undefined { + const line = firstLine(text); + if (!line.startsWith("#!")) return undefined; + const interpreter = line.slice(2).trim().split(/\s+/)[0] ?? ""; + const base = api(platform).basename(interpreter).toLowerCase(); + if (!api(platform).isAbsolute(interpreter)) return undefined; + return /^python(\d+(\.\d+)*)?w?(\.exe)?$/.test(base) ? interpreter : undefined; +} + +const INSTALLERS: Record = { + MSI: "MSI", + PIP: "PIP", + DEB: "DEB", + RPM: "RPM", + HOMEBREW: "HOMEBREW", +}; + +/** + * The interpreter a bash launcher execs, e.g. the deb package's + * `AZ_INSTALLER=DEB "$bin_dir"/../../opt/az/bin/python3 -Im azure.cli "$@"` + * or the MSI's Git Bash `wbin/az`. Undefined when the text has another shape, or + * needs PYTHONPATH (which `-I` ignores); such launchers are spawned as they are. + */ +export function parseBashLauncher( + text: string, + launcherDir: string, + platform: NodeJS.Platform +): { python: string; installer?: AzExecutable["azInstaller"] } | undefined { + const pathApi = api(platform); + for (const rawLine of text.split("\n")) { + const line = rawLine.replace(/\r$/, ""); + if (!/\bazure\.cli\b/.test(line) || /PYTHONPATH=/.test(line) || /^\s*#/.test(line)) continue; + const match = + /^\s*(?:exec\s+)?(?:AZ_INSTALLER=(\w+)\s+)?(?:exec\s+)?(.+?)\s+-[A-Za-z]*m\s+azure\.cli\b/.exec( + line + ); + if (!match) continue; + const interpreter = match[2] + .replace(/\$\(dirname\s+"?\$\{?BASH_SOURCE(?:\[0\])?\}?"?\)/g, launcherDir) + .replace(/\$\{?bin_dir\}?/g, launcherDir) + .replace(/"/g, ""); + if (interpreter.includes("$") || interpreter.includes("`")) return undefined; + return { + python: pathApi.resolve(launcherDir, interpreter), + installer: match[1] ? INSTALLERS[match[1].toUpperCase()] : undefined, + }; + } + return undefined; +} + +function msiPython(dir: string, ctx: ResolveAzContext): Mapped { + const python = api(ctx.platform).join(dir, "..", "python.exe"); + if (!ctx.fs.isFile(python)) { + return { ok: false, reason: `${dir}: broken MSI install (${python} is missing)` }; + } + return { + ok: true, + exe: { + file: python, + prefixArgs: pythonPrefix(ctx), + installer: "msi", + azInstaller: "MSI", + }, + }; +} + +function pipPython(dir: string, ctx: ResolveAzContext): Mapped { + const pathApi = api(ctx.platform); + // The venv layout, then the base install; the sibling `az` script's shebang covers + // the user-site layout. az.bat's own fallback is "python from PATH": never used. + const tried = [pathApi.join(dir, "python.exe"), pathApi.join(dir, "..", "python.exe")]; + const sibling = ctx.fs.readText(pathApi.join(dir, "az"), 4096); + const fromShebang = sibling ? shebangPython(sibling, ctx.platform) : undefined; + if (fromShebang) tried.push(fromShebang); + const python = tried.find((p) => ctx.fs.isFile(p)); + if (!python) { + return { + ok: false, + reason: `${dir}: pip launcher without a usable python.exe (tried ${tried.join(", ")})`, + }; + } + return { + ok: true, + exe: { + file: pathApi.normalize(python), + prefixArgs: pythonPrefix(ctx), + installer: "pip", + azInstaller: "PIP", + }, + }; +} + +/** Map one Windows launcher file to its Python. */ +function mapWindowsLauncher(file: string, ctx: ResolveAzContext): Mapped { + const pathApi = path.win32; + const dir = pathApi.dirname(file); + const name = pathApi.basename(file).toLowerCase(); + if (/^python(\d+(\.\d+)*)?w?\.exe$/.test(name)) { + return { + ok: true, + exe: { + file, + prefixArgs: pythonPrefix(ctx), + installer: "explicit", + azInstaller: /microsoft sdks[\\/]azure[\\/]cli2/i.test(file) ? "MSI" : "PIP", + }, + }; + } + if (name === "az.exe") { + return { + ok: false, + reason: `${file}: an az.exe shim cannot be mapped to its Python; set LOCALSTACK_AZ_PATH to the CLI's python.exe`, + }; + } + const text = ctx.fs.readText(file, 16 * 1024) ?? ""; + const isMsi = /\.\.[\\/]python\.exe/i.test(text) && /-IBm\s+azure\.cli/.test(text); + if ((name === "az.cmd" || name === "az") && isMsi) return msiPython(dir, ctx); + if (name === "az.bat" && /-m\s+azure\.cli/.test(text)) return pipPython(dir, ctx); + if (name === "az" && shebangPython(text, "win32") && /azure\.cli/.test(text)) { + return pipPython(dir, ctx); + } + return { + ok: false, + reason: `${file}: not a known Azure CLI launcher; set LOCALSTACK_AZ_PATH to the CLI's python.exe`, + }; +} + +/** + * WSL interop does not forward AZURE_CONFIG_DIR, so a Windows az would use the user's real + * ~/.azure. There is deliberately no escape hatch. + */ +const windowsAzThroughWsl = (file: string): Mapped => ({ + ok: false, + reason: `${file}: a Windows Azure CLI seen through WSL; it would use your real Windows ~/.azure profile, so it is never used`, +}); + +/** Map one POSIX `az` to its Python, or to itself when its shape is unknown. */ +function mapPosixLauncher(file: string, ctx: ResolveAzContext): Mapped { + const pathApi = path.posix; + const real = ctx.fs.realpath(file) ?? file; + if ( + isWsl(ctx) && + [file, real].some((p) => /^\/mnt\/[a-z]\//i.test(p) || /\.(exe|cmd|bat)$/i.test(p)) + ) { + return windowsAzThroughWsl(file); + } + // A launcher elsewhere can still run a Windows Python: drives mount at /c with + // `[automount] root = /`, and not only under /mnt. + const windowsPython = (python: string) => isWsl(ctx) && /\.exe$/i.test(python); + if (/^python(\d+(\.\d+)*)?$/.test(pathApi.basename(real))) { + // By the path as given: a venv's python links to the system Python, and Python finds the + // venv (pyvenv.cfg) next to the path it was started by. + return { ok: true, exe: { file, prefixArgs: pythonPrefix(ctx), installer: "explicit" } }; + } + const text = ctx.fs.readText(real, 16 * 1024); + if (text === undefined) return { ok: false, reason: `${file}: unreadable` }; + const python = shebangPython(text, "linux"); + if (python) { + if (windowsPython(python)) return windowsAzThroughWsl(file); + if (!ctx.fs.isFile(python)) + return { ok: false, reason: `${file}: its interpreter ${python} is missing` }; + return { + ok: true, + exe: { file: python, prefixArgs: pythonPrefix(ctx), installer: "pip", azInstaller: "PIP" }, + }; + } + const line = firstLine(text); + if (/^#!\s*\S*env\s+python/.test(line)) { + return { + ok: false, + reason: `${file}: its shebang runs python through env, which would pick python from PATH; set LOCALSTACK_AZ_PATH to the CLI's python`, + }; + } + if (!/^#!.*\b(ba)?sh\b/.test(line)) + return { ok: false, reason: `${file}: not a known Azure CLI launcher` }; + // `$BASH_SOURCE` is the path as invoked, so try the unresolved directory first. + for (const dir of [pathApi.dirname(file), pathApi.dirname(real)]) { + const parsed = parseBashLauncher(text, dir, "linux"); + if (parsed && ctx.fs.isFile(parsed.python)) { + if (windowsPython(parsed.python)) return windowsAzThroughWsl(file); + const installer = parsed.installer + ? (parsed.installer.toLowerCase() as AzExecutable["installer"]) + : "script"; + return { + ok: true, + exe: { + file: parsed.python, + prefixArgs: pythonPrefix(ctx), + installer, + azInstaller: parsed.installer, + }, + }; + } + } + // Unknown bash launcher (rpm and Homebrew texts are pinned once CI meets them): + // spawned as-is, with LC_ALL=C.UTF-8 from the child environment. + return { ok: true, exe: { file: real, prefixArgs: [], installer: "launcher-as-is" } }; +} + +function mapCandidate(file: string, ctx: ResolveAzContext): Mapped { + return ctx.platform === "win32" ? mapWindowsLauncher(file, ctx) : mapPosixLauncher(file, ctx); +} + +/** + * Locate `az` without running it: LOCALSTACK_AZ_PATH, then PATH in order, then the + * known install locations. Throws AzResolveError with every skipped candidate. + */ +export function locateAz(ctx: ResolveAzContext): LocatedAz { + const pathApi = api(ctx.platform); + const reasons: string[] = []; + const explicit = envValue(ctx.env, "LOCALSTACK_AZ_PATH", ctx.platform)?.trim(); + if (explicit) { + // No silent fallback: an explicit path that does not work is a hard error. + if (!pathApi.isAbsolute(explicit) || !ctx.fs.isFile(explicit)) { + throw new AzResolveError( + `LOCALSTACK_AZ_PATH must be an absolute path to an existing file (the az launcher or the CLI's Python); got "${explicit}".` + ); + } + const mapped = mapCandidate(explicit, ctx); + if (!mapped.ok) + throw new AzResolveError(`LOCALSTACK_AZ_PATH: ${mapped.reason}.`, [mapped.reason]); + return { + ...mapped.exe, + installer: mapped.exe.installer === "launcher-as-is" ? "launcher-as-is" : "explicit", + }; + } + + // One launcher per directory: az and az.cmd side by side map once, + // and the first that maps wins, so the extensionless file is never the target. + const names = ctx.platform === "win32" ? ["az.cmd", "az.bat", "az", "az.exe"] : ["az"]; + for (const dir of pathEntries(ctx.env, ctx.platform)) { + for (const name of names) { + const file = pathApi.join(dir, name); + const present = ctx.platform === "win32" ? ctx.fs.isFile(file) : ctx.fs.isExecutable(file); + if (!present) continue; + const mapped = mapCandidate(file, ctx); + if (mapped.ok) return mapped.exe; + if (!reasons.includes(mapped.reason)) reasons.push(mapped.reason); + } + } + + const known = + ctx.platform === "win32" + ? ["ProgramFiles", "ProgramFiles(x86)"] + .map((v) => envValue(ctx.env, v, ctx.platform)) + .filter((v): v is string => Boolean(v)) + .map((root) => path.win32.join(root, "Microsoft SDKs", "Azure", "CLI2", "python.exe")) + : [ + "/usr/bin/az", + "/usr/local/bin/az", + "/opt/homebrew/bin/az", + path.posix.join(ctx.homedir, ".local", "bin", "az"), + ]; + for (const file of known) { + if (!(ctx.platform === "win32" ? ctx.fs.isFile(file) : ctx.fs.isExecutable(file))) continue; + const mapped = + ctx.platform === "win32" + ? ({ + ok: true, + exe: { file, prefixArgs: pythonPrefix(ctx), installer: "msi", azInstaller: "MSI" }, + } as Mapped) + : mapCandidate(file, ctx); + if (mapped.ok) return mapped.exe; + reasons.push(mapped.reason); + } + + const wslNote = reasons.some((r) => r.includes("through WSL")) + ? " Only a Windows Azure CLI was found, and it cannot be used from WSL: WSL does not forward AZURE_CONFIG_DIR, so it would use your real Windows ~/.azure profile. Install the Linux Azure CLI inside WSL." + : ""; + throw new AzResolveError( + `The Azure CLI (az) was not found.${wslNote} ${INSTALL_ADVICE}`, + reasons + ); +} + +/** The result of probing an interpreter or an as-is launcher. */ +export interface AzProbeOutcome { + ok: boolean; + /** Python version, e.g. "3.13.5" (interpreter probes only). */ + python?: string; + /** azure-cli-core version. */ + core?: string; + error?: string; +} + +/** + * Probe code: prints the Python version first, so a failed import still tells the + * resolver whether the `-P` fallback exists (Python 3.11+). ~0.17 s against ~1.3 s + * for `az version`. + */ +export const AZ_PROBE_CODE = + "import sys, json\n" + + "out = {'python': '%d.%d.%d' % sys.version_info[:3]}\n" + + "try:\n" + + " import azure.cli.core as c\n" + + " out['core'] = c.__version__\n" + + "except Exception as e:\n" + + " out['error'] = repr(e)[:300]\n" + + "print(json.dumps(out))\n"; + +/** The argv of the interpreter probe, matching the spawn prefix's isolation. */ +export function probeArgs(safePathFallback: boolean) { + return ["-X", "utf8", ...(safePathFallback ? ["-P", "-B"] : ["-IB"]), "-c", AZ_PROBE_CODE]; +} + +export function compareVersions(a: string, b: string): number { + const pa = a.split(/[.+-]/).map((p) => parseInt(p, 10) || 0); + const pb = b.split(/[.+-]/).map((p) => parseInt(p, 10) || 0); + for (let i = 0; i < Math.max(pa.length, pb.length, 3); i++) { + const d = (pa[i] ?? 0) - (pb[i] ?? 0); + if (d !== 0) return d; + } + return 0; +} + +/** + * Locate `az`, then probe it once: `-I` first; if the import fails under `-I` and + * the interpreter is Python 3.11+, retry with `-P` (a user-site install needs it). + */ +export async function resolveAz( + ctx: ResolveAzContext, + probe: (exe: LocatedAz, safePathFallback: boolean) => Promise +): Promise { + const located = locateAz(ctx); + let exe: LocatedAz = located; + let outcome = await probe(located, false); + if (!outcome.ok && located.installer !== "launcher-as-is") { + const python = outcome.python ?? "0"; + if (compareVersions(python, "3.11.0") >= 0) { + const fallback = { + ...located, + prefixArgs: pythonPrefix({ pycacheDir: ctx.pycacheDir, safePathFallback: true }), + }; + const retry = await probe(fallback, true); + if (retry.ok) { + exe = fallback; + outcome = retry; + } + } + } + if (!outcome.ok || !outcome.core) { + throw new AzResolveError( + `The Azure CLI at ${located.file} could not be started: ${outcome.error ?? "azure-cli is not importable"}. ${INSTALL_ADVICE}` + ); + } + if (compareVersions(outcome.core, MIN_AZ_VERSION) < 0) { + throw new AzResolveError( + `The Azure CLI at ${located.file} is version ${outcome.core}; the LocalStack Azure tool needs ${MIN_AZ_VERSION} or newer. Upgrade it (https://learn.microsoft.com/cli/azure/update-azure-cli).` + ); + } + return { ...exe, version: outcome.core }; +} + +// --------------------------------------------------------------------------- +// Bicep +// --------------------------------------------------------------------------- + +export interface BicepResolution { + path: string; + /** Prepended to the child's PATH: az finds Bicep with `shutil.which("bicep")`. */ + dir: string; + source: "explicit" | "tool" | "path"; + version?: string; + /** False when the probed version is older than 0.14.85. */ + supportsBicepparam: boolean; +} + +export class BicepPathError extends Error { + constructor(message: string) { + super(message); + this.name = "BicepPathError"; + } +} + +export interface ResolveBicepContext { + env: NodeJS.ProcessEnv; + platform: NodeJS.Platform; + homedir: string; + fs: ResolveFs; + /** Canonicalises a path for comparison (see core/config canonicalPath). */ + canonical: (p: string) => string; + /** Directories never searched: the real ~/.azure and the parent's AZURE_CONFIG_DIR. */ + excludeDirs: string[]; +} + +/** + * The Bicep binary, in order: LOCALSTACK_AZ_BICEP_PATH (a hard error when missing or + * misnamed), the tool's own ~/.localstack/azure/bin, then PATH. It + * never picks up `~/.azure/bin/bicep` (where `az bicep install` puts it) on its own. + */ +export function locateBicep( + ctx: ResolveBicepContext +): Omit | undefined { + const pathApi = api(ctx.platform); + const exeName = ctx.platform === "win32" ? "bicep.exe" : "bicep"; + const explicit = envValue(ctx.env, "LOCALSTACK_AZ_BICEP_PATH", ctx.platform)?.trim(); + if (explicit) { + const base = pathApi.basename(explicit); + const named = + ctx.platform === "win32" ? /^bicep(\.exe)?$/i.test(base) : /^bicep(\.exe)?$/.test(base); + if (!pathApi.isAbsolute(explicit) || !named || !ctx.fs.isFile(explicit)) { + throw new BicepPathError( + `LOCALSTACK_AZ_BICEP_PATH must be an existing file named bicep or bicep.exe (az looks the binary up by that name); got "${explicit}".` + ); + } + return { path: explicit, dir: pathApi.dirname(explicit), source: "explicit" }; + } + const own = pathApi.join(ctx.homedir, ".localstack", "azure", "bin", exeName); + if (ctx.fs.isFile(own)) return { path: own, dir: pathApi.dirname(own), source: "tool" }; + const excluded = ctx.excludeDirs.map((d) => ctx.canonical(d)); + for (const dir of pathEntries(ctx.env, ctx.platform)) { + const canonicalDir = ctx.canonical(dir); + const inExcluded = excluded.some((ex) => { + const rel = pathApi.relative(ex, canonicalDir); + return rel === "" || (!rel.startsWith("..") && !pathApi.isAbsolute(rel)); + }); + if (inExcluded) continue; + const file = pathApi.join(dir, exeName); + if (ctx.platform === "win32" ? ctx.fs.isFile(file) : ctx.fs.isExecutable(file)) { + return { path: file, dir, source: "path" }; + } + } + return undefined; +} + +export function parseBicepVersion(output: string): string | undefined { + return /(\d+\.\d+\.\d+)/.exec(output)?.[1]; +} + +export async function resolveBicep( + ctx: ResolveBicepContext, + probeVersion: (bicepPath: string) => Promise +): Promise { + const located = locateBicep(ctx); + if (!located) return undefined; + const version = parseBicepVersion((await probeVersion(located.path)) ?? ""); + return { + ...located, + version, + // An unknown version is given the benefit of the doubt: az reports the real error. + supportsBicepparam: !version || compareVersions(version, MIN_BICEPPARAM_VERSION) >= 0, + }; +} + +// --------------------------------------------------------------------------- +// `version`, answered locally (`az version` calls Microsoft) +// --------------------------------------------------------------------------- + +export interface InstalledExtension { + name: string; + version: string; +} + +/** Extensions in an AZURE_EXTENSION_DIR, read from their `*.dist-info/METADATA`. */ +export function listInstalledExtensions( + extensionDir: string, + fs: ResolveFs, + platform: NodeJS.Platform +) { + const pathApi = api(platform); + const found: InstalledExtension[] = []; + for (const entry of fs.listDir(extensionDir).sort()) { + const dir = pathApi.join(extensionDir, entry); + const info = fs.listDir(dir).find((d) => /\.(dist|egg)-info$/.test(d)); + if (!info) continue; + const metadata = + fs.readText(pathApi.join(dir, info, "METADATA"), 8192) ?? + fs.readText(pathApi.join(dir, info, "PKG-INFO"), 8192); + const version = metadata && /^Version:\s*(\S+)/m.exec(metadata)?.[1]; + if (version) found.push({ name: entry, version }); + } + return found; +} + +/** + * `az version -o json`'s shape, from the probe: `azure-cli` and `azure-cli-core` are released + * in lockstep, and tools such as Terraform's azurerm provider read `azure-cli` (L4 found it). + */ +export function localVersionJson(az: AzExecutable, extensions: InstalledExtension[]): string { + const version = az.version ?? "unknown"; + const body = { + "azure-cli": version, + "azure-cli-core": version, + extensions: Object.fromEntries(extensions.map((e) => [e.name, e.version])), + }; + return JSON.stringify(body, null, 2); +} + +export function localVersionNote(az: AzExecutable): string { + return ( + "Answered by the tool without running `az version`, which would call Microsoft's update service. The CLI runs from " + + `${az.file} (${az.installer}).` + ); +} + +export function localVersionText(az: AzExecutable, extensions: InstalledExtension[]): string { + return `${localVersionJson(az, extensions)}\n\n${localVersionNote(az)}`; +} diff --git a/src/lib/azure/runner.test.ts b/src/lib/azure/runner.test.ts new file mode 100644 index 0000000..5e1b664 --- /dev/null +++ b/src/lib/azure/runner.test.ts @@ -0,0 +1,760 @@ +// The raw module object: the namespace that `import * as` builds has fixed properties, +// which jest.spyOn cannot replace. +// eslint-disable-next-line @typescript-eslint/no-require-imports +const childProcess: typeof import("child_process") = require("child_process"); +import { + existsSync, + mkdirSync, + mkdtempSync, + readFileSync, + realpathSync, + rmSync, + writeFileSync, +} from "fs"; +import os from "os"; +import path from "path"; +import { + activeRuns, + HostRunner, + killAllLiveTreesSync, + liveChildCount, + progressSender, + quoteWindowsArg, + Semaphore, + taskkillPath, + windowsCommandLineLength, + type KillStep, +} from "./runner"; +import { pythonPrefix } from "./resolve-az"; +import type { AzExecutable, EgressEvent, EgressProxy, EgressRecords } from "./types"; + +// The runner, case by case. The fake +// `az` is a Node script; the cases that need a real interpreter use Python. + +jest.setTimeout(60_000); + +const FIXTURES = path.resolve(__dirname, "../../../tests/fixtures/azure/fake-az"); +const FAKE_AZ = path.join(FIXTURES, "fake-az.mjs"); +const isWin = process.platform === "win32"; +const fakeExe: AzExecutable = { + file: process.execPath, + prefixArgs: [FAKE_AZ], + installer: "explicit", +}; + +let root: string; +let workdir: string; +let baseEnv: Record; + +beforeAll(() => { + root = mkdtempSync(path.join(os.tmpdir(), "lsaz-runner-")); + workdir = path.join(root, "work"); + mkdirSync(workdir); + baseEnv = { + AZURE_CONFIG_DIR: path.join(root, "cfg"), + PATH: process.env.PATH ?? "", + ...(isWin ? { SYSTEMROOT: process.env.SystemRoot ?? "C:\\Windows" } : {}), + }; +}); + +afterAll(() => { + rmSync(root, { recursive: true, force: true }); +}); + +const runner = (over: Partial[0]> = {}) => + new HostRunner({ exe: fakeExe, env: baseEnv, ...over }); + +const run = ( + argv: string[], + over: Partial[0]> = {}, + timeoutMs = 20_000 +) => runner(over).run(argv, { timeoutMs, cwd: workdir }); + +function isAlive(pid: number): boolean { + try { + process.kill(pid, 0); + return true; + } catch { + return false; + } +} + +async function waitUntil(check: () => boolean, ms: number): Promise { + const deadline = Date.now() + ms; + while (Date.now() < deadline) { + if (check()) return true; + await new Promise((r) => setTimeout(r, 50)); + } + return check(); +} + +/** A Python for the real-interpreter cases: PYTHON, else python3, python or py -3. */ +function findPython(): string | undefined { + const candidates: Array<[string, string[]]> = [ + ...(process.env.PYTHON ? [[process.env.PYTHON, []] as [string, string[]]] : []), + ["python3", []], + ["python", []], + ...(isWin ? [["py", ["-3"]] as [string, string[]]] : []), + ]; + for (const [file, pre] of candidates) { + const probe = childProcess.spawnSync( + file, + [...pre, "-c", "import sys; print(sys.executable)"], + { + encoding: "utf8", + timeout: 20_000, + windowsHide: true, + } + ); + const exe = probe.status === 0 ? probe.stdout.trim() : ""; + if (exe && existsSync(exe)) return exe; + } + return undefined; +} +const PYTHON = findPython(); +const withPython = PYTHON ? test : test.skip; +if (!PYTHON) + console.warn("runner.test: no Python found (set PYTHON); the real-interpreter cases are skipped"); + +describe("spawn options and stdin", () => { + test("shell:false, windowsHide, three pipes, the workdir as cwd, and stdin ended at once", async () => { + const spy = jest.spyOn(childProcess, "spawn"); + try { + const result = await run(["echo"]); + const call = spy.mock.calls.find((c) => c[0] === process.execPath); + expect(call?.[2]).toMatchObject({ + shell: false, + windowsHide: true, + stdio: ["pipe", "pipe", "pipe"], + cwd: workdir, + detached: !isWin, + }); + const echoed = JSON.parse(result.stdout); + expect(echoed.stdinIsTTY).toBe(false); + expect(echoed.stdinEnded).toBe(true); + // Real paths: a temp dir behind a link (macOS /var -> /private/var) reports its target. + expect(realpathSync.native(echoed.cwd).toLowerCase()).toBe( + realpathSync.native(workdir).toLowerCase() + ); + expect(result.exitCode).toBe(0); + } finally { + spy.mockRestore(); + } + }); +}); + +describe("argv fidelity", () => { + // The 15 required values and a few extras; none may be expanded or re-split. + const values = [ + "[?name=='a'].id | [0]", + '{"a":"b c","d":[1,2]}', + "C:\\dir with space\\", + "ends\\\\", + 'he said "hi"', + "a;b&c|df", + "$(whoami)", + "%PATH%", + "^caret", + "!bang!", + "", + "unicode ✓ ü é", + "a\tb", + "--set", + "properties.x='{\"a\":1}'", + '"', + '\\"', + 'a\\\\"b', + "\\server\\share\\", + "trailing ", + " leading", + "a b\\\\", + "line1\nline2", + "*", + "?", + "~", + "%%", + "%C03VAR%", + "!C03VAR!", + "${C03VAR}", + "`whoami`", + "\u00a0nbsp", + "emoji 😀", + ]; + + test("through the Node fake", async () => { + const result = await run(["echo", ...values], { env: { ...baseEnv, C03VAR: "EXPANDED" } }); + expect(JSON.parse(result.stdout).argv).toEqual(values); + }); + + withPython("through a real Python", async () => { + const exe: AzExecutable = { + file: PYTHON!, + prefixArgs: ["-X", "utf8", "-IB", path.join(FIXTURES, "echo_argv.py")], + installer: "explicit", + }; + const result = await run(values, { exe, env: { ...baseEnv, C03VAR: "EXPANDED" } }); + expect(result.exitCode).toBe(0); + const received: string[] = JSON.parse(result.stdout); + expect(received).toEqual(values); + expect(received.join("")).not.toContain("EXPANDED"); + }); +}); + +describe("spawn failures", () => { + test("a synchronous throw becomes a spawn error, not an exception", async () => { + const result = await run([], { + exe: { file: "bad\0file", prefixArgs: [], installer: "explicit" }, + }); + expect(result.spawnError).toMatch(/ERR_INVALID_ARG_VALUE|null bytes/); + expect(liveChildCount()).toBe(0); + }); + + (isWin ? test : test.skip)("a .cmd file throws EINVAL synchronously on Windows", async () => { + const cmd = path.join(root, "az.cmd"); + writeFileSync(cmd, "@echo off\r\n"); + const result = await run([], { exe: { file: cmd, prefixArgs: [], installer: "explicit" } }); + expect(result.spawnError).toMatch(/EINVAL/); + }); + + test("an async ENOENT gives a clean error and no hang", async () => { + const started = Date.now(); + const result = await run([], { + exe: { file: path.join(root, "no-such-az"), prefixArgs: [], installer: "explicit" }, + }); + expect(result.spawnError).toMatch(/ENOENT/); + expect(Date.now() - started).toBeLessThan(5000); + }); + + (isWin ? test : test.skip)( + "an extensionless script gives ENOENT on Windows although it exists", + async () => { + const script = path.join(root, "az"); + writeFileSync(script, "#!/usr/bin/env bash\necho hi\n"); + const result = await run([], { + exe: { file: script, prefixArgs: [], installer: "explicit" }, + }); + expect(result.spawnError).toMatch(/ENOENT/); + } + ); + + test("a Windows command line over the limit is refused before spawning, with tooLong", async () => { + const spy = jest.spyOn(childProcess, "spawn"); + try { + const result = await run(["x".repeat(33_000)], { platform: "win32" }); + expect(result.tooLong).toBe(true); + expect(spy).not.toHaveBeenCalled(); + } finally { + spy.mockRestore(); + } + }); + + test("quoteWindowsArg follows libuv's quoting", () => { + expect(quoteWindowsArg("")).toBe('""'); + expect(quoteWindowsArg("plain")).toBe("plain"); + expect(quoteWindowsArg("a b")).toBe('"a b"'); + expect(quoteWindowsArg('he said "hi"')).toBe('"he said \\"hi\\""'); + expect(quoteWindowsArg("a b\\")).toBe('"a b\\\\"'); + expect(quoteWindowsArg("C:\\x\\y")).toBe("C:\\x\\y"); + expect(windowsCommandLineLength("C:\\p.exe", ["a b", "c"])).toBe('C:\\p.exe "a b" c'.length); + }); +}); + +describe("UTF-8", () => { + test("non-ASCII output comes back exact on both streams", async () => { + const result = await run(["utf8"]); + expect(result.stdout.replace(/\r\n/g, "\n")).toBe("ü✓é\n"); + expect(result.stderr).toContain("WARNING: ü✓é"); + }); + + test("a ✓ straddling byte 65,536 comes back exact", async () => { + const result = await run(["utf8-boundary"]); + expect(result.stdout).toBe( + "a".repeat(65535) + "✓".repeat(5) + "b".repeat(70000) + "✓".repeat(5) + ); + expect(result.stdout).not.toContain("\ufffd"); + }); +}); + +describe("the planted azure/cli/__main__.py", () => { + withPython( + "is NOT executed with the -IBm prefix, and IS with plain -m (the positive control)", + async () => { + const plant = path.join(root, "plant"); + mkdirSync(path.join(plant, "azure", "cli"), { recursive: true }); + writeFileSync( + path.join(plant, "azure", "cli", "__main__.py"), + 'open("MARKER", "w").write("shadowed")\n' + ); + // A dead proxy keeps a real azure-cli, if installed, from reaching the internet. + const env = { + ...baseEnv, + HTTPS_PROXY: "http://127.0.0.1:9", + HTTP_PROXY: "http://127.0.0.1:9", + }; + const guarded = new HostRunner({ + exe: { file: PYTHON!, prefixArgs: pythonPrefix({}), installer: "pip" }, + env, + }); + await guarded.run(["--version"], { timeoutMs: 50_000, cwd: plant }); + expect(existsSync(path.join(plant, "MARKER"))).toBe(false); + + const unguarded = new HostRunner({ + exe: { file: PYTHON!, prefixArgs: ["-Bm", "azure.cli"], installer: "pip" }, + env, + }); + await unguarded.run(["--version"], { timeoutMs: 50_000, cwd: plant }); + expect(existsSync(path.join(plant, "MARKER"))).toBe(true); + } + ); +}); + +describe("timeouts and tree kills", () => { + test("a timeout sets timedOut and resolves within timeout + grace", async () => { + const started = Date.now(); + const result = await run(["sleep", "20000"], {}, 700); + expect(result.timedOut).toBe(true); + expect(Date.now() - started).toBeLessThan(700 + 5000); + if (isWin) expect(result.exitCode).toBe(1); // after taskkill: code=1, signal=null + }); + + test("a grandchild holding the pipe is killed, and the promise resolves", async () => { + const pids = path.join(root, "tree.json"); + const started = Date.now(); + // 4 s, not 1.5 s: under a loaded full-suite run on Linux the helper had not written its + // pids yet when the timeout fired (WSL). The grandchild sleeps 15 s, so it still times out. + const result = await run(["grandchild", pids, "15000"], {}, 4000); + expect(result.timedOut).toBe(true); + expect(Date.now() - started).toBeLessThan(4000 + 6000); + const { root: rootPid, grandchild } = JSON.parse(readFileSync(pids, "utf8")); + expect(await waitUntil(() => !isAlive(rootPid) && !isAlive(grandchild), 5000)).toBe(true); + }); + + test("a root that exits while a grandchild holds the pipe resolves after the drain grace, not on 'close'", async () => { + const pids = path.join(root, "orphan.json"); + const started = Date.now(); + const result = await run(["orphan", pids, "4000"], { timings: { drainGraceMs: 300 } }); + expect(result.exitCode).toBe(0); + expect(result.stdout).toContain("root done"); + expect(Date.now() - started).toBeLessThan(3500); // the sleeper lives 4 s; it ends on its own + }); + + (isWin ? test : test.skip)("Windows: taskkill runs first, on the live root only", async () => { + const steps: Array<[KillStep, number | undefined]> = []; + const spy = jest.spyOn(childProcess, "spawn"); + try { + const pids = path.join(root, "order.json"); + await run( + ["grandchild", pids, "15000"], + { onKillStep: (s, status) => steps.push([s, status]) }, + 1000 + ); + const { root: rootPid } = JSON.parse(readFileSync(pids, "utf8")); + const taskkillCalls = spy.mock.calls.filter((c) => + String(c[0]).toLowerCase().endsWith("taskkill.exe") + ); + expect(taskkillCalls).toHaveLength(1); + expect(taskkillCalls[0][1]).toEqual(["/PID", String(rootPid), "/T", "/F"]); + // The tree can be gone (and the run resolved) before taskkill's own exit arrives. + await waitUntil(() => steps.length > 0, 5000); + expect(steps).toEqual([["taskkill", 0]]); + } finally { + spy.mockRestore(); + } + }); + + (isWin ? test : test.skip)("Windows: a failing taskkill falls back to child.kill()", async () => { + const steps: KillStep[] = []; + // node.exe as "taskkill": it cannot run a script named /PID and exits non-zero. + const result = await run( + ["sleep", "20000"], + { taskkill: process.execPath, onKillStep: (s) => steps.push(s) }, + 700 + ); + expect(result.timedOut).toBe(true); + expect(steps).toEqual(["taskkill", "kill"]); + }); + + (isWin ? test.skip : test)( + "POSIX: SIGTERM to the group, then SIGKILL after the grace", + async () => { + const steps: KillStep[] = []; + const result = await run( + ["sleep", "20000"], + { onKillStep: (s) => steps.push(s), timings: { posixKillGraceMs: 200 } }, + 500 + ); + expect(result.timedOut).toBe(true); + await waitUntil(() => steps.includes("sigkill"), 2000); + expect(steps).toEqual(["sigterm", "sigkill"]); + } + ); + + test("taskkill's path falls back to C:\\Windows when SystemRoot is missing", () => { + expect(taskkillPath({})).toBe("C:\\Windows\\System32\\taskkill.exe"); + expect(taskkillPath({ SystemRoot: "D:\\Win" })).toBe("D:\\Win\\System32\\taskkill.exe"); + }); +}); + +describe("prompts, environment, caps, exit codes", () => { + test("a prompt without --yes fails fast with exit 1, and nothing on stdout", async () => { + const started = Date.now(); + const result = await run(["prompt"]); + expect(result.exitCode).toBe(1); + expect(result.stdout).toBe(""); + expect(result.stderr).toMatch(/no tty available\. Use --yes\./); + expect(Date.now() - started).toBeLessThan(10_000); + }); + + // libuv's uv_spawn on Windows copies these from the PARENT when the env block + // lacks them (process.c, required_vars); child-env.ts therefore sets the home and + // temp ones itself. + const LIBUV_REQUIRED = [ + "HOMEDRIVE", + "HOMEPATH", + "LOGONSERVER", + "PATH", + "SYSTEMDRIVE", + "SYSTEMROOT", + "TEMP", + "USERDOMAIN", + "USERNAME", + "USERPROFILE", + "WINDIR", + ]; + + test("the child gets the environment it is given, and nothing else (Windows: plus libuv's required set)", async () => { + const given = { ...baseEnv, AZ_ONLY_THIS: "1" }; + const result = await run(["echo"], { env: given }); + const echoed = JSON.parse(result.stdout); + const norm = (k: string) => (isWin ? k.toUpperCase() : k); + const keys = (echoed.envKeys as string[]).map(norm); + for (const [k, v] of Object.entries(given)) + expect(echoed.env[k] ?? echoed.env[norm(k)]).toBe(v); + const extra = keys.filter((k) => !Object.keys(given).map(norm).includes(k)); + if (isWin) expect(extra.every((k) => LIBUV_REQUIRED.includes(k))).toBe(true); + // macOS adds __CF_USER_TEXT_ENCODING to every process's environment. + else expect(extra.filter((k) => k !== "__CF_USER_TEXT_ENCODING")).toEqual([]); + }); + + (isWin ? test : test.skip)( + "Windows: a private USERPROFILE, TEMP, HOMEDRIVE and HOMEPATH reach the child, not the parent's", + async () => { + const privateHome = path.join(root, "private-home"); + const env = { + ...baseEnv, + USERPROFILE: privateHome, + HOME: privateHome, + HOMEDRIVE: privateHome.slice(0, 2), + HOMEPATH: privateHome.slice(2), + TEMP: path.join(root, "private-tmp"), + TMP: path.join(root, "private-tmp"), + }; + const echoed = JSON.parse((await run(["echo"], { env })).stdout).env; + expect(echoed.USERPROFILE).toBe(privateHome); + expect(echoed.HOMEDRIVE + echoed.HOMEPATH).toBe(privateHome); + expect(echoed.TEMP).toBe(path.join(root, "private-tmp")); + expect(echoed.USERPROFILE).not.toBe(process.env.USERPROFILE); + } + ); + + test("the runner refuses to start without AZURE_CONFIG_DIR", async () => { + const { AZURE_CONFIG_DIR: _drop, ...noConfig } = baseEnv; + await expect(run(["echo"], { env: noConfig })).rejects.toThrow(/isolated AZURE_CONFIG_DIR/); + }); + + test("byte caps count bytes, kill the tree and mark the output truncated", async () => { + const result = await run(["flood"], { maxStreamBytes: 30_000 }); + expect(result.truncated).toBe(true); + expect(Buffer.byteLength(result.stdout)).toBeLessThanOrEqual(30_000); + expect(result.stdout.length).toBe(10_000); // 3-byte characters, none split + expect(result.stdout).not.toContain("\ufffd"); + }); + + test.each([1, 3])("exit code %i is passed through, and CRLF is kept", async (code) => { + const result = await run(["exit", String(code)]); + expect(result.exitCode).toBe(code); + expect(result.stdout).toBe("line1\r\nline2\r\n"); + expect(result.stderr).toContain("(ResourceNotFound)"); + }); +}); + +describe("concurrency, progress and cancellation", () => { + test("five parallel runs have at most four children at once", async () => { + const r = runner(); + const results = await Promise.all( + Array.from({ length: 5 }, () => + r.run(["timestamps", "1200"], { timeoutMs: 30_000, cwd: workdir }) + ) + ); + const spans = results.map((res) => JSON.parse(res.stdout) as { start: number; end: number }); + const maxOverlap = Math.max( + ...spans.map((s) => spans.filter((o) => o.start <= s.start && s.start < o.end).length) + ); + expect(maxOverlap).toBeLessThanOrEqual(4); + expect(activeRuns()).toBe(0); + }); + + test("a queued run whose signal aborts never spawns", async () => { + const slots = new Semaphore(1); + expect(await slots.acquire()).toBe(true); + const controller = new AbortController(); + const waiting = slots.acquire(controller.signal); + controller.abort(); + expect(await waiting).toBe(false); + slots.release(); + expect(slots.inUse).toBe(0); + }); + + test("onProgress is called on every interval while az runs", async () => { + const calls: number[] = []; + await runner({ timings: { progressIntervalMs: 150 } }).run(["sleep", "700"], { + timeoutMs: 10_000, + cwd: workdir, + onProgress: (ms) => calls.push(ms), + }); + expect(calls.length).toBeGreaterThanOrEqual(3); + expect(calls).toEqual([...calls].sort((a, b) => a - b)); + }); + + test("an aborted signal kills the tree and sets aborted", async () => { + const controller = new AbortController(); + setTimeout(() => controller.abort(), 400); + const started = Date.now(); + const result = await runner().run(["sleep", "20000"], { + timeoutMs: 30_000, + cwd: workdir, + signal: controller.signal, + }); + expect(result.aborted).toBe(true); + expect(result.timedOut).toBe(false); + expect(Date.now() - started).toBeLessThan(6000); + }); + + test("a signal aborted before the run never spawns", async () => { + const controller = new AbortController(); + controller.abort(); + const spy = jest.spyOn(childProcess, "spawn"); + try { + const result = await runner().run(["echo"], { + timeoutMs: 5000, + cwd: workdir, + signal: controller.signal, + }); + expect(result.aborted).toBe(true); + expect(spy).not.toHaveBeenCalled(); + } finally { + spy.mockRestore(); + } + }); + + test("progressSender sends notifications/progress with the token, and nothing without one", async () => { + const sendNotification = jest.fn().mockResolvedValue(undefined); + const send = progressSender({ _meta: { progressToken: "tok-1" }, sendNotification }); + send!(20_400); + expect(sendNotification).toHaveBeenCalledWith({ + method: "notifications/progress", + params: { progressToken: "tok-1", progress: 20, message: "az is still running (20 s)" }, + }); + expect(progressSender({ sendNotification })).toBeUndefined(); + expect(progressSender(undefined)).toBeUndefined(); + }); +}); + +describe("the egress guard's fail-fast (with a fake guard)", () => { + function fakeProxy() { + const listeners: Array<(id: string, e: EgressEvent) => void> = []; + const ids: string[] = []; + const records: EgressRecords = { + refused: ["api.loganalytics.io"], + upstream: [], + housekeeping: [], + allowed: 0, + }; + const proxy: EgressProxy = { + port: 9, + envFor: (id) => { + ids.push(id); + return { + HTTPS_PROXY: `http://${id}:x@127.0.0.1:9`, + HTTP_PROXY: `http://${id}:x@127.0.0.1:9`, + }; + }, + takeRecords: () => records, + onEvent: (l) => { + listeners.push(l); + return () => listeners.splice(listeners.indexOf(l), 1); + }, + close: async () => undefined, + }; + return { + proxy, + ids, + emit: (e: EgressEvent) => listeners.forEach((l) => l(ids[0], e)), + listeners, + }; + } + + test("a refused host kills the call after the delay; the records come back", async () => { + const fake = fakeProxy(); + const r = runner({ proxy: fake.proxy, timings: { failFastDelayMs: 100 } }); + setTimeout(() => fake.emit({ kind: "refused", host: "api.loganalytics.io" }), 300); + const started = Date.now(); + const result = await r.run(["sleep", "20000"], { timeoutMs: 30_000, cwd: workdir }); + expect(result.failFast).toBe("refused"); + expect(result.timedOut).toBe(false); + expect(result.egress.refused).toEqual(["api.loganalytics.io"]); + expect(Date.now() - started).toBeLessThan(6000); + expect(fake.listeners).toHaveLength(0); // unsubscribed + }); + + test("a housekeeping refusal is not a trigger", async () => { + const fake = fakeProxy(); + const r = runner({ proxy: fake.proxy, timings: { failFastDelayMs: 50 } }); + setTimeout( + () => fake.emit({ kind: "housekeeping", host: "azcliprod.blob.core.windows.net" }), + 100 + ); + const result = await r.run(["sleep", "600"], { timeoutMs: 30_000, cwd: workdir }); + expect(result.failFast).toBeUndefined(); + expect(result.exitCode).toBe(0); + }); + + test("the child's proxy variables carry the call tag", async () => { + const fake = fakeProxy(); + const result = await runner({ proxy: fake.proxy }).run(["echo"], { + timeoutMs: 10_000, + cwd: workdir, + }); + const env = JSON.parse(result.stdout).env; + expect(env.HTTPS_PROXY).toBe(`http://${fake.ids[0]}:x@127.0.0.1:9`); + expect(env.NO_PROXY).toBeUndefined(); + }); +}); + +describe("the exit hook", () => { + // The tree sleeps a minute, far longer than the death wait: a pass means the kill worked, + // never that the processes ended on their own. The generous waits only absorb a loaded + // machine (the full suite, parallel workers). + const SLEEP_MS = "60000"; + const DEATH_WAIT_MS = 15_000; + + test("killAllLiveTreesSync kills a live tree synchronously", async () => { + const pids = path.join(root, "hook.json"); + const pending = run(["grandchild", pids, SLEEP_MS], {}, 120_000); + expect(await waitUntil(() => existsSync(pids), 20_000)).toBe(true); + const { root: rootPid, grandchild } = JSON.parse(readFileSync(pids, "utf8")); + expect(process.listeners("exit")).toContain(killAllLiveTreesSync); + killAllLiveTreesSync(); + await pending; + expect(await waitUntil(() => !isAlive(rootPid) && !isAlive(grandchild), DEATH_WAIT_MS)).toBe( + true + ); + }, 90_000); + + test("the server process exiting mid-run leaves no descendant alive", async () => { + // Bundle the runner and drive it from a separate Node process that exits the way + // src/cli/lifecycle.ts does (process.exit while az runs). + const { buildSync } = require("esbuild") as typeof import("esbuild"); + const bundle = path.join(root, "runner.bundle.js"); + buildSync({ + entryPoints: [path.join(__dirname, "runner.ts")], + bundle: true, + platform: "node", + format: "cjs", + outfile: bundle, + logLevel: "silent", + }); + const pids = path.join(root, "exit.json"); + const driver = path.join(root, "driver.js"); + writeFileSync( + driver, + `const fs = require("fs"); +const { HostRunner } = require(${JSON.stringify(bundle)}); +const runner = new HostRunner({ exe: { file: process.execPath, prefixArgs: [${JSON.stringify(FAKE_AZ)}], installer: "explicit" }, env: ${JSON.stringify(baseEnv)} }); +runner.run(["grandchild", ${JSON.stringify(pids)}, ${JSON.stringify(SLEEP_MS)}], { timeoutMs: 120000, cwd: ${JSON.stringify(workdir)} }); +const t = setInterval(() => { if (fs.existsSync(${JSON.stringify(pids)})) { clearInterval(t); setTimeout(() => process.exit(0), 300); } }, 50); +` + ); + const exited = childProcess.spawnSync(process.execPath, [driver], { + timeout: 60_000, + windowsHide: true, + }); + expect(exited.status).toBe(0); + const { root: rootPid, grandchild } = JSON.parse(readFileSync(pids, "utf8")); + expect(await waitUntil(() => !isAlive(rootPid) && !isAlive(grandchild), DEATH_WAIT_MS)).toBe( + true + ); + }, 120_000); +}); + +describe("fail-fast with the real egress guard (the runner integration)", () => { + // eslint-disable-next-line @typescript-eslint/no-require-imports + const { startEgressProxy } = require("./egress-proxy") as typeof import("./egress-proxy"); + + async function closedPort(): Promise { + const net = await import("net"); + const probe = net.createServer(); + await new Promise((r) => probe.listen(0, "127.0.0.1", r)); + const port = (probe.address() as { port: number }).port; + await new Promise((r) => probe.close(r)); + return port; + } + + test("a CONNECT to a refused host is killed within about 2 s, and the records name the host", async () => { + const proxy = await startEgressProxy(); + try { + const started = Date.now(); + const result = await runner({ proxy }).run(["connect", "api.loganalytics.io:443", "30000"], { + timeoutMs: 60_000, + cwd: workdir, + }); + const elapsed = Date.now() - started; + expect(result.failFast).toBe("refused"); + expect(result.egress.refused).toEqual(["api.loganalytics.io"]); + expect(result.stderr).toContain("403"); + expect(elapsed).toBeLessThan(6000); + } finally { + await proxy.close(); + } + }); + + test("a CONNECT to the emulator's name on a dead port is an upstream failure, killed fast", async () => { + const proxy = await startEgressProxy(); + try { + const port = await closedPort(); + const started = Date.now(); + const result = await runner({ proxy }).run( + ["connect", `azure.localhost.localstack.cloud:${port}`, "30000"], + { + timeoutMs: 60_000, + cwd: workdir, + } + ); + expect(result.failFast).toBe("upstream"); + expect(result.egress.upstream.join(" ")).toContain("azure.localhost.localstack.cloud"); + expect(result.stderr).toContain("502"); + expect(Date.now() - started).toBeLessThan(6000); + } finally { + await proxy.close(); + } + }); + + test("a housekeeping host is refused but never triggers the fail-fast", async () => { + const proxy = await startEgressProxy(); + try { + const result = await runner({ proxy }).run( + ["connect", "azcliprod.blob.core.windows.net:443", "1500"], + { + timeoutMs: 60_000, + cwd: workdir, + } + ); + expect(result.failFast).toBeUndefined(); + expect(result.exitCode).toBe(0); + expect(result.egress.housekeeping).toEqual(["azcliprod.blob.core.windows.net"]); + expect(result.egress.refused).toEqual([]); + } finally { + await proxy.close(); + } + }); +}); diff --git a/src/lib/azure/runner.ts b/src/lib/azure/runner.ts new file mode 100644 index 0000000..83811fa --- /dev/null +++ b/src/lib/azure/runner.ts @@ -0,0 +1,577 @@ +import { spawn, spawnSync, type ChildProcess } from "child_process"; +import { randomUUID } from "crypto"; +import path from "path"; +import { StringDecoder } from "string_decoder"; +import type { + AzExecutable, + AzRunner, + AzRunOptions, + AzRunResult, + EgressProxy, + EgressRecords, +} from "./types"; + +/** + * The dedicated `az` runner. The shared + * `core/command-runner.ts` is not used: it decodes per chunk (a character split at a + * 64 KB chunk boundary becomes U+FFFD), resolves on `'close'` (a grandchild holding + * the pipe makes it hang past its timeout), and its SIGKILL escalation never runs. + */ + +/** Byte cap per stream; a breach kills the tree and marks the result truncated. */ +export const MAX_STREAM_BYTES = 10 * 1024 * 1024; +/** `spawn()` throws ENAMETOOLONG at 32,767 UTF-16 units for the whole command line. */ +export const WINDOWS_COMMAND_LINE_LIMIT = 32_000; +/** At most four `az` runs at once per server process. */ +export const MAX_CONCURRENT_RUNS = 4; + +export interface RunnerTimings { + /** After `'exit'`, how long to wait for the pipes before destroying them. */ + drainGraceMs: number; + /** POSIX: SIGTERM to the group, then SIGKILL after this. */ + posixKillGraceMs: number; + /** After a refused egress or a failed upstream, how long before the tree is killed. */ + failFastDelayMs: number; + progressIntervalMs: number; +} + +export const DEFAULT_TIMINGS: RunnerTimings = { + drainGraceMs: 1000, + posixKillGraceMs: 2000, + failFastDelayMs: 1000, + progressIntervalMs: 10_000, +}; + +export const emptyEgressRecords = (): EgressRecords => ({ + refused: [], + upstream: [], + housekeeping: [], + allowed: 0, +}); + +/** A counting semaphore; `acquire` resolves false when the signal aborts first. */ +export class Semaphore { + private active = 0; + private readonly waiters: Array<() => void> = []; + + constructor(private readonly limit: number) {} + + get inUse(): number { + return this.active; + } + + acquire(signal?: AbortSignal): Promise { + if (signal?.aborted) return Promise.resolve(false); + if (this.active < this.limit) { + this.active++; + return Promise.resolve(true); + } + return new Promise((resolve) => { + const grant = () => { + signal?.removeEventListener("abort", onAbort); + this.active++; + resolve(true); + }; + const onAbort = () => { + const index = this.waiters.indexOf(grant); + if (index >= 0) this.waiters.splice(index, 1); + resolve(false); + }; + signal?.addEventListener("abort", onAbort, { once: true }); + this.waiters.push(grant); + }); + } + + release(): void { + this.active--; + const next = this.waiters.shift(); + if (next) next(); + } +} + +/** Shared by every runner (the worker too): at most four az runs at once. */ +export const runSlots = new Semaphore(MAX_CONCURRENT_RUNS); + +/** Runs in flight in this process (tests). */ +export function activeRuns(): number { + return runSlots.inUse; +} + +// --------------------------------------------------------------------------- +// Killing process trees +// --------------------------------------------------------------------------- + +/** Absolute, so PATH cannot substitute it; a reduced environment may lack SystemRoot. */ +export function taskkillPath(env: NodeJS.ProcessEnv = process.env): string { + const root = env.SystemRoot || env.SYSTEMROOT || "C:\\Windows"; + return path.win32.join(root, "System32", "taskkill.exe"); +} + +export type KillStep = "taskkill" | "kill" | "sigterm" | "sigkill"; + +export interface KillOptions { + platform: NodeJS.Platform; + taskkill?: string; + posixKillGraceMs?: number; + onKillStep?: (step: KillStep, status?: number) => void; +} + +function runTaskkill(file: string, pid: number): Promise { + return new Promise((resolve) => { + let child: ChildProcess; + try { + child = spawn(file, ["/PID", String(pid), "/T", "/F"], { + shell: false, + windowsHide: true, + stdio: "ignore", + }); + } catch { + resolve(-1); + return; + } + const timer = setTimeout(() => { + child.kill(); + resolve(-1); + }, 10_000); + child.on("error", () => { + clearTimeout(timer); + resolve(-1); + }); + child.on("exit", (code) => { + clearTimeout(timer); + resolve(code ?? -1); + }); + }); +} + +function signalGroup(pid: number, signal: NodeJS.Signals): boolean { + try { + process.kill(-pid, signal); + return true; + } catch { + return false; + } +} + +/** + * Kill a child and everything it started. + * - Windows: `taskkill /PID /T /F` FIRST, while the root is alive; once the + * root is gone taskkill can no longer find the tree (exit 128) and orphans keep + * the pipe open. It runs through an awaited asynchronous spawn, so the event loop, + * and the in-process egress guard with it, is not blocked for its ~0.4 s. Then + * `child.kill()` only if taskkill failed. The kill list is never built by walking + * parent PIDs, which is unsafe under PID reuse. + * - POSIX: the child leads its own process group (`detached`): SIGTERM to the group, + * then SIGKILL after a grace period. + */ +export async function killTree(child: ChildProcess, opts: KillOptions): Promise { + const pid = child.pid; + if (!pid) return; + if (opts.platform === "win32") { + const status = await runTaskkill(opts.taskkill ?? taskkillPath(), pid); + opts.onKillStep?.("taskkill", status); + if (status !== 0) { + opts.onKillStep?.("kill"); + try { + child.kill(); + } catch { + // already gone + } + } + return; + } + opts.onKillStep?.("sigterm"); + if (!signalGroup(pid, "SIGTERM")) child.kill("SIGTERM"); + setTimeout(() => { + opts.onKillStep?.("sigkill"); + // The group id cannot be reused while any member lives, so this reaches only + // the tree this runner started. + signalGroup(pid, "SIGKILL"); + }, opts.posixKillGraceMs ?? DEFAULT_TIMINGS.posixKillGraceMs).unref(); +} + +/** The synchronous form, for `process.on("exit")`, where nothing asynchronous runs. */ +export function killTreeSync(child: ChildProcess, platform: NodeJS.Platform): void { + const pid = child.pid; + if (!pid || child.exitCode !== null || child.signalCode !== null) return; + if (platform === "win32") { + const result = spawnSync(taskkillPath(), ["/PID", String(pid), "/T", "/F"], { + windowsHide: true, + stdio: "ignore", + timeout: 5000, + }); + if (result.status !== 0) { + try { + child.kill(); + } catch { + // already gone + } + } + return; + } + if (!signalGroup(pid, "SIGKILL")) { + try { + child.kill("SIGKILL"); + } catch { + // already gone + } + } +} + +// A registry of live children. The server exits 1.25 s after the client closes +// stdin (src/cli/lifecycle.ts); an interrupted `aks create` would otherwise keep +// changing the emulator. +const liveChildren = new Map(); +const TERMINATING_SIGNALS: NodeJS.Signals[] = ["SIGTERM", "SIGINT", "SIGHUP"]; +let exitHookInstalled = false; +let signalHooksInstalled = false; + +/** Kill every live `az` tree now (the exit hook; exported for tests). */ +export function killAllLiveTreesSync(): void { + for (const [child, platform] of liveChildren) killTreeSync(child, platform); + liveChildren.clear(); +} + +function onTerminatingSignal(signal: NodeJS.Signals) { + killAllLiveTreesSync(); + removeSignalHooks(); + // Re-raise, so the default action (or another handler) still ends the server. + process.kill(process.pid, signal); +} + +function removeSignalHooks() { + if (!signalHooksInstalled) return; + signalHooksInstalled = false; + for (const signal of TERMINATING_SIGNALS) process.removeListener(signal, onTerminatingSignal); +} + +export function trackChild(child: ChildProcess, platform: NodeJS.Platform) { + liveChildren.set(child, platform); + if (!exitHookInstalled) { + exitHookInstalled = true; + process.on("exit", killAllLiveTreesSync); + } + // POSIX children lead their own process group, so a signal that ends the server + // would not reach them. The hooks exist only while a child is live, because a + // listener suppresses a signal's default action. + if (platform !== "win32" && !signalHooksInstalled) { + signalHooksInstalled = true; + for (const signal of TERMINATING_SIGNALS) process.on(signal, onTerminatingSignal); + } +} + +export function untrackChild(child: ChildProcess) { + liveChildren.delete(child); + if (liveChildren.size === 0) removeSignalHooks(); +} + +/** Live children in this process (tests). */ +export function liveChildCount(): number { + return liveChildren.size; +} + +// --------------------------------------------------------------------------- +// Spawning +// --------------------------------------------------------------------------- + +/** libuv's quoting of one argument for the Windows command line (quote_cmd_arg). */ +export function quoteWindowsArg(arg: string): string { + if (arg.length === 0) return '""'; + if (!/[\t "]/.test(arg)) return arg; + if (!/["\\]/.test(arg)) return `"${arg}"`; + let quoted = ""; + let backslashes = 0; + for (const ch of arg) { + if (ch === "\\") { + backslashes++; + continue; + } + if (ch === '"') { + quoted += "\\".repeat(backslashes * 2 + 1) + '"'; + } else { + quoted += "\\".repeat(backslashes) + ch; + } + backslashes = 0; + } + return `"${quoted}${"\\".repeat(backslashes * 2)}"`; +} + +export function windowsCommandLineLength(file: string, args: string[]): number { + return [file, ...args].map(quoteWindowsArg).join(" ").length; +} + +function describeSpawnError(error: unknown): string { + const e = error as NodeJS.ErrnoException; + return e?.code ? `${e.code}: ${e.message}` : String(error); +} + +export interface HostRunnerOptions { + exe: AzExecutable; + /** The child environment from buildAzChildEnv(); must carry AZURE_CONFIG_DIR. */ + env: Record; + /** Containment layer 4; undefined only with LOCALSTACK_AZ_EGRESS_GUARD=0. */ + proxy?: EgressProxy; + platform?: NodeJS.Platform; + timings?: Partial; + maxStreamBytes?: number; + /** Tests: another taskkill binary, and a hook that records the kill steps. */ + taskkill?: string; + onKillStep?: (step: KillStep, status?: number) => void; +} + +export class HostRunner implements AzRunner { + private readonly timings: RunnerTimings; + + constructor(private readonly opts: HostRunnerOptions) { + this.timings = { ...DEFAULT_TIMINGS, ...opts.timings }; + } + + async run(argv: string[], o: AzRunOptions): Promise { + if (!this.opts.env.AZURE_CONFIG_DIR) { + // Containment layer 1: never run az against the user's own profile. + throw new Error("Refusing to run az without an isolated AZURE_CONFIG_DIR."); + } + const acquired = await runSlots.acquire(o.signal); + if (!acquired) return this.result({ aborted: true }); + try { + return await this.runOnce(argv, o); + } finally { + runSlots.release(); + } + } + + private result(fields: Partial): AzRunResult { + return { + exitCode: null, + stdout: "", + stderr: "", + timedOut: false, + aborted: false, + truncated: false, + durationMs: 0, + egress: emptyEgressRecords(), + ...fields, + }; + } + + private runOnce(argv: string[], o: AzRunOptions): Promise { + const { exe, proxy } = this.opts; + const platform = this.opts.platform ?? process.platform; + const maxBytes = this.opts.maxStreamBytes ?? MAX_STREAM_BYTES; + const started = Date.now(); + if (o.signal?.aborted) return Promise.resolve(this.result({ aborted: true })); + + const args = [...exe.prefixArgs, ...argv]; + if ( + platform === "win32" && + windowsCommandLineLength(exe.file, args) > WINDOWS_COMMAND_LINE_LIMIT + ) { + return Promise.resolve(this.result({ tooLong: true })); + } + + // Not a secret: Bicep's BCP192 error prints the proxy URL, tag included. + const callId = randomUUID(); + const env = proxy ? { ...this.opts.env, ...proxy.envFor(callId) } : this.opts.env; + const takeEgress = () => (proxy ? proxy.takeRecords(callId) : emptyEgressRecords()); + + let child: ChildProcess; + try { + child = spawn(exe.file, args, { + shell: false, + windowsHide: true, + // Not "ignore": on Windows that is NUL, which Python treats as a tty, so + // prompts would leak into stdout. + stdio: ["pipe", "pipe", "pipe"], + env, + cwd: o.cwd, + detached: platform !== "win32", + }); + } catch (error) { + // EINVAL (a .cmd/.bat file) and ENAMETOOLONG are thrown synchronously. + return Promise.resolve( + this.result({ + spawnError: describeSpawnError(error), + durationMs: Date.now() - started, + egress: takeEgress(), + }) + ); + } + + return new Promise((resolve) => { + const timers: NodeJS.Timeout[] = []; + let settled = false; + let exited = false; + let exitCode: number | null = null; + let killing = false; + let timedOut = false; + let aborted = false; + let failFast: AzRunResult["failFast"]; + let unsubscribe: (() => void) | undefined; + const streams = { + stdout: { + decoder: new StringDecoder("utf8"), + text: "", + bytes: 0, + ended: false, + capped: false, + }, + stderr: { + decoder: new StringDecoder("utf8"), + text: "", + bytes: 0, + ended: false, + capped: false, + }, + }; + + const kill = () => { + if (killing || exited) return; + killing = true; + void killTree(child, { + platform, + taskkill: this.opts.taskkill, + posixKillGraceMs: this.timings.posixKillGraceMs, + onKillStep: this.opts.onKillStep, + }); + }; + + const finish = (fields: Partial = {}) => { + if (settled) return; + settled = true; + for (const t of timers) clearTimeout(t); + o.signal?.removeEventListener("abort", onAbort); + unsubscribe?.(); + untrackChild(child); + for (const s of Object.values(streams)) { + // A capped stream ends mid-character: its tail is dropped, not decoded as U+FFFD. + if (!s.capped) s.text += s.decoder.end(); + } + child.stdout?.destroy(); + child.stderr?.destroy(); + child.stdin?.destroy(); + resolve( + this.result({ + exitCode, + stdout: streams.stdout.text, + stderr: streams.stderr.text, + timedOut, + aborted, + truncated: streams.stdout.capped || streams.stderr.capped, + durationMs: Date.now() - started, + egress: takeEgress(), + failFast, + ...fields, + }) + ); + }; + + const maybeFinish = () => { + if (exited && streams.stdout.ended && streams.stderr.ended) finish(); + }; + + const onAbort = () => { + aborted = true; + kill(); + }; + + trackChild(child, platform); + + child.on("error", (error) => { + // The async form of a failed start (ENOENT for an extensionless script); + // "file missing" is not implied. + if (child.pid === undefined) finish({ spawnError: describeSpawnError(error) }); + }); + child.on("exit", (code) => { + exited = true; + exitCode = code; + // Resolve once the pipes drain, or after the grace: a grandchild holding the + // pipe would otherwise keep 'close' from ever firing. + timers.push(setTimeout(() => finish(), this.timings.drainGraceMs)); + maybeFinish(); + }); + + for (const [name, stream] of [ + ["stdout", child.stdout], + ["stderr", child.stderr], + ] as const) { + const s = streams[name]; + stream?.on("error", () => undefined); + stream?.on("end", () => { + s.ended = true; + maybeFinish(); + }); + stream?.on("data", (chunk: Buffer) => { + if (s.capped) return; + const room = maxBytes - s.bytes; + if (chunk.length > room) { + s.text += s.decoder.write(chunk.subarray(0, room)); + s.bytes = maxBytes; + s.capped = true; + kill(); + return; + } + s.bytes += chunk.length; + s.text += s.decoder.write(chunk); + }); + } + + // The child sees a non-tty stdin already at EOF: it can neither block on a + // prompt nor read the server's JSON-RPC stream. + child.stdin?.on("error", () => undefined); + child.stdin?.end(); + + timers.push( + setTimeout(() => { + timedOut = true; + kill(); + }, o.timeoutMs) + ); + o.signal?.addEventListener("abort", onAbort, { once: true }); + if (o.onProgress) { + const onProgress = o.onProgress; + timers.push( + setInterval(() => onProgress(Date.now() - started), this.timings.progressIntervalMs) + ); + } + + // Fail fast: after a refused host or a failed upstream, SDK paths + // retry for about 87 s and then print an error without the host in it. + unsubscribe = proxy?.onEvent((id, event) => { + if (id !== callId || failFast || (event.kind !== "refused" && event.kind !== "upstream")) + return; + const kind = event.kind; + timers.push( + setTimeout(() => { + if (exited) return; + failFast = kind; + kill(); + }, this.timings.failFastDelayMs) + ); + }); + }); + } +} + +/** + * `notifications/progress` every 10 s while `az` runs, when the client asked for + * progress. Undefined without a progress token. + */ +export function progressSender(extra?: { + _meta?: { progressToken?: string | number }; + sendNotification?: (notification: unknown) => Promise; +}): ((elapsedMs: number) => void) | undefined { + const token = extra?._meta?.progressToken; + const send = extra?.sendNotification; + if (token === undefined || !send) return undefined; + return (elapsedMs) => { + const seconds = Math.round(elapsedMs / 1000); + void send({ + method: "notifications/progress", + params: { + progressToken: token, + progress: seconds, + message: `az is still running (${seconds} s)`, + }, + }).catch(() => undefined); + }; +} diff --git a/src/lib/azure/runtime-status.test.ts b/src/lib/azure/runtime-status.test.ts new file mode 100644 index 0000000..ad377d1 --- /dev/null +++ b/src/lib/azure/runtime-status.test.ts @@ -0,0 +1,95 @@ +import { azureStartedCheck, getAzureRuntimeStatus } from "./runtime-status"; +import { getAzureEmulatorStatus } from "./emulator"; +import { ensureLoopbackForwarder } from "./loopback-forwarder"; +import { azureConfig } from "./services"; + +// localstack-management's view of the Azure emulator. +jest.mock("./emulator", () => ({ getAzureEmulatorStatus: jest.fn() })); +jest.mock("./loopback-forwarder", () => ({ + ensureLoopbackForwarder: jest.fn().mockResolvedValue(undefined), +})); +jest.mock("./services", () => ({ azureConfig: jest.fn() })); + +const mockedStatus = getAzureEmulatorStatus as jest.MockedFunction; +const mockedConfig = azureConfig as jest.MockedFunction; +const mockedForwarder = ensureLoopbackForwarder as jest.MockedFunction< + typeof ensureLoopbackForwarder +>; + +const config = (inDocker = false) => + ({ inDocker, port: 4566, healthBaseUrl: "http://127.0.0.1:4566" }) as never; + +describe("getAzureRuntimeStatus", () => { + beforeEach(() => jest.clearAllMocks()); + + test("ready: the emulator answered as Azure, with HTTPS; a single HTTPS probe is asked for", async () => { + mockedConfig.mockReturnValue(config()); + mockedStatus.mockResolvedValue({ + ok: true, + edition: "azure-alpha", + license: true, + version: "2026.9.0", + }); + await expect(getAzureRuntimeStatus()).resolves.toMatchObject({ + isRunning: true, + isReady: true, + statusOutput: "edition: azure-alpha, license: true, version: 2026.9.0", + }); + expect(mockedStatus.mock.calls[0][2]).toEqual({ httpsWaitMs: 0 }); + expect(mockedForwarder).not.toHaveBeenCalled(); + }); + + test.each(["https-not-ready", "license", "not-responding"] as const)( + "%s: running but not ready", + async (problem) => { + mockedConfig.mockReturnValue(config()); + mockedStatus.mockResolvedValue({ ok: false, problem, edition: "azure-alpha" }); + await expect(getAzureRuntimeStatus()).resolves.toMatchObject({ + isRunning: true, + isReady: false, + }); + } + ); + + test("not-responding (an open port, no answer) is marked unresponsive; the others are not", async () => { + mockedConfig.mockReturnValue(config()); + mockedStatus.mockResolvedValueOnce({ ok: false, problem: "not-responding" }); + await expect(getAzureRuntimeStatus()).resolves.toMatchObject({ + isRunning: true, + unresponsive: true, + }); + for (const problem of ["https-not-ready", "license"] as const) { + mockedStatus.mockResolvedValueOnce({ ok: false, problem, edition: "azure-alpha" }); + await expect(getAzureRuntimeStatus()).resolves.toMatchObject({ + isRunning: true, + unresponsive: false, + }); + } + }); + + test.each(["not-running", "wrong-edition"] as const)("%s: not running", async (problem) => { + mockedConfig.mockReturnValue(config()); + mockedStatus.mockResolvedValue({ ok: false, problem }); + await expect(getAzureRuntimeStatus()).resolves.toMatchObject({ + isRunning: false, + isReady: false, + }); + }); + + test("in Docker, the loopback forwarder is tried first", async () => { + mockedConfig.mockReturnValue(config(true)); + mockedStatus.mockResolvedValue({ ok: false, problem: "not-running" }); + await getAzureRuntimeStatus(); + expect(mockedForwarder).toHaveBeenCalled(); + }); +}); + +describe("azureStartedCheck", () => { + test("an inactive licence is an error; HTTPS still starting is not", async () => { + mockedConfig.mockReturnValue(config()); + mockedStatus.mockResolvedValueOnce({ ok: false, problem: "license", message: "no licence" }); + expect((await azureStartedCheck())?.content[0].text).toMatch(/License Not Active/); + mockedStatus.mockResolvedValueOnce({ ok: false, problem: "https-not-ready", message: "later" }); + expect(await azureStartedCheck()).toBeNull(); + }); +}); diff --git a/src/lib/azure/runtime-status.ts b/src/lib/azure/runtime-status.ts new file mode 100644 index 0000000..2640898 --- /dev/null +++ b/src/lib/azure/runtime-status.ts @@ -0,0 +1,62 @@ +import { ResponseBuilder } from "../../core/response-builder"; +import { getAzureEmulatorStatus, type AzureEmulatorStatus } from "./emulator"; +import { ensureLoopbackForwarder } from "./loopback-forwarder"; +import { azureConfig } from "./services"; + +/** + * `localstack-management`'s view of the Azure emulator. It + * is "running" once the gateway answers as the Azure edition, and "ready" once the + * ARM endpoint's HTTPS listener answers too, which comes a few seconds later. + */ +export interface AzureRuntimeStatus { + isRunning: boolean; + isReady: boolean; + /** The port accepts connections but the health check gets no answer (RuntimeStatus). */ + unresponsive?: boolean; + statusOutput?: string; + status: AzureEmulatorStatus; +} + +export async function getAzureRuntimeStatus(): Promise { + const config = azureConfig(); + // In the Docker image the emulator is reached through the loopback forwarder, + // which retries its targets on every call until the emulator is up. + if (config.inDocker) await ensureLoopbackForwarder().catch(() => undefined); + const status = await getAzureEmulatorStatus(config, undefined, { httpsWaitMs: 0 }); + // A busy emulator that does not answer its health check still runs: `start` must not + // launch a second one beside it. + const isRunning = + status.ok || + status.problem === "https-not-ready" || + status.problem === "license" || + status.problem === "not-responding"; + const details = [ + status.edition && `edition: ${status.edition}`, + status.license !== undefined && `license: ${status.license}`, + status.version && `version: ${status.version}`, + ].filter(Boolean); + return { + isRunning, + isReady: status.ok, + // Running for the pre-start check, but a start waiting for its own container keeps + // polling: a just-started container looks exactly like this. + unresponsive: status.problem === "not-responding", + statusOutput: details.length ? details.join(", ") : undefined, + status, + }; +} + +/** + * The start action's last step: wait for HTTPS (up to 10 s) and refuse an emulator + * without a licence. HTTPS that is still starting is not a failure: the Azure tool + * waits for it on its first call. + */ +export async function azureStartedCheck(): Promise | null> { + const status = await getAzureEmulatorStatus(azureConfig()); + if (status.problem === "license") { + return ResponseBuilder.error("LocalStack Azure License Not Active", status.message); + } + return null; +} diff --git a/src/lib/azure/services.test.ts b/src/lib/azure/services.test.ts new file mode 100644 index 0000000..f1dde0c --- /dev/null +++ b/src/lib/azure/services.test.ts @@ -0,0 +1,554 @@ +import { mkdirSync, mkdtempSync, writeFileSync } from "fs"; +import os from "os"; +import path from "path"; +import { ensureAzureCliConfigured, runWithSelfHeal } from "./bootstrap"; +import { buildAzChildEnv } from "./child-env"; +import { startEgressProxy } from "./egress-proxy"; +import { + listInstalledExtensions, + probeArgs, + resolveAz, + resolveBicep, + type AzProbeOutcome, + type LocatedAz, +} from "./resolve-az"; +import { HostRunner } from "./runner"; +import * as services from "./services"; +import type { AzExecutable, AzRunResult } from "./types"; +import { WorkerRunner } from "./worker-runner"; + +// The composition root's wiring: what each service is built from, what is +// cached, and what a failure resets. Every process-spawning piece is a stand-in; the +// probes' parsing and the runner choice are the real code. + +jest.mock("./runner", () => ({ HostRunner: jest.fn() })); +jest.mock("./worker-runner", () => ({ WorkerRunner: jest.fn() })); +jest.mock("./egress-proxy", () => ({ startEgressProxy: jest.fn() })); +jest.mock("./child-env", () => ({ + buildAzChildEnv: jest.fn((_env: unknown, opts: { bicepDir?: string }) => ({ + CHILD_ENV: "1", + BICEP_DIR: opts.bicepDir ?? "", + })), + ensurePrivateDirs: jest.fn(), +})); +jest.mock("./bootstrap", () => ({ + ...jest.requireActual("./bootstrap"), + ensureAzureCliConfigured: jest.fn(), + runWithSelfHeal: jest.fn(), +})); +jest.mock("./resolve-az", () => ({ + ...jest.requireActual("./resolve-az"), + resolveAz: jest.fn(), + resolveBicep: jest.fn(), + listInstalledExtensions: jest.fn(() => [{ name: "fleet", version: "1.4.0" }]), +})); + +const mockedHost = HostRunner as unknown as jest.Mock; +const mockedWorker = WorkerRunner as unknown as jest.Mock; +const mockedGuard = startEgressProxy as jest.MockedFunction; +const mockedResolveAz = resolveAz as jest.MockedFunction; +const mockedResolveBicep = resolveBicep as jest.MockedFunction; +const mockedChildEnv = buildAzChildEnv as unknown as jest.Mock; +const mockedEnsure = ensureAzureCliConfigured as jest.MockedFunction< + typeof ensureAzureCliConfigured +>; +const mockedSelfHeal = runWithSelfHeal as jest.MockedFunction; + +const root = mkdtempSync(path.join(os.tmpdir(), "lsaz-services-")); +const workdir = path.join(root, "work"); +const configDir = path.join(root, "mcp-config"); +const extensionDir = path.join(root, "extensions"); +mkdirSync(workdir); +mkdirSync(extensionDir); + +const PY: AzExecutable = { + file: "/opt/az/bin/python3", + prefixArgs: ["-X", "utf8", "-IBm", "azure.cli"], + installer: "pip", + azInstaller: "PIP", + version: "2.90.0", +}; +const LAUNCHER: AzExecutable = { + file: "/usr/local/bin/az", + prefixArgs: [], + installer: "launcher-as-is", +}; + +function result(over: Partial = {}): AzRunResult { + return { + exitCode: 0, + stdout: "", + stderr: "", + timedOut: false, + aborted: false, + truncated: false, + durationMs: 1, + egress: { refused: [], upstream: [], housekeeping: [], allowed: 0 }, + ...over, + }; +} + +/** Every HostRunner the services built, with the calls made on it. */ +let hosts: Array<{ + opts: Record; + calls: Array<{ argv: string[]; opts: unknown }>; +}>; +let respond: (argv: string[]) => AzRunResult; +const guard = { close: jest.fn().mockResolvedValue(undefined) }; +const worker = { close: jest.fn().mockResolvedValue(undefined), run: jest.fn() }; +const savedEnv = { ...process.env }; + +beforeEach(async () => { + await services.resetAzureServices(); + jest.clearAllMocks(); + process.env = { + ...savedEnv, + LOCALSTACK_AZ_WORKDIR: workdir, + LOCALSTACK_AZ_CONFIG_DIR: configDir, + LOCALSTACK_AZ_EXTENSION_DIR: extensionDir, + }; + for (const name of [ + "LOCALSTACK_AZ_EGRESS_GUARD", + "LOCALSTACK_AZ_RUNNER", + "LOCALSTACK_AZ_PATH", + "LOCALSTACK_AZ_DENYLIST_FILE", + "LOCALSTACK_AZ_PYCACHE_DIR", + "LOCALSTACK_AZ_TIMEOUT_SECONDS", + "AZURE_CONFIG_DIR", + ]) { + delete process.env[name]; + } + hosts = []; + respond = () => result(); + mockedHost.mockImplementation((opts: Record) => { + const record = { opts, calls: [] as Array<{ argv: string[]; opts: unknown }> }; + hosts.push(record); + return { + run: jest.fn(async (argv: string[], runOpts: unknown) => { + record.calls.push({ argv, opts: runOpts }); + return respond(argv); + }), + }; + }); + mockedWorker.mockImplementation(() => worker); + mockedGuard.mockResolvedValue(guard as never); + mockedResolveAz.mockResolvedValue(PY); + mockedResolveBicep.mockResolvedValue(undefined); +}); + +afterAll(async () => { + await services.resetAzureServices(); + process.env = savedEnv; +}); + +describe("configuration and policy inputs", () => { + test("the config is read once, and a reset reads it again", async () => { + const first = services.azureConfig(); + expect(services.azureConfig()).toBe(first); + process.env.LOCALSTACK_AZ_TIMEOUT_SECONDS = "77"; + expect(services.azureConfig().timeoutMs).toBe(first.timeoutMs); + await services.resetAzureServices(); + expect(services.azureConfig().timeoutMs).toBe(77_000); + }); + + test("the policy gets the workdir, the private home, the protected dirs and the denylist", () => { + const denylist = path.join(root, "denylist.txt"); + writeFileSync( + denylist, + "# the user's own rules\nwebapp deploy\n\nvm run-command # no remote shells\n" + ); + process.env.LOCALSTACK_AZ_DENYLIST_FILE = denylist; + const options = services.policyOptions(); + expect(options.extraDenied).toEqual([ + ["webapp", "deploy"], + ["vm", "run-command"], + ]); + expect(options.workdir).toBe(services.azureConfig().workdir); + expect(options.homeDir).toBe(services.azureConfig().homeDir); + expect(options.platform).toBe(process.platform); + expect(options.protectedDirs).toEqual(services.protectedDirs()); + expect(options.protectedDirs).toContain(path.join(os.homedir(), ".azure")); + expect(services.policyOptions()).toBe(options); + }); + + test("a denylist file that vanished gives an empty list (the start check reports it)", () => { + process.env.LOCALSTACK_AZ_DENYLIST_FILE = path.join(root, "no-such-denylist.txt"); + expect(services.policyOptions().extraDenied).toEqual([]); + }); + + test("the extension lists come from the tool's extension dir", () => { + mkdirSync(path.join(extensionDir, "fleet", "fleet-1.4.0.dist-info"), { recursive: true }); + expect([...services.installedExtensionNames()]).toEqual(["fleet"]); + expect(services.installedExtensions()).toEqual([{ name: "fleet", version: "1.4.0" }]); + expect(listInstalledExtensions).toHaveBeenCalledWith( + services.azureConfig().extensionDir, + expect.anything(), + process.platform + ); + }); + + test("the emulator session is recorded for the bootstrap", async () => { + expect(services.emulatorSessionId()).toBeUndefined(); + services.recordEmulatorSession("session-1"); + expect(services.emulatorSessionId()).toBe("session-1"); + const config = services.azureConfig(); + await expect(services.bootstrapTarget()).resolves.toEqual({ + configDir: config.configDir, + endpoint: config.endpoint, + sessionId: "session-1", + azVersion: "2.90.0", + cwd: config.workdir, + timeoutMs: config.timeoutMs, + }); + }); +}); + +describe("the egress guard", () => { + test("starts once, on first use, and logs to stderr only", async () => { + const first = await services.egressGuard(); + expect(await services.egressGuard()).toBe(first); + expect(first).toBe(guard); + expect(mockedGuard).toHaveBeenCalledTimes(1); + const write = jest.spyOn(process.stderr, "write").mockImplementation(() => true); + try { + mockedGuard.mock.calls[0][0]!.log!("refused example.com"); + expect(write).toHaveBeenCalledWith("[localstack-azure-client] refused example.com\n"); + } finally { + write.mockRestore(); + } + }); + + test("LOCALSTACK_AZ_EGRESS_GUARD=0 starts none", async () => { + process.env.LOCALSTACK_AZ_EGRESS_GUARD = "0"; + await expect(services.egressGuard()).resolves.toBeUndefined(); + expect(mockedGuard).not.toHaveBeenCalled(); + }); + + test("a reset closes it", async () => { + await services.egressGuard(); + await services.resetAzureServices(); + expect(guard.close).toHaveBeenCalledTimes(1); + }); +}); + +describe("resolving az: the probe resolveAz calls", () => { + const located: LocatedAz = { file: "/opt/az/bin/python3", prefixArgs: [], installer: "pip" }; + + async function probeWith(exe: LocatedAz, safePathFallback = false): Promise { + let outcome: AzProbeOutcome | undefined; + mockedResolveAz.mockImplementationOnce(async (_ctx, probe) => { + outcome = await probe(exe, safePathFallback); + return PY; + }); + await services.azCli(); + await services.resetAzureServices(); + return outcome!; + } + + test("an interpreter is probed with the import code, in the private temp dir", async () => { + respond = () => result({ stdout: 'a warning line\n{"python": "3.12.4", "core": "2.90.0"}\n' }); + await expect(probeWith(located, true)).resolves.toEqual({ + ok: true, + python: "3.12.4", + core: "2.90.0", + error: undefined, + }); + expect(hosts[0].opts.exe).toEqual({ + file: located.file, + prefixArgs: probeArgs(true), + installer: "pip", + }); + expect(hosts[0].opts.proxy).toBe(guard); + expect(hosts[0].calls[0]).toEqual({ + argv: [], + opts: { timeoutMs: 60_000, cwd: services.azureConfig().tmpDir }, + }); + }); + + test("a failed import reports the Python version and the error", async () => { + respond = () => + result({ stdout: '{"python": "3.10.2", "error": "ModuleNotFoundError()"}', exitCode: 0 }); + await expect(probeWith(located)).resolves.toEqual({ + ok: false, + python: "3.10.2", + core: undefined, + error: "ModuleNotFoundError()", + }); + }); + + test("no JSON: the spawn error, else stderr, else the exit code", async () => { + respond = () => result({ exitCode: null, spawnError: "spawn ENOENT" }); + expect((await probeWith(located)).error).toBe("spawn ENOENT"); + respond = () => result({ exitCode: 1, stderr: " Traceback: boom " }); + expect((await probeWith(located)).error).toBe("Traceback: boom"); + respond = () => result({ exitCode: 9 }); + expect(await probeWith(located)).toEqual({ ok: false, error: "exit 9" }); + }); + + test("a launcher run as-is is probed with `az version`", async () => { + const launcher: LocatedAz = { + file: LAUNCHER.file, + prefixArgs: [], + installer: "launcher-as-is", + }; + respond = () => result({ stdout: '{"azure-cli": "2.90.0", "azure-cli-core": "2.90.0"}' }); + await expect(probeWith(launcher)).resolves.toEqual({ ok: true, core: "2.90.0" }); + expect(hosts[0].calls[0].argv).toEqual(["version", "-o", "json"]); + expect(hosts[0].calls[0].opts).toMatchObject({ timeoutMs: 120_000 }); + + respond = () => result({ exitCode: 2, stderr: "az: broken install" }); + await expect(probeWith(launcher)).resolves.toEqual({ ok: false, error: "az: broken install" }); + respond = () => result({ exitCode: 2, spawnError: "spawn EACCES" }); + await expect(probeWith(launcher)).resolves.toEqual({ ok: false, error: "spawn EACCES" }); + respond = () => result({ stdout: "not json" }); + await expect(probeWith(launcher)).resolves.toEqual({ + ok: false, + error: "`az version -o json` did not print JSON", + }); + }); + + test("az is resolved once; LOCALSTACK_AZ_PATH and the pycache dir reach the resolver", async () => { + process.env.LOCALSTACK_AZ_PATH = "/custom/az"; + process.env.LOCALSTACK_AZ_PYCACHE_DIR = path.join(root, "pycache-ok"); + await expect(services.azCli()).resolves.toBe(PY); + await services.azCli(); + expect(mockedResolveAz).toHaveBeenCalledTimes(1); + const ctx = mockedResolveAz.mock.calls[0][0]; + expect(ctx.env.LOCALSTACK_AZ_PATH).toBe("/custom/az"); + expect(ctx.platform).toBe(process.platform); + expect(ctx.homedir).toBe(os.homedir()); + expect(ctx.pycacheDir).toBe(path.join(root, "pycache-ok")); + // Windows has no /proc/version to read (macOS has none either, Linux does). + if (process.platform === "win32") expect(ctx.procVersion).toBeUndefined(); + }); + + test("a failed resolution is not cached", async () => { + mockedResolveAz.mockRejectedValueOnce(new Error("az not found")); + await expect(services.azCli()).rejects.toThrow("az not found"); + await expect(services.azCli()).resolves.toBe(PY); + expect(mockedResolveAz).toHaveBeenCalledTimes(2); + }); +}); + +describe("safePycacheDir", () => { + test("no setting, no dir", () => { + expect(services.safePycacheDir(undefined)).toBeUndefined(); + }); + + test("the dir is created and used", () => { + const dir = path.join(root, "pycache-new"); + expect(services.safePycacheDir(dir)).toBe(dir); + }); + + test("a path that cannot be a directory is not used", () => { + const file = path.join(root, "a-file"); + writeFileSync(file, "x"); + expect(services.safePycacheDir(path.join(file, "cache"))).toBeUndefined(); + }); + + (process.platform === "win32" ? test.skip : test)( + "a dir others can write is not used, and the log says why", + () => { + const dir = path.join(root, "pycache-shared"); + mkdirSync(dir, { mode: 0o777 }); + // mkdirSync's mode is masked by the umask: set the bits explicitly. + // eslint-disable-next-line @typescript-eslint/no-require-imports + require("fs").chmodSync(dir, 0o777); + const write = jest.spyOn(process.stderr, "write").mockImplementation(() => true); + try { + expect(services.safePycacheDir(dir)).toBeUndefined(); + expect(String(write.mock.calls[0][0])).toContain("writable by others"); + } finally { + write.mockRestore(); + } + } + ); +}); + +describe("Bicep", () => { + test("resolved once, with the Azure profiles excluded, probed with --version", async () => { + process.env.AZURE_CONFIG_DIR = path.join(root, "parent-azure"); + let probed: string | undefined; + mockedResolveBicep.mockImplementationOnce(async (ctx, probe) => { + expect(ctx.excludeDirs).toEqual([`${os.homedir()}/.azure`, path.join(root, "parent-azure")]); + expect(typeof ctx.canonical("x")).toBe("string"); + respond = () => result({ stdout: "Bicep CLI version 0.47.16 (abc)" }); + probed = await probe("/opt/bicep/bicep"); + respond = () => result({ exitCode: 1, stdout: "ignored" }); + expect(await probe("/opt/bicep/bicep")).toBeUndefined(); + return { + path: "/opt/bicep/bicep", + dir: "/opt/bicep", + source: "explicit", + supportsBicepparam: true, + }; + }); + await expect(services.bicep()).resolves.toMatchObject({ dir: "/opt/bicep" }); + await services.bicep(); + expect(mockedResolveBicep).toHaveBeenCalledTimes(1); + expect(probed).toBe("Bicep CLI version 0.47.16 (abc)"); + expect(hosts[0].opts.exe).toEqual({ + file: "/opt/bicep/bicep", + prefixArgs: [], + installer: "pip", + }); + expect(hosts[0].calls[0].argv).toEqual(["--version"]); + }); + + test("without AZURE_CONFIG_DIR only ~/.azure is excluded; a failure is retried", async () => { + mockedResolveBicep.mockImplementationOnce(async (ctx) => { + expect(ctx.excludeDirs).toEqual([`${os.homedir()}/.azure`]); + throw new Error("LOCALSTACK_AZ_BICEP_PATH does not exist"); + }); + await expect(services.bicep()).rejects.toThrow("does not exist"); + await expect(services.bicep()).resolves.toBeUndefined(); + expect(mockedResolveBicep).toHaveBeenCalledTimes(2); + }); + + test("a miss is not cached: a Bicep installed meanwhile is found on the next call", async () => { + // As the Snowflake tool checks for `snow` on each call: installing Bicep with + // install-azure-addons while the server runs needs no restart. + // A flag, not queued mockResolvedValueOnce values: beforeEach replaces this implementation, + // so nothing leaks into the next test even if this one fails midway. + let installed = false; + mockedResolveBicep.mockImplementation(async () => + installed + ? { + path: "/home/u/.localstack/azure/bin/bicep", + dir: "/home/u/.localstack/azure/bin", + source: "tool", + supportsBicepparam: true, + } + : undefined + ); + await expect(services.bicep()).resolves.toBeUndefined(); + installed = true; + await expect(services.bicep()).resolves.toMatchObject({ source: "tool" }); + await services.bicep(); + expect(mockedResolveBicep).toHaveBeenCalledTimes(2); // a hit is still resolved once + }); +}); + +describe("readLocalVersions (the update-check seed)", () => { + test("an interpreter runs azure.cli.core's own version code", async () => { + respond = () => + result({ stdout: 'noise\n{"core": {"local": "2.90.0"}, "telemetry": {"local": "1.1.0"}}\n' }); + await expect(services.readLocalVersions(PY)).resolves.toEqual({ + core: "2.90.0", + telemetry: "1.1.0", + }); + const exe = hosts[0].opts.exe as AzExecutable; + expect(exe.prefixArgs.slice(-2)[0]).toBe("-c"); + expect(exe.prefixArgs.slice(-1)[0]).toContain("_get_local_versions"); + expect(exe.prefixArgs).not.toContain("azure.cli"); + }); + + test("unparseable output gives no versions", async () => { + respond = () => result({ stdout: "Traceback (most recent call last)" }); + await expect(services.readLocalVersions(PY)).resolves.toEqual({}); + }); + + test("a launcher run as-is uses `az version`", async () => { + respond = () => result({ stdout: '{"azure-cli": "2.88.0", "azure-cli-core": "2.88.0"}' }); + await expect(services.readLocalVersions(LAUNCHER)).resolves.toEqual({ + "azure-cli": "2.88.0", + core: "2.88.0", + }); + respond = () => result({ exitCode: 1 }); + await expect(services.readLocalVersions(LAUNCHER)).resolves.toEqual({}); + }); +}); + +describe("the runners", () => { + test("the host runner gets the child env with Bicep's dir, and the guard", async () => { + mockedResolveBicep.mockResolvedValue({ + path: "/opt/bicep/bicep", + dir: "/opt/bicep", + source: "tool", + supportsBicepparam: true, + }); + await services.azRunner(); + const last = hosts[hosts.length - 1]; + expect(last.opts).toEqual({ + exe: PY, + env: { CHILD_ENV: "1", BICEP_DIR: "/opt/bicep" }, + proxy: guard, + }); + expect(mockedChildEnv).toHaveBeenLastCalledWith( + process.env, + expect.objectContaining({ platform: process.platform, az: PY, bicepDir: "/opt/bicep" }) + ); + }); + + test("a bad Bicep path does not stop az commands", async () => { + mockedResolveBicep.mockRejectedValue(new Error("bad LOCALSTACK_AZ_BICEP_PATH")); + await services.azRunner(); + expect(hosts[hosts.length - 1].opts.env).toEqual({ CHILD_ENV: "1", BICEP_DIR: "" }); + }); + + test("by default the bootstrap and commands use the host runner", async () => { + await services.ensureProfile(); + const [target, deps] = mockedEnsure.mock.calls[0]; + expect(target.azVersion).toBe("2.90.0"); + expect(deps.commandRunner).toBe(deps.runner); + respond = () => result({ stdout: '{"core": {"local": "2.90.0"}}' }); + await expect(deps.readLocalVersions()).resolves.toEqual({ core: "2.90.0" }); + expect(mockedWorker).not.toHaveBeenCalled(); + }); + + test("LOCALSTACK_AZ_RUNNER=worker: one warm worker, falling back to the host runner", async () => { + process.env.LOCALSTACK_AZ_RUNNER = "worker"; + mockedResolveBicep.mockRejectedValue(new Error("bad LOCALSTACK_AZ_BICEP_PATH")); + mockedSelfHeal.mockResolvedValue(result({ stdout: "[]" })); + await expect(services.runAzCommand(["group", "list"], {})).resolves.toMatchObject({ + stdout: "[]", + }); + await services.runAzCommand(["group", "list"], {}); + expect(mockedWorker).toHaveBeenCalledTimes(1); + const options = mockedWorker.mock.calls[0][0]; + const config = services.azureConfig(); + expect(options.exe).toBe(PY); + expect(options.env).toEqual({ CHILD_ENV: "1", BICEP_DIR: "" }); + expect(options.proxy).toBe(guard); + expect(options.stateFiles).toEqual( + ["config", "clouds.config", "azureProfile.json"].map((f) => path.join(config.configDir, f)) + ); + expect(options.fallback).toBe(mockedSelfHeal.mock.calls[0][1].runner); + const [, deps, argv, opts] = mockedSelfHeal.mock.calls[0]; + expect(deps.commandRunner).toBe(worker); + expect(argv).toEqual(["group", "list"]); + expect(opts).toEqual({ timeoutMs: config.timeoutMs, cwd: config.workdir }); + + const write = jest.spyOn(process.stderr, "write").mockImplementation(() => true); + try { + options.log("worker replaced"); + expect(write).toHaveBeenCalledWith("[localstack-azure-client] worker replaced\n"); + } finally { + write.mockRestore(); + } + + await services.resetAzureServices(); + expect(worker.close).toHaveBeenCalledTimes(1); + }); + + test("the worker gets Bicep's dir when there is one", async () => { + process.env.LOCALSTACK_AZ_RUNNER = "worker"; + mockedResolveBicep.mockResolvedValue({ + path: "/opt/bicep/bicep", + dir: "/opt/bicep", + source: "tool", + supportsBicepparam: true, + }); + await services.ensureProfile(); + expect(mockedWorker.mock.calls[0][0].env).toEqual({ CHILD_ENV: "1", BICEP_DIR: "/opt/bicep" }); + }); + + test("a launcher run as-is names no Python for the worker: the host runner stays", async () => { + process.env.LOCALSTACK_AZ_RUNNER = "worker"; + mockedResolveAz.mockResolvedValue(LAUNCHER); + await services.ensureProfile(); + const [, deps] = mockedEnsure.mock.calls[0]; + expect(deps.commandRunner).toBe(deps.runner); + expect(mockedWorker).not.toHaveBeenCalled(); + }); +}); diff --git a/src/lib/azure/services.ts b/src/lib/azure/services.ts new file mode 100644 index 0000000..5793168 --- /dev/null +++ b/src/lib/azure/services.ts @@ -0,0 +1,411 @@ +import { mkdirSync, readFileSync, statSync } from "fs"; +import os from "os"; +import path from "path"; +import { azureProtectedDirs, canonicalPath, getAzureConfig } from "../../core/config"; +import { + BootstrapError, + ensureAzureCliConfigured, + runWithSelfHeal, + versionsFromAzVersionJson, + versionsFromLocalVersionsJson, + type BootstrapDeps, + type BootstrapTarget, +} from "./bootstrap"; +import { buildAzChildEnv, ensurePrivateDirs } from "./child-env"; +import { startEgressProxy } from "./egress-proxy"; +import { emulatorPorts } from "./local-hosts"; +import { + interpreterFlags, + listInstalledExtensions, + nodeResolveFs, + probeArgs, + resolveAz, + resolveBicep, + type AzProbeOutcome, + type BicepResolution, + type InstalledExtension, + type LocatedAz, +} from "./resolve-az"; +import { listInstalledExtensions as listExtensionNames } from "./extension-map"; +import { parseDenylistFile } from "./policy"; +import { HostRunner } from "./runner"; +import { WorkerRunner } from "./worker-runner"; +import type { + AzExecutable, + AzRunner, + AzRunOptions, + AzRunResult, + AzureConfig, + EgressProxy, + PolicyOptions, +} from "./types"; + +/** + * Process-wide state of the Azure client tool: the configuration, the resolved `az` + * and Bicep, the egress guard and the runner. Each is created once, on first use, so + * a server that never runs an Azure command never starts any of them. + */ + +let configCache: AzureConfig | undefined; +let azPromise: Promise | undefined; +let bicepPromise: Promise | undefined; +let guardPromise: Promise | undefined; +let lastSessionId: string | undefined; +let workerRunner: WorkerRunner | undefined; + +/** Drop every cached service (tests). */ +export async function resetAzureServices(): Promise { + await workerRunner?.close(); + workerRunner = undefined; + const guard = guardPromise ? await guardPromise.catch(() => undefined) : undefined; + await guard?.close(); + configCache = undefined; + policyCache = undefined; + denylistCache = undefined; + azPromise = undefined; + bicepPromise = undefined; + guardPromise = undefined; + lastSessionId = undefined; +} + +export function azureConfig(): AzureConfig { + configCache ??= getAzureConfig(process.env); + return configCache; +} + +/** The directories the file rule protects even inside the workdir. */ +export function protectedDirs(): string[] { + return azureProtectedDirs(azureConfig(), process.env); +} + +let denylistCache: string[][] | undefined; +let policyCache: PolicyOptions | undefined; + +/** The policy's inputs: workdir, private home, protected dirs and the user's denylist. */ +export function policyOptions(): PolicyOptions { + if (policyCache) return policyCache; + const config = azureConfig(); + if (!denylistCache) { + let text = ""; + try { + text = config.denylistFile ? readFileSync(config.denylistFile, "utf8") : ""; + } catch { + text = ""; // reported by requireAzureConfig at start (the file must exist) + } + denylistCache = parseDenylistFile(text); + } + policyCache = { + workdir: config.workdir, + homeDir: config.homeDir, + extraDenied: denylistCache, + protectedDirs: protectedDirs(), + platform: process.platform, + localPorts: emulatorPorts(gatewayPorts(config)), + }; + return policyCache; +} + +/** Names of the installed extensions, for the missing-extension hint. */ +export function installedExtensionNames(): Set { + return listExtensionNames(azureConfig().extensionDir); +} + +/** The Azure port and the port of the endpoint az calls (LOCALSTACK_AZURE_ENDPOINT may name another). */ +function gatewayPorts(config: { port: number; endpoint: string }): number[] { + try { + const url = new URL(config.endpoint); + return [config.port, Number(url.port) || (url.protocol === "https:" ? 443 : 80)]; + } catch { + return [config.port]; + } +} + +/** Containment layer 4; undefined only with LOCALSTACK_AZ_EGRESS_GUARD=0. */ +export function egressGuard(): Promise { + const config = azureConfig(); + if (!config.egressGuard) return Promise.resolve(undefined); + guardPromise ??= startEgressProxy({ + allowedPorts: emulatorPorts(gatewayPorts(config)), + // stderr only: stdout carries the MCP server's JSON-RPC. + log: (line) => process.stderr.write(`[localstack-azure-client] ${line}\n`), + }); + return guardPromise; +} + +/** The emulator session last seen by requireAzureEmulatorRunning(); the bootstrap keys on it. */ +export function recordEmulatorSession(sessionId: string | undefined): void { + lastSessionId = sessionId; +} + +/** The session recorded above (the test envelope shows it). */ +export function emulatorSessionId(): string | undefined { + return lastSessionId; +} + +function childEnvFor(az: Pick, bicepDir?: string) { + const config = azureConfig(); + ensurePrivateDirs({ platform: process.platform, config }); + return buildAzChildEnv(process.env, { platform: process.platform, config, az, bicepDir }); +} + +async function runOnce(exe: AzExecutable, argv: string[], timeoutMs: number): Promise { + const config = azureConfig(); + const runner = new HostRunner({ exe, env: childEnvFor(exe), proxy: await egressGuard() }); + // Probes run in the private temp dir: nothing the agent wrote can shadow them. + return runner.run(argv, { timeoutMs, cwd: config.tmpDir }); +} + +function lastJsonLine(text: string): unknown { + const line = text.trim().split(/\r?\n/).pop() ?? ""; + try { + return JSON.parse(line); + } catch { + return undefined; + } +} + +/** The probe resolveAz() calls: an import of azure.cli.core, or `az version` for a launcher as-is. */ +async function probeAz(exe: LocatedAz, safePathFallback: boolean): Promise { + if (exe.installer === "launcher-as-is") { + const result = await runOnce({ ...exe }, ["version", "-o", "json"], 120_000); + if (result.exitCode !== 0) + return { ok: false, error: result.stderr.trim().slice(0, 300) || result.spawnError }; + try { + const core = versionsFromAzVersionJson(result.stdout).core; + return { ok: Boolean(core), core }; + } catch { + return { ok: false, error: "`az version -o json` did not print JSON" }; + } + } + const result = await runOnce( + { file: exe.file, prefixArgs: probeArgs(safePathFallback), installer: exe.installer }, + [], + 60_000 + ); + const parsed = lastJsonLine(result.stdout) as + { python?: string; core?: string; error?: string } | undefined; + if (!parsed) { + return { + ok: false, + error: result.spawnError ?? (result.stderr.trim().slice(0, 300) || `exit ${result.exitCode}`), + }; + } + return { + ok: Boolean(parsed.core), + python: parsed.python, + core: parsed.core, + error: parsed.error, + }; +} + +function readProcVersion(): string | undefined { + try { + return readFileSync("/proc/version", "utf8"); + } catch { + return undefined; + } +} + +/** + * The bytecode cache dir (LOCALSTACK_AZ_PYCACHE_DIR, set by the image), created + * owner-only. A writable bytecode cache can be poisoned, so a directory that others can + * write, or that another user owns, is not used. + */ +export function safePycacheDir(dir: string | undefined): string | undefined { + if (!dir) return undefined; + try { + mkdirSync(dir, { recursive: true, mode: 0o700 }); + if (process.platform !== "win32") { + const stat = statSync(dir); + const foreign = typeof process.getuid === "function" && stat.uid !== process.getuid(); + if (foreign || (stat.mode & 0o022) !== 0) { + process.stderr.write( + `[localstack-azure-client] LOCALSTACK_AZ_PYCACHE_DIR ${dir} is writable by others or owned by another user; not used\n` + ); + return undefined; + } + } + return dir; + } catch { + return undefined; + } +} + +/** Resolve and probe `az` once per process (throws AzResolveError). A failure is retried next call. */ +export function azCli(): Promise { + const config = azureConfig(); + azPromise ??= resolveAz( + { + env: config.azPath ? { ...process.env, LOCALSTACK_AZ_PATH: config.azPath } : process.env, + platform: process.platform, + homedir: os.homedir(), + fs: nodeResolveFs, + procVersion: process.platform === "win32" ? undefined : readProcVersion(), + pycacheDir: safePycacheDir(config.pycacheDir), + }, + probeAz + ).catch((error) => { + azPromise = undefined; + throw error; + }); + return azPromise; +} + +/** + * Resolve Bicep once per process when found (throws BicepPathError for a bad explicit path). A + * miss is looked up again on the next call, so a Bicep installed meanwhile (install-azure-addons) + * needs no restart, as the Snowflake tool checks for `snow` on each call. + */ +export function bicep(): Promise { + const config = azureConfig(); + bicepPromise ??= (async () => { + const az = await azCli(); + const home = os.homedir(); + const platform = process.platform; + return resolveBicep( + { + env: process.env, + platform, + homedir: home, + fs: nodeResolveFs, + canonical: (p) => canonicalPath(p, platform), + excludeDirs: [ + `${home}/.azure`, + ...(process.env.AZURE_CONFIG_DIR ? [process.env.AZURE_CONFIG_DIR] : []), + ], + }, + async (bicepPath) => { + const result = await runOnce( + { file: bicepPath, prefixArgs: [], installer: az.installer }, + ["--version"], + 120_000 + ); + return result.exitCode === 0 ? result.stdout : undefined; + } + ); + })() + .then((resolved) => { + if (!resolved) bicepPromise = undefined; + return resolved; + }) + .catch((error) => { + bicepPromise = undefined; + throw error; + }); + return bicepPromise; +} + +export function installedExtensions(): InstalledExtension[] { + return listInstalledExtensions(azureConfig().extensionDir, nodeResolveFs, process.platform); +} + +const LOCAL_VERSIONS_CODE = + "import json\nfrom azure.cli.core.util import _get_local_versions as v\nprint(json.dumps(v()))\n"; + +/** The update-check seed's versions (exported for the spawn smoke). */ +export async function readLocalVersions(az: AzExecutable): Promise> { + if (az.installer === "launcher-as-is") { + const result = await runOnce(az, ["version", "-o", "json"], 120_000); + return result.exitCode === 0 ? versionsFromAzVersionJson(result.stdout) : {}; + } + const result = await runOnce( + { + file: az.file, + prefixArgs: [...interpreterFlags(az.prefixArgs), "-c", LOCAL_VERSIONS_CODE], + installer: az.installer, + }, + [], + 60_000 + ); + // An empty answer is fine: the seed falls back to the probed core version. + try { + return versionsFromLocalVersionsJson(result.stdout.trim().split(/\r?\n/).pop() ?? "{}"); + } catch { + return {}; + } +} + +/** The runner for agent commands; Bicep's directory goes first on PATH when there is one. */ +export async function azRunner(): Promise { + const az = await azCli(); + let bicepDir: string | undefined; + try { + bicepDir = (await bicep())?.dir; + } catch { + bicepDir = undefined; // a bad LOCALSTACK_AZ_BICEP_PATH is reported by the Bicep step + } + return new HostRunner({ exe: az, env: childEnvFor(az, bicepDir), proxy: await egressGuard() }); +} + +/** + * The runner for the agent's commands: the warm worker with LOCALSTACK_AZ_RUNNER=worker, + * else the subprocess runner. A launcher run as-is names no Python to + * run the worker with, and a Python that cannot start it falls back to subprocesses. + */ +async function commandRunner(host: HostRunner): Promise { + const config = azureConfig(); + if (config.runner !== "worker") return host; + const az = await azCli(); + if (az.installer === "launcher-as-is") return host; + if (!workerRunner) { + let bicepDir: string | undefined; + try { + bicepDir = (await bicep())?.dir; + } catch { + bicepDir = undefined; // reported by the Bicep step + } + workerRunner = new WorkerRunner({ + exe: az, + env: childEnvFor(az, bicepDir), + proxy: await egressGuard(), + // A bootstrap or self-heal rewrites these: a worker started before holds stale config. + stateFiles: ["config", "clouds.config", "azureProfile.json"].map((f) => + path.join(config.configDir, f) + ), + fallback: host, + log: (line) => process.stderr.write(`[localstack-azure-client] ${line}\n`), + }); + } + return workerRunner; +} + +export async function bootstrapTarget(): Promise { + const config = azureConfig(); + const az = await azCli(); + return { + configDir: config.configDir, + endpoint: config.endpoint, + sessionId: lastSessionId, + azVersion: az.version, + cwd: config.workdir, + timeoutMs: config.timeoutMs, + }; +} + +async function bootstrapDeps(): Promise { + const az = await azCli(); + const runner = await azRunner(); + return { + runner, + commandRunner: await commandRunner(runner), + readLocalVersions: () => readLocalVersions(az), + }; +} + +export async function ensureProfile(): Promise { + await ensureAzureCliConfigured(await bootstrapTarget(), await bootstrapDeps()); +} + +export async function runAzCommand( + argv: string[], + opts: Omit +): Promise { + const config = azureConfig(); + return runWithSelfHeal(await bootstrapTarget(), await bootstrapDeps(), argv, { + ...opts, + timeoutMs: config.timeoutMs, + cwd: config.workdir, + }); +} + +export { BootstrapError, interpreterFlags }; diff --git a/src/lib/azure/spawn.smoke.test.ts b/src/lib/azure/spawn.smoke.test.ts new file mode 100644 index 0000000..943caca --- /dev/null +++ b/src/lib/azure/spawn.smoke.test.ts @@ -0,0 +1,166 @@ +// The runner's real `az`, no emulator. +// Opt-in with AZ_SMOKE=1: it spawns the installed Azure CLI. Everything runs in a +// temporary config dir, behind the egress guard, after the versionCheck.json seed, and +// the test asserts that nothing but the housekeeping hosts was refused, so the smoke +// itself never calls Microsoft from a developer's machine. +import { copyFileSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "fs"; +import os from "os"; +import path from "path"; +import type { AzExecutable, AzRunResult } from "./types"; + +const SMOKE = process.env.AZ_SMOKE === "1"; +const describeSmoke = SMOKE ? describe : describe.skip; +jest.setTimeout(300_000); + +const root = mkdtempSync(path.join(os.tmpdir(), "lsaz-smoke-")); +const configDir = path.join(root, "mcp-config-smoke"); +const workdir = path.join(root, "work"); +mkdirSync(workdir, { recursive: true }); +process.env.LOCALSTACK_AZ_CONFIG_DIR = configDir; +process.env.LOCALSTACK_AZ_WORKDIR = workdir; +process.env.LOCALSTACK_AZ_EXTENSION_DIR = path.join(root, "extensions"); +delete process.env.LOCALSTACK_AZ_EGRESS_GUARD; + +/* eslint-disable @typescript-eslint/no-require-imports */ +const services = require("./services") as typeof import("./services"); +const { versionCheckSeed, CLI_CONFIG, VERSION_CHECK_FILE } = + require("./bootstrap") as typeof import("./bootstrap"); +const { formatAzResult } = require("./output") as typeof import("./output"); +const { localVersionText } = require("./resolve-az") as typeof import("./resolve-az"); +/* eslint-enable @typescript-eslint/no-require-imports */ + +const results: Array<{ argv: string[]; result: AzRunResult }> = []; +let az: AzExecutable; + +async function run(argv: string[], timeoutMs = 120_000): Promise { + const runner = await services.azRunner(); + const result = await runner.run(argv, { timeoutMs, cwd: workdir }); + results.push({ argv, result }); + return result; +} + +const format = (result: AzRunResult, argv: string[], isHelp = false) => + formatAzResult(result, { + argv, + notes: [], + isHelp, + guardOn: true, + port: 4566, + timeoutSeconds: 120, + maxOutputChars: 30000, + maxHelpChars: 30000, + envelope: false, + inDocker: false, + }).content[0].text; + +afterAll(async () => { + await services.resetAzureServices(); + rmSync(root, { recursive: true, force: true }); +}); + +describeSmoke("the real az (AZ_SMOKE=1)", () => { + test("resolves and probes: the CLI's own Python with -X utf8 and -I (or the -P fallback)", async () => { + az = await services.azCli(); + console.log(`az: ${az.file} ${az.prefixArgs.join(" ")} (${az.installer}, ${az.version})`); + expect(az.version).toMatch(/^\d+\.\d+\.\d+/); + if (az.installer !== "launcher-as-is") { + expect(az.prefixArgs.slice(0, 2)).toEqual(["-X", "utf8"]); + expect(az.prefixArgs.join(" ")).toMatch(/-I|-P/); + } + }); + + test("the seed is written before the first az call, then the config set line runs once", async () => { + mkdirSync(configDir, { recursive: true }); + const seed = versionCheckSeed(await services.readLocalVersions(az), az.version); + writeFileSync(path.join(configDir, VERSION_CHECK_FILE), JSON.stringify(seed)); + expect(seed.versions.core.local).toBe(az.version); + + const set = await run(["config", "set", ...CLI_CONFIG, "--only-show-errors"]); + expect(set.exitCode).toBe(0); + const got = await run(["config", "get", "--only-show-errors", "-o", "json"]); + expect(got.exitCode).toBe(0); + const sections = JSON.parse(got.stdout) as Record< + string, + Array<{ name: string; value: string; source: string }> + >; + for (const entry of CLI_CONFIG) { + const [key, value] = entry.split("="); + const [section, name] = [key.slice(0, key.indexOf(".")), key.slice(key.indexOf(".") + 1)]; + const found = sections[section]?.find((e) => e.name === name); + // `config get` shows effective values: the child's AZURE_CORE_COLLECT_TELEMETRY=no + // wins over the file's `false`, with the same effect. + if (found?.source === "AZURE_CORE_COLLECT_TELEMETRY") expect(found.value).toBe("no"); + else expect(found?.value).toBe(value); + } + }); + + test("UTF-8 round trip (-X utf8)", async () => { + const result = await run(["cloud", "list", "--query", "'ü✓é'", "-o", "json"]); + expect(result.exitCode).toBe(0); + expect(JSON.parse(result.stdout)).toBe("ü✓é"); + }); + + test("--help comes back, re-flowed", async () => { + const argv = ["group", "--help"]; + const result = await run(argv); + expect(result.exitCode).toBe(0); + const text = format(result, argv, true); + expect(text).toMatch(/resource groups/i); + expect(text.split("\n").some((line) => /\S {3,}\S/.test(line))).toBe(false); + }); + + test("the local version answer", async () => { + const text = localVersionText(az, services.installedExtensions()); + expect(JSON.parse(text.split("\n\n")[0])["azure-cli-core"]).toBe(az.version); + }); + + test("group list fails offline with `login` (not logged in)", async () => { + const argv = ["group", "list"]; + const result = await run(argv); + expect(result.exitCode).not.toBe(0); + expect(result.stderr).toMatch(/Please run 'az login' to setup account\./); + expect(format(result, argv).split("\n")[0]).toMatch(/\(exit \d+, login\)$/); + }); + + test("az accepts abbreviated long flags (so the policy matches prefixes)", async () => { + const result = await run(["cloud", "list", "--out", "tsv", "--que", "[0].name"]); + console.log(`abbreviated flags: exit ${result.exitCode}`); + expect(result.exitCode).toBe(0); + }); + + test("@~/ expands into the private home, not the user's", async () => { + const privateHome = services.azureConfig().homeDir; + mkdirSync(privateHome, { recursive: true }); + writeFileSync(path.join(privateHome, "marker.txt"), "'marker-ok'"); + const result = await run(["cloud", "list", "--query", "@~/marker.txt", "-o", "json"]); + expect(result.exitCode).toBe(0); + expect(JSON.parse(result.stdout)).toBe("marker-ok"); + }); + + test("bicep build through the runner and the guard, when a Bicep binary resolves", async () => { + const bicep = await services.bicep(); + if (!bicep) { + console.log("no Bicep binary resolved: the Bicep case is skipped"); + return; + } + copyFileSync( + path.resolve(__dirname, "../../../tests/fixtures/azure/bicep/storage.bicep"), + path.join(workdir, "storage.bicep") + ); + const result = await run(["bicep", "build", "--file", "storage.bicep", "--stdout"], 240_000); + expect(result.exitCode).toBe(0); + expect(JSON.parse(result.stdout).resources).toBeDefined(); + // Bicep writes nothing next to its input with --stdout. + expect(() => readFileSync(path.join(workdir, "storage.json"))).toThrow(); + }); + + test("nothing but the housekeeping hosts was refused during the smoke", () => { + const refused = results.flatMap((r) => + r.result.egress.refused.map((h) => `${r.argv.join(" ")} -> ${h}`) + ); + const housekeeping = [...new Set(results.flatMap((r) => r.result.egress.housekeeping))]; + console.log(`housekeeping refusals: ${housekeeping.join(", ") || "none"}`); + expect(refused).toEqual([]); + expect(results.every((r) => r.result.egress.upstream.length === 0)).toBe(true); + }); +}); diff --git a/src/lib/azure/testing/tls.ts b/src/lib/azure/testing/tls.ts new file mode 100644 index 0000000..356f033 --- /dev/null +++ b/src/lib/azure/testing/tls.ts @@ -0,0 +1,46 @@ +import { spawnSync } from "child_process"; +import { mkdtempSync, readFileSync, rmSync } from "fs"; +import os from "os"; +import path from "path"; + +/** + * A throwaway self-signed certificate for tests that need a local TLS server. It is + * generated at test time, so no private key is ever checked in. Undefined when + * openssl is not available (the calling test then skips). + */ +export function makeTestCertificate( + commonName = "azure.localhost.localstack.cloud" +): { key: string; cert: string } | undefined { + const dir = mkdtempSync(path.join(os.tmpdir(), "lsaz-tls-")); + try { + const keyFile = path.join(dir, "key.pem"); + const certFile = path.join(dir, "cert.pem"); + const result = spawnSync( + "openssl", + [ + "req", + "-x509", + "-newkey", + "rsa:2048", + "-nodes", + "-keyout", + keyFile, + "-out", + certFile, + "-days", + "1", + "-subj", + `/CN=${commonName}`, + "-addext", + `subjectAltName=DNS:${commonName},DNS:localhost,IP:127.0.0.1`, + ], + { windowsHide: true, timeout: 30_000, env: { ...process.env, MSYS_NO_PATHCONV: "1" } } + ); + if (result.status !== 0) return undefined; + return { key: readFileSync(keyFile, "utf8"), cert: readFileSync(certFile, "utf8") }; + } catch { + return undefined; + } finally { + rmSync(dir, { recursive: true, force: true }); + } +} diff --git a/src/lib/azure/types.ts b/src/lib/azure/types.ts new file mode 100644 index 0000000..129430b --- /dev/null +++ b/src/lib/azure/types.ts @@ -0,0 +1,219 @@ +/** + * Shared contracts of the Azure client modules. + * Every module in src/lib/azure/ builds against these types, so they can be + * implemented and tested independently. + */ + +/** Resolved settings of the Azure client tool (built by getAzureConfig). */ +export interface AzureConfig { + /** Gateway port of the Azure emulator (LOCALSTACK_AZURE_PORT, default LOCALSTACK_PORT, 4566). */ + port: number; + /** Plain-HTTP health base, always IPv4: `http://127.0.0.1:`. */ + healthBaseUrl: string; + /** ARM endpoint: `https://azure.localhost.localstack.cloud:` unless overridden. */ + endpoint: string; + /** Host name of `endpoint`, lower-case. */ + endpointHost: string; + /** Isolated CLI config dir (AZURE_CONFIG_DIR of the child); one per port by default. */ + configDir: string; + /** Private home of the child: `/home`. */ + homeDir: string; + /** Private temp of the child: `/tmp`. */ + tmpDir: string; + /** Extension dir passed to the child as AZURE_EXTENSION_DIR. */ + extensionDir: string; + /** Explicit `az` launcher or its Python (LOCALSTACK_AZ_PATH). */ + azPath?: string; + /** Explicit Bicep binary (LOCALSTACK_AZ_BICEP_PATH). */ + bicepPath?: string; + /** + * The server environment variables a `.bicepparam` may read with `readEnvironmentVariable()` + * (LOCALSTACK_AZ_BICEP_ENV). az's environment is an allow-list, so nothing else ever reaches + * Bicep; the token and the variables that steer az are refused even when listed. + */ + bicepEnv: string[]; + /** Per-command timeout in milliseconds (LOCALSTACK_AZ_TIMEOUT_SECONDS x 1000). */ + timeoutMs: number; + maxOutputChars: number; + maxHelpChars: number; + /** Root of the file policy and the runner's cwd (LOCALSTACK_AZ_WORKDIR). */ + workdir: string; + /** Containment layer 4 on (default) or off (LOCALSTACK_AZ_EGRESS_GUARD=0, debugging only). */ + egressGuard: boolean; + denylistFile?: string; + runner: "host" | "worker"; + /** Bytecode cache dir for installs without .pyc (LOCALSTACK_AZ_PYCACHE_DIR; the image sets it). */ + pycacheDir?: string; + /** Docker only: loopback forwarder target host (LOCALSTACK_AZURE_FORWARD_TARGET). */ + forwardTarget?: string; + /** Tests only: append the JSON result envelope (LOCALSTACK_AZ_TEST_ENVELOPE=1). */ + testEnvelope: boolean; + /** The server runs inside a container. */ + inDocker: boolean; + /** Invalid values that fell back to a default, with the reason. */ + warnings: string[]; + /** Hard configuration errors; the tool refuses to run while any is present. */ + errors: string[]; +} + +/** How to start `az`. */ +export interface AzExecutable { + /** The executable spawned: the CLI's Python, or a launcher run as-is. */ + file: string; + /** Arguments before the az argv, e.g. ["-X","utf8","-W","ignore::SyntaxWarning","-IBm","azure.cli"]; [] for a launcher as-is. */ + prefixArgs: string[]; + /** `script`: a bash launcher whose interpreter was parsed but that names no AZ_INSTALLER. */ + installer: "msi" | "pip" | "deb" | "rpm" | "homebrew" | "script" | "launcher-as-is" | "explicit"; + /** Value for the child's AZ_INSTALLER, when known. */ + azInstaller?: "MSI" | "PIP" | "DEB" | "RPM" | "HOMEBREW"; + /** azure-cli-core version from the import probe (or `az version` for a launcher as-is). */ + version?: string; +} + +/** Records the egress guard keeps per call. */ +export interface EgressRecords { + /** Non-housekeeping hosts refused. */ + refused: string[]; + /** Allowed hosts whose upstream connection failed (`conn-refused` with the guard on). */ + upstream: string[]; + /** Housekeeping hosts refused (never a failure). */ + housekeeping: string[]; + /** Number of CONNECTs relayed to the emulator for this call. */ + allowed: number; +} + +export interface EgressEvent { + kind: "refused" | "upstream" | "housekeeping" | "allowed"; + host: string; +} + +/** Containment layer 4: the CONNECT allow-list proxy. */ +export interface EgressProxy { + /** The loopback port the guard listens on. */ + readonly port: number; + /** Proxy variables for one call: `http://:x@127.0.0.1:`. Registers the tag. */ + envFor(callId: string): { HTTPS_PROXY: string; HTTP_PROXY: string }; + /** Records of a call, removed on read. */ + takeRecords(callId: string): EgressRecords; + /** Subscribe to events of all calls; returns the unsubscribe function. */ + onEvent(listener: (callId: string, event: EgressEvent) => void): () => void; + close(): Promise; +} + +export interface AzRunOptions { + timeoutMs: number; + /** Always the workdir. */ + cwd: string; + signal?: AbortSignal; + /** Called about every 10 s while `az` runs. */ + onProgress?: (elapsedMs: number) => void; +} + +export interface AzRunResult { + exitCode: number | null; + stdout: string; + stderr: string; + timedOut: boolean; + aborted: boolean; + truncated: boolean; + durationMs: number; + /** From the egress guard (empty with the guard off). */ + egress: EgressRecords; + /** The spawn itself failed (ENOENT, EINVAL, ENAMETOOLONG, ...). */ + spawnError?: string; + /** Refused before spawning because the Windows command line would be too long (`too-long`). */ + tooLong?: boolean; + /** Killed early by the fail-fast rule after a refusal or an upstream failure. */ + failFast?: "refused" | "upstream"; +} + +export interface AzRunner { + run(argv: string[], opts: AzRunOptions): Promise; +} + +/** Outcome of the pure command policy. */ +export type PolicyResult = + | { + ok: true; + local?: undefined; + /** Argv to spawn, without the leading `az`. */ + argv: string[]; + /** User-facing notes, e.g. a URL rewrite. */ + notes: string[]; + isHelp: boolean; + needsBicep: boolean; + /** `rewritten` when a management.azure.com URL was rewritten. */ + outcome: "ok" | "rewritten"; + } + | { + ok: true; + /** Answered by the handler from the CLI probe (`version`, `--version`). */ + local: "version"; + argv: string[]; + notes: string[]; + isHelp: false; + needsBicep: false; + outcome: "local"; + } + | { + ok: false; + title: string; + message: string; + /** Stable id for analytics, e.g. `denied:cloud-set`, `syntax`, `file:outside-workdir`. */ + ruleId: string; + /** The argv, when tokenizing succeeded (for analytics fields). */ + argv?: string[]; + }; + +export interface PolicyOptions { + workdir: string; + /** The child's private home; `~` paths resolve here. */ + homeDir: string; + /** Extra denied prefixes from LOCALSTACK_AZ_DENYLIST_FILE (already parsed). */ + extraDenied?: string[][]; + /** + * Directories the file rule refuses even inside the workdir: the user's real + * `~/.azure`, `~/.ssh`, `~/.kube`, `~/.docker`, the parent's AZURE_CONFIG_DIR and + * the tool's own config dir. They matter when the workdir is the user's home or + * one of its ancestors. A path inside `homeDir` (the private home) stays allowed. + */ + protectedDirs?: string[]; + platform?: NodeJS.Platform; + /** + * The emulator's ports (its gateway, 443 and its service range): a URL for a local host on + * any other port, such as Docker's API on localhost:2375, is refused. Unset: any port. + */ + localPorts?: ReadonlySet; +} + +/** The class of a failure, as its first line names it. */ +export type AzFailureClass = + | "login" + | "conn-refused" + | "dns" + | "discovery" + | "not-implemented" + | "provider" + | "no-route" + | "extension" + | "unknown-command" + | "argument" + | "not-found" + | "cli-error" + | "emulator-error" + | "needs-yes" + | "egress-refused" + | "too-long" + | "timeout" + | "guard-down" + | "bicep-missing" + | "bicep-registry" + | "bicep-env" + | "azcopy" + | "cancelled" + | "spawn-error" + | "other"; + +export interface ToolTextResponse { + content: Array<{ type: "text"; text: string }>; +} diff --git a/src/lib/azure/worker-runner.test.ts b/src/lib/azure/worker-runner.test.ts new file mode 100644 index 0000000..0902564 --- /dev/null +++ b/src/lib/azure/worker-runner.test.ts @@ -0,0 +1,331 @@ +import * as fs from "fs"; +import * as os from "os"; +import * as path from "path"; +import { startEgressProxy } from "./egress-proxy"; +import type { AzRunner, AzRunOptions, AzRunResult } from "./types"; +import { WorkerRunner, writesCliState, type WorkerRunnerOptions } from "./worker-runner"; + +// The runner's cases against the warm worker, plus its own: state isolation, +// SystemExit, the fd 2 capture. The REAL worker code (worker-script.ts) runs under a real +// Python, with a stand-in azure.cli.core (tests/fixtures/azure/fake-worker), so no +// azure-cli is needed. Skipped when no Python is on PATH. + +function findPython(): string | undefined { + const names = + process.platform === "win32" ? ["python.exe", "python3.exe"] : ["python3", "python"]; + for (const dir of (process.env.PATH ?? "").split(path.delimiter)) { + // The Microsoft Store's python.exe aliases exist but open the Store instead. + if (!dir || /WindowsApps/i.test(dir)) continue; + for (const name of names) { + const candidate = path.join(dir, name); + if (fs.existsSync(candidate)) return candidate; + } + } + return undefined; +} + +const PYTHON = findPython(); +const STUB = path.join(__dirname, "../../../tests/fixtures/azure/fake-worker"); +const describeIfPython = PYTHON ? describe : describe.skip; + +describe("writesCliState", () => { + test("state writers are matched by command prefix, readers are not", () => { + expect(writesCliState(["account", "show"])).toBe(false); + expect(writesCliState(["account", "list"])).toBe(false); + expect(writesCliState(["account", "set", "--subscription", "x"])).toBe(true); + expect(writesCliState(["config", "get"])).toBe(true); + expect(writesCliState(["extension", "list"])).toBe(true); + expect(writesCliState(["bicep", "build", "--file", "x"])).toBe(false); + expect(writesCliState(["bicep", "install"])).toBe(true); + expect(writesCliState(["group", "list"])).toBe(false); + }); +}); + +describeIfPython("WorkerRunner, with the real worker code and a stand-in azure.cli.core", () => { + let root: string; + let configDir: string; + const runners: WorkerRunner[] = []; + + beforeEach(() => { + root = fs.mkdtempSync(path.join(os.tmpdir(), "lsmcp-worker-")); + configDir = path.join(root, "config"); + fs.mkdirSync(configDir); + }); + afterEach(async () => { + await Promise.all(runners.splice(0).map((r) => r.close())); + fs.rmSync(root, { recursive: true, force: true }); + }); + + function runner(over: Partial = {}): WorkerRunner { + const r = new WorkerRunner({ + // `-m azure.cli` becomes `-c `; no -I here, so PYTHONPATH finds the stand-in. + exe: { file: PYTHON!, prefixArgs: ["-X", "utf8", "-m", "azure.cli"], installer: "explicit" }, + env: { + ...(process.env as Record), + PYTHONPATH: STUB, + PYTHONDONTWRITEBYTECODE: "1", + AZURE_CONFIG_DIR: configDir, + }, + ...over, + }); + runners.push(r); + return r; + } + const opts = (over: Partial = {}): AzRunOptions => ({ + timeoutMs: 60_000, + cwd: root, + ...over, + }); + const out = (r: AzRunResult) => r.stdout.trim(); + + test("one warm process: its state survives from one command to the next", async () => { + const r = runner(); + const first = await r.run(["count"], opts()); + const second = await r.run(["count"], opts()); + expect([out(first), out(second)]).toEqual(["1", "2"]); + expect(out(await r.run(["pid"], opts()))).toBe(out(await r.run(["pid"], opts()))); + expect(first.exitCode).toBe(0); + expect(r.workerCount).toBe(1); + }); + + test("each command gets a fresh CLI context: no change to it leaks into the next", async () => { + // `az rest` unregisters az's global result transforms on its CLI context. With one context + // for the worker's life, every later command in that worker answered resourceGroup: null. + const r = runner(); + await r.run(["unregister-transforms"], opts()); + expect(out(await r.run(["transforms"], opts()))).toBe("on"); + expect(r.workerCount).toBe(1); // still the same warm process + }); + + test("stderr: both a log handler bound at import and raw fd 2 writes are captured", async () => { + const r = runner(); + const logged = await r.run(["log", "a", "warning"], opts()); + expect(logged.stderr).toContain("a warning"); + expect(logged.stdout).toBe(""); + const raw = await r.run(["stderr", "raw-err"], opts()); + expect(raw.stderr).toBe("raw-err"); + }); + + test("bytes written straight to fd 1 land in stdout, never in the protocol stream", async () => { + const r = runner(); + const raw = await r.run(["rawfd", "raw-out"], opts()); + expect(raw.stdout).toContain("raw-out"); + expect(out(await r.run(["echo", "still", "fine"], opts()))).toBe('["still", "fine"]'); + }); + + test("SystemExit gives its code; an exception is exit 1 with its message; the worker lives on", async () => { + const r = runner(); + const pid = out(await r.run(["pid"], opts())); + expect((await r.run(["exit", "3"], opts())).exitCode).toBe(3); + const raised = await r.run(["raise"], opts()); + expect(raised.exitCode).toBe(1); + expect(raised.stderr).toContain("RuntimeError: boom"); + expect(out(await r.run(["pid"], opts()))).toBe(pid); + }); + + test("a prompt sees no tty and an empty stdin, as in a subprocess", async () => { + const r = runner(); + expect(out(await r.run(["prompt"], opts()))).toBe("no tty\n''"); + }); + + test("each command runs in its own cwd, with its own call tag in the proxy variables", async () => { + const proxy = await startEgressProxy(); + try { + const r = runner({ proxy }); + const sub = path.join(root, "sub"); + fs.mkdirSync(sub); + expect(fs.realpathSync(out(await r.run(["cwd"], opts({ cwd: sub }))))).toBe( + fs.realpathSync(sub) + ); + const a = out(await r.run(["env", "HTTPS_PROXY"], opts())); + const b = out(await r.run(["env", "HTTPS_PROXY"], opts())); + expect(a).toMatch(/^http:\/\/[0-9a-f-]{36}:x@127\.0\.0\.1:\d+$/); + expect(b).not.toBe(a); + } finally { + await proxy.close(); + } + }); + + test("a command that writes CLI state replaces the worker", async () => { + const r = runner(); + expect(out(await r.run(["count"], opts()))).toBe("1"); + await r.run(["config", "set", "core.x=y"], opts()); + expect(out(await r.run(["count"], opts()))).toBe("1"); + }); + + test("a changed profile file (the bootstrap, a self-heal) replaces the worker", async () => { + const profile = path.join(configDir, "config"); + fs.writeFileSync(profile, "[core]\n"); + const r = runner({ stateFiles: [profile] }); + expect(out(await r.run(["count"], opts()))).toBe("1"); + expect(out(await r.run(["count"], opts()))).toBe("2"); + fs.writeFileSync(profile, "[core]\noutput = json\n"); + expect(out(await r.run(["count"], opts()))).toBe("1"); + }); + + test("recycled after the command limit", async () => { + const r = runner({ maxCommandsPerWorker: 2 }); + const counts = []; + for (let i = 0; i < 3; i++) counts.push(out(await r.run(["count"], opts()))); + expect(counts).toEqual(["1", "2", "1"]); + }); + + test("a timeout kills the worker's tree; the next command gets a fresh worker", async () => { + const r = runner(); + const pid = out(await r.run(["pid"], opts())); + const started = Date.now(); + const slow = await r.run(["sleep", "30"], opts({ timeoutMs: 1500 })); + expect(slow.timedOut).toBe(true); + expect(slow.exitCode).toBeNull(); + expect(Date.now() - started).toBeLessThan(15_000); + expect(out(await r.run(["pid"], opts()))).not.toBe(pid); + }); + + test("a cancelled call kills the worker too", async () => { + const r = runner(); + await r.run(["pid"], opts()); + const controller = new AbortController(); + setTimeout(() => controller.abort(), 500); + const result = await r.run(["sleep", "30"], opts({ signal: controller.signal })); + expect(result.aborted).toBe(true); + expect(result.exitCode).toBeNull(); + }); + + test("a cancel during the worker's cold start aborts the call, and the command never runs", async () => { + const r = runner({ + env: { + ...(process.env as Record), + PYTHONPATH: STUB, + PYTHONDONTWRITEBYTECODE: "1", + AZURE_CONFIG_DIR: configDir, + FAKE_AZ_IMPORT_DELAY: "2", + }, + }); + const marker = path.join(root, "ran.txt"); + const controller = new AbortController(); + setTimeout(() => controller.abort(), 300); + const started = Date.now(); + const result = await r.run(["touch", marker], opts({ signal: controller.signal })); + expect(result.aborted).toBe(true); + expect(Date.now() - started).toBeLessThan(1500); // not held until the worker is up + // Well after the worker would have come up: the command still never ran. + await new Promise((resolve) => setTimeout(resolve, 3000)); + expect(fs.existsSync(marker)).toBe(false); + }); + + test("each command gets its own log level: a first --debug does not stick", async () => { + const r = runner(); + const first = await r.run(["debuglog", "--debug"], opts()); + expect(first.stderr).toContain("DEBUG: a debug line"); + const pid = out(await r.run(["pid"], opts())); + const second = await r.run(["debuglog"], opts()); + expect(out(await r.run(["pid"], opts()))).toBe(pid); // the same warm worker + expect(second.stderr).not.toContain("DEBUG: a debug line"); + expect((await r.run(["debuglog", "--debug"], opts())).stderr).toContain("DEBUG: a debug line"); + }); + + test("fail-fast: a refused CONNECT through the real guard ends the call and names the host", async () => { + const proxy = await startEgressProxy(); + try { + const r = runner({ proxy }); + const started = Date.now(); + const result = await r.run(["connect", "api.loganalytics.io:443"], opts()); + expect(result.failFast).toBe("refused"); + expect(result.egress.refused).toEqual(["api.loganalytics.io"]); + expect(Date.now() - started).toBeLessThan(10_000); + // The next command runs in a fresh worker. + expect(out(await r.run(["count"], opts()))).toBe("1"); + } finally { + await proxy.close(); + } + }); + + test("a crash mid-command is a failure, and the next command starts a new worker", async () => { + const r = runner(); + const crashed = await r.run(["crash"], opts()); + expect(crashed.exitCode).toBeNull(); + expect(crashed.timedOut || crashed.aborted || crashed.failFast).toBeFalsy(); + expect(crashed.stderr).toMatch(/worker exited/); + expect(out(await r.run(["count"], opts()))).toBe("1"); + }); + + test("output over the cap is truncated, and that worker is replaced", async () => { + const r = runner({ maxStreamBytes: 1000 }); + expect(out(await r.run(["count"], opts()))).toBe("1"); + const big = await r.run(["big", "5000"], opts()); + expect(big.truncated).toBe(true); + expect(Buffer.byteLength(big.stdout)).toBeLessThanOrEqual(1000); + expect(out(await r.run(["count"], opts()))).toBe("1"); + }); + + test("a Python that cannot run the worker: this and every later call go to the fallback", async () => { + const calls: string[][] = []; + const fallback: AzRunner = { + run: async (argv) => { + calls.push(argv); + return { + exitCode: 0, + stdout: "from the subprocess runner", + stderr: "", + timedOut: false, + aborted: false, + truncated: false, + durationMs: 1, + egress: { refused: [], upstream: [], housekeeping: [], allowed: 0 }, + }; + }, + }; + const logs: string[] = []; + const r = runner({ + workerArgs: ["-c", "import sys; sys.stderr.write('no azure.cli here'); sys.exit(3)"], + fallback, + log: (line) => logs.push(line), + }); + expect((await r.run(["group", "list"], opts())).stdout).toBe("from the subprocess runner"); + expect((await r.run(["group", "show"], opts())).stdout).toBe("from the subprocess runner"); + expect(calls).toEqual([ + ["group", "list"], + ["group", "show"], + ]); + expect(logs.join("\n")).toMatch(/no azure\.cli here[\s\S]*subprocess/); + }); + + test("never without an isolated AZURE_CONFIG_DIR", async () => { + const env: Record = { ...(process.env as Record) }; + env.PYTHONPATH = STUB; + delete env.AZURE_CONFIG_DIR; + const r = runner({ env }); + await expect(r.run(["count"], opts())).rejects.toThrow(/isolated AZURE_CONFIG_DIR/); + }); + + test("parallel commands run in separate workers at once; close() stops them all", async () => { + const r = runner(); + const started = Date.now(); + const results = await Promise.all([1, 2, 3].map(() => r.run(["sleep", "1.5"], opts()))); + expect(results.every((x) => x.exitCode === 0)).toBe(true); + expect(Date.now() - started).toBeLessThan(4500 + 3000); // not 3 x 1.5 s in a row, plus starts + expect(r.workerCount).toBe(3); + await r.close(); + expect(r.workerCount).toBe(0); + }); + + test("close() resolves only once every worker process has exited", async () => { + // Under load on Windows, close() used to resolve when taskkill returned, before the worker's + // exit: the afterEach's rmSync of the worker's directory then failed with EPERM. + const r = runner(); + const pids = (await Promise.all([1, 2].map(() => r.run(["pid"], opts())))).map((x) => + Number(x.stdout.trim()) + ); + expect(pids.every((pid) => pid > 0)).toBe(true); + await r.close(); + const alive = (pid: number) => { + try { + process.kill(pid, 0); + return true; + } catch { + return false; + } + }; + expect(pids.filter(alive)).toEqual([]); + }); +}); diff --git a/src/lib/azure/worker-runner.ts b/src/lib/azure/worker-runner.ts new file mode 100644 index 0000000..16a9d5b --- /dev/null +++ b/src/lib/azure/worker-runner.ts @@ -0,0 +1,472 @@ +import { spawn, type ChildProcess } from "child_process"; +import { randomUUID } from "crypto"; +import { statSync } from "fs"; +import { + DEFAULT_TIMINGS, + emptyEgressRecords, + killTree, + MAX_STREAM_BYTES, + runSlots, + trackChild, + untrackChild, + type KillStep, + type RunnerTimings, +} from "./runner"; +import { interpreterFlags } from "./resolve-az"; +import type { AzExecutable, AzRunner, AzRunOptions, AzRunResult, EgressProxy } from "./types"; +import { WORKER_CODE } from "./worker-script"; + +/** + * The warm az worker (optional; LOCALSTACK_AZ_RUNNER=worker). A long-running Python child + * imports azure-cli once and runs each command in-process, so a call costs the command, not + * the interpreter start and the imports. + * + * The containment is the subprocess runner's: the same child environment (private home, + * the tool's own AZURE_CONFIG_DIR), one egress-guard call tag per command, the same timeout, + * cancel and fail-fast (by killing the worker's tree), the same output caps and the same + * limit of four runs at once. Worker state never outlives what a fresh process would read + * from disk: a worker is replaced after 200 commands, after any command that writes CLI + * state, and whenever a profile file changed since it started (the bootstrap, a self-heal). + */ + +/** + * Command prefixes that write the CLI's own state; a worker that ran one is replaced. The + * policy refuses most of them anyway: this is the second line. + */ +const STATE_WRITERS = [ + "account clear", + "account set", + "bicep install", + "bicep uninstall", + "bicep upgrade", + "cloud", + "config", + "configure", + "extension", + "init", + "login", + "logout", +]; + +export const WORKER_MAX_COMMANDS = 200; + +export interface WorkerRunnerOptions { + /** The CLI's Python and its spawn prefix; never a launcher run as-is. */ + exe: AzExecutable; + env: Record; + proxy?: EgressProxy; + platform?: NodeJS.Platform; + timings?: Partial; + maxStreamBytes?: number; + maxCommandsPerWorker?: number; + /** Profile files whose change replaces the workers (config, clouds.config, azureProfile.json). */ + stateFiles?: string[]; + /** How long a new worker may take to import azure-cli. */ + startTimeoutMs?: number; + /** Tests: the worker program instead of `-c WORKER_CODE`. */ + workerArgs?: string[]; + /** Used from the first worker that cannot start on: this Python cannot run the worker. */ + fallback?: AzRunner; + taskkill?: string; + onKillStep?: (step: KillStep, status?: number) => void; + log?: (line: string) => void; +} + +interface Reply { + id: string; + exitCode: number; + stdout: string; + stderr: string; + truncated: boolean; +} + +export function writesCliState(argv: string[]): boolean { + return STATE_WRITERS.some((prefix) => { + const words = prefix.split(" "); + return words.every((word, i) => argv[i] === word); + }); +} + +/** How long kill() waits for a killed worker's exit event before it gives up waiting. */ +const KILL_EXIT_WAIT_MS = 5_000; + +/** `ready`, or a rejection as soon as `signal` aborts: a cancel during a cold start. */ +function readyOrAborted(ready: Promise, signal?: AbortSignal): Promise { + if (!signal) return ready; + return new Promise((resolve, reject) => { + if (signal.aborted) return reject(new Error("aborted")); + const onAbort = () => reject(new Error("aborted")); + signal.addEventListener("abort", onAbort, { once: true }); + ready.then(resolve, reject).finally(() => signal.removeEventListener("abort", onAbort)); + }); +} + +class Worker { + readonly child: ChildProcess; + readonly ready: Promise; + /** Settles when the process has exited (or failed to spawn). */ + readonly exited: Promise; + commands = 0; + dead = false; + private buffer = ""; + private stderrTail = ""; + private onReply?: (reply: Reply) => void; + private onDeath?: (why: string) => void; + + constructor( + readonly stamp: string, + spawnArgs: { file: string; args: string[]; env: Record; cwd: string }, + private readonly platform: NodeJS.Platform, + startTimeoutMs: number + ) { + this.child = spawn(spawnArgs.file, spawnArgs.args, { + shell: false, + windowsHide: true, + stdio: ["pipe", "pipe", "pipe"], + env: spawnArgs.env, + cwd: spawnArgs.cwd, + detached: platform !== "win32", + }); + trackChild(this.child, platform); + this.child.stdin?.on("error", () => undefined); + this.child.stdout?.on("error", () => undefined); + this.child.stderr?.on("error", () => undefined); + this.child.stderr?.on("data", (d: Buffer) => { + // Outside a command only the worker itself writes here (an import error, a crash). + this.stderrTail = (this.stderrTail + d.toString("utf8")).slice(-2000); + }); + + let readyResolve: () => void; + let readyReject: (error: Error) => void; + this.ready = new Promise((resolve, reject) => { + readyResolve = resolve; + readyReject = reject; + }); + let exitedResolve: () => void; + this.exited = new Promise((resolve) => { + exitedResolve = resolve; + }); + // A worker that dies after starting rejects a promise nobody awaits any more. + this.ready.catch(() => undefined); + const startTimer = setTimeout(() => { + readyReject(new Error(`the az worker did not start within ${startTimeoutMs} ms`)); + this.stop(); + }, startTimeoutMs); + + this.child.stdout?.on("data", (d: Buffer) => { + this.buffer += d.toString("utf8"); + let nl: number; + while ((nl = this.buffer.indexOf("\n")) >= 0) { + const line = this.buffer.slice(0, nl); + this.buffer = this.buffer.slice(nl + 1); + if (!line.trim()) continue; + let message: Record; + try { + message = JSON.parse(line) as Record; + } catch { + continue; + } + if (message.ready === true) { + clearTimeout(startTimer); + readyResolve(); + } else if (typeof message.id === "string") { + this.onReply?.(message as unknown as Reply); + } + } + }); + const died = (why: string) => { + if (this.dead) return; + this.dead = true; + exitedResolve(); + clearTimeout(startTimer); + untrackChild(this.child); + const detail = this.stderrTail.trim(); + readyReject(new Error(`the az worker exited (${why})${detail ? `: ${detail}` : ""}`)); + this.onDeath?.(why); + }; + this.child.on("error", (error) => died(error.message)); + this.child.on("exit", (code, signal) => died(signal ? `signal ${signal}` : `exit ${code}`)); + } + + /** Sends one command; resolves with the reply, or rejects when the worker dies first. */ + send(request: { id: string; argv: string[]; cwd: string; env: Record }) { + return new Promise((resolve, reject) => { + this.onReply = (reply) => { + if (reply.id !== request.id) return; + this.onReply = undefined; + this.onDeath = undefined; + resolve(reply); + }; + this.onDeath = (why) => reject(new Error(`the az worker exited (${why})`)); + this.child.stdin?.write(JSON.stringify(request) + "\n"); + }); + } + + /** + * Kills the tree and waits (bounded) for the exit: on POSIX killTree only sends SIGTERM, and on + * Windows taskkill's own exit can come first, so "closed" would otherwise not mean gone. + */ + async kill(opts: { + taskkill?: string; + posixKillGraceMs: number; + onKillStep?: WorkerRunnerOptions["onKillStep"]; + }): Promise { + if (this.dead) return; + await killTree(this.child, { platform: this.platform, ...opts }); + let timer: NodeJS.Timeout | undefined; + const bound = Math.max(KILL_EXIT_WAIT_MS, opts.posixKillGraceMs + 2_000); + await Promise.race([ + this.exited, + new Promise((resolve) => { + timer = setTimeout(resolve, bound); + timer.unref(); + }), + ]); + clearTimeout(timer); + } + + /** A clean stop: end stdin, so the command loop ends and Python exits. */ + stop() { + if (this.dead) return; + this.child.stdin?.end(); + const timer = setTimeout(() => { + if (!this.dead) this.child.kill(); + }, 5000); + timer.unref(); + } +} + +export class WorkerRunner implements AzRunner { + private readonly timings: RunnerTimings; + private readonly idle: Worker[] = []; + private readonly all = new Set(); + private disabled = false; + + constructor(private readonly opts: WorkerRunnerOptions) { + this.timings = { ...DEFAULT_TIMINGS, ...opts.timings }; + } + + /** Live workers (tests). */ + get workerCount(): number { + return [...this.all].filter((w) => !w.dead).length; + } + + async run(argv: string[], o: AzRunOptions): Promise { + if (!this.opts.env.AZURE_CONFIG_DIR) { + // Containment layer 1: never run az against the user's own profile. + throw new Error("Refusing to run az without an isolated AZURE_CONFIG_DIR."); + } + if (this.disabled && this.opts.fallback) return this.opts.fallback.run(argv, o); + const acquired = await runSlots.acquire(o.signal); + if (!acquired) return this.result({ aborted: true }); + let result: AzRunResult | "fallback"; + try { + result = await this.runOnce(argv, o); + } finally { + runSlots.release(); + } + return result === "fallback" ? this.opts.fallback!.run(argv, o) : result; + } + + /** Stops every worker (tests, and resetAzureServices). */ + async close(): Promise { + const workers = [...this.all]; + this.idle.length = 0; + this.all.clear(); + await Promise.all(workers.map((w) => w.kill(this.killOpts()))); + } + + private killOpts() { + return { + taskkill: this.opts.taskkill, + posixKillGraceMs: this.timings.posixKillGraceMs, + onKillStep: this.opts.onKillStep, + }; + } + + private result(fields: Partial): AzRunResult { + return { + exitCode: null, + stdout: "", + stderr: "", + timedOut: false, + aborted: false, + truncated: false, + durationMs: 0, + egress: emptyEgressRecords(), + ...fields, + }; + } + + /** Modification stamp of the profile files: a change means a worker is out of date. */ + private stateStamp(): string { + return (this.opts.stateFiles ?? []) + .map((file) => { + try { + const s = statSync(file); + return `${s.mtimeMs}:${s.size}`; + } catch { + return "-"; + } + }) + .join("|"); + } + + private retire(worker: Worker) { + this.all.delete(worker); + const index = this.idle.indexOf(worker); + if (index >= 0) this.idle.splice(index, 1); + worker.stop(); + } + + private async acquireWorker(cwd: string): Promise { + const stamp = this.stateStamp(); + while (this.idle.length > 0) { + const worker = this.idle.pop()!; + if (!worker.dead && worker.stamp === stamp) return worker; + this.retire(worker); + } + const platform = this.opts.platform ?? process.platform; + const worker = new Worker( + stamp, + { + file: this.opts.exe.file, + args: this.opts.workerArgs ?? [ + ...interpreterFlags(this.opts.exe.prefixArgs), + "-c", + WORKER_CODE, + ], + env: { + ...this.opts.env, + LOCALSTACK_AZ_WORKER_MAX_BYTES: String(this.opts.maxStreamBytes ?? MAX_STREAM_BYTES), + }, + cwd, + }, + platform, + this.opts.startTimeoutMs ?? 120_000 + ); + this.all.add(worker); + return worker; + } + + private async runOnce(argv: string[], o: AzRunOptions): Promise { + const started = Date.now(); + if (o.signal?.aborted) return this.result({ aborted: true }); + const { proxy } = this.opts; + const callId = randomUUID(); + const env = proxy ? proxy.envFor(callId) : {}; + const takeEgress = () => (proxy ? proxy.takeRecords(callId) : emptyEgressRecords()); + + let worker: Worker | undefined; + try { + worker = await this.acquireWorker(o.cwd); + // A cancel during a cold start must not leave the command to run once the worker is up. + await readyOrAborted(worker.ready, o.signal); + if (o.signal?.aborted) throw new Error("aborted"); + } catch (error) { + if (o.signal?.aborted) { + if (worker) { + this.all.delete(worker); + void worker.kill(this.killOpts()); + } + return this.result({ + aborted: true, + durationMs: Date.now() - started, + egress: takeEgress(), + }); + } + const message = error instanceof Error ? error.message : String(error); + if (this.opts.fallback) { + // This Python cannot run the worker: every later command runs as a subprocess. + this.disabled = true; + this.opts.log?.(`${message}; running az as a subprocess from now on`); + takeEgress(); + return "fallback"; + } + return this.result({ + spawnError: message, + durationMs: Date.now() - started, + egress: takeEgress(), + }); + } + + const timers: NodeJS.Timeout[] = []; + let timedOut = false; + let aborted = false; + let failFast: AzRunResult["failFast"]; + let killed = false; + const kill = () => { + if (killed) return; + killed = true; + this.all.delete(worker); + void worker.kill(this.killOpts()); + }; + const onAbort = () => { + aborted = true; + kill(); + }; + o.signal?.addEventListener("abort", onAbort, { once: true }); + timers.push( + setTimeout(() => { + timedOut = true; + kill(); + }, o.timeoutMs) + ); + if (o.onProgress) { + const onProgress = o.onProgress; + timers.push( + setInterval(() => onProgress(Date.now() - started), this.timings.progressIntervalMs) + ); + } + // Fail fast, as the subprocess runner does: after a refused host or a + // failed upstream, kill the worker's tree instead of waiting out the SDK's retries. + const unsubscribe = proxy?.onEvent((id, event) => { + if (id !== callId || failFast || (event.kind !== "refused" && event.kind !== "upstream")) + return; + const kind = event.kind; + timers.push( + setTimeout(() => { + failFast = kind; + kill(); + }, this.timings.failFastDelayMs) + ); + }); + + try { + const reply = await worker.send({ id: callId, argv, cwd: o.cwd, env }); + worker.commands++; + const replace = + worker.commands >= (this.opts.maxCommandsPerWorker ?? WORKER_MAX_COMMANDS) || + writesCliState(argv) || + reply.truncated; + if (replace) this.retire(worker); + else this.idle.push(worker); + return this.result({ + exitCode: reply.exitCode, + stdout: reply.stdout, + stderr: reply.stderr, + truncated: reply.truncated, + durationMs: Date.now() - started, + egress: takeEgress(), + failFast, + }); + } catch (error) { + // The worker died mid-command: killed by the tool (timeout, cancel, fail-fast) or crashed. + this.all.delete(worker); + const stoppedByTool = timedOut || aborted || failFast !== undefined; + return this.result({ + exitCode: null, + stderr: stoppedByTool ? "" : error instanceof Error ? error.message : String(error), + timedOut, + aborted, + failFast, + durationMs: Date.now() - started, + egress: takeEgress(), + }); + } finally { + for (const t of timers) clearTimeout(t); + o.signal?.removeEventListener("abort", onAbort); + unsubscribe?.(); + } + } +} diff --git a/src/lib/azure/worker-script.ts b/src/lib/azure/worker-script.ts new file mode 100644 index 0000000..29da961 --- /dev/null +++ b/src/lib/azure/worker-script.ts @@ -0,0 +1,143 @@ +/** + * The Python side of the warm az worker. It imports azure-cli once and runs each command + * in-process with `get_default_cli().invoke(argv)`. + * + * Protocol: one JSON line per command on stdin (`{id, argv, cwd, env}`), one JSON line per + * answer on a private copy of the original stdout (`{id, exitCode, stdout, stderr, + * truncated}`), after a first `{"ready": true}`. While a command runs, file descriptors 1 + * and 2 point at temp files: az's log handlers keep the stream they were built with, so + * redirecting `sys.stderr` alone would miss its errors, and nothing az or a child (Bicep) + * prints can reach the protocol stream. `sys.stdin` is an empty stream meanwhile, so a + * prompt fails as it does in a subprocess whose stdin is at EOF. + */ +export const WORKER_CODE = String.raw` +import contextlib, io, json, os, sys, tempfile + +_proto = os.fdopen(os.dup(1), "w", encoding="utf-8", newline="\n") +LIMIT = int(os.environ.get("LOCALSTACK_AZ_WORKER_MAX_BYTES", "10485760")) + + +def _reply(obj): + _proto.write(json.dumps(obj) + "\n") + _proto.flush() + + +class _Capped(io.TextIOBase): + """az's out_file: keeps at most LIMIT bytes of UTF-8, like the subprocess runner.""" + + encoding = "utf-8" + errors = "strict" + + def __init__(self): + self.parts, self.size, self.capped = [], 0, False + + def writable(self): + return True + + def write(self, s): + if self.capped: + return len(s) + data = s.encode("utf-8", "surrogatepass") + if self.size + len(data) > LIMIT: + room = LIMIT - self.size + self.parts.append(data[:room].decode("utf-8", "ignore")) + self.size, self.capped = LIMIT, True + else: + self.parts.append(s) + self.size += len(data) + return len(s) + + def getvalue(self): + return "".join(self.parts) + + +def _flush_std(): + for stream in (sys.stdout, sys.stderr): + try: + stream.flush() + except Exception: + pass + + +@contextlib.contextmanager +def _fd_to_file(fd): + _flush_std() + saved = os.dup(fd) + tmp = tempfile.TemporaryFile(mode="w+b") + os.dup2(tmp.fileno(), fd) + try: + yield tmp + finally: + _flush_std() + os.dup2(saved, fd) + os.close(saved) + + +def _read_capped(tmp): + tmp.seek(0) + data = tmp.read(LIMIT + 1) + tmp.close() + if len(data) > LIMIT: + return data[:LIMIT].decode("utf-8", "ignore"), True + return data.decode("utf-8", "replace"), False + + +def _reset_logging(): + import logging + + try: + from knack.log import cli_logger_names + except ImportError: # the tests' stand-in azure.cli.core comes without knack + cli_logger_names = [] + for logger in [logging.getLogger(), *(logging.getLogger(n) for n in cli_logger_names)]: + for handler in list(logger.handlers): + logger.removeHandler(handler) + handler.close() + + +def main(): + from azure.cli.core import get_default_cli + + get_default_cli() # warms the imports and the config before the first command + _reply({"ready": True}) + for line in sys.stdin: + if not line.strip(): + continue + req = json.loads(line) + for key, value in req.get("env", {}).items(): + os.environ[key] = value + os.chdir(req["cwd"]) + # A fresh CLI context per command, as a subprocess gets: a command may change its own + # context, e.g. az rest unregisters the global result transforms, which with one shared + # context left every later command without resourceGroup. Imports stay warm. + cli = get_default_cli() + # knack configures logging only while the root logger has no handlers: drop the + # previous command's, so each command gets its own level, as a subprocess does. + _reset_logging() + out = _Capped() + rc = 1 + saved_stdin = sys.stdin + with _fd_to_file(2) as err_file, _fd_to_file(1) as raw_file: + sys.stdin = io.StringIO("") + try: + rc = cli.invoke(req["argv"], out_file=out) + except SystemExit as e: + rc = e.code if isinstance(e.code, int) else (0 if e.code is None else 1) + except BaseException as e: + sys.stderr.write("ERROR: the az worker caught %s: %s\n" % (type(e).__name__, e)) + rc = 1 + finally: + sys.stdin = saved_stdin + err, err_capped = _read_capped(err_file) + raw, raw_capped = _read_capped(raw_file) + _reply({ + "id": req["id"], + "exitCode": int(rc or 0), + "stdout": out.getvalue() + raw, + "stderr": err, + "truncated": out.capped or err_capped or raw_capped, + }) + + +main() +`; diff --git a/src/lib/cli/argv.test.ts b/src/lib/cli/argv.test.ts new file mode 100644 index 0000000..12b1a8f --- /dev/null +++ b/src/lib/cli/argv.test.ts @@ -0,0 +1,282 @@ +import { CliSyntaxError, splitCliArgs, type CliArgsOptions } from "./argv"; + +// The options the Azure client passes. +const AZURE: CliArgsOptions = { + quotedControlChars: true, + keepEmptyQuoted: true, + bashDoubleQuoteEscapes: true, +}; + +const errorOf = (fn: () => unknown): CliSyntaxError => { + try { + fn(); + } catch (error) { + return error as CliSyntaxError; + } + throw new Error("expected splitCliArgs to throw"); +}; + +describe("splitCliArgs: the AWS behaviour (no options)", () => { + test.each([ + ["s3 ls", ["s3", "ls"]], + ["s3\tls", ["s3", "ls"]], + [ + "ec2 describe-instances --filters 'Name=tag:Name,Values=test instance'", + ["ec2", "describe-instances", "--filters", "Name=tag:Name,Values=test instance"], + ], + [ + "s3api head-object --key 'price$2026.txt'", + ["s3api", "head-object", "--key", "price$2026.txt"], + ], + [ + String.raw`s3api head-object --key 'folder\file$2026.txt'`, + ["s3api", "head-object", "--key", String.raw`folder\file$2026.txt`], + ], + [ + `dynamodb put-item --item '{":value":{"S":"a|b;$c"}}'`, + ["dynamodb", "put-item", "--item", '{":value":{"S":"a|b;$c"}}'], + ], + ])("splits %s", (command, expected) => { + expect(splitCliArgs(command)).toEqual(expected); + }); + + test("parses escaped JSON, dollar signs, backslashes and tabs as argv", () => { + const command = String.raw`s3api put-object --cli-input-json "{\"Bucket\":\"test\",\"Key\":\"folder\\price$2026.txt\"}"`; + expect(splitCliArgs(command)).toEqual([ + "s3api", + "put-object", + "--cli-input-json", + '{"Bucket":"test","Key":"folder\\price$2026.txt"}', + ]); + }); + + test.each([ + "s3 ls || echo injected", + "s3 ls && echo injected", + "s3 ls; echo injected", + "s3 ls | tee output", + "s3 ls &", + "s3 ls `whoami`", + "s3 ls $(whoami)", + "s3 ls ${HOME}", + "s3 ls > output", + "s3 ls < input", + "s3 ls\necho injected", + "s3 ls\recho injected", + ])("refuses shell syntax: %s", (command) => { + const error = errorOf(() => splitCliArgs(command)); + expect(error).toBeInstanceOf(CliSyntaxError); + expect(error.code).toBe("shell-syntax"); + expect(error.message).toBe("Command contains forbidden shell syntax."); + }); + + test.each(["s3 ls 'unterminated", 's3 ls "unterminated'])( + "refuses unterminated quotes: %s", + (command) => { + const error = errorOf(() => splitCliArgs(command)); + expect(error.code).toBe("unterminated-quote"); + expect(error.message).toBe("Command contains an unterminated quote."); + } + ); + + // The old behaviour is pinned: the options only change things when passed. + test("drops quoted empty arguments", () => { + expect(splitCliArgs('keyvault secret set --value ""')).toEqual([ + "keyvault", + "secret", + "set", + "--value", + ]); + }); + + test("keeps the backslash before $ inside double quotes", () => { + expect(splitCliArgs('x --consumer-group "\\$Default"')).toEqual([ + "x", + "--consumer-group", + "\\$Default", + ]); + }); + + test.each([ + "monitor diagnostic-settings create --logs '[\n {\"enabled\": true}\n]'", + "x --query '{a: join(`,`, b)}'", + ])("refuses newlines and backticks even inside quotes: %s", (command) => { + expect(errorOf(() => splitCliArgs(command)).code).toBe("shell-syntax"); + }); +}); + +describe("splitCliArgs: the Azure options", () => { + test("accepts multi-line JSON inside single quotes", () => { + const command = "monitor diagnostic-settings create --logs '[\n {\"enabled\": true}\n]'"; + expect(splitCliArgs(command, AZURE)).toEqual([ + "monitor", + "diagnostic-settings", + "create", + "--logs", + '[\n {"enabled": true}\n]', + ]); + }); + + test("accepts CRLF inside quotes and keeps it as data", () => { + expect(splitCliArgs("x --policy '{\r\n}'", AZURE)).toEqual(["x", "--policy", "{\r\n}"]); + }); + + test("accepts JMESPath backtick literals inside single and double quotes", () => { + expect(splitCliArgs("x --query '{a: join(`,`, b)}'", AZURE)).toEqual([ + "x", + "--query", + "{a: join(`,`, b)}", + ]); + // Inside double quotes a backtick stays literal: there is no shell to substitute it. + expect(splitCliArgs('x --query "[?enabled==`true`]"', AZURE)).toEqual([ + "x", + "--query", + "[?enabled==`true`]", + ]); + }); + + test.each(["x\ny", "x\ry", "x `whoami`"])( + "still refuses control characters outside quotes: %j", + (command) => { + expect(errorOf(() => splitCliArgs(command, AZURE)).code).toBe("shell-syntax"); + } + ); + + test("keeps quoted empty arguments, in both quote styles", () => { + expect(splitCliArgs('keyvault secret set --value ""', AZURE)).toEqual([ + "keyvault", + "secret", + "set", + "--value", + "", + ]); + expect(splitCliArgs("group update --tags ''", AZURE)).toEqual([ + "group", + "update", + "--tags", + "", + ]); + expect(splitCliArgs('x """"', AZURE)).toEqual(["x", ""]); + }); + + test("joins adjacent quoted and unquoted parts into one word, as bash does", () => { + expect(splitCliArgs("x a\"b c\"d 'e'f", AZURE)).toEqual(["x", "ab cd", "ef"]); + }); + + test("unescapes \\$ and \\` inside double quotes, as bash does", () => { + expect( + splitCliArgs('eventhubs eventhub consumer-group show --name "\\$Default"', AZURE) + ).toEqual(["eventhubs", "eventhub", "consumer-group", "show", "--name", "$Default"]); + expect(splitCliArgs('x "a\\`b"', AZURE)).toEqual(["x", "a`b"]); + }); + + test("drops a line continuation inside double quotes", () => { + expect(splitCliArgs('x "a\\\nb"', AZURE)).toEqual(["x", "ab"]); + }); + + test("keeps other backslashes inside double quotes literally", () => { + expect(splitCliArgs('x "C:\\Program Files\\app"', AZURE)).toEqual([ + "x", + "C:\\Program Files\\app", + ]); + }); +}); + +describe("splitCliArgs: Azure command shapes (inside quotes)", () => { + test.each([ + [ + `vm list --query "[?location=='westeurope'] | [0].name"`, + ["vm", "list", "--query", "[?location=='westeurope'] | [0].name"], + ], + [ + `appconfig kv list --query "[?key=='A' || key=='B'].key"`, + ["appconfig", "kv", "list", "--query", "[?key=='A' || key=='B'].key"], + ], + [ + `graph query -q "Resources | where type =~ 'microsoft.storage/storageaccounts' | project name"`, + [ + "graph", + "query", + "-q", + "Resources | where type =~ 'microsoft.storage/storageaccounts' | project name", + ], + ], + [ + `rest --method put --body '{"a":"b c"}'`, + ["rest", "--method", "put", "--body", '{"a":"b c"}'], + ], + [ + `resource update --set properties.x='{"a":1}'`, + ["resource", "update", "--set", 'properties.x={"a":1}'], + ], + [ + `webapp config connection-string set --settings "Db=Server=tcp:x;Database=y;User Id=z"`, + [ + "webapp", + "config", + "connection-string", + "set", + "--settings", + "Db=Server=tcp:x;Database=y;User Id=z", + ], + ], + [ + `apim api policy create --value ''`, + [ + "apim", + "api", + "policy", + "create", + "--value", + "", + ], + ], + [ + `storage blob upload --file "C:\\Users\\me\\My Files\\a.txt"`, + ["storage", "blob", "upload", "--file", "C:\\Users\\me\\My Files\\a.txt"], + ], + [`x --name "tab\there"`, ["x", "--name", "tab\there"]], + [`group create --name "gruppe-müller-✓"`, ["group", "create", "--name", "gruppe-müller-✓"]], + [`group update --tags ""`, ["group", "update", "--tags", ""]], + ])("splits %s", (command, expected) => { + expect(splitCliArgs(command, AZURE)).toEqual(expected); + }); +}); + +describe("splitCliArgs: deliberate differences from bash, pinned in both modes", () => { + test.each([{}, AZURE])("an unquoted backslash stays literal (options: %j)", (opts) => { + expect(splitCliArgs(String.raw`storage blob upload --file C:\x\y.txt`, opts)).toEqual([ + "storage", + "blob", + "upload", + "--file", + String.raw`C:\x\y.txt`, + ]); + }); + + test.each([{}, AZURE])("the POSIX '\\'' idiom is refused (options: %j)", (opts) => { + expect(errorOf(() => splitCliArgs("x --name 'it'\\''s'", opts)).code).toBe( + "unterminated-quote" + ); + }); + + test.each([{}, AZURE])("U+00A0 splits like a space (options: %j)", (opts) => { + expect(splitCliArgs("group\u00a0list", opts)).toEqual(["group", "list"]); + }); + + test.each([{}, AZURE])( + "an unquoted #word is an argument, not a comment (options: %j)", + (opts) => { + expect(splitCliArgs("x --name #tag", opts)).toEqual(["x", "--name", "#tag"]); + } + ); + + test.each([{}, AZURE])("$VAR outside quotes is passed literally (options: %j)", (opts) => { + expect(splitCliArgs("x --name $NAME", opts)).toEqual(["x", "--name", "$NAME"]); + }); + + test.each([{}, AZURE])("$( and ${ outside quotes are refused (options: %j)", (opts) => { + expect(errorOf(() => splitCliArgs("x $(id)", opts)).code).toBe("shell-syntax"); + expect(errorOf(() => splitCliArgs("x ${HOME}", opts)).code).toBe("shell-syntax"); + }); +}); diff --git a/src/lib/cli/argv.ts b/src/lib/cli/argv.ts new file mode 100644 index 0000000..bea1cc4 --- /dev/null +++ b/src/lib/cli/argv.ts @@ -0,0 +1,113 @@ +/** + * Quote-aware splitting of a CLI command string into an argv array, shared by the + * AWS and Azure client tools. + * + * No shell is ever involved: the result goes to exec/spawn as an argument vector, so + * this only has to reproduce POSIX word splitting for a safe subset, and refuse + * anything that would need a shell (pipes, chaining, redirects, substitutions). + * + * Deliberate differences from bash, pinned by tests: + * - an unquoted backslash is literal (Windows paths such as C:\x\y work); + * - `$VAR` outside quotes is passed literally (there is no expansion); + * - an unquoted `#word` is an argument, not a comment; + * - the POSIX `'\''` idiom is refused as an unterminated quote. + */ + +export class CliSyntaxError extends Error { + constructor( + public readonly code: "shell-syntax" | "unterminated-quote", + message: string + ) { + super(message); + this.name = "CliSyntaxError"; + } +} + +/** + * Opt-in fixes that the Azure client turns on. All default to false, so callers + * that pass no options (the AWS client) keep their original behaviour exactly. + */ +export interface CliArgsOptions { + /** Refuse newline, CR and backtick only outside quotes; inside quotes they are data. */ + quotedControlChars?: boolean; + /** `""` and `''` produce an empty argument instead of disappearing. */ + keepEmptyQuoted?: boolean; + /** Inside double quotes, `\$`, `` \` `` and `\` behave as in bash. */ + bashDoubleQuoteEscapes?: boolean; +} + +const SHELL_SYNTAX_MESSAGE = "Command contains forbidden shell syntax."; +const UNTERMINATED_QUOTE_MESSAGE = "Command contains an unterminated quote."; + +export function splitCliArgs(command: string, opts: CliArgsOptions = {}): string[] { + const args: string[] = []; + let current = ""; + // True once the current word has started, even if it holds no characters yet (a + // quoted empty string). Only consulted when keepEmptyQuoted is on. + let wordStarted = false; + let quote: "'" | '"' | undefined; + + const endWord = () => { + if (current || (opts.keepEmptyQuoted && wordStarted)) { + args.push(current); + } + current = ""; + wordStarted = false; + }; + + for (let index = 0; index < command.length; index++) { + const character = command[index]; + + if (character === "\n" || character === "\r" || character === "`") { + if (!opts.quotedControlChars || !quote) { + throw new CliSyntaxError("shell-syntax", SHELL_SYNTAX_MESSAGE); + } + } + + if (character === "\\" && quote === '"') { + const escaped = command[index + 1]; + if (escaped === '"' || escaped === "\\") { + current += escaped; + index++; + } else if (opts.bashDoubleQuoteEscapes && (escaped === "$" || escaped === "`")) { + current += escaped; + index++; + } else if (opts.bashDoubleQuoteEscapes && escaped === "\n") { + // Line continuation inside double quotes: bash drops both characters. + index++; + } else { + current += character; + } + continue; + } + + if ((character === "'" || character === '"') && (!quote || quote === character)) { + quote = quote ? undefined : character; + wordStarted = true; + continue; + } + + if (!quote) { + if ( + ";&|<>".includes(character) || + command.startsWith("$(", index) || + command.startsWith("${", index) + ) { + throw new CliSyntaxError("shell-syntax", SHELL_SYNTAX_MESSAGE); + } + if (/\s/.test(character)) { + endWord(); + continue; + } + } + + current += character; + wordStarted = true; + } + + if (quote) { + throw new CliSyntaxError("unterminated-quote", UNTERMINATED_QUOTE_MESSAGE); + } + endWord(); + return args; +} diff --git a/src/lib/docker/docker.client.test.ts b/src/lib/docker/docker.client.test.ts index 6284e90..4efc0f9 100644 --- a/src/lib/docker/docker.client.test.ts +++ b/src/lib/docker/docker.client.test.ts @@ -279,6 +279,218 @@ describe("DockerApiClient", () => { ); }); + // --- Azure as a third stack --- + + // A port-shifted test emulator as the spec builder really publishes it: + // GATEWAY_LISTEN=:4666 gives 4666 -> 4666, the service range 4610-4660, and no 443. + const recipeTestContainer = { + Id: "test-4666", + Names: ["/localstack-azure-mcp-test"], + Image: "localstack/localstack-azure:latest", + Ports: [ + { PrivatePort: 4666, PublicPort: 4666, Type: "tcp" }, + { PrivatePort: 4610, PublicPort: 4610, Type: "tcp" }, + { PrivatePort: 4660, PublicPort: 4660, Type: "tcp" }, + ], + }; + const sharedAzure = { + Id: "shared-4566", + Names: ["/localstack-azure"], + Image: "f91897f1de85", + Labels: { description: "LocalStack for Azure" }, + Ports: [{ PrivatePort: 4566, PublicPort: 4566, Type: "tcp" }], + }; + + test("findLocalStackContainer accepts the localstack-azure image by image match", async () => { + const mocks = getDockerMocks(); + mocks.listContainers.mockResolvedValueOnce([ + { Id: "img-azure", Names: ["/some-azure"], Image: "localstack/localstack-azure:latest" }, + ]); + await expect(new DockerApiClient().findLocalStackContainer()).resolves.toBe("img-azure"); + }); + + test("findLocalStackContainer finds the known name localstack-azure", async () => { + const mocks = getDockerMocks(); + mocks.listContainers.mockResolvedValueOnce([ + { Id: "az1", Names: ["/localstack-azure"], Image: "localstack/localstack-azure:latest" }, + ]); + await expect(new DockerApiClient().findLocalStackContainer()).resolves.toBe("az1"); + }); + + test("findLocalStackContainer treats lstk's tagged localstack-azure- as a known name", async () => { + const mocks = getDockerMocks(); + mocks.listContainers.mockResolvedValueOnce([ + { Id: "az-tag", Names: ["/localstack-azure-2026.9"], Image: "registry/x:1" }, + ]); + await expect(new DockerApiClient().findLocalStackContainer()).resolves.toBe("az-tag"); + }); + + test("with LOCALSTACK_PORT=4666 the lookup picks the test container, never the shared localstack-azure", async () => { + process.env.LOCALSTACK_PORT = "4666"; + const mocks = getDockerMocks(); + mocks.listContainers.mockResolvedValueOnce([sharedAzure, recipeTestContainer]); + await expect(new DockerApiClient().findLocalStackContainer()).resolves.toBe("test-4666"); + }); + + test("with LOCALSTACK_PORT=4666 and only the shared emulator running, the lookup fails instead of picking it", async () => { + process.env.LOCALSTACK_PORT = "4666"; + const mocks = getDockerMocks(); + mocks.listContainers.mockResolvedValueOnce([sharedAzure]); + await expect(new DockerApiClient().findLocalStackContainer()).rejects.toThrow( + /Could not find a running LocalStack container/i + ); + }); + + test("a known name that does not publish the explicit port is not picked", async () => { + process.env.LOCALSTACK_PORT = "4567"; + const mocks = getDockerMocks(); + mocks.listContainers.mockResolvedValueOnce([ + { + Id: "main-4566", + Names: ["/localstack-main"], + Image: "localstack/localstack-pro:latest", + Ports: [{ PrivatePort: 4566, PublicPort: 4566, Type: "tcp" }], + }, + ]); + await expect(new DockerApiClient().findLocalStackContainer()).rejects.toThrow( + /none publishes the configured gateway port 4567/i + ); + }); + + test("with LOCALSTACK_PORT set, a known name that publishes no port (host or compose network) is found", async () => { + process.env.LOCALSTACK_PORT = "4566"; + const mocks = getDockerMocks(); + mocks.listContainers.mockResolvedValueOnce([ + { + Id: "host-network", + Names: ["/localstack-main"], + Image: "localstack/localstack-pro:latest", + Ports: [], + }, + ]); + await expect(new DockerApiClient().findLocalStackContainer()).resolves.toBe("host-network"); + }); + + test("{stack, port} looks on that port: the Azure emulator on 4566 beside an AWS one on LOCALSTACK_PORT", async () => { + process.env.LOCALSTACK_PORT = "4666"; + const mocks = getDockerMocks(); + const aws = { + Id: "aws-4666", + Names: ["/localstack-main"], + Image: "localstack/localstack-pro:latest", + Ports: [{ PrivatePort: 4566, PublicPort: 4666, Type: "tcp" }], + }; + mocks.listContainers.mockResolvedValue([aws, sharedAzure]); + const client = new DockerApiClient(); + await expect(client.findLocalStackContainer({ stack: "azure", port: "4566" })).resolves.toBe( + sharedAzure.Id + ); + // Without the port, LOCALSTACK_PORT decides, and it names the AWS emulator. + await expect(client.findLocalStackContainer()).resolves.toBe("aws-4666"); + mocks.listContainers.mockReset(); + }); + + test("the port rule accepts the default 4566 -> layout", async () => { + process.env.LOCALSTACK_PORT = "4666"; + const mocks = getDockerMocks(); + mocks.listContainers.mockResolvedValueOnce([ + sharedAzure, + { + Id: "remapped", + Names: ["/localstack-main"], + Image: "localstack/localstack-azure:latest", + Ports: [{ PrivatePort: 4566, PublicPort: 4666, Type: "tcp" }], + }, + ]); + await expect(new DockerApiClient().findLocalStackContainer()).resolves.toBe("remapped"); + }); + + test("two known-name candidates on the configured port are an ambiguity error naming both", async () => { + process.env.LOCALSTACK_PORT = "4666"; + const mocks = getDockerMocks(); + mocks.listContainers.mockResolvedValueOnce([ + { ...recipeTestContainer, Id: "a", Names: ["/localstack-main"] }, + { ...recipeTestContainer, Id: "b", Names: ["/localstack-azure"] }, + ]); + await expect(new DockerApiClient().findLocalStackContainer()).rejects.toThrow( + /several LocalStack containers publishing the configured gateway port 4666: "localstack-main", "localstack-azure"/ + ); + }); + + test("{stack: 'aws'} with only an Azure container throws a not-found error naming the Azure container", async () => { + const mocks = getDockerMocks(); + mocks.listContainers.mockResolvedValue([ + { Id: "az1", Names: ["/localstack-azure"], Image: "localstack/localstack-azure:latest" }, + ]); + const client = new DockerApiClient(); + const error = await client.findLocalStackContainer({ stack: "aws" }).catch((e) => e); + expect(error.name).toBe("LocalStackContainerNotFoundError"); + expect(error.message).toMatch(/"localstack-azure".*is the Azure stack, not the AWS stack/); + mocks.listContainers.mockReset(); + }); + + test("{stack: 'aws'} finds the Snowflake emulator's container: it serves the AWS APIs too", async () => { + const mocks = getDockerMocks(); + mocks.listContainers.mockResolvedValue([ + { Id: "snow1", Names: ["/localstack-main"], Image: "localstack/snowflake:latest" }, + ]); + await expect(new DockerApiClient().findLocalStackContainer({ stack: "aws" })).resolves.toBe( + "snow1" + ); + mocks.listContainers.mockReset(); + }); + + test("{stack: 'azure'} finds the Azure container", async () => { + const mocks = getDockerMocks(); + mocks.listContainers.mockResolvedValueOnce([ + { Id: "aws1", Names: ["/localstack-main"], Image: "localstack/localstack-pro:latest" }, + { Id: "az1", Names: ["/localstack-azure"], Image: "localstack/localstack-azure:latest" }, + ]); + await expect(new DockerApiClient().findLocalStackContainer({ stack: "azure" })).resolves.toBe( + "az1" + ); + }); + + test("F38: a bare-ID container with the Azure label counts as Azure for the stack filter", async () => { + const mocks = getDockerMocks(); + mocks.listContainers.mockResolvedValue([sharedAzure]); + const client = new DockerApiClient(); + await expect(client.findLocalStackContainer({ stack: "azure" })).resolves.toBe("shared-4566"); + await expect(client.findLocalStackContainer({ stack: "aws" })).rejects.toThrow( + /is the Azure stack/ + ); + mocks.listContainers.mockReset(); + }); + + test("a bare-ID container without a label is unknown, so the stack filter keeps it", async () => { + const mocks = getDockerMocks(); + mocks.listContainers.mockResolvedValueOnce([ + { Id: "bare", Names: ["/localstack-main"], Image: "0123456789ab" }, + ]); + await expect(new DockerApiClient().findLocalStackContainer({ stack: "aws" })).resolves.toBe( + "bare" + ); + }); + + test("without a stack filter the lookup keeps today's behaviour for an Azure container", async () => { + const mocks = getDockerMocks(); + mocks.listContainers.mockResolvedValueOnce([ + { Id: "az1", Names: ["/localstack-aws"], Image: "localstack/localstack-azure-alpha:latest" }, + ]); + await expect(new DockerApiClient().findLocalStackContainer()).resolves.toBe("az1"); + }); + + test("inspectContainer returns the container labels", async () => { + const mocks = getDockerMocks(); + mocks.containerInspect.mockResolvedValueOnce({ + Name: "/localstack-azure", + Config: { Image: "f91897f1de85", Labels: { description: "LocalStack for Azure" }, Env: [] }, + Mounts: [], + }); + const metadata = await new DockerApiClient().inspectContainer("x"); + expect(metadata.labels).toEqual({ description: "LocalStack for Azure" }); + }); + test("stopContainer stops the container via the Docker API", async () => { const mocks = getDockerMocks(); mocks.stop.mockResolvedValueOnce(undefined); diff --git a/src/lib/docker/docker.client.ts b/src/lib/docker/docker.client.ts index fb512ee..c74c316 100644 --- a/src/lib/docker/docker.client.ts +++ b/src/lib/docker/docker.client.ts @@ -1,7 +1,11 @@ import { PassThrough } from "stream"; import { createRequire } from "node:module"; import { LOCALSTACK_PORT } from "../../core/config"; -import type { LocalStackContainerSpec } from "../localstack/container-spec.logic"; +import { + stackFromImage, + type LocalStackContainerSpec, + type LocalStackStack, +} from "../localstack/container-spec.logic"; export interface ContainerExecResult { stdout: string; @@ -104,10 +108,35 @@ export interface ContainerMetadata { id: string; name?: string; image?: string; + /** `Config.Labels`; the Azure image's `description` label identifies bare-ID containers. */ + labels?: Record; env?: string[]; mounts?: ContainerMountInfo[]; + /** `HostConfig.PortBindings`: container port ("4566/tcp") to its host bindings. */ + portBindings?: Record | null>; + /** The container's IP on each network it is attached to. */ + ipAddresses?: string[]; } +interface RunningContainerSummary { + Id: string; + Names?: string[]; + Image?: string; + Labels?: Record; + Ports?: Array<{ PrivatePort?: number; PublicPort?: number; Type?: string }>; +} + +const STACK_LABELS: Record = { + aws: "AWS", + snowflake: "Snowflake", + azure: "Azure", +}; + +/** Container names LocalStack tooling uses, in lookup priority order. */ +const KNOWN_CONTAINER_NAMES = ["localstack-main", "localstack-aws", "localstack-azure"]; +/** lstk names the Azure container `localstack-azure-` when the tag is not `latest`. */ +const LSTK_AZURE_TAGGED_NAME = /^localstack-azure-[A-Za-z0-9._-]+$/; + export class LocalStackContainerNotFoundError extends Error { constructor(message: string) { super(message); @@ -144,18 +173,29 @@ export class DockerApiClient { return (container.Names || []).some((n) => this.normalizeContainerName(n) === configuredName); } - private publishesConfiguredGatewayPort(container: { - Ports?: Array<{ PrivatePort?: number; PublicPort?: number; Type?: string }>; - }): boolean { - const configuredPort = Number(process.env.LOCALSTACK_PORT || LOCALSTACK_PORT); + /** + * Whether the container publishes the configured gateway port on the host. The + * container side is 4566 by default, but the spec builder publishes GATEWAY_LISTEN + * entries 1:1 (container 4666 -> host 4666), so the container port may also equal + * the configured one. + */ + private publishesConfiguredGatewayPort( + container: { + Ports?: Array<{ PrivatePort?: number; PublicPort?: number; Type?: string }>; + }, + port?: string + ): boolean { + const configuredPort = Number(port || process.env.LOCALSTACK_PORT || LOCALSTACK_PORT); return (container.Ports || []).some( (port) => - port.Type === "tcp" && port.PrivatePort === 4566 && port.PublicPort === configuredPort + port.Type === "tcp" && + port.PublicPort === configuredPort && + (port.PrivatePort === 4566 || port.PrivatePort === configuredPort) ); } private hasLocalStackImage(container: { Image?: string }): boolean { - return /^(?:[^/]+\/)?localstack\/(?:localstack(?:-pro)?|snowflake|localstack-azure-alpha)(?::|@|$)/.test( + return /^(?:[^/]+\/)?localstack\/(?:localstack(?:-pro)?|snowflake|localstack-azure(?:-alpha)?)(?::|@|$)/.test( container.Image || "" ); } @@ -191,24 +231,78 @@ export class DockerApiClient { ); } - private findByKnownLocalStackName( - containers: T[] - ): T | undefined { - return ["localstack-main", "localstack-aws"] - .map((name) => containers.find((c) => this.matchesConfiguredContainerName(c, name))) - .find(Boolean); + /** Containers with a known LocalStack name, in priority order. */ + private findByKnownLocalStackNames(containers: T[]): T[] { + const byExactName = KNOWN_CONTAINER_NAMES.map((name) => + containers.find((c) => this.matchesConfiguredContainerName(c, name)) + ).filter((c): c is T => Boolean(c)); + const byLstkTag = containers.filter( + (c) => + !byExactName.includes(c) && + (c.Names || []).some((n) => LSTK_AZURE_TAGGED_NAME.test(this.normalizeContainerName(n))) + ); + return [...byExactName, ...byLstkTag]; + } + + private containerLabel(container: RunningContainerSummary): string { + const name = this.normalizeContainerName(container.Names?.[0]); + return `"${name || container.Id}"`; } - async findLocalStackContainer(): Promise { - const running = (await (this.docker.listContainers as any)({ + /** + * A candidate of a known, different stack is skipped; an unknown one is kept. The Snowflake + * image also serves the AWS APIs and ships awslocal (its stages and Snowpipe read S3), so it + * counts for AWS. + */ + private isStackCompatible(container: RunningContainerSummary, stack: LocalStackStack): boolean { + const actual = stackFromImage(container.Image, container.Labels); + return actual === undefined || actual === stack || (stack === "aws" && actual === "snowflake"); + } + + /** + * Find the running LocalStack container. With `stack`, containers that belong to a + * different stack are skipped, and a failure names the container that was found. With + * `port`, that port stands in for LOCALSTACK_PORT: the Azure emulator beside an AWS one + * sits on the Azure tool's own port. + */ + async findLocalStackContainer( + opts: { stack?: LocalStackStack; port?: string } = {} + ): Promise { + const running = ((await (this.docker.listContainers as any)({ filters: { status: ["running"] }, - })) as Array<{ - Id: string; - Names?: string[]; - Image?: string; - Ports?: Array<{ PrivatePort?: number; PublicPort?: number; Type?: string }>; - }>; + })) || []) as RunningContainerSummary[]; + + if (!opts.stack) return this.selectLocalStackContainer(running, opts.port); + const expected = opts.stack; + try { + return this.selectLocalStackContainer( + running.filter((container) => this.isStackCompatible(container, expected)), + opts.port + ); + } catch (error) { + if (!isLocalStackContainerNotFoundError(error)) throw error; + let otherId: string | undefined; + try { + otherId = this.selectLocalStackContainer(running, opts.port); + } catch { + otherId = undefined; + } + const other = running.find((container) => container.Id === otherId); + const otherStack = other ? stackFromImage(other.Image, other.Labels) : undefined; + if (other && otherStack && otherStack !== expected) { + throw new LocalStackContainerNotFoundError( + `The running LocalStack container ${this.containerLabel(other)} (image: ${other.Image}) is the ` + + `${STACK_LABELS[otherStack]} stack, not the ${STACK_LABELS[expected]} stack this tool needs.` + ); + } + throw error; + } + } + + private selectLocalStackContainer(running: RunningContainerSummary[], port?: string): string { + const portOverride = port?.trim() || undefined; + const configuredPort = portOverride || process.env.LOCALSTACK_PORT?.trim(); const explicitName = ( process.env.MAIN_CONTAINER_NAME || process.env.LOCALSTACK_MAIN_CONTAINER_NAME || @@ -224,17 +318,38 @@ export class DockerApiClient { } if (!explicitName) { - const byKnownName = this.findByKnownLocalStackName(running || []); - if (byKnownName) return byKnownName.Id as string; + const explicitPort = Boolean(configuredPort); + const knownNames = this.findByKnownLocalStackNames(running || []); + if (knownNames.length > 0 && !explicitPort) return knownNames[0].Id as string; + if (knownNames.length > 0 && explicitPort) { + // With an explicit port, a known name alone is not enough: a shared + // `localstack-azure` on 4566 must never be picked by a server configured for a + // test emulator on another port. + const onPort = knownNames.filter((c) => + this.publishesConfiguredGatewayPort(c, portOverride) + ); + if (onPort.length === 1) return onPort[0].Id as string; + if (onPort.length > 1) { + throw new LocalStackContainerNotFoundError( + `Found several LocalStack containers publishing the configured gateway port ${configuredPort}: ` + + `${onPort.map((c) => this.containerLabel(c)).join(", ")}. Set MAIN_CONTAINER_NAME to the one to use.` + ); + } + // A container that publishes no port at all (host network, a compose network) cannot + // sit on another port, so its known name still identifies it. + const unpublished = knownNames.filter((c) => !(c.Ports || []).some((p) => p.PublicPort)); + if (unpublished.length === 1) return unpublished[0].Id as string; + } const localstackImages = (running || []).filter((c) => this.hasLocalStackImage(c)); - const byGatewayPort = localstackImages.find((c) => this.publishesConfiguredGatewayPort(c)); + const byGatewayPort = localstackImages.find((c) => + this.publishesConfiguredGatewayPort(c, portOverride) + ); if (byGatewayPort) return byGatewayPort.Id as string; - const explicitPort = Boolean(process.env.LOCALSTACK_PORT?.trim()); if (explicitPort && localstackImages.length > 0) { throw new LocalStackContainerNotFoundError( - `Found running LocalStack containers, but none publishes the configured gateway port ${process.env.LOCALSTACK_PORT}. ` + + `Found running LocalStack containers, but none publishes the configured gateway port ${configuredPort}. ` + `Set MAIN_CONTAINER_NAME to the container name to use.` ); } @@ -242,7 +357,7 @@ export class DockerApiClient { if (localstackImages.length === 1) return localstackImages[0].Id as string; if (localstackImages.length > 1) { throw new LocalStackContainerNotFoundError( - `Found multiple running LocalStack containers but none publishes the configured gateway port ${process.env.LOCALSTACK_PORT || LOCALSTACK_PORT}. ` + + `Found multiple running LocalStack containers but none publishes the configured gateway port ${configuredPort || LOCALSTACK_PORT}. ` + `Set MAIN_CONTAINER_NAME to the container name to use.` ); } @@ -261,6 +376,7 @@ export class DockerApiClient { id: containerId, name: this.normalizeContainerName(inspect?.Name), image: inspect?.Config?.Image, + labels: inspect?.Config?.Labels || undefined, env: inspect?.Config?.Env, mounts: (inspect?.Mounts || []).map( (mount: { Type?: string; Name?: string; Source?: string; Destination?: string }) => ({ @@ -270,9 +386,19 @@ export class DockerApiClient { destination: mount.Destination, }) ), + portBindings: inspect?.HostConfig?.PortBindings || undefined, + ipAddresses: this.containerIps(inspect?.NetworkSettings?.Networks), }; } + /** The container's IP on each attached network; undefined when there is none. */ + private containerIps(networks: unknown): string[] | undefined { + const ips = Object.values((networks || {}) as Record) + .map((network) => network?.IPAddress) + .filter((ip): ip is string => Boolean(ip)); + return ips.length ? ips : undefined; + } + /** * Stop a container via the Docker API (graceful SIGTERM, then SIGKILL after the * timeout). Provenance-agnostic — works regardless of which CLI started it (or none) @@ -400,6 +526,25 @@ export class DockerApiClient { ); } + /** + * The image's baked-in `Config.Env` (an array of `KEY=value`). Used on restart to tell a + * container's operator-set env (the flags to carry over) from the image's own defaults. + * Best-effort: returns [] if the image cannot be inspected, so the caller falls + * back to its system-var backstop rather than failing the restart. + */ + async imageConfigEnv(imageName: string, timeoutMs = 30000): Promise { + try { + const info = (await this.withDockerRequestTimeout( + this.docker.getImage(imageName).inspect(), + timeoutMs, + "Docker image inspect" + )) as { Config?: { Env?: string[] } }; + return info?.Config?.Env ?? []; + } catch { + return []; + } + } + /** * Pull an image, consuming the progress stream manually. docker-modem's * followProgress reports success even when the stream carries an `error` event and diff --git a/src/lib/localstack/container-spec.logic.test.ts b/src/lib/localstack/container-spec.logic.test.ts index 6dd7ab0..bd37da8 100644 --- a/src/lib/localstack/container-spec.logic.test.ts +++ b/src/lib/localstack/container-spec.logic.test.ts @@ -1,10 +1,12 @@ import { + AZURE_CLIENT_ENV_KEYS, buildLocalStackContainerSpec, detectAiAgent, parseGatewayListen, resolveContainerName, resolveImage, resolveVolume, + stackFromEdition, type ContainerSpecInput, } from "./container-spec.logic"; @@ -43,6 +45,17 @@ describe("resolveImage", () => { "localstack/snowflake:latest" ); }); + + test("azure stack defaults to the azure image and honors only LOCALSTACK_AZURE_IMAGE_NAME", () => { + expect(resolveImage("azure", {})).toBe("localstack/localstack-azure:latest"); + expect(resolveImage("azure", { LOCALSTACK_AZURE_IMAGE_NAME: " my/azure:1 " })).toBe( + "my/azure:1" + ); + // An AWS image override must not hijack the Azure stack. + expect( + resolveImage("azure", { LOCALSTACK_IMAGE_NAME: "my/img:1", IMAGE_NAME: "other/img:2" }) + ).toBe("localstack/localstack-azure:latest"); + }); }); describe("resolveContainerName", () => { @@ -178,6 +191,49 @@ describe("buildLocalStackContainerSpec", () => { expect(Object.keys(spec.ExposedPorts)).toHaveLength(53); }); + test("stack azure uses LOCALSTACK_AZURE_IMAGE_NAME, and an AWS IMAGE_NAME never", () => { + const azure = buildLocalStackContainerSpec( + baseInput({ + stack: "azure", + hostEnv: { + LOCALSTACK_AZURE_IMAGE_NAME: "localstack/localstack-azure:2026.9", + IMAGE_NAME: "other/aws:1", + }, + }) + ); + expect(azure.Image).toBe("localstack/localstack-azure:2026.9"); + const byDefault = buildLocalStackContainerSpec( + baseInput({ stack: "azure", hostEnv: { IMAGE_NAME: "other/aws:1" } }) + ); + expect(byDefault.Image).toBe("localstack/localstack-azure:latest"); + }); + + test("stack azure forwards the Azure emulator's own settings from the env block", () => { + const env = envMap( + buildLocalStackContainerSpec( + baseInput({ + stack: "azure", + hostEnv: { + LS_AZURE_ENFORCE_RBAC: "1", + LS_AZURE_PORTAL: "1", + MSSQL_ACCEPT_EULA: "Y", + FRONT_DOOR_CLASSIC_ALLOW_CREATE: "1", + CDN_CLASSIC_ALLOW_CREATE: "1", + UNRELATED_HOST_VAR: "x", + }, + }) + ) + ); + expect(env).toMatchObject({ + LS_AZURE_ENFORCE_RBAC: "1", + LS_AZURE_PORTAL: "1", + MSSQL_ACCEPT_EULA: "Y", + FRONT_DOOR_CLASSIC_ALLOW_CREATE: "1", + CDN_CLASSIC_ALLOW_CREATE: "1", + }); + expect(env.UNRELATED_HOST_VAR).toBeUndefined(); + }); + test("binds to 0.0.0.0 when the server runs inside Docker (DooD reachability)", () => { const spec = buildLocalStackContainerSpec(baseInput({ isInDocker: true })); expect(spec.HostConfig.PortBindings["4566/tcp"][0].HostIp).toBe("0.0.0.0"); @@ -445,4 +501,78 @@ describe("stackFromImage", () => { expect(stackFromImage("localstack/localstack:4.5")).toBe("aws"); expect(stackFromImage(undefined)).toBeUndefined(); }); + + test.each([ + "localstack/localstack-azure:latest", + "localstack/localstack-azure-alpha:latest", + "localstack/localstack-azure@sha256:0123abcd", + "registry.example.com/localstack/localstack-azure:2026.9", + "localstack-azure:dev", + ])("classifies azure image %s", (image) => { + expect(stackFromImage(image)).toBe("azure"); + }); + + test("keeps the aws default for other named images", () => { + expect(stackFromImage("my-registry/custom-runtime:1")).toBe("aws"); + expect(stackFromImage("localstack/localstack-azure-tools:1")).toBe("aws"); + }); + + test("uses the Azure description label for a container started from a bare image ID", () => { + expect(stackFromImage("f91897f1de85", { description: "LocalStack for Azure emulator" })).toBe( + "azure" + ); + expect(stackFromImage("sha256:" + "a".repeat(64), { description: "Azure" })).toBe("azure"); + }); + + test("reports a bare image ID without the label as unknown, not aws", () => { + expect(stackFromImage("f91897f1de85")).toBeUndefined(); + expect(stackFromImage("f91897f1de85", { description: "something else" })).toBeUndefined(); + }); +}); + +describe("stackFromEdition", () => { + test.each([ + ["azure-alpha", "azure"], + ["AZURE-ALPHA", "azure"], + ["pro", "aws"], + ["bigdata-pro", "aws"], + ["community", "aws"], + ["enterprise", "aws"], + ["snowflake", "snowflake"], + ["snowflake-pro", "snowflake"], + ])("maps %s to %s", (edition, stack) => { + expect(stackFromEdition(edition)).toBe(stack); + }); + + test.each(["unknown", "", " ", undefined])("reports %p as unknown", (edition) => { + expect(stackFromEdition(edition as string | undefined)).toBeUndefined(); + }); +}); + +describe("Azure client settings are client-only", () => { + test("none of them, nor their unprefixed aliases, reaches the emulator container", () => { + const hostEnv = Object.fromEntries( + AZURE_CLIENT_ENV_KEYS.map((key) => [key, `value-of-${key}`]) + ); + const spec = buildLocalStackContainerSpec(baseInput({ stack: "azure", hostEnv })); + const env = envMap(spec); + for (const key of AZURE_CLIENT_ENV_KEYS) { + expect(env[key]).toBeUndefined(); + expect(env[key.replace(/^LOCALSTACK_/, "")]).toBeUndefined(); + } + expect(spec.Image).toBe("value-of-LOCALSTACK_AZURE_IMAGE_NAME"); + }); + + test("an emulator setting passed as LOCALSTACK_AZURE_* still reaches the container", () => { + const spec = buildLocalStackContainerSpec( + baseInput({ stack: "azure", hostEnv: { LOCALSTACK_AZURE_SOME_EMULATOR_FLAG: "1" } }) + ); + expect(envMap(spec).LOCALSTACK_AZURE_SOME_EMULATOR_FLAG).toBe("1"); + }); + + test("the Azure stack builds the default azure image spec", () => { + const spec = buildLocalStackContainerSpec(baseInput({ stack: "azure" })); + expect(spec.Image).toBe("localstack/localstack-azure:latest"); + expect(spec.name).toBe("localstack-main"); + }); }); diff --git a/src/lib/localstack/container-spec.logic.ts b/src/lib/localstack/container-spec.logic.ts index c17c85e..fc1814a 100644 --- a/src/lib/localstack/container-spec.logic.ts +++ b/src/lib/localstack/container-spec.logic.ts @@ -17,19 +17,52 @@ export const DEFAULT_CONTAINER_NAME = "localstack-main"; export const DEFAULT_AWS_IMAGE = "localstack/localstack-pro:latest"; export const DEFAULT_SNOWFLAKE_IMAGE = "localstack/snowflake:latest"; +export const DEFAULT_AZURE_IMAGE = "localstack/localstack-azure:latest"; export const DEFAULT_GATEWAY_CONTAINER_PORT = 4566; export const DEFAULT_HTTPS_GATEWAY_PORT = 443; export const DEFAULT_SERVICE_PORT_START = 4510; export const DEFAULT_SERVICE_PORT_END = 4560; // inclusive — 51 ports, CLI parity export const NAMED_VOLUME_NAME = "localstack-mcp"; +/** + * The LOCALSTACK_CLIENT_NAME this server stamps on every container it starts. A restart reads it + * back to tell its own containers from externally started ones, so it lives here once. + */ +export const MCP_CLIENT_NAME = "localstack-mcp-server"; const VOLUME_CONTAINER_PATH = "/var/lib/localstack"; const DOCKER_SOCKET_CONTAINER_PATH = "/var/run/docker.sock"; +/** + * Settings of the Azure client tool. They configure this server, not the emulator. + * Listed by name (not by prefix), so that other `LOCALSTACK_AZURE_*` values still reach the + * container; the emulator's own `LS_AZURE_*` settings are forwarded as they are. + */ +export const AZURE_CLIENT_ENV_KEYS = [ + "LOCALSTACK_AZURE_PORT", + "LOCALSTACK_AZURE_ENDPOINT", + "LOCALSTACK_AZURE_IMAGE_NAME", // selects the image; not runtime config + "LOCALSTACK_AZURE_FORWARD_TARGET", + "LOCALSTACK_AZ_PATH", + "LOCALSTACK_AZ_CONFIG_DIR", + "LOCALSTACK_AZ_EXTENSION_DIR", + "LOCALSTACK_AZ_BICEP_PATH", + "LOCALSTACK_AZ_TIMEOUT_SECONDS", + "LOCALSTACK_AZ_MAX_OUTPUT_CHARS", + "LOCALSTACK_AZ_MAX_HELP_CHARS", + "LOCALSTACK_AZ_WORKDIR", + "LOCALSTACK_AZ_EGRESS_GUARD", + "LOCALSTACK_AZ_DENYLIST_FILE", + "LOCALSTACK_AZ_RUNNER", + "LOCALSTACK_AZ_PYCACHE_DIR", + "LOCALSTACK_AZ_TEST_ENVELOPE", +] as const; + /** * Env vars that configure how MCP clients/tools reach LocalStack. They must never * leak into the runtime container, where they would corrupt its own URL generation. + * The container entrypoint re-exports `LOCALSTACK_X` as `X`, so an Azure client + * setting such as `LOCALSTACK_AZ_CONFIG_DIR` would otherwise arrive as a host path. */ -const CLIENT_ONLY_ENV_KEYS = new Set([ +const CLIENT_ONLY_ENV_KEYS = new Set([ "HOSTNAME", "LOCALSTACK_HOSTNAME", "LOCALSTACK_PORT", @@ -41,6 +74,7 @@ const CLIENT_ONLY_ENV_KEYS = new Set([ "LOCALSTACK_API_KEY", "LOCALSTACK_IMAGE_NAME", // selects the image; not runtime config "LOCALSTACK_VOLUME_DIR", // host-side path; meaningless inside the container + ...AZURE_CLIENT_ENV_KEYS, ]); /** @@ -60,8 +94,13 @@ const FORWARDED_CONFIG_ENV_NAMES = new Set([ "APP_INSPECTOR", "DNS_ADDRESS", "MAIN_DOCKER_NETWORK", + // The Azure emulator's settings that have no LS_AZURE_ prefix. + "MSSQL_ACCEPT_EULA", + "FRONT_DOOR_CLASSIC_ALLOW_CREATE", + "CDN_CLASSIC_ALLOW_CREATE", ]); -const FORWARDED_CONFIG_ENV_PREFIXES = ["LAMBDA_", "CFN_", "SNOWFLAKE_", "SF_"]; +// LS_AZURE_ is the Azure emulator's own settings (LS_AZURE_ENFORCE_RBAC, LS_AZURE_PORTAL, ...). +const FORWARDED_CONFIG_ENV_PREFIXES = ["LAMBDA_", "CFN_", "SNOWFLAKE_", "SF_", "LS_AZURE_"]; /** Same detector list the localstack CLI uses to tag agent-driven starts. */ const AI_AGENT_DETECTORS: Array<[string, string[]]> = [ @@ -80,7 +119,7 @@ const AI_AGENT_DETECTORS: Array<[string, string[]]> = [ ["replit", ["REPL_ID"]], ]; -export type LocalStackStack = "aws" | "snowflake"; +export type LocalStackStack = "aws" | "snowflake" | "azure"; export type VolumeResolution = { type: "bind"; source: string } | { type: "volume"; name: string }; @@ -128,17 +167,49 @@ export function resolveImage( hostEnv: Record ): string { if (stack === "snowflake") return DEFAULT_SNOWFLAKE_IMAGE; + if (stack === "azure") { + // Only the Azure-specific override: an AWS IMAGE_NAME must not hijack the Azure stack. + return hostEnv.LOCALSTACK_AZURE_IMAGE_NAME?.trim() || DEFAULT_AZURE_IMAGE; + } const override = hostEnv.LOCALSTACK_IMAGE_NAME?.trim() || hostEnv.IMAGE_NAME?.trim(); return override || DEFAULT_AWS_IMAGE; } -/** Which stack a LocalStack container image belongs to (undefined when unknown). */ -export function stackFromImage(image?: string): LocalStackStack | undefined { - if (!image) return undefined; - if (/\/snowflake(:|@|$)/.test(image)) return "snowflake"; +const AZURE_IMAGE = /(^|\/)localstack-azure(-alpha)?(:|@|$)/; +const BARE_IMAGE_ID = /^(sha256:)?[0-9a-f]{12,64}$/; + +/** + * Which stack a LocalStack container image belongs to (undefined when unknown). + * + * `labels` are the container's `Config.Labels`: the Azure image's `description` label + * names Azure, which identifies a container started from a bare image ID (as `docker + * ps` shows once the tag has moved on). A bare ID without that label is unknown, not + * AWS; any other named image keeps the historical "aws" default. + */ +export function stackFromImage( + image?: string, + labels?: Record +): LocalStackStack | undefined { + if (image && /\/snowflake(:|@|$)/.test(image)) return "snowflake"; + if (image && AZURE_IMAGE.test(image)) return "azure"; + if (/azure/i.test(labels?.description ?? "")) return "azure"; + if (!image || BARE_IMAGE_ID.test(image)) return undefined; return "aws"; } +/** + * Which stack a gateway's health `edition` names (undefined when unknown). Verified + * values: `pro` and `bigdata-pro` (AWS), `azure-alpha` (Azure). The Snowflake value + * is unconfirmed, so anything with "snowflake" in it counts. + */ +export function stackFromEdition(edition?: string): LocalStackStack | undefined { + const value = (edition || "").trim().toLowerCase(); + if (!value || value === "unknown") return undefined; + if (value.includes("azure")) return "azure"; + if (value.includes("snowflake")) return "snowflake"; + return "aws"; // pro, bigdata-pro, community, enterprise, ... +} + export function resolveContainerName(hostEnv: Record): string { return ( hostEnv.MAIN_CONTAINER_NAME?.trim() || @@ -389,6 +460,35 @@ function resolvePorts(input: ContainerSpecInput): ResolvedPorts { return { bindings, containerGatewayListen, servicePortStart, servicePortEnd }; } +/** Step (a) of buildEnv: a curated unprefixed config var. */ +function isForwardedConfigName(key: string): boolean { + return ( + FORWARDED_CONFIG_ENV_NAMES.has(key) || + FORWARDED_CONFIG_ENV_PREFIXES.some((prefix) => key.startsWith(prefix)) + ); +} + +/** Step (b) of buildEnv: a LOCALSTACK_* / PROVIDER_OVERRIDE_* var that is not client-only. */ +function isForwardedPrefixed(key: string): boolean { + return ( + !CLIENT_ONLY_ENV_KEYS.has(key) && + (key.startsWith("LOCALSTACK_") || key.startsWith("PROVIDER_OVERRIDE_")) + ); +} + +/** + * Whether a start forwards the host's own value for `key` (the MCP server's `env` block), + * directly or as its `LOCALSTACK_` alias, which the container re-exports under the plain name. + */ +export function forwardsHostEnvKey(key: string, hostEnv: NodeJS.ProcessEnv): boolean { + const forwardsFromHost = (k: string) => + hostEnv[k] !== undefined && (isForwardedConfigName(k) || isForwardedPrefixed(k)); + return ( + forwardsFromHost(key) || + (!key.startsWith("LOCALSTACK_") && forwardsFromHost(`LOCALSTACK_${key}`)) + ); +} + function buildEnv(input: ContainerSpecInput, ports: ResolvedPorts, name: string): string[] { const { hostEnv, envVars = {} } = input; const env = new Map(); @@ -396,18 +496,13 @@ function buildEnv(input: ContainerSpecInput, ports: ResolvedPorts, name: string) // (a) curated unprefixed config vars set on the host for (const [key, value] of Object.entries(hostEnv)) { if (value === undefined) continue; - const forwarded = - FORWARDED_CONFIG_ENV_NAMES.has(key) || - FORWARDED_CONFIG_ENV_PREFIXES.some((prefix) => key.startsWith(prefix)); - if (forwarded) env.set(key, value); + if (isForwardedConfigName(key)) env.set(key, value); } // (b) all LOCALSTACK_* / PROVIDER_OVERRIDE_* host vars minus client-only keys for (const [key, value] of Object.entries(hostEnv)) { - if (value === undefined || CLIENT_ONLY_ENV_KEYS.has(key)) continue; - if (key.startsWith("LOCALSTACK_") || key.startsWith("PROVIDER_OVERRIDE_")) { - env.set(key, value); - } + if (value === undefined) continue; + if (isForwardedPrefixed(key)) env.set(key, value); } // (c) explicit tool envVars win over anything host-derived @@ -424,7 +519,7 @@ function buildEnv(input: ContainerSpecInput, ports: ResolvedPorts, name: string) env.set("EXTERNAL_SERVICE_PORTS_END", String(ports.servicePortEnd)); env.set("DOCKER_HOST", `unix://${DOCKER_SOCKET_CONTAINER_PATH}`); env.set("LOCALSTACK_AUTH_TOKEN", input.authToken); - env.set("LOCALSTACK_CLIENT_NAME", "localstack-mcp-server"); + env.set("LOCALSTACK_CLIENT_NAME", MCP_CLIENT_NAME); env.set("LOCALSTACK_CLIENT_VERSION", input.serverVersion); const aiAgent = detectAiAgent(hostEnv); if (aiAgent) env.set("AI_AGENT", aiAgent); diff --git a/src/lib/localstack/localstack.utils.test.ts b/src/lib/localstack/localstack.utils.test.ts index 8a436f8..ac10e5d 100644 --- a/src/lib/localstack/localstack.utils.test.ts +++ b/src/lib/localstack/localstack.utils.test.ts @@ -1,4 +1,5 @@ import { + carriedRestartEnv, deriveRecreateOverrides, getGatewayHealth, getLocalStackStatus, @@ -6,6 +7,7 @@ import { launchRuntime, recreateRunningContainer, restartRuntimeInPlace, + unmountableBindSource, } from "./localstack.utils"; import { httpClient } from "../../core/http-client"; import { request as httpRequest } from "http"; @@ -272,6 +274,36 @@ describe("localstack.utils", () => { expect(client.createAndStartContainer).not.toHaveBeenCalled(); }); + test("an Azure start on a named volume says that app deployments need LOCALSTACK_VOLUME_DIR", async () => { + // The emulator shares files with the containers it starts for Function and Web Apps from + // its state folder, and refuses to unless that folder is a bind mount. + const ready = { isRunning: true, isReady: true, statusOutput: "healthy" }; + const start = async (stack: "azure" | "aws", volumeOverride: any) => { + const { client } = mockDockerClient(); + const getStatus = jest + .fn() + .mockResolvedValueOnce({ isRunning: false }) + .mockResolvedValue(ready); + const result = await launchRuntime({ + ...launchDefaults, + stack, + volumeOverride, + getStatus, + dockerClient: client, + }); + return result.content[0].text; + }; + const named = { type: "volume", name: "localstack-mcp" }; + const bind = { type: "bind", source: "/host/localstack-volume" }; + const onNamed = await start("azure", named); + expect(onNamed).toContain("started successfully"); + expect(onNamed).toMatch(/Function App and Web App deployments need/); + expect(onNamed).toMatch(/LOCALSTACK_VOLUME_DIR/); + expect(onNamed).toMatch(/stop and start the emulator/); + expect(await start("azure", bind)).not.toMatch(/LOCALSTACK_VOLUME_DIR/); + expect(await start("aws", named)).not.toMatch(/LOCALSTACK_VOLUME_DIR/); + }); + test("starts the container and succeeds once the gateway becomes reachable", async () => { const { client } = mockDockerClient(); const getStatus = jest @@ -297,6 +329,49 @@ describe("localstack.utils", () => { expect(spec.name).toBe("localstack-main"); }); + test("a runtime that accepts connections but does not answer yet is not started: the start keeps polling", async () => { + // Docker's port proxy accepts connections before the runtime listens, so a + // just-started container looks unresponsive for a while; the start must wait for it. + const { client } = mockDockerClient(); + const onReady = jest.fn().mockResolvedValue(null); + const getStatus = jest + .fn() + .mockResolvedValueOnce({ isRunning: false }) + .mockResolvedValueOnce({ isRunning: true, isReady: false, unresponsive: true }) + .mockResolvedValueOnce({ isRunning: true, isReady: false, unresponsive: true }) + .mockResolvedValue({ + isRunning: true, + isReady: true, + statusOutput: "edition: azure-alpha", + }); + + const result = await launchRuntime({ + ...launchDefaults, + getStatus, + onReady, + dockerClient: client, + }); + + expect(result.content[0].text).toContain("started successfully"); + expect(result.content[0].text).toContain("edition: azure-alpha"); + // The pre-start check, two unresponsive polls, then the one that answered. + expect(getStatus).toHaveBeenCalledTimes(4); + expect(onReady).toHaveBeenCalledTimes(1); + }); + + test("an unresponsive runtime still counts as running before a start: no second one is launched", async () => { + const { client } = mockDockerClient(); + const result = await launchRuntime({ + ...launchDefaults, + getStatus: jest + .fn() + .mockResolvedValue({ isRunning: true, isReady: false, unresponsive: true }), + dockerClient: client, + }); + expect(result.content[0].text).toBe("already running"); + expect(client.createAndStartContainer).not.toHaveBeenCalled(); + }); + test("pulls the image when missing", async () => { const { client } = mockDockerClient({ imageExists: jest.fn().mockResolvedValue(false), @@ -377,6 +452,62 @@ describe("localstack.utils", () => { expect(logHandle.destroy).toHaveBeenCalled(); }); + test("names the licence when the container exits with code 55 before printing why", async () => { + // The real tail from a bad token: the container exits before the reason is flushed. + let exitCallback: (() => void) | undefined; + const logHandle = { + getBuffered: jest.fn( + () => + "LocalStack version: 2026.9.0.dev336\nLocalStack build date: 2026-09-23\n" + + "Localstack returning with exit code 55. Reason: \n==============================\n" + ), + hasExited: jest.fn(() => true), + onExit: jest.fn((cb: () => void) => { + exitCallback = cb; + }), + destroy: jest.fn(), + }; + const { client } = mockDockerClient({ + attachLogBuffer: jest.fn().mockResolvedValue(logHandle), + }); + const getStatus = jest.fn().mockResolvedValue({ isRunning: false }); + + const resultPromise = launchRuntime({ ...launchDefaults, getStatus, dockerClient: client }); + await new Promise((resolve) => setTimeout(resolve, 5)); + exitCallback?.(); + + const text = (await resultPromise).content[0].text; + expect(text).toContain("exited unexpectedly"); + expect(text).toMatch(/could not activate its licen[cs]e/); + expect(text).toContain("LOCALSTACK_AUTH_TOKEN"); + // The log tail still follows, for the details. + expect(text).toContain("exit code 55"); + }); + + test("never creates the bind source of a recreated container (C:\\home on Windows)", async () => { + // A restart passes the old container's bind as volumeOverride; creating a missing one + // made C:\home\o\.cache\... on Windows for a WSL path. + // eslint-disable-next-line @typescript-eslint/no-require-imports + const { existsSync } = require("fs"); + // eslint-disable-next-line @typescript-eslint/no-require-imports + const source = require("path").join( + require("os").tmpdir(), + `lsmcp-test-no-mkdir-${Date.now()}` + ); + const { client } = mockDockerClient(); + const getStatus = jest + .fn() + .mockResolvedValueOnce({ isRunning: false }) + .mockResolvedValue({ isRunning: true, isReady: true }); + await launchRuntime({ + ...launchDefaults, + getStatus, + dockerClient: client, + volumeOverride: { type: "bind", source }, + }); + expect(existsSync(source)).toBe(false); + }); + test("times out with the timeout message when readiness never arrives", async () => { const { client } = mockDockerClient(); const getStatus = jest.fn().mockResolvedValue({ isRunning: false }); @@ -582,6 +713,45 @@ describe("localstack.utils", () => { }); }); + describe("unmountableBindSource", () => { + // eslint-disable-next-line @typescript-eslint/no-require-imports + const os = require("os"); + // eslint-disable-next-line @typescript-eslint/no-require-imports + const path = require("path"); + const bind = (source: string) => ({ volumeOverride: { type: "bind" as const, source } }); + + test("a Linux path seen from a Windows host is unmountable, even if C:\\ happens to hold it", () => { + expect( + unmountableBindSource(bind("/home/dev/.cache/lstk/volume/localstack-azure"), { + platform: "win32", + inDocker: false, + }) + ).toBe("/home/dev/.cache/lstk/volume/localstack-azure"); + }); + + test("an existing folder is mountable; a missing one is not", () => { + const platform = process.platform === "win32" ? "win32" : "linux"; + expect( + unmountableBindSource(bind(os.tmpdir()), { platform, inDocker: false }) + ).toBeUndefined(); + const missing = path.join(os.tmpdir(), `lsmcp-test-missing-${Date.now()}`); + expect(unmountableBindSource(bind(missing), { platform, inDocker: false })).toBe(missing); + }); + + test("named volumes, no overrides and servers inside Docker are not checked", () => { + expect( + unmountableBindSource( + { volumeOverride: { type: "volume", name: "v" } }, + { inDocker: false } + ) + ).toBeUndefined(); + expect(unmountableBindSource(undefined)).toBeUndefined(); + expect( + unmountableBindSource(bind("/nope"), { platform: "linux", inDocker: true }) + ).toBeUndefined(); + }); + }); + describe("deriveRecreateOverrides", () => { test("reuses image/name/volume of a matching-stack container", () => { expect( @@ -607,6 +777,27 @@ describe("localstack.utils", () => { ).toBeUndefined(); }); + test("uses the labels: a bare-ID Azure container keeps its identity for an azure restart", () => { + const metadata = { + id: "x", + name: "localstack-azure", + image: "f91897f1de85", + labels: { description: "LocalStack for Azure" }, + }; + expect(deriveRecreateOverrides(metadata, "azure")).toMatchObject({ + imageOverride: "f91897f1de85", + containerNameOverride: "localstack-azure", + }); + // Restarting it as AWS is a deliberate stack switch: no overrides. + expect(deriveRecreateOverrides(metadata, "aws")).toBeUndefined(); + }); + + test("keeps the AWS default for an unlabelled bare-ID image", () => { + expect(deriveRecreateOverrides({ id: "x", image: "0123456789ab" }, "aws")).toMatchObject({ + imageOverride: "0123456789ab", + }); + }); + test("returns undefined without image metadata", () => { expect(deriveRecreateOverrides(undefined, "aws")).toBeUndefined(); expect(deriveRecreateOverrides({ id: "x" }, "aws")).toBeUndefined(); @@ -655,3 +846,36 @@ describe("localstack.utils", () => { }); }); }); + +describe("carriedRestartEnv", () => { + test("keeps operator flags, drops unchanged image pairs, reserved keys and the token", () => { + const carried = carriedRestartEnv( + [ + "MSSQL_ACCEPT_EULA=Y", + "PATH=/usr/bin", + "GATEWAY_LISTEN=0.0.0.0:4566", + "LOCALSTACK_AUTH_TOKEN=secret", + "LOCALSTACK_HOST=localhost.localstack.cloud:4566", + ], + ["PATH=/usr/bin"] + ); + expect(carried).toEqual({ + MSSQL_ACCEPT_EULA: "Y", + LOCALSTACK_HOST: "localhost.localstack.cloud:4566", + }); + }); + + test("an image default the operator changed IS carried (the pair differs)", () => { + expect(carriedRestartEnv(["DEBUG=1"], ["DEBUG=0"])).toEqual({ DEBUG: "1" }); + }); + + test("a value containing '=' keeps everything after the first '='", () => { + expect(carriedRestartEnv(["CONN=a=b;c=d"], [])).toEqual({ CONN: "a=b;c=d" }); + }); + + test("malformed entries and system vars are skipped even without the image env", () => { + expect(carriedRestartEnv(["NOEQUALS", "=novalue", "HOME=/root", "FLAG="], undefined)).toEqual({ + FLAG: "", + }); + }); +}); diff --git a/src/lib/localstack/localstack.utils.ts b/src/lib/localstack/localstack.utils.ts index 5d59ff7..8a250fd 100644 --- a/src/lib/localstack/localstack.utils.ts +++ b/src/lib/localstack/localstack.utils.ts @@ -120,6 +120,13 @@ export interface RuntimeStatus { isRunning: boolean; isReady?: boolean; statusOutput?: string; + /** + * Something accepts connections on the port but the runtime does not answer (yet). + * Before a start it counts as running, so no second runtime is launched beside a busy + * one. After this start launched its own container it is NOT started: + * Docker's port proxy accepts connections before the runtime listens. + */ + unresponsive?: boolean; } const SNOWFLAKE_ROUTING_HOST = "snowflake.localhost.localstack.cloud"; @@ -146,14 +153,21 @@ const READY_SERVICE_STATES = new Set(["available", "running"]); * container is named. * * This is the source of truth for "is LocalStack running?". + * + * `baseUrl` targets another gateway than the configured one, for example the Azure + * client's own port when AWS and Azure run side by side. */ -export async function getGatewayHealth(): Promise { +export async function getGatewayHealth(baseUrl?: string): Promise { try { const data = await httpClient.request<{ services?: Record; edition?: string; version?: string; - }>("/_localstack/health", { method: "GET", timeout: GATEWAY_HEALTH_TIMEOUT }); + }>("/_localstack/health", { + method: "GET", + timeout: GATEWAY_HEALTH_TIMEOUT, + ...(baseUrl ? { baseUrl } : {}), + }); if (!data || typeof data !== "object" || Array.isArray(data)) { return { reachable: false, ready: false }; @@ -178,11 +192,12 @@ export async function getGatewayHealth(): Promise { } /** Best-effort read of `/_localstack/info` for status enrichment and restart detection. */ -export async function getSessionInfo(): Promise { +export async function getSessionInfo(baseUrl?: string): Promise { try { const info = await httpClient.request("/_localstack/info", { method: "GET", timeout: SESSION_INFO_TIMEOUT, + ...(baseUrl ? { baseUrl } : {}), }); if (!info || typeof info !== "object") return null; return info; @@ -191,9 +206,13 @@ export async function getSessionInfo(): Promise { } } -function describeGatewayHealth(health: GatewayHealth, info?: SessionInfo | null): string { +function describeGatewayHealth( + health: GatewayHealth, + info?: SessionInfo | null, + baseUrl: string = LOCALSTACK_BASE_URL +): string { const lines = [ - `LocalStack gateway is reachable at ${LOCALSTACK_BASE_URL} (detected via /_localstack/health).`, + `LocalStack gateway is reachable at ${baseUrl} (detected via /_localstack/health).`, ]; const edition = info?.edition || health.edition; const version = info?.version || health.version; @@ -219,22 +238,23 @@ function describeGatewayHealth(health: GatewayHealth, info?: SessionInfo | null) * Get LocalStack status information. Running state is decided by the gateway probe; * display detail is enriched from `/_localstack/info` when available. */ -export async function getLocalStackStatus(): Promise { - const health = await getGatewayHealth(); +/** The gateway's status; `baseUrl` probes another gateway than LOCALSTACK_PORT's (side by side). */ +export async function getLocalStackStatus(baseUrl?: string): Promise { + const health = await getGatewayHealth(baseUrl); if (!health.reachable) { return { isRunning: false, isReady: false, - statusOutput: `LocalStack is not running — the gateway at ${LOCALSTACK_BASE_URL} is not reachable.`, + statusOutput: `LocalStack is not running — the gateway at ${baseUrl ?? LOCALSTACK_BASE_URL} is not reachable.`, }; } - const info = await getSessionInfo(); + const info = await getSessionInfo(baseUrl); return { isRunning: true, isReady: health.ready, - statusOutput: describeGatewayHealth(health, info), + statusOutput: describeGatewayHealth(health, info, baseUrl), }; } @@ -369,6 +389,15 @@ function conflictResponse(processLabel: string, containerName: string, image?: s * Start a LocalStack runtime flavor directly through the Docker Engine API (no * localstack/lstk CLI involved) and poll until it becomes available. */ +/** + * The Azure emulator shares files with the containers it starts for Function and Web Apps from + * its state folder, and refuses to ("Mount to /var/lib/localstack needs to be a bind mount") when + * that folder is a named volume, as it is when this server runs in Docker. A restart keeps the + * container's volume, so the way out is a stop and a start. + */ +const AZURE_VOLUME_NOTE = + "ℹ️ Function App and Web App deployments need the emulator's state in a host folder, from which it shares files with the apps' containers; this emulator uses a named Docker volume. To deploy apps, set LOCALSTACK_VOLUME_DIR in this MCP server's configuration to an absolute host path, then stop and start the emulator (a restart keeps the current volume)."; + export async function launchRuntime( options: LaunchRuntimeOptions ): Promise> { @@ -432,7 +461,9 @@ async function launchRuntimeInner( platform: process.platform, homedir: homedir(), }); - if (volume.type === "bind" && !inDocker) { + // Only this server's own default folder is created: a recreated container's bind must + // already exist (creating a WSL path on Windows made C:\home\\... there). + if (volume.type === "bind" && !inDocker && !options.volumeOverride) { try { await mkdir(volume.source, { recursive: true }); } catch (error) { @@ -548,12 +579,21 @@ async function launchRuntimeInner( ? `\n\nContainer logs (last ${CRASH_LOG_TAIL_LINES} lines):\n${buffered}` : ""; }; + // LocalStack exits with code 55 when it cannot activate its licence, and the container + // can be gone before the reason reaches the log stream (the tail ended at + // "exit code 55. Reason:"), so name the cause and the setting to check. + const exitHint = () => + /exit code 55\b/.test(logBuffer?.getBuffered() ?? "") + ? "\n\nExit code 55: LocalStack could not activate its licence. Check that LOCALSTACK_AUTH_TOKEN in this MCP server's configuration is a valid auth token." + : ""; const successResponse = (status: RuntimeStatus) => { let resultMessage = `${successTitle}\n\n`; if (envVars) resultMessage += `✅ Custom environment variables passed to the LocalStack container: ${Object.keys(envVars).join(", ")}\n`; if (status.statusOutput) resultMessage += `\n**${statusHeading}:**\n${status.statusOutput}`; + if (stack === "azure" && volume.type === "volume") + resultMessage += `\n\n${AZURE_VOLUME_NOTE}`; return ResponseBuilder.markdown(resultMessage); }; @@ -561,7 +601,8 @@ async function launchRuntimeInner( if (resolved) return true; const status = await getStatus(); if (resolved) return true; - if (!(status.isReady || status.isRunning)) return false; + // A runtime that does not answer yet is still starting (see RuntimeStatus.unresponsive). + if (!(status.isReady || (status.isRunning && !status.unresponsive))) return false; if (onReady) { const preflight = await onReady(); @@ -595,7 +636,7 @@ async function launchRuntimeInner( } finish( ResponseBuilder.markdown( - `❌ ${processLabel} container exited unexpectedly before becoming ready.${failureDetails()}` + `❌ ${processLabel} container exited unexpectedly before becoming ready.${exitHint()}${failureDetails()}` ) ); }); @@ -643,7 +684,9 @@ export function deriveRecreateOverrides( stack: LocalStackStack ): RecreateOverrides | undefined { if (!metadata?.image) return undefined; - if ((stackFromImage(metadata.image) ?? "aws") !== stack) return undefined; + // Pass the labels: a bare-ID Azure container is identified by its description label. + // An image of unknown stack keeps the historical AWS default. + if ((stackFromImage(metadata.image, metadata.labels) ?? "aws") !== stack) return undefined; const overrides: RecreateOverrides = { imageOverride: metadata.image, @@ -665,6 +708,131 @@ export function deriveRecreateOverrides( const AWS_ALREADY_RUNNING = "⚠️ LocalStack is already running."; const SNOWFLAKE_ALREADY_RUNNING = "⚠️ Snowflake emulator is already running."; +// Keys the launcher sets authoritatively for a new container (buildEnv step (d)); carrying stale +// values from the old container is pointless, and one of them is the auth token, which must never +// be echoed back. So a restart never carries these over. +/** + * The env key a restart writes on the new container, listing the settings it carried over from an + * externally started one. The new container carries this server's LOCALSTACK_CLIENT_NAME, + * so without the list a SECOND restart would see a server-started container and drop them all. + */ +export const CARRIED_ENV_KEY = "MCP_CARRIED_ENV"; + +/** `KEY=value` entries of `containerEnv` for exactly `keys` (the settings a restart keeps carrying). */ +export function pickCarriedEnv( + containerEnv: string[] | undefined, + keys: string[] +): Record { + const wanted = new Set(keys); + const picked: Record = {}; + for (const entry of containerEnv ?? []) { + const eq = entry.indexOf("="); + if (eq <= 0) continue; + const key = entry.slice(0, eq); + if (wanted.has(key) && !RESERVED_RESTART_ENV_KEYS.has(key)) picked[key] = entry.slice(eq + 1); + } + return picked; +} + +const RESERVED_RESTART_ENV_KEYS = new Set([ + CARRIED_ENV_KEY, // written by the restart itself, never carried as a flag + "MAIN_CONTAINER_NAME", + "GATEWAY_LISTEN", + "EXTERNAL_SERVICE_PORTS_START", + "EXTERNAL_SERVICE_PORTS_END", + "DOCKER_HOST", + "LOCALSTACK_AUTH_TOKEN", + "LOCALSTACK_CLIENT_NAME", + "LOCALSTACK_CLIENT_VERSION", + "AI_AGENT", +]); + +// A backstop for when the image's baked env cannot be read: base-image/system variables that are +// never an operator's emulator flag, so they must not be pushed as `-e` onto the new container +// (passing PATH/HOME could break it). +const SYSTEM_ENV_KEYS = new Set([ + "PATH", + "HOME", + "HOSTNAME", + "TERM", + "LANG", + "LANGUAGE", + "LC_ALL", + "LC_CTYPE", + "PWD", + "OLDPWD", + "SHLVL", + "SHELL", + "USER", + "TZ", + "container", + "GPG_KEY", + "DEBIAN_FRONTEND", + "PYTHONPATH", + "PYTHONUNBUFFERED", + "PYTHONDONTWRITEBYTECODE", + "VIRTUAL_ENV", +]); + +/** + * The environment a restart must carry from the old container to the new one: the + * flags the operator set with `-e` (for example MSSQL_ACCEPT_EULA, DISABLE_EVENTS, LS_AZURE_PORTAL). + * Without this a recreate keeps only this server's own settings and silently drops the rest, so a + * shared Azure emulator loses its EULA acceptance and starts emitting telemetry again. + * + * `containerEnv` is the old container's full `Config.Env`; `imageEnv` is the image's baked env. A + * pair present unchanged in the image is a default, not an operator choice, so it is dropped (this + * is what removes PATH and the like). Launcher-owned reserved keys are dropped too — the launcher + * sets them for the new container, and one is the auth token. A system-var backstop keeps + * base-image variables out even when `imageEnv` could not be read. + */ +export function carriedRestartEnv( + containerEnv: string[] | undefined, + imageEnv: string[] | undefined +): Record { + const imagePairs = new Set(imageEnv ?? []); + const carried: Record = {}; + for (const entry of containerEnv ?? []) { + const eq = entry.indexOf("="); + if (eq <= 0) continue; // no key, or an empty key + const key = entry.slice(0, eq); + if (imagePairs.has(entry)) continue; // an image default, unchanged by the operator + if (RESERVED_RESTART_ENV_KEYS.has(key) || SYSTEM_ENV_KEYS.has(key)) continue; + carried[key] = entry.slice(eq + 1); + } + return carried; +} + +/** + * The bind source of a container about to be recreated, when this machine cannot mount it: + * stopping the container would remove it with no way to recreate it (an lstk + * emulator started from WSL binds ~/.cache/lstk/volume/... as a Linux path, which a Windows + * server cannot mount). Inside Docker the path is host-side and cannot be checked. + */ +export function unmountableBindSource( + overrides: RecreateOverrides | undefined, + opts: { platform?: NodeJS.Platform; inDocker?: boolean } = {} +): string | undefined { + const volume = overrides?.volumeOverride; + if (volume?.type !== "bind" || (opts.inDocker ?? isRunningInDocker())) return undefined; + // A Linux path seen from a Windows host belongs to WSL or another machine. + if ((opts.platform ?? process.platform) === "win32" && volume.source.startsWith("/")) { + return volume.source; + } + return existsSync(volume.source) ? undefined : volume.source; +} + +/** The refusal for unmountableBindSource: nothing is stopped. */ +export function cannotRecreateResponse(source: string, containerName: string) { + return ResponseBuilder.error( + "Cannot restart this LocalStack container from here", + `Its state folder is bound from \`${source}\`, which this machine cannot mount: "${containerName}" was ` + + "probably started from WSL or another host. Stopping it here would remove it with no way to " + + "recreate it, so nothing was stopped. Restart it where it was started (for example `lstk restart` " + + "in WSL), or stop it and start a new one with the start action." + ); +} + /** * Recreate the running LocalStack container: inspect it, stop + wait for removal, then * launch a fresh one preserving its image/name/volume. Used as the reliable fallback @@ -688,6 +856,11 @@ export async function recreateRunningContainer({ try { const id = await docker.findLocalStackContainer(); metadata = await docker.inspectContainer(id); + // Before stopping it: one this machine could not recreate stays as it is. + const unmountable = unmountableBindSource( + deriveRecreateOverrides(metadata, stackFromImage(metadata?.image, metadata?.labels) ?? "aws") + ); + if (unmountable) return cannotRecreateResponse(unmountable, metadata?.name ?? id); await docker.stopContainer(id); await docker.waitForRemoval(id); } catch (error) { @@ -702,7 +875,7 @@ export async function recreateRunningContainer({ // Nothing running — fall through to a fresh AWS start. } - const stack = stackFromImage(metadata?.image) ?? "aws"; + const stack = stackFromImage(metadata?.image, metadata?.labels) ?? "aws"; const overrides = deriveRecreateOverrides(metadata, stack); return launchRuntime({ diff --git a/src/lib/wizard/azure-steps.test.ts b/src/lib/wizard/azure-steps.test.ts new file mode 100644 index 0000000..ccbb3bd --- /dev/null +++ b/src/lib/wizard/azure-steps.test.ts @@ -0,0 +1,105 @@ +import * as fs from "fs"; +import * as os from "os"; +import * as path from "path"; +import type { RunAz } from "../azure/extension-install"; +import { AZURE_EXTENSION_PINS } from "../azure/extension-pins"; +import { installAzureExtensionsStep, installBicepStep, type AzureStepDeps } from "./azure-steps"; + +// The wizard's Azure steps, with az and the download injected and a +// temporary home, so the real ~/.localstack and ~/.azure are never touched. + +describe("the wizard's Azure steps", () => { + let home: string; + beforeEach(() => { + home = fs.mkdtempSync(path.join(os.tmpdir(), "lsmcp-wizard-azure-")); + }); + afterEach(() => fs.rmSync(home, { recursive: true, force: true })); + + function deps(over: Partial = {}) { + const calls: Array<{ file: string; args: string[] }> = []; + const run: RunAz = async (file, args) => { + calls.push({ file, args }); + return { status: 0 }; + }; + const d: AzureStepDeps = { + platform: process.platform, + arch: process.arch, + homedir: home, + env: {}, + locate: () => ({ + file: "/usr/bin/python3", + prefixArgs: ["-X", "utf8", "-IBm", "azure.cli"], + installer: "pip", + }), + run, + pins: AZURE_EXTENSION_PINS, + musl: false, + installBicep: jest.fn(async () => ({ path: "/x/bicep", asset: "bicep-linux-x64" })), + ...over, + }; + return { d, calls }; + } + + test("extensions: the 26 pins into ~/.localstack/azure/mcp-extensions, through the located az", async () => { + const { d, calls } = deps(); + const result = await installAzureExtensionsStep(d); + expect(result).toMatchObject({ step: "extensions", status: "installed" }); + const dir = path.join(home, ".localstack", "azure", "mcp-extensions"); + expect(result.detail).toContain(dir); + expect(fs.existsSync(dir)).toBe(true); + expect(calls).toHaveLength(26); + expect(calls.every((c) => c.file === "/usr/bin/python3")).toBe(true); + }); + + test("extensions: a no-op when az is missing", async () => { + const { d, calls } = deps({ + locate: () => { + throw new Error("no az"); + }, + }); + const result = await installAzureExtensionsStep(d); + expect(result).toMatchObject({ step: "extensions", status: "skipped" }); + expect(result.detail).toMatch(/no Azure CLI found/); + expect(calls).toHaveLength(0); + expect(fs.readdirSync(home)).toEqual([]); + }); + + test("extensions: an extension dir inside ~/.azure is refused before az runs", async () => { + const { d, calls } = deps({ + env: { LOCALSTACK_AZ_EXTENSION_DIR: path.join(home, ".azure", "cliextensions") }, + }); + const result = await installAzureExtensionsStep(d); + expect(result).toMatchObject({ step: "extensions", status: "failed" }); + expect(result.detail).toMatch(/refusing to install into/); + expect(calls).toHaveLength(0); + }); + + test("extensions: a Windows .cmd launcher run as-is is skipped with the way out", async () => { + const { d, calls } = deps({ + platform: "win32", + locate: () => ({ file: "C:\\tools\\az.cmd", prefixArgs: [], installer: "launcher-as-is" }), + }); + const result = await installAzureExtensionsStep(d); + expect(result).toMatchObject({ status: "skipped" }); + expect(result.detail).toMatch(/LOCALSTACK_AZ_PATH/); + expect(calls).toHaveLength(0); + }); + + test("bicep: installed, or failed with the installer's message", async () => { + const ok = deps(); + expect(await installBicepStep(ok.d)).toMatchObject({ step: "bicep", status: "installed" }); + expect(ok.d.installBicep).toHaveBeenCalledWith( + expect.objectContaining({ homedir: home, platform: process.platform, musl: false }) + ); + const failing = deps({ + installBicep: jest.fn(async () => { + throw new Error("the downloaded bicep has sha256 abc, not the pinned def"); + }), + }); + expect(await installBicepStep(failing.d)).toEqual({ + step: "bicep", + status: "failed", + detail: "the downloaded bicep has sha256 abc, not the pinned def", + }); + }); +}); diff --git a/src/lib/wizard/azure-steps.ts b/src/lib/wizard/azure-steps.ts new file mode 100644 index 0000000..4625522 --- /dev/null +++ b/src/lib/wizard/azure-steps.ts @@ -0,0 +1,141 @@ +import { readFileSync } from "fs"; +import os from "os"; +import { installBicep, isMusl, BICEP_VERSION } from "../azure/bicep-install"; +import { + assertNotInProfile, + defaultExtensionDir, + installExtensions, + spawnAz, + type ExtensionPin, + type RunAz, +} from "../azure/extension-install"; +import { AZURE_EXTENSION_PINS } from "../azure/extension-pins"; +import { locateAz, nodeResolveFs, type LocatedAz } from "../azure/resolve-az"; + +/** + * The steps of `install-azure-addons`: the curated Azure CLI extensions into + * the tool's extension dir, and the pinned Bicep CLI into ~/.localstack/azure/bin. The user runs + * that command, as they install the Snowflake CLI for the Snowflake tool; the setup wizard never + * installs a tool's CLI. The Docker image bundles both. + */ + +/** /proc/version, which tells WSL apart for the Azure CLI lookup (undefined off Linux). */ +export function procVersion(): string | undefined { + try { + return readFileSync("/proc/version", "utf8"); + } catch { + return undefined; + } +} + +export interface AzureStepResult { + step: "extensions" | "bicep"; + status: "installed" | "skipped" | "failed"; + detail: string; +} + +export interface AzureStepDeps { + platform: NodeJS.Platform; + arch: string; + homedir: string; + env: NodeJS.ProcessEnv; + /** Throws when no Azure CLI is found (the tool's own lookup, without running az). */ + locate: () => LocatedAz; + run: RunAz; + pins: readonly ExtensionPin[]; + musl: boolean; + installBicep: typeof installBicep; +} + +export function defaultAzureStepDeps(): AzureStepDeps { + return { + platform: process.platform, + arch: process.arch, + homedir: os.homedir(), + env: process.env, + locate: () => + locateAz({ + env: process.env, + platform: process.platform, + homedir: os.homedir(), + fs: nodeResolveFs, + procVersion: process.platform === "win32" ? undefined : procVersion(), + }), + // Captured, so az's output does not bury the command's progress lines; shown on failure. + run: spawnAz("pipe"), + pins: AZURE_EXTENSION_PINS, + musl: isMusl(), + installBicep, + }; +} + +/** Installs the pinned extensions; a no-op, reported as skipped, when az is missing. */ +export async function installAzureExtensionsStep( + deps: AzureStepDeps, + onProgress?: (index: number, total: number, pin: ExtensionPin) => void +): Promise { + let az: LocatedAz; + try { + az = deps.locate(); + } catch { + return { + step: "extensions", + status: "skipped", + detail: + "no Azure CLI found; install it (az 2.85 or newer), then run install-azure-addons again", + }; + } + if (az.prefixArgs.length === 0 && deps.platform === "win32" && /\.(cmd|bat)$/i.test(az.file)) { + return { + step: "extensions", + status: "skipped", + detail: `${az.file} cannot be run without a shell; set LOCALSTACK_AZ_PATH to the Azure CLI's python.exe and re-run`, + }; + } + const dir = defaultExtensionDir(deps.env, deps.homedir); + try { + assertNotInProfile(dir, { homedir: deps.homedir, env: deps.env, platform: deps.platform }); + const result = await installExtensions({ + pins: deps.pins, + dir, + az: { file: az.file, prefix: az.prefixArgs }, + run: deps.run, + env: deps.env, + onProgress, + }); + return { + step: "extensions", + status: "installed", + detail: `${result.installed} Azure CLI extensions in ${result.dir}`, + }; + } catch (error) { + return { + step: "extensions", + status: "failed", + detail: error instanceof Error ? error.message : String(error), + }; + } +} + +/** Downloads, checks and installs the pinned Bicep CLI. */ +export async function installBicepStep(deps: AzureStepDeps): Promise { + try { + const result = await deps.installBicep({ + platform: deps.platform, + arch: deps.arch, + homedir: deps.homedir, + musl: deps.musl, + }); + return { + step: "bicep", + status: "installed", + detail: `Bicep ${BICEP_VERSION} (${result.asset}, sha256 checked) at ${result.path}`, + }; + } catch (error) { + return { + step: "bicep", + status: "failed", + detail: error instanceof Error ? error.message : String(error), + }; + } +} diff --git a/src/lib/wizard/cli-args.logic.test.ts b/src/lib/wizard/cli-args.logic.test.ts index 75323b1..b0eef74 100644 --- a/src/lib/wizard/cli-args.logic.test.ts +++ b/src/lib/wizard/cli-args.logic.test.ts @@ -1,6 +1,16 @@ import { parseInitFlags, parseRemoveFlags } from "./cli-args.logic"; describe("parseInitFlags", () => { + it("has no flag that installs a tool's CLI: the Azure add-ons are not init options", () => { + // Like the Snowflake CLI, the Azure tool's extensions and Bicep are the user's to install, + // with their own command (install-azure-addons), never through the setup wizard. + for (const flag of ["--azure-extensions", "--bicep"]) { + const { flags, errors } = parseInitFlags([flag]); + expect(flags).toBeUndefined(); + expect(errors.join(" ")).toMatch(/Unknown option/); + } + }); + it("parses a full non-interactive invocation", () => { const { flags, errors } = parseInitFlags([ "--method", diff --git a/src/lib/wizard/prereqs.test.ts b/src/lib/wizard/prereqs.test.ts new file mode 100644 index 0000000..c1e7c8d --- /dev/null +++ b/src/lib/wizard/prereqs.test.ts @@ -0,0 +1,35 @@ +import { checkNodeVersion, checkPrereqs } from "./prereqs"; +import { runCommand } from "../../core/command-runner"; + +// The wizard's prerequisites: Node and Docker, and nothing else. A tool's own +// CLI, the Snowflake tool's `snow` or the Azure tool's `az`, is the user's to install, as the +// README says, and each tool names what is missing when it is used: the wizard neither checks +// nor installs it. Docker is never run: the command runner is mocked. +jest.mock("../../core/command-runner", () => ({ runCommand: jest.fn() })); +const mockedRun = runCommand as jest.MockedFunction; + +describe("checkPrereqs", () => { + beforeEach(() => { + mockedRun.mockReset(); + mockedRun.mockResolvedValue({ stdout: "", stderr: "", exitCode: 0 } as never); + }); + + test("npx: Node and Docker only, with no tool CLI (neither az nor snow)", async () => { + const names = (await checkPrereqs("npx")).map((r) => r.name); + expect(names[0]).toMatch(/^Node\.js /); + expect(names.slice(1)).toEqual(["Docker CLI", "Docker daemon"]); + expect(names.join(" ")).not.toMatch(/azure|\baz\b|bicep|snow/i); + }); + + test("docker: Docker only, and it stays fatal", async () => { + mockedRun.mockResolvedValue({ stdout: "", stderr: "", exitCode: 1 } as never); + const results = await checkPrereqs("docker"); + expect(results.map((r) => r.name)).toEqual(["Docker CLI"]); + expect(results[0]).toMatchObject({ ok: false, fatal: true }); + }); + + test("the Node check is unchanged", () => { + expect(checkNodeVersion("v24.11.1")).toMatchObject({ ok: true, fatal: false }); + expect(checkNodeVersion("v18.0.0").ok).toBe(false); + }); +}); diff --git a/src/lib/wizard/setup-parity.test.ts b/src/lib/wizard/setup-parity.test.ts new file mode 100644 index 0000000..86ae777 --- /dev/null +++ b/src/lib/wizard/setup-parity.test.ts @@ -0,0 +1,20 @@ +import fs from "fs"; +import path from "path"; + +// The setup wizard treats every tool alike: it configures MCP clients and never checks, asks +// about or installs a tool's own CLI. The Snowflake tool's `snow` never appears in it, and +// neither do the Azure tool's `az`, its extensions or Bicep (those have their own command, +// `install-azure-addons`, which the README and the Azure tool's answers name). +const REPO = path.resolve(__dirname, "..", "..", ".."); +const WIZARD_SOURCES = [ + "src/cli/init.ts", + "src/lib/wizard/prereqs.ts", + "src/lib/wizard/cli-args.logic.ts", +]; + +describe("the setup wizard has no tool-specific step, as for Snowflake", () => { + test.each(WIZARD_SOURCES)("%s mentions neither Azure, Bicep nor Snowflake's CLI", (file) => { + const text = fs.readFileSync(path.join(REPO, file), "utf8"); + expect(text).not.toMatch(/azure|bicep|snowflake-cli|\bsnow\b/i); + }); +}); diff --git a/src/tools-tests/localstack-azure-client.test.ts b/src/tools-tests/localstack-azure-client.test.ts new file mode 100644 index 0000000..03d99f3 --- /dev/null +++ b/src/tools-tests/localstack-azure-client.test.ts @@ -0,0 +1,529 @@ +// The handler, the preflight precedence and the value-free analytics of +// localstack-azure-client. The real policy, output, preflight +// gates and analytics run; only the edges are mocked: the emulator status, gateway +// health, the resolved az, Bicep, the bootstrap, the runner and PostHog. +import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "fs"; +import os from "os"; +import path from "path"; +import type { AzRunResult } from "../lib/azure/types"; + +const mockCapture = jest.fn(); +jest.mock("posthog-node", () => ({ + PostHog: jest.fn().mockImplementation(() => ({ + capture: mockCapture, + flush: jest.fn().mockResolvedValue(undefined), + shutdown: jest.fn().mockResolvedValue(undefined), + })), +})); + +jest.mock("../lib/localstack/localstack.utils", () => ({ + ...jest.requireActual("../lib/localstack/localstack.utils"), + getGatewayHealth: jest.fn(), +})); + +jest.mock("../lib/azure/emulator", () => ({ + ...jest.requireActual("../lib/azure/emulator"), + getAzureEmulatorStatus: jest.fn(), +})); + +jest.mock("../lib/azure/loopback-forwarder", () => ({ + ensureLoopbackForwarder: jest.fn().mockResolvedValue(undefined), +})); + +jest.mock("../lib/azure/services", () => { + const actual = jest.requireActual("../lib/azure/services"); + return { + ...actual, + azCli: jest.fn(), + bicep: jest.fn(), + runAzCommand: jest.fn(), + ensureProfile: jest.fn(), + installedExtensions: jest.fn(() => []), + installedExtensionNames: jest.fn(() => new Set()), + }; +}); + +const root = mkdtempSync(path.join(os.tmpdir(), "lsaz-tool-")); +const workdir = path.join(root, "work"); +mkdirSync(workdir); +process.env.LOCALSTACK_AZ_WORKDIR = workdir; +process.env.LOCALSTACK_AZ_CONFIG_DIR = path.join(root, "azure-config"); +process.env.LOCALSTACK_AZURE_PORT = "4566"; +process.env.MCP_ANALYTICS_DISTINCT_ID = "azure-client-u11"; +delete process.env.MCP_ANALYTICS_DISABLED; +delete process.env.LOCALSTACK_AZ_EGRESS_GUARD; +delete process.env.LOCALSTACK_AZ_TEST_ENVELOPE; + +/* eslint-disable @typescript-eslint/no-require-imports */ +const tool = + require("../tools/localstack-azure-client") as typeof import("../tools/localstack-azure-client"); +const services = require("../lib/azure/services") as typeof import("../lib/azure/services"); +const emulator = require("../lib/azure/emulator") as typeof import("../lib/azure/emulator"); +const utils = + require("../lib/localstack/localstack.utils") as typeof import("../lib/localstack/localstack.utils"); +const forwarder = + require("../lib/azure/loopback-forwarder") as typeof import("../lib/azure/loopback-forwarder"); +const { AzResolveError, BicepPathError } = + require("../lib/azure/resolve-az") as typeof import("../lib/azure/resolve-az"); +const { BootstrapError } = + require("../lib/azure/bootstrap") as typeof import("../lib/azure/bootstrap"); +const { resetStackDetectionCache } = + require("../core/preflight") as typeof import("../core/preflight"); +/* eslint-enable @typescript-eslint/no-require-imports */ + +const azure = tool.default; +const mocked = { + azCli: services.azCli as jest.MockedFunction, + bicep: services.bicep as jest.MockedFunction, + runAzCommand: services.runAzCommand as jest.MockedFunction, + ensureProfile: services.ensureProfile as jest.MockedFunction, + status: emulator.getAzureEmulatorStatus as jest.MockedFunction< + typeof emulator.getAzureEmulatorStatus + >, + health: utils.getGatewayHealth as jest.MockedFunction, + forwarder: forwarder.ensureLoopbackForwarder as jest.MockedFunction< + typeof forwarder.ensureLoopbackForwarder + >, +}; + +const AZ = { + file: "/opt/az/bin/python3", + prefixArgs: ["-X", "utf8", "-IBm", "azure.cli"], + installer: "deb" as const, + version: "2.90.0", +}; + +const result = (fields: Partial = {}): AzRunResult => ({ + exitCode: 0, + stdout: "", + stderr: "", + timedOut: false, + aborted: false, + truncated: false, + durationMs: 5, + egress: { refused: [], upstream: [], housekeeping: [], allowed: 1 }, + ...fields, +}); + +const text = (r: { content: Array<{ text: string }> }) => r.content[0].text; + +afterAll(() => rmSync(root, { recursive: true, force: true })); + +beforeEach(() => { + jest.clearAllMocks(); + resetStackDetectionCache(); + process.env.LOCALSTACK_AUTH_TOKEN = "ls-test-token"; + mocked.health.mockResolvedValue({ + reachable: true, + ready: true, + edition: "azure-alpha", + } as never); + mocked.status.mockResolvedValue({ + ok: true, + edition: "azure-alpha", + license: true, + sessionId: "s-1", + }); + mocked.azCli.mockResolvedValue(AZ); + mocked.ensureProfile.mockResolvedValue(undefined); + mocked.runAzCommand.mockResolvedValue(result({ stdout: '[{"name": "rg1"}]' })); + mocked.bicep.mockResolvedValue({ + path: "/usr/local/bin/bicep", + dir: "/usr/local/bin", + source: "path", + version: "0.47.16", + supportsBicepparam: true, + }); +}); + +describe("the handler", () => { + test("the happy path runs the policy's argv in the workdir and returns az's output", async () => { + const answer = await azure({ command: "az group list --query \"[?name=='rg1']\"" }); + expect(text(answer)).toContain("rg1"); + expect(mocked.runAzCommand).toHaveBeenCalledWith( + ["group", "list", "--query", "[?name=='rg1']"], + { + signal: undefined, + onProgress: undefined, + } + ); + }); + + test("a policy refusal never spawns and needs no emulator (the real gates)", async () => { + const answer = await azure({ command: "login" }); + expect(text(answer)).toMatch(/^❌ /); + expect(mocked.status).not.toHaveBeenCalled(); + expect(mocked.health).not.toHaveBeenCalled(); + expect(mocked.azCli).not.toHaveBeenCalled(); + expect(mocked.runAzCommand).not.toHaveBeenCalled(); + }); + + test("version is answered locally: the CLI probe only, no emulator, no az run", async () => { + const answer = await azure({ command: "version" }); + expect(text(answer)).toContain('"azure-cli-core": "2.90.0"'); + expect(mocked.azCli).toHaveBeenCalled(); + expect(mocked.status).not.toHaveBeenCalled(); + expect(mocked.runAzCommand).not.toHaveBeenCalled(); + }); + + test("postgres flexible-server create --version 16 is a normal command", async () => { + await azure({ command: "postgres flexible-server create -g rg -n pg --version 16" }); + expect(mocked.runAzCommand).toHaveBeenCalled(); + }); + + test("a runner timeout gives the timeout class and hint", async () => { + mocked.runAzCommand.mockResolvedValue(result({ exitCode: 1, timedOut: true })); + const answer = text(await azure({ command: "group list" })); + // A killed process's code differs by OS, so the tool reports `none` when it stopped az. + expect(answer.split("\n")[0]).toMatch(/^❌ \*\*Command Failed\*\* \(exit none, timeout\)$/); + expect(answer).toMatch(/--no-wait/); + }); + + test("blocked hosts appear in the response, after the first line", async () => { + mocked.runAzCommand.mockResolvedValue( + result({ + exitCode: 1, + failFast: "refused", + egress: { refused: ["api.loganalytics.io"], upstream: [], housekeeping: [], allowed: 0 }, + }) + ); + const answer = text( + await azure({ command: "monitor log-analytics query -w x --analytics-query y" }) + ); + const [first, ...rest] = answer.split("\n"); + expect(first).toMatch(/egress-refused/); + expect(first).not.toContain("loganalytics"); + expect(rest.join("\n")).toContain("api.loganalytics.io"); + }); + + test("an aborted signal reaches the runner, and the answer says the call was cancelled", async () => { + const controller = new AbortController(); + mocked.runAzCommand.mockImplementation(async (_argv, opts) => { + expect(opts.signal).toBe(controller.signal); + return result({ exitCode: null, aborted: true }); + }); + const answer = text( + await azure({ command: "group list" }, { signal: controller.signal } as never) + ); + expect(answer.split("\n")[0]).toMatch(/cancelled/); + }); + + test("with a progress token, a 25 s run sends 2 progress notifications; none without one", async () => { + const sendNotification = jest.fn().mockResolvedValue(undefined); + mocked.runAzCommand.mockImplementation(async (_argv, opts) => { + opts.onProgress?.(10_000); + opts.onProgress?.(20_000); + return result({ stdout: "[]" }); + }); + await azure({ command: "group list" }, { + signal: new AbortController().signal, + _meta: { progressToken: 7 }, + sendNotification, + } as never); + expect(sendNotification).toHaveBeenCalledTimes(2); + expect(sendNotification.mock.calls[1][0]).toMatchObject({ + method: "notifications/progress", + params: { progressToken: 7, progress: 20 }, + }); + + sendNotification.mockClear(); + let sawCallback = true; + mocked.runAzCommand.mockImplementation(async (_argv, opts) => { + sawCallback = Boolean(opts.onProgress); + return result({ stdout: "[]" }); + }); + await azure({ command: "group list" }, { + signal: new AbortController().signal, + sendNotification, + } as never); + expect(sawCallback).toBe(false); + expect(sendNotification).not.toHaveBeenCalled(); + }); + + test("a bootstrap failure is an az failure with a class id and the setup note", async () => { + mocked.ensureProfile.mockRejectedValue( + new BootstrapError( + "login", + result({ + exitCode: 1, + stderr: "ERROR: Unable to get endpoints from the cloud.\nError detail: boom", + }) + ) + ); + const answer = text(await azure({ command: "group list" })); + expect(answer.split("\n")[0]).toMatch(/^❌ \*\*Command Failed\*\* \(exit 1, [a-z-]+\)$/); + expect(answer).toContain("before your command ran"); + expect(mocked.runAzCommand).not.toHaveBeenCalled(); + }); + + test("an unexpected exception becomes a constant-titled error, never a thrown message", async () => { + mocked.runAzCommand.mockRejectedValue(new Error("boom at C:\\Users\\me\\secret-path")); + const answer = text(await azure({ command: "group list" })); + expect(answer.split("\n")[0]).toBe("❌ **Azure Tool Error**"); + expect(answer).toContain("secret-path"); + }); +}); + +describe("preflights and their precedence", () => { + test("the token check comes first", async () => { + delete process.env.LOCALSTACK_AUTH_TOKEN; + const answer = text(await azure({ command: "login" })); + expect(answer).toMatch(/Auth Token Required/); + expect(mocked.azCli).not.toHaveBeenCalled(); + }); + + test("the emulator not running short-circuits", async () => { + mocked.status.mockResolvedValue({ + ok: false, + problem: "not-running", + message: "The LocalStack Azure emulator is not running at http://127.0.0.1:4566.", + }); + const answer = text(await azure({ command: "group list" })); + expect(answer).toMatch(/^❌ \*\*LocalStack Azure Emulator Not Ready\*\*/); + expect(mocked.ensureProfile).not.toHaveBeenCalled(); + expect(mocked.runAzCommand).not.toHaveBeenCalled(); + }); + + test("a missing az short-circuits, listing the skipped candidates", async () => { + mocked.azCli.mockRejectedValue( + new AzResolveError("The Azure CLI (az) was not found.", ["C:\\shims\\az.exe: an az.exe shim"]) + ); + const answer = text(await azure({ command: "group list" })); + expect(answer).toMatch(/Azure CLI Not Available/); + expect(answer).toContain("az.exe shim"); + expect(mocked.runAzCommand).not.toHaveBeenCalled(); + }); + + test("the first failure in array order wins: requireStack before the emulator and the CLI", async () => { + mocked.health.mockResolvedValue({ reachable: true, ready: true, edition: "pro" } as never); + mocked.status.mockResolvedValue({ ok: false, problem: "wrong-edition", message: "x" }); + mocked.azCli.mockRejectedValue(new AzResolveError("missing")); + const answer = text(await azure({ command: "group list" })); + expect(answer).toMatch(/Wrong emulator for this tool/); + expect(answer).toContain("http://127.0.0.1:4566"); + }); + + test("the stack check reads health on the Azure port's base URL", async () => { + await azure({ command: "group list" }); + expect(mocked.health).toHaveBeenCalledWith("http://127.0.0.1:4566"); + }); + + test("in Docker, the forwarder is awaited before the preflight group", async () => { + const real = services.azureConfig(); + const spy = jest.spyOn(services, "azureConfig").mockReturnValue({ ...real, inDocker: true }); + const order: string[] = []; + mocked.forwarder.mockImplementation(async () => { + await new Promise((r) => setTimeout(r, 20)); + order.push("forwarder"); + return undefined; + }); + mocked.health.mockImplementation(async () => { + order.push("health"); + return { reachable: true, ready: true, edition: "azure-alpha" } as never; + }); + try { + await azure({ command: "group list" }); + expect(order).toEqual(["forwarder", "health"]); + } finally { + spy.mockRestore(); + } + }); +}); + +describe("Bicep", () => { + test("no Bicep binary: bicep-missing, without spawning az", async () => { + mocked.bicep.mockResolvedValue(undefined); + const answer = text( + await azure({ command: "deployment group create -g rg --template-file main.bicep" }) + ); + expect(answer.split("\n")[0]).toMatch(/bicep-missing/); + expect(answer).toMatch(/--template-file main\.json/); + expect(mocked.runAzCommand).not.toHaveBeenCalled(); + }); + + test("a bad LOCALSTACK_AZ_BICEP_PATH is its own hard error", async () => { + mocked.bicep.mockRejectedValue( + new BicepPathError( + "LOCALSTACK_AZ_BICEP_PATH must be an existing file named bicep or bicep.exe" + ) + ); + const answer = text(await azure({ command: "bicep build --file main.bicep" })); + expect(answer).toMatch(/Bicep CLI Not Usable/); + expect(mocked.runAzCommand).not.toHaveBeenCalled(); + }); + + test("a registry module in the template: bicep-registry, without spawning", async () => { + writeFileSync( + path.join(workdir, "reg.bicep"), + "module st 'br/public:avm/res/storage/storage-account:0.9.0' = {\n name: 'st'\n}\n" + ); + const answer = text( + await azure({ command: "deployment group create -g rg --template-file reg.bicep" }) + ); + expect(answer.split("\n")[0]).toMatch(/bicep-registry/); + expect(mocked.runAzCommand).not.toHaveBeenCalled(); + }); + + test("an old Bicep refuses .bicepparam", async () => { + mocked.bicep.mockResolvedValue({ + path: "/usr/bin/bicep", + dir: "/usr/bin", + source: "path", + version: "0.13.1", + supportsBicepparam: false, + }); + writeFileSync(path.join(workdir, "main.bicep"), "param location string\n"); + const answer = text( + await azure({ + command: + "deployment group create -g rg --template-file main.bicep --parameters main.bicepparam", + }) + ); + expect(answer).toMatch(/Bicep CLI Too Old/); + }); + + test("a compiled ARM template does not need Bicep", async () => { + await azure({ command: "deployment group create -g rg --template-file main.json" }); + expect(mocked.bicep).not.toHaveBeenCalled(); + expect(mocked.runAzCommand).toHaveBeenCalled(); + }); +}); + +describe("analytics carry no values", () => { + const events = () => + mockCapture.mock.calls.map( + (c) => c[0] as { event: string; properties: Record } + ); + + function randomSecret() { + return `S3cr3t${Math.random().toString(36).slice(2, 12)}X${Date.now().toString(36)}`; + } + + const shapes: Array<(secret: string) => string> = [ + (s) => `keyvault secret set --vault-name kv1 --name pw --value ${s}`, + (s) => `vm create -g rg -n vm1 --image Ubuntu2204 --admin-password ${s}`, + (s) => `vm create -g rg -n vm1 --image Ubuntu2204 --admin-password=${s}`, + (s) => + `storage blob upload --account-name st --container-name c --name b --file f.txt --sas-token ${s}`, + (s) => + `webapp config connection-string set -g rg -n app --settings "Db=Server=tcp:x;Password=${s}"`, + (s) => `keyvault secret set --vault-name kv1 --name pw --file @${s}.txt`, + (s) => `rest --method get --url https://${s}.blob.core.windows.net/c`, + (s) => `login --username ${s}`, + (s) => `group list; echo ${s}`, + ]; + + test.each(shapes.map((shape, i) => [i, shape] as const))( + "shape %i: no secret reaches captureToolEvent", + async (_i, shape) => { + for (let round = 0; round < 3; round++) { + mockCapture.mockClear(); + const secret = randomSecret(); + mocked.runAzCommand.mockResolvedValue( + round === 1 + ? result({ exitCode: 1, stderr: `ERROR: (BadRequest) value ${secret} rejected` }) + : round === 2 + ? result({ + exitCode: 1, + egress: { + refused: [`${secret}.blob.core.windows.net`], + upstream: [], + housekeeping: [], + allowed: 0, + }, + }) + : result({ stdout: `{"value": "${secret}"}` }) + ); + await azure({ command: shape(secret) }); + const captured = JSON.stringify(events()); + expect(events().length).toBeGreaterThan(0); + expect(captured).not.toContain(secret); + expect(captured.toLowerCase()).not.toContain(secret.toLowerCase()); + for (const e of events()) { + const args = e.properties.args as Record; + expect(Object.keys(args).sort()).toEqual( + ["command_path", "flag_names", "policy_outcome"].filter((k) => k in args).sort() + ); + } + } + } + ); + + test("every failure path records success === false, with a value-free first line", async () => { + const failures: Array<() => void> = [ + () => + mocked.runAzCommand.mockResolvedValue( + result({ + exitCode: 3, + stderr: "ERROR: (ResourceNotFound) Resource group 'x' could not be found.", + }) + ), + () => mocked.status.mockResolvedValue({ ok: false, problem: "not-running", message: "down" }), + () => mocked.azCli.mockRejectedValue(new AzResolveError("missing")), + () => mocked.runAzCommand.mockResolvedValue(result({ exitCode: 1, timedOut: true })), + ]; + for (const arrange of failures) { + mockCapture.mockClear(); + mocked.status.mockResolvedValue({ ok: true, sessionId: "s-1" }); + mocked.azCli.mockResolvedValue(AZ); + arrange(); + await azure({ command: "group show -n x" }); + const executed = events().find((e) => e.event === "mcp_tool_executed"); + expect(executed?.properties.success).toBe(false); + expect(String(executed?.properties.error_message)).not.toMatch(/'x'/); + } + mockCapture.mockClear(); + await azure({ command: "login" }); + expect(events().find((e) => e.event === "mcp_tool_executed")?.properties).toMatchObject({ + success: false, + args: { command_path: "login", policy_outcome: expect.stringMatching(/^denied:/) }, + }); + }); + + test("the recorded fields for a normal command", async () => { + mockCapture.mockClear(); + await azure({ + command: "storage account create --name st1abc --resource-group rg1 --sku=Standard_LRS", + }); + expect(events()[0].properties).toMatchObject({ + success: true, + args: { + command_path: "storage account create", + flag_names: "--name,--resource-group,--sku", + policy_outcome: "ok", + }, + }); + }); +}); + +describe("configuration warnings reach the user", () => { + afterEach(async () => { + delete process.env.LOCALSTACK_AZ_TIMEOUT_SECONDS; + delete process.env.LOCALSTACK_AZ_RUNNER; + await services.resetAzureServices(); + }); + + test("an invalid setting's fallback is a note in the first answer, not in every answer", async () => { + process.env.LOCALSTACK_AZ_TIMEOUT_SECONDS = "2"; + process.env.LOCALSTACK_AZ_RUNNER = "fast"; + await services.resetAzureServices(); + const first = text(await azure({ command: "group list" })); + expect(first).toContain( + "Note: LOCALSTACK_AZ_TIMEOUT_SECONDS=2 is not a whole number from 5 to 3600; using 300." + ); + expect(first).toContain("Note: LOCALSTACK_AZ_RUNNER=fast is not a runner"); + const second = text(await azure({ command: "group list" })); + expect(second).not.toContain("LOCALSTACK_AZ_TIMEOUT_SECONDS"); + }); + + test("a refusal can carry them too, and valid settings add nothing", async () => { + process.env.LOCALSTACK_AZ_TIMEOUT_SECONDS = "abc"; + await services.resetAzureServices(); + expect(text(await azure({ command: "login" }))).toContain( + "Note: LOCALSTACK_AZ_TIMEOUT_SECONDS=abc is not a whole number" + ); + delete process.env.LOCALSTACK_AZ_TIMEOUT_SECONDS; + await services.resetAzureServices(); + expect(text(await azure({ command: "group list" }))).not.toContain("Note:"); + }); +}); diff --git a/src/tools-tests/localstack-management-ports.test.ts b/src/tools-tests/localstack-management-ports.test.ts new file mode 100644 index 0000000..b29da91 --- /dev/null +++ b/src/tools-tests/localstack-management-ports.test.ts @@ -0,0 +1,102 @@ +// Through the tool: with LOCALSTACK_PORT=4666, stop and restart never select a +// container that does not publish 4666, so a test run beside a shared emulator on 4566 +// can never stop it. The real DockerApiClient +// runs; only dockerode is mocked. +import localstackManagement from "../tools/localstack-management"; + +jest.mock("../core/analytics", () => ({ + withToolAnalytics: (_name: string, _args: unknown, fn: () => unknown) => fn(), +})); + +jest.mock("../core/preflight", () => { + const actual = jest.requireActual("../core/preflight"); + return { ...actual, requireDockerDaemon: jest.fn().mockResolvedValue(null) }; +}); + +jest.mock("../lib/localstack/localstack.utils", () => { + const actual = jest.requireActual("../lib/localstack/localstack.utils"); + return { + ...actual, + getLocalStackStatus: jest.fn().mockResolvedValue({ isRunning: false, isReady: false }), + launchRuntime: jest.fn().mockResolvedValue({ content: [{ type: "text", text: "launched" }] }), + }; +}); + +jest.mock("dockerode", () => { + const listContainers = jest.fn(); + const stop = jest.fn(); + const remove = jest.fn(); + const inspect = jest.fn(); + const getContainer = jest.fn((id: string) => ({ + id, + stop: () => stop(id), + remove: () => remove(id), + inspect: () => inspect(id), + })); + class DockerMock { + static __mocks = { listContainers, getContainer, stop, remove, inspect }; + modem = {}; + listContainers = listContainers; + getContainer = getContainer; + } + return DockerMock as any; +}); + +// eslint-disable-next-line @typescript-eslint/no-require-imports +const mocks = () => (require("dockerode") as any).__mocks; +const text = (r: { content: Array<{ text: string }> }) => r.content[0].text; + +const sharedAzure = { + Id: "shared-4566", + Names: ["/localstack-azure"], + Image: "f91897f1de85", + Labels: { description: "LocalStack for Azure" }, + Ports: [{ PrivatePort: 4566, PublicPort: 4566, Type: "tcp" }], + State: "running", +}; +const testContainer = { + Id: "test-4666", + Names: ["/localstack-azure-mcp-test"], + Image: "localstack/localstack-azure:latest", + Ports: [{ PrivatePort: 4666, PublicPort: 4666, Type: "tcp" }], + State: "running", +}; + +describe("stop and restart with LOCALSTACK_PORT=4666, through the tool", () => { + beforeEach(() => { + jest.clearAllMocks(); + process.env.LOCALSTACK_AUTH_TOKEN = "ls-test-token"; + process.env.LOCALSTACK_PORT = "4666"; + process.env.MAIN_CONTAINER_NAME = "localstack-azure-mcp-test"; + mocks().stop.mockResolvedValue(undefined); + mocks().remove.mockResolvedValue(undefined); + }); + + afterAll(() => { + delete process.env.LOCALSTACK_PORT; + delete process.env.MAIN_CONTAINER_NAME; + }); + + test("stop stops the test container, never the shared emulator", async () => { + mocks().listContainers.mockResolvedValue([sharedAzure, testContainer]); + const result = await localstackManagement({ action: "stop", service: "azure" } as any); + expect(text(result)).toMatch(/stopped successfully/); + expect(mocks().stop).toHaveBeenCalledWith("test-4666"); + expect(mocks().stop).not.toHaveBeenCalledWith("shared-4566"); + }); + + test("with only the shared emulator running, stop touches nothing", async () => { + mocks().listContainers.mockResolvedValue([sharedAzure]); + const result = await localstackManagement({ action: "stop", service: "azure" } as any); + expect(mocks().stop).not.toHaveBeenCalled(); + expect(mocks().remove).not.toHaveBeenCalled(); + expect(text(result)).not.toMatch(/stopped successfully/); + }); + + test("with only the shared emulator running, restart never stops it", async () => { + mocks().listContainers.mockResolvedValue([sharedAzure]); + await localstackManagement({ action: "restart", service: "azure" } as any); + expect(mocks().stop).not.toHaveBeenCalled(); + expect(mocks().remove).not.toHaveBeenCalled(); + }); +}); diff --git a/src/tools-tests/localstack-management.test.ts b/src/tools-tests/localstack-management.test.ts index c939cac..3e92845 100644 --- a/src/tools-tests/localstack-management.test.ts +++ b/src/tools-tests/localstack-management.test.ts @@ -3,7 +3,8 @@ // below resolve to the same modules the tool imports (this dir is a sibling of tools/). import localstackManagement from "../tools/localstack-management"; import { DockerApiClient } from "../lib/docker/docker.client"; -import { getLocalStackStatus } from "../lib/localstack/localstack.utils"; +import { getLocalStackStatus, launchRuntime } from "../lib/localstack/localstack.utils"; +import { azureStartedCheck, getAzureRuntimeStatus } from "../lib/azure/runtime-status"; jest.mock("../core/analytics", () => ({ withToolAnalytics: (_name: string, _args: unknown, fn: () => unknown) => fn(), @@ -35,6 +36,19 @@ jest.mock("../lib/localstack/localstack.utils", () => { }; }); +jest.mock("../lib/azure/runtime-status", () => ({ + getAzureRuntimeStatus: jest.fn(), + azureStartedCheck: jest.fn().mockResolvedValue(null), +})); + +jest.mock("../lib/azure/loopback-forwarder", () => ({ + ensureLoopbackForwarder: jest.fn().mockResolvedValue(undefined), +})); + +jest.mock("../lib/azure/services", () => ({ + azureConfig: jest.fn(() => ({ inDocker: false })), +})); + const MockedDocker = DockerApiClient as jest.MockedClass; const mockedGetStatus = getLocalStackStatus as jest.MockedFunction; @@ -110,9 +124,646 @@ describe("localstack-management", () => { expect(text(result)).toMatch(/not running — no container to stop/); }); + test("status service=aws with an Azure container says it is the Azure stack", async () => { + mockDocker({ + findLocalStackContainer: jest.fn().mockResolvedValue("id-az"), + inspectContainer: jest.fn().mockResolvedValue({ + id: "id-az", + name: "localstack-azure", + image: "localstack/localstack-azure:latest", + }), + }); + mockedGetStatus.mockResolvedValue({ + isRunning: true, + isReady: false, + statusOutput: "gateway reachable", + }); + + const result = await localstackManagement({ action: "status", service: "aws" } as any); + expect(text(result)).toContain('The running LocalStack container ("localstack-azure"'); + expect(text(result)).toContain("is the Azure stack"); + expect(text(result)).toContain("the AWS emulator is not running"); + }); + + test("status recognises a bare-ID container with the Azure label as the Azure stack", async () => { + mockDocker({ + findLocalStackContainer: jest.fn().mockResolvedValue("id-bare"), + inspectContainer: jest.fn().mockResolvedValue({ + id: "id-bare", + name: "localstack-azure", + image: "f91897f1de85", + labels: { description: "LocalStack for Azure" }, + }), + }); + mockedGetStatus.mockResolvedValue({ + isRunning: true, + isReady: false, + statusOutput: "gateway reachable", + }); + + const result = await localstackManagement({ action: "status", service: "aws" } as any); + expect(text(result)).toContain("is the Azure stack"); + }); + + test("status service=snowflake with an Azure container names the Azure stack", async () => { + mockDocker({ + findLocalStackContainer: jest.fn().mockResolvedValue("id-az"), + inspectContainer: jest.fn().mockResolvedValue({ + id: "id-az", + name: "localstack-azure", + image: "localstack/localstack-azure:latest", + }), + }); + mockedGetStatus.mockResolvedValue({ + isRunning: true, + isReady: false, + statusOutput: "gateway reachable", + }); + + const result = await localstackManagement({ action: "status", service: "snowflake" } as any); + expect(text(result)).toContain("is the Azure stack"); + expect(text(result)).toContain("the Snowflake emulator is not running"); + }); + + test("status service=aws with an AWS container is unchanged", async () => { + mockDocker({ + findLocalStackContainer: jest.fn().mockResolvedValue("id-aws"), + inspectContainer: jest.fn().mockResolvedValue({ + id: "id-aws", + name: "localstack-main", + image: "localstack/localstack-pro:latest", + }), + }); + mockedGetStatus.mockResolvedValue({ + isRunning: true, + isReady: true, + statusOutput: "gateway reachable", + }); + + const result = await localstackManagement({ action: "status", service: "aws" } as any); + expect(text(result)).toContain("currently running and ready"); + expect(text(result)).not.toMatch(/stack —/); + }); + test("requires the auth token", async () => { delete process.env.LOCALSTACK_AUTH_TOKEN; const result = await localstackManagement({ action: "status", service: "aws" } as any); expect(text(result)).toContain("LOCALSTACK_AUTH_TOKEN"); }); }); + +// The Azure stack (service "azure"). +describe("localstack-management service=azure", () => { + const mockedLaunch = launchRuntime as jest.MockedFunction; + const mockedAzureStatus = getAzureRuntimeStatus as jest.MockedFunction< + typeof getAzureRuntimeStatus + >; + const azureContainer = (over: Record = {}) => ({ + id: "id-az", + name: "localstack-azure", + image: "localstack/localstack-azure:latest", + portBindings: { "4566/tcp": [{ HostIp: "127.0.0.1", HostPort: "4566" }] }, + ...over, + }); + + beforeEach(() => { + jest.clearAllMocks(); + process.env.LOCALSTACK_AUTH_TOKEN = "ls-test-token"; + delete process.env.MAIN_CONTAINER_NAME; + delete process.env.LOCALSTACK_PORT; + delete process.env.LOCALSTACK_AZURE_PORT; + mockedLaunch.mockResolvedValue({ content: [{ type: "text", text: "launched" }] } as any); + }); + + test("start passes stack azure, the Azure labels and the Azure status function", async () => { + await localstackManagement({ action: "start", service: "azure" } as any); + expect(mockedLaunch).toHaveBeenCalledWith( + expect.objectContaining({ + stack: "azure", + processLabel: "LocalStack Azure emulator", + successTitle: "🚀 LocalStack Azure emulator started successfully!", + statusHeading: "Health check", + getStatus: getAzureRuntimeStatus, + onReady: azureStartedCheck, + }) + ); + }); + + test("start refuses a LOCALSTACK_AZURE_PORT that differs from the published LOCALSTACK_PORT", async () => { + process.env.LOCALSTACK_AZURE_PORT = "4666"; + const result = await localstackManagement({ action: "start", service: "azure" } as any); + expect(text(result)).toMatch(/Conflicting Azure port settings/); + expect(mockedLaunch).not.toHaveBeenCalled(); + }); + + test("status: a healthy Azure emulator passes, and the Container line names it", async () => { + mockDocker({ + findLocalStackContainer: jest.fn().mockResolvedValue("id-az"), + inspectContainer: jest.fn().mockResolvedValue(azureContainer()), + }); + mockedGetStatus.mockResolvedValue({ + isRunning: true, + isReady: true, + statusOutput: "gateway reachable", + }); + mockedAzureStatus.mockResolvedValue({ + isRunning: true, + isReady: true, + statusOutput: "edition: azure-alpha", + status: { ok: true }, + }); + const result = text(await localstackManagement({ action: "status", service: "azure" } as any)); + expect(result).toContain("Azure emulator health check passed"); + expect(result).toContain( + 'Container: "localstack-azure" (image localstack/localstack-azure:latest, gateway 127.0.0.1:4566)' + ); + }); + + test("status side by side (AWS on LOCALSTACK_PORT=4666, Azure on 4566) reports the Azure emulator", async () => { + process.env.LOCALSTACK_PORT = "4666"; + process.env.LOCALSTACK_AZURE_PORT = "4566"; + // eslint-disable-next-line @typescript-eslint/no-require-imports + const { azureConfig } = require("../lib/azure/services"); + (azureConfig as jest.Mock).mockReturnValue({ + inDocker: false, + port: 4566, + healthBaseUrl: "http://127.0.0.1:4566", + }); + const awsContainer = { + id: "id-aws", + name: "localstack-uat-aws", + image: "localstack/localstack-pro:latest", + }; + // As the real lookup does: it follows LOCALSTACK_PORT (4666: an emulator on another port is never picked), so + // no Azure container is found there; the AWS one must never be taken for it. + mockDocker({ + findLocalStackContainer: jest.fn(async (opts?: { stack?: string }) => { + if (opts?.stack === "azure") throw new Error("no Azure container publishes 4666"); + return "id-aws"; + }), + inspectContainer: jest.fn(async () => awsContainer), + }); + mockedGetStatus.mockImplementation(async (baseUrl?: string) => ({ + isRunning: true, + isReady: true, + statusOutput: `gateway reachable at ${baseUrl ?? "http://localhost:4666"}`, + })); + mockedAzureStatus.mockResolvedValue({ + isRunning: true, + isReady: true, + statusOutput: "edition: azure-alpha", + status: { ok: true }, + }); + const result = text(await localstackManagement({ action: "status", service: "azure" } as any)); + expect(mockedGetStatus).toHaveBeenCalledWith("http://127.0.0.1:4566"); + expect(result).toContain("gateway reachable at http://127.0.0.1:4566"); + expect(result).toContain("Azure emulator health check passed"); + // Neither the AWS gateway nor the AWS container stands in for the Azure emulator. + expect(result).not.toContain("localhost:4666"); + expect(result).not.toContain("localstack-uat-aws"); + expect(result).not.toContain("is the AWS stack"); + (azureConfig as jest.Mock).mockReturnValue({ inDocker: false }); + }); + + describe("side by side (AWS on LOCALSTACK_PORT=4666, Azure on LOCALSTACK_AZURE_PORT=4566)", () => { + const sideBySide = () => { + process.env.LOCALSTACK_PORT = "4666"; + process.env.LOCALSTACK_AZURE_PORT = "4566"; + // eslint-disable-next-line @typescript-eslint/no-require-imports + const { azureConfig } = require("../lib/azure/services"); + (azureConfig as jest.Mock).mockReturnValue({ + inDocker: false, + port: 4566, + healthBaseUrl: "http://127.0.0.1:4566", + }); + const docker = { + // The Azure container is found only by stack and the Azure port; without them the + // lookup follows LOCALSTACK_PORT to the AWS emulator. + findLocalStackContainer: jest.fn(async (opts?: { stack?: string; port?: string }) => + opts?.stack === "azure" && opts?.port === "4566" ? "id-azure" : "id-aws" + ), + inspectContainer: jest.fn(), + stopContainer: jest.fn().mockResolvedValue(undefined), + removeContainer: jest.fn().mockResolvedValue(undefined), + waitForRemoval: jest.fn().mockResolvedValue(undefined), + findContainerByNameAnyState: jest.fn().mockResolvedValue(undefined), + }; + mockDocker(docker); + return docker; + }; + afterEach(() => { + // eslint-disable-next-line @typescript-eslint/no-require-imports + const { azureConfig } = require("../lib/azure/services"); + (azureConfig as jest.Mock).mockReturnValue({ inDocker: false }); + }); + + test("stop service: azure stops the Azure container, never the AWS one", async () => { + const docker = sideBySide(); + const result = await localstackManagement({ action: "stop", service: "azure" } as any); + expect(text(result)).toMatch(/stopped successfully/); + expect(docker.stopContainer).toHaveBeenCalledWith("id-azure"); + expect(docker.stopContainer).not.toHaveBeenCalledWith("id-aws"); + }); + + test("restart service: azure refuses before anything is stopped", async () => { + const docker = sideBySide(); + const result = await localstackManagement({ action: "restart", service: "azure" } as any); + expect(text(result)).toMatch(/Conflicting Azure port settings/); + expect(text(result)).toMatch(/Nothing was stopped/); + expect(docker.stopContainer).not.toHaveBeenCalled(); + expect(docker.removeContainer).not.toHaveBeenCalled(); + expect(mockedLaunch).not.toHaveBeenCalled(); + }); + }); + + test("status: an AWS container is reported as the foreign stack", async () => { + mockDocker({ + findLocalStackContainer: jest.fn().mockResolvedValue("id-aws"), + inspectContainer: jest.fn().mockResolvedValue({ + id: "id-aws", + name: "localstack-main", + image: "localstack/localstack-pro:latest", + }), + }); + mockedGetStatus.mockResolvedValue({ + isRunning: true, + isReady: true, + statusOutput: "gateway reachable", + }); + const result = text(await localstackManagement({ action: "status", service: "azure" } as any)); + expect(result).toContain("is the AWS stack"); + expect(result).toContain("the Azure emulator is not running"); + expect(result).toContain("start with service: azure"); + expect(mockedAzureStatus).not.toHaveBeenCalled(); + }); + + test("status: Azure reachable but unhealthy gives the diagnostics line", async () => { + mockDocker({ + findLocalStackContainer: jest.fn().mockResolvedValue("id-az"), + inspectContainer: jest.fn().mockResolvedValue(azureContainer()), + }); + mockedGetStatus.mockResolvedValue({ + isRunning: true, + isReady: false, + statusOutput: "gateway reachable", + }); + mockedAzureStatus.mockResolvedValue({ + isRunning: true, + isReady: false, + statusOutput: "edition: azure-alpha, license: true", + status: { ok: false, problem: "https-not-ready", message: "HTTPS did not become ready" }, + }); + const result = text(await localstackManagement({ action: "status", service: "azure" } as any)); + expect(result).toMatch( + /Azure emulator health check did not pass\. \(edition: azure-alpha, license: true \| HTTPS did not become ready\)/ + ); + }); + + test("status: nothing running gives the standard message", async () => { + mockedGetStatus.mockResolvedValue({ isRunning: false, isReady: false }); + const result = text(await localstackManagement({ action: "status", service: "azure" } as any)); + expect(result).toContain("LocalStack is not currently running"); + expect(result).not.toContain("Container:"); + }); + + test("status: a healthy test container on 4666 gets its own Container line", async () => { + process.env.LOCALSTACK_PORT = "4666"; + mockDocker({ + findLocalStackContainer: jest.fn().mockResolvedValue("id-test"), + inspectContainer: jest.fn().mockResolvedValue( + azureContainer({ + id: "id-test", + name: "localstack-azure-mcp-test", + portBindings: { + "4666/tcp": [{ HostIp: "", HostPort: "4666" }], + "4610/tcp": [{ HostIp: "", HostPort: "4610" }], + }, + }) + ), + }); + mockedGetStatus.mockResolvedValue({ + isRunning: true, + isReady: true, + statusOutput: "gateway reachable", + }); + mockedAzureStatus.mockResolvedValue({ isRunning: true, isReady: true, status: { ok: true } }); + const result = text(await localstackManagement({ action: "status", service: "azure" } as any)); + expect(result).toContain( + 'Container: "localstack-azure-mcp-test" (image localstack/localstack-azure:latest, gateway 0.0.0.0:4666)' + ); + }); + + test("restart of an lstk-named localstack-azure keeps its name, image and volume", async () => { + const stopContainer = jest.fn().mockResolvedValue(undefined); + mockDocker({ + findLocalStackContainer: jest.fn().mockResolvedValue("id-az"), + inspectContainer: jest.fn().mockResolvedValue( + azureContainer({ + mounts: [ + { type: "volume", name: "localstack-azure-vol", destination: "/var/lib/localstack" }, + ], + }) + ), + stopContainer, + waitForRemoval: jest.fn().mockResolvedValue(undefined), + }); + await localstackManagement({ action: "restart", service: "azure" } as any); + expect(stopContainer).toHaveBeenCalledWith("id-az"); + expect(mockedLaunch).toHaveBeenCalledWith( + expect.objectContaining({ + stack: "azure", + imageOverride: "localstack/localstack-azure:latest", + containerNameOverride: "localstack-azure", + volumeOverride: { type: "volume", name: "localstack-azure-vol" }, + }) + ); + }); + + test("restart refuses, before stopping, a container whose state folder this machine cannot mount", async () => { + // An lstk emulator started from WSL has a WSL path as its bind. A Windows server that + // stopped it (--rm deletes it) could not recreate it. + const stopContainer = jest.fn().mockResolvedValue(undefined); + const source = + process.platform === "win32" + ? "/home/dev/.cache/lstk/volume/localstack-azure" + : "/nonexistent-uat-life02/volume"; + mockDocker({ + findLocalStackContainer: jest.fn().mockResolvedValue("id-az"), + inspectContainer: jest + .fn() + .mockResolvedValue( + azureContainer({ mounts: [{ type: "bind", source, destination: "/var/lib/localstack" }] }) + ), + stopContainer, + waitForRemoval: jest.fn().mockResolvedValue(undefined), + }); + const result = text(await localstackManagement({ action: "restart", service: "azure" } as any)); + expect(stopContainer).not.toHaveBeenCalled(); + expect(mockedLaunch).not.toHaveBeenCalled(); + expect(result).toMatch(/Cannot restart this LocalStack container from here/); + expect(result).toContain(source); + expect(result).toMatch(/nothing was stopped/i); + }); + + test("restart still recreates a container whose state folder is on this machine", async () => { + const stopContainer = jest.fn().mockResolvedValue(undefined); + // eslint-disable-next-line @typescript-eslint/no-require-imports + const source = require("os").tmpdir(); + mockDocker({ + findLocalStackContainer: jest.fn().mockResolvedValue("id-az"), + inspectContainer: jest + .fn() + .mockResolvedValue( + azureContainer({ mounts: [{ type: "bind", source, destination: "/var/lib/localstack" }] }) + ), + stopContainer, + waitForRemoval: jest.fn().mockResolvedValue(undefined), + }); + await localstackManagement({ action: "restart", service: "azure" } as any); + expect(stopContainer).toHaveBeenCalledWith("id-az"); + expect(mockedLaunch).toHaveBeenCalledWith( + expect.objectContaining({ volumeOverride: { type: "bind", source } }) + ); + }); + + test("restart carries the container's own env flags across, minus image/reserved/token", async () => { + const stopContainer = jest.fn().mockResolvedValue(undefined); + mockDocker({ + findLocalStackContainer: jest.fn().mockResolvedValue("id-az"), + inspectContainer: jest.fn().mockResolvedValue( + azureContainer({ + env: [ + "MSSQL_ACCEPT_EULA=Y", // operator flag, not in the image → carried + "DISABLE_EVENTS=1", // operator flag → carried + "LS_AZURE_PORTAL=1", // operator flag → carried + "PATH=/usr/local/bin:/usr/bin", // image-baked (same pair) → dropped + "GATEWAY_LISTEN=0.0.0.0:4566", // launcher-owned reserved key → dropped + "LOCALSTACK_AUTH_TOKEN=stale-token", // sensitive + reserved → dropped + ], + }) + ), + imageConfigEnv: jest.fn().mockResolvedValue(["PATH=/usr/local/bin:/usr/bin", "LANG=C.UTF-8"]), + stopContainer, + waitForRemoval: jest.fn().mockResolvedValue(undefined), + }); + + const result = text(await localstackManagement({ action: "restart", service: "azure" } as any)); + expect(stopContainer).toHaveBeenCalledWith("id-az"); + const options = mockedLaunch.mock.calls[0][0]; + expect(options.envVars).toMatchObject({ + MSSQL_ACCEPT_EULA: "Y", + DISABLE_EVENTS: "1", + LS_AZURE_PORTAL: "1", + }); + expect(options.envVars).not.toHaveProperty("PATH"); + expect(options.envVars).not.toHaveProperty("GATEWAY_LISTEN"); + expect(options.envVars).not.toHaveProperty("LOCALSTACK_AUTH_TOKEN"); + // The answer names the carried keys, and never the token's value. + expect(result).toMatch(/MSSQL_ACCEPT_EULA/); + expect(result).not.toMatch(/stale-token/); + }); + + test("restart: this server's own env block wins over a carried value, directly or as LOCALSTACK_*", async () => { + // The old container ran with PERSISTENCE=0 and DEBUG=0; the MCP config now sets + // PERSISTENCE=1 and LOCALSTACK_DEBUG=1, which the start forwards itself. + process.env.PERSISTENCE = "1"; + process.env.LOCALSTACK_DEBUG = "1"; + try { + mockDocker({ + findLocalStackContainer: jest.fn().mockResolvedValue("id-az"), + inspectContainer: jest + .fn() + .mockResolvedValue( + azureContainer({ env: ["PERSISTENCE=0", "DEBUG=0", "MSSQL_ACCEPT_EULA=Y"] }) + ), + imageConfigEnv: jest.fn().mockResolvedValue([]), + stopContainer: jest.fn().mockResolvedValue(undefined), + waitForRemoval: jest.fn().mockResolvedValue(undefined), + }); + const result = text( + await localstackManagement({ action: "restart", service: "azure" } as any) + ); + const options = mockedLaunch.mock.calls[0][0]; + expect(options.envVars).not.toHaveProperty("PERSISTENCE"); + expect(options.envVars).not.toHaveProperty("DEBUG"); + expect(options.envVars).toMatchObject({ MSSQL_ACCEPT_EULA: "Y" }); // the host does not set it + expect(result).not.toMatch(/PERSISTENCE/); + } finally { + delete process.env.PERSISTENCE; + delete process.env.LOCALSTACK_DEBUG; + } + }); + + test("restart: the caller's own envVars win over a carried value", async () => { + mockDocker({ + findLocalStackContainer: jest.fn().mockResolvedValue("id-az"), + inspectContainer: jest + .fn() + .mockResolvedValue(azureContainer({ env: ["DISABLE_EVENTS=1", "MSSQL_ACCEPT_EULA=Y"] })), + imageConfigEnv: jest.fn().mockResolvedValue([]), + stopContainer: jest.fn().mockResolvedValue(undefined), + waitForRemoval: jest.fn().mockResolvedValue(undefined), + }); + await localstackManagement({ + action: "restart", + service: "azure", + envVars: { DISABLE_EVENTS: "0" }, + } as any); + const options = mockedLaunch.mock.calls[0][0]; + expect(options.envVars).toMatchObject({ DISABLE_EVENTS: "0", MSSQL_ACCEPT_EULA: "Y" }); + }); + + test("restart of a container THIS server started carries nothing: restart applies new config", async () => { + // Our own starts stamp LOCALSTACK_CLIENT_NAME=localstack-mcp-server; their env came from this + // server's (previous) config, so the upstream "restart applies new configuration" holds and a + // restart must still be able to drop a setting. Only externally started containers carry. + const imageConfigEnv = jest.fn().mockResolvedValue([]); + mockDocker({ + findLocalStackContainer: jest.fn().mockResolvedValue("id-az"), + inspectContainer: jest.fn().mockResolvedValue( + azureContainer({ + env: ["LOCALSTACK_CLIENT_NAME=localstack-mcp-server", "MSSQL_ACCEPT_EULA=Y"], + }) + ), + imageConfigEnv, + stopContainer: jest.fn().mockResolvedValue(undefined), + waitForRemoval: jest.fn().mockResolvedValue(undefined), + }); + const result = text(await localstackManagement({ action: "restart", service: "azure" } as any)); + const options = mockedLaunch.mock.calls[0][0]; + expect(options.envVars).toBeUndefined(); + expect(result).not.toMatch(/Carried/); + expect(imageConfigEnv).not.toHaveBeenCalled(); + }); + + test("a restart records what it carried, so a SECOND restart keeps carrying it", async () => { + // Live on 4f: the first MCP restart of the lstk-started emulator kept its six flags, but the new + // container carries this server's LOCALSTACK_CLIENT_NAME, so the second restart saw a + // server-started container, carried nothing and lost them all. The carried keys now travel with + // the container in MCP_CARRIED_ENV. + mockDocker({ + findLocalStackContainer: jest.fn().mockResolvedValue("id-az"), + inspectContainer: jest + .fn() + .mockResolvedValue( + azureContainer({ env: ["MSSQL_ACCEPT_EULA=Y", "DISABLE_EVENTS=1", "PATH=/usr/bin"] }) + ), + imageConfigEnv: jest.fn().mockResolvedValue(["PATH=/usr/bin"]), + stopContainer: jest.fn().mockResolvedValue(undefined), + waitForRemoval: jest.fn().mockResolvedValue(undefined), + }); + await localstackManagement({ action: "restart", service: "azure" } as any); + expect(mockedLaunch.mock.calls[0][0].envVars).toMatchObject({ + MSSQL_ACCEPT_EULA: "Y", + DISABLE_EVENTS: "1", + MCP_CARRIED_ENV: "DISABLE_EVENTS,MSSQL_ACCEPT_EULA", + }); + + // The second restart: the container now looks server-started, but it lists what it carries. + mockedLaunch.mockClear(); + mockDocker({ + findLocalStackContainer: jest.fn().mockResolvedValue("id-az2"), + inspectContainer: jest.fn().mockResolvedValue( + azureContainer({ + id: "id-az2", + env: [ + "LOCALSTACK_CLIENT_NAME=localstack-mcp-server", + "MSSQL_ACCEPT_EULA=Y", + "DISABLE_EVENTS=1", + "SOME_START_ENV=from-an-earlier-mcp-start", + "MCP_CARRIED_ENV=DISABLE_EVENTS,MSSQL_ACCEPT_EULA", + ], + }) + ), + imageConfigEnv: jest.fn().mockResolvedValue([]), + stopContainer: jest.fn().mockResolvedValue(undefined), + waitForRemoval: jest.fn().mockResolvedValue(undefined), + }); + await localstackManagement({ action: "restart", service: "azure" } as any); + const second = mockedLaunch.mock.calls[0][0].envVars; + expect(second).toMatchObject({ + MSSQL_ACCEPT_EULA: "Y", + DISABLE_EVENTS: "1", + MCP_CARRIED_ENV: "DISABLE_EVENTS,MSSQL_ACCEPT_EULA", + }); + // What an earlier MCP start set is still NOT carried: restart applies new configuration there. + expect(second).not.toHaveProperty("SOME_START_ENV"); + }); + + test("restart tolerates an image inspect failure and still carries the operator flags", async () => { + mockDocker({ + findLocalStackContainer: jest.fn().mockResolvedValue("id-az"), + inspectContainer: jest + .fn() + .mockResolvedValue( + azureContainer({ env: ["MSSQL_ACCEPT_EULA=Y", "PATH=/usr/bin", "HOME=/root"] }) + ), + imageConfigEnv: jest.fn().mockRejectedValue(new Error("image gone")), + stopContainer: jest.fn().mockResolvedValue(undefined), + waitForRemoval: jest.fn().mockResolvedValue(undefined), + }); + await localstackManagement({ action: "restart", service: "azure" } as any); + const options = mockedLaunch.mock.calls[0][0]; + // Without the image env, the system-var backstop still keeps PATH/HOME out. + expect(options.envVars).toMatchObject({ MSSQL_ACCEPT_EULA: "Y" }); + expect(options.envVars).not.toHaveProperty("PATH"); + expect(options.envVars).not.toHaveProperty("HOME"); + }); + + test("restart with service azure over an AWS container switches stacks (no overrides)", async () => { + mockDocker({ + findLocalStackContainer: jest.fn().mockResolvedValue("id-aws"), + inspectContainer: jest.fn().mockResolvedValue({ + id: "id-aws", + name: "localstack-main", + image: "localstack/localstack-pro:latest", + }), + stopContainer: jest.fn().mockResolvedValue(undefined), + waitForRemoval: jest.fn().mockResolvedValue(undefined), + }); + await localstackManagement({ action: "restart", service: "azure" } as any); + const options = mockedLaunch.mock.calls[0][0]; + expect(options.stack).toBe("azure"); + expect(options.imageOverride).toBeUndefined(); + expect(options.containerNameOverride).toBeUndefined(); + }); +}); + +describe("status in the Docker image", () => { + /* eslint-disable @typescript-eslint/no-require-imports */ + const { ensureLoopbackForwarder } = + require("../lib/azure/loopback-forwarder") as typeof import("../lib/azure/loopback-forwarder"); + const { azureConfig } = + require("../lib/azure/services") as typeof import("../lib/azure/services"); + /* eslint-enable @typescript-eslint/no-require-imports */ + const forwarder = ensureLoopbackForwarder as jest.Mock; + const config = azureConfig as jest.Mock; + + beforeEach(() => { + jest.clearAllMocks(); + process.env.LOCALSTACK_AUTH_TOKEN = "ls-test-token"; + }); + afterEach(() => config.mockReturnValue({ inDocker: false })); + + test("service azure starts the loopback forwarder before the gateway is probed", async () => { + config.mockReturnValue({ inDocker: true }); + const order: string[] = []; + forwarder.mockImplementation(async () => { + order.push("forwarder"); + }); + mockedGetStatus.mockImplementation(async () => { + order.push("gateway"); + return { isRunning: false, isReady: false, statusOutput: "not reachable" }; + }); + await localstackManagement({ action: "status", service: "azure" } as any); + expect(order).toEqual(["forwarder", "gateway"]); + }); + + test("outside Docker, or for another stack, no forwarder is started", async () => { + mockedGetStatus.mockResolvedValue({ isRunning: false, isReady: false, statusOutput: "x" }); + await localstackManagement({ action: "status", service: "azure" } as any); + config.mockReturnValue({ inDocker: true }); + await localstackManagement({ action: "status", service: "aws" } as any); + expect(forwarder).not.toHaveBeenCalled(); + }); +}); diff --git a/src/tools-tests/stack-guards.test.ts b/src/tools-tests/stack-guards.test.ts new file mode 100644 index 0000000..a7f65f4 --- /dev/null +++ b/src/tools-tests/stack-guards.test.ts @@ -0,0 +1,172 @@ +// Tool tests live outside src/tools/ because xmcp registers every src/tools/*.ts as a +// tool. These tests exercise the real tools with the real runPreflights/requireStack; +// only the network, Docker and CLI edges are stubbed. +import { readFileSync } from "fs"; +import { join } from "path"; +import localstackAwsClient from "../tools/localstack-aws-client"; +import localstackChaosInjector from "../tools/localstack-chaos-injector"; +import localstackLogsAnalysis from "../tools/localstack-logs-analysis"; +import localstackSnowflakeClient from "../tools/localstack-snowflake-client"; +import { DockerApiClient } from "../lib/docker/docker.client"; +import { getGatewayHealth } from "../lib/localstack/localstack.utils"; +import { checkProFeature } from "../lib/localstack/license-checker"; +import { LocalStackLogRetriever } from "../lib/logs/log-retriever"; +import { resetStackDetectionCache } from "../core/preflight"; + +jest.mock("../core/analytics", () => ({ + withToolAnalytics: (_name: string, _args: unknown, fn: () => unknown) => fn(), +})); + +jest.mock("../lib/localstack/localstack.utils", () => { + const actual = jest.requireActual("../lib/localstack/localstack.utils"); + return { + ...actual, + getGatewayHealth: jest.fn(), + ensureSnowflakeCli: jest.fn().mockResolvedValue(null), + }; +}); + +jest.mock("../lib/localstack/license-checker", () => { + const actual = jest.requireActual("../lib/localstack/license-checker"); + return { ...actual, checkProFeature: jest.fn() }; +}); + +jest.mock("../lib/docker/docker.client", () => { + const actual = jest.requireActual("../lib/docker/docker.client"); + return { ...actual, DockerApiClient: jest.fn() }; +}); + +jest.mock("../lib/logs/log-retriever", () => ({ LocalStackLogRetriever: jest.fn() })); + +jest.mock("../core/command-runner", () => ({ + runCommand: jest + .fn() + .mockResolvedValue({ stdout: "localstack connected", stderr: "", exitCode: 0 }), +})); + +const mockedHealth = getGatewayHealth as jest.MockedFunction; +const mockedProFeature = checkProFeature as jest.MockedFunction; +const MockedDocker = DockerApiClient as jest.MockedClass; +const MockedRetriever = LocalStackLogRetriever as jest.MockedClass; + +const text = (r: { content: Array<{ text: string }> }) => r.content[0].text; + +const AZURE = { reachable: true, ready: true, edition: "azure-alpha" }; +const PRO = { reachable: true, ready: true, edition: "pro", services: { s3: "available" } }; + +function mockDocker(image?: string) { + const docker = { + findLocalStackContainer: jest.fn().mockResolvedValue("id-1"), + inspectContainer: jest.fn().mockResolvedValue({ id: "id-1", image }), + executeInContainer: jest.fn().mockResolvedValue({ stdout: "ok", stderr: "", exitCode: 0 }), + }; + MockedDocker.mockImplementation(() => docker as any); + return docker; +} + +describe("stack guards in the AWS-only tools, logs analysis and the Snowflake client", () => { + beforeEach(() => { + jest.clearAllMocks(); + resetStackDetectionCache(); + process.env.LOCALSTACK_AUTH_TOKEN = "ls-test-token"; + mockedProFeature.mockResolvedValue({ isSupported: true } as any); + }); + + test("localstack-aws-client on the Azure emulator is refused and never execs in the container", async () => { + mockedHealth.mockResolvedValue(AZURE); + const docker = mockDocker("localstack/localstack-azure:latest"); + + const result = await localstackAwsClient({ command: "s3 ls" }); + + expect(text(result)).toMatch(/Wrong emulator for this tool/); + expect(text(result)).toContain("Use `localstack-azure-client`"); + expect(docker.executeInContainer).not.toHaveBeenCalled(); + }); + + test("localstack-aws-client on AWS looks up only an AWS container", async () => { + mockedHealth.mockResolvedValue(PRO); + const docker = mockDocker("localstack/localstack-pro:latest"); + + await localstackAwsClient({ command: "s3 ls" }); + + expect(docker.findLocalStackContainer).toHaveBeenCalledWith({ stack: "aws" }); + expect(docker.executeInContainer).toHaveBeenCalled(); + }); + + test("a Pro-feature tool on Azure: the stack guard's error wins over requireProFeature", async () => { + mockedHealth.mockResolvedValue(AZURE); + mockDocker("localstack/localstack-azure:latest"); + mockedProFeature.mockResolvedValue({ + isSupported: false, + errorMessage: "license does not include chaos", + } as any); + + const result = await localstackChaosInjector({ action: "get-faults" } as any); + + expect(text(result)).toMatch(/Wrong emulator for this tool/); + expect(text(result)).not.toMatch(/Feature Not Available/); + }); + + test("logs analysis in `logs` mode reaches the log retriever on the Azure emulator", async () => { + mockedHealth.mockResolvedValue(AZURE); + const retrieveLogs = jest + .fn() + .mockResolvedValue({ success: true, logs: [], totalLines: 0, filteredLines: 0 }); + MockedRetriever.mockImplementation(() => ({ retrieveLogs }) as any); + + const result = await localstackLogsAnalysis({ analysisType: "logs", lines: 100 } as any); + + expect(retrieveLogs).toHaveBeenCalled(); + expect(text(result)).not.toMatch(/Wrong emulator/); + }); + + test("logs analysis in `errors` mode is refused on Azure with the analysisType: logs hint", async () => { + mockedHealth.mockResolvedValue(AZURE); + const retrieveLogs = jest.fn(); + MockedRetriever.mockImplementation(() => ({ retrieveLogs }) as any); + + const result = await localstackLogsAnalysis({ analysisType: "errors", lines: 100 } as any); + + expect(text(result)).toMatch(/Wrong emulator for this tool/); + expect(text(result)).toContain("use analysisType: logs"); + expect(retrieveLogs).not.toHaveBeenCalled(); + }); + + test("the Snowflake client is refused on the Azure emulator before requireProFeature", async () => { + mockedHealth.mockResolvedValue(AZURE); + mockDocker("localstack/localstack-azure:latest"); + mockedProFeature.mockResolvedValue({ isSupported: false, errorMessage: "no snowflake" } as any); + + const result = await localstackSnowflakeClient({ action: "check-connection" } as any); + + expect(text(result)).toMatch(/Wrong emulator for this tool/); + expect(text(result)).toContain("is LocalStack Azure"); + }); + + test("the Snowflake client is refused on `pro` when the running container is the AWS image", async () => { + mockedHealth.mockResolvedValue(PRO); + mockDocker("localstack/localstack-pro:latest"); + + const result = await localstackSnowflakeClient({ action: "check-connection" } as any); + + expect(text(result)).toMatch(/Wrong emulator for this tool/); + expect(text(result)).toContain("is LocalStack AWS"); + }); + + test("the Snowflake client is NOT refused on `pro` when the container is the Snowflake image (fail-open)", async () => { + mockedHealth.mockResolvedValue(PRO); + mockDocker("localstack/snowflake:latest"); + + const result = await localstackSnowflakeClient({ action: "check-connection" } as any); + + expect(text(result)).not.toMatch(/Wrong emulator/); + }); + + test.each(["localstack-docs.ts", "localstack-ephemeral-instances.ts"])( + "%s never calls requireStack (it does not talk to a local emulator)", + (file) => { + const source = readFileSync(join(__dirname, "..", "tools", file), "utf8"); + expect(source).not.toContain("requireStack"); + } + ); +}); diff --git a/src/tools/localstack-app-inspector.ts b/src/tools/localstack-app-inspector.ts index 59d6948..9aac95d 100644 --- a/src/tools/localstack-app-inspector.ts +++ b/src/tools/localstack-app-inspector.ts @@ -5,6 +5,7 @@ import { requireAuthToken, requireLocalStackRunning, requireProFeature, + requireStack, } from "../core/preflight"; import { ResponseBuilder } from "../core/response-builder"; import { withToolAnalytics } from "../core/analytics"; @@ -139,6 +140,7 @@ export default async function localstackAppInspector(params: AppInspectorArgs) { async () => { const preflightError = await runPreflights([ requireAuthToken(), + requireStack("aws", "localstack-app-inspector"), requireLocalStackRunning(), requireProFeature(ProFeature.APP_INSPECTOR), ]); diff --git a/src/tools/localstack-aws-client.ts b/src/tools/localstack-aws-client.ts index 3f5ea24..9f80f21 100644 --- a/src/tools/localstack-aws-client.ts +++ b/src/tools/localstack-aws-client.ts @@ -1,6 +1,11 @@ import { z } from "zod"; import { type ToolMetadata, type InferSchema } from "xmcp"; -import { runPreflights, requireLocalStackRunning, requireAuthToken } from "../core/preflight"; +import { + runPreflights, + requireLocalStackRunning, + requireAuthToken, + requireStack, +} from "../core/preflight"; import { ResponseBuilder } from "../core/response-builder"; import { withToolAnalytics } from "../core/analytics"; import { DockerApiClient } from "../lib/docker/docker.client"; @@ -29,12 +34,16 @@ export const metadata: ToolMetadata = { export default async function localstackAwsClient({ command }: InferSchema) { return withToolAnalytics("localstack-aws-client", { command }, async () => { - const preflightError = await runPreflights([requireAuthToken(), requireLocalStackRunning()]); + const preflightError = await runPreflights([ + requireAuthToken(), + requireStack("aws", "localstack-aws-client"), + requireLocalStackRunning(), + ]); if (preflightError) return preflightError; try { const dockerClient = new DockerApiClient(); - const containerId = await dockerClient.findLocalStackContainer(); + const containerId = await dockerClient.findLocalStackContainer({ stack: "aws" }); const sanitized = sanitizeAwsCliCommand(command); diff --git a/src/tools/localstack-aws-replicator.ts b/src/tools/localstack-aws-replicator.ts index 410d4a3..a10a7ff 100644 --- a/src/tools/localstack-aws-replicator.ts +++ b/src/tools/localstack-aws-replicator.ts @@ -6,6 +6,7 @@ import { requireAuthToken, requireLocalStackRunning, requireProFeature, + requireStack, } from "../core/preflight"; import { withToolAnalytics } from "../core/analytics"; import { ProFeature } from "../lib/localstack/license-checker"; @@ -98,6 +99,7 @@ export default async function localstackAwsReplicator(args: AwsReplicatorArgs) { async () => { const preflightError = await runPreflights([ requireAuthToken(), + requireStack("aws", "localstack-aws-replicator"), requireLocalStackRunning(), requireProFeature(ProFeature.REPLICATOR), ]); diff --git a/src/tools/localstack-azure-client.ts b/src/tools/localstack-azure-client.ts new file mode 100644 index 0000000..39496c5 --- /dev/null +++ b/src/tools/localstack-azure-client.ts @@ -0,0 +1,251 @@ +import { z } from "zod"; +import { type InferSchema, type ToolExtraArguments, type ToolMetadata } from "xmcp"; +import { + requireAuthToken, + requireAzureCli, + requireAzureCliConfigured, + requireAzureConfig, + requireAzureEmulatorRunning, + requireStack, + runPreflights, +} from "../core/preflight"; +import { ResponseBuilder } from "../core/response-builder"; +import { withToolAnalytics } from "../core/analytics"; +import { AZURE_COMMAND_DESCRIPTION, buildAzureClientDescription } from "../lib/azure/description"; +import { extensionFor } from "../lib/azure/extension-map"; +import { + activeForwarder, + ensureLoopbackForwarder, + totalForwarderConnections, +} from "../lib/azure/loopback-forwarder"; +import { + bicepMissing, + bicepRegistryUnsupported, + formatAzResult, + GUARD_OFF_NOTE, + type FormatOptions, +} from "../lib/azure/output"; +import { analyticsFields, evaluateAzCommand, scanBicepModules } from "../lib/azure/policy"; +import { + BicepPathError, + localVersionJson, + localVersionNote, + localVersionText, +} from "../lib/azure/resolve-az"; +import { progressSender } from "../lib/azure/runner"; +import { + azCli, + azureConfig, + bicep, + emulatorSessionId, + installedExtensionNames, + installedExtensions, + policyOptions, + runAzCommand, +} from "../lib/azure/services"; +import type { AzureConfig, ToolTextResponse } from "../lib/azure/types"; + +const TOOL = "localstack-azure-client"; + +export const schema = { + command: z + .string() + .trim() + .min(1, { message: "The command string cannot be empty." }) + .describe(AZURE_COMMAND_DESCRIPTION), +}; + +export const metadata: ToolMetadata = { + name: "localstack-azure-client", + description: buildAzureClientDescription({ + workdir: azureConfig().workdir, + maxOutputChars: azureConfig().maxOutputChars, + }), + annotations: { + title: "LocalStack Azure Client", + readOnlyHint: false, + destructiveHint: true, + idempotentHint: false, + openWorldHint: false, + }, +}; + +/** The configuration whose warnings were last shown (see withGuardNote). */ +let warnedConfig: AzureConfig | undefined; + +/** + * With the guard off, every response says so. The configuration's warnings (an + * invalid setting fell back to its default, the workdir contains the home directory) go into + * the first response of each configuration: they were computed but shown nowhere. + */ +function withGuardNote(response: ToolTextResponse): ToolTextResponse { + const config = azureConfig(); + const notes: string[] = []; + if (!config.egressGuard) notes.push(GUARD_OFF_NOTE); + if (warnedConfig !== config && config.warnings.length > 0) { + warnedConfig = config; + notes.push(...config.warnings.map((warning) => `Note: ${warning}`)); + } + const [first, ...rest] = response.content; + const missing = first ? notes.filter((note) => !first.text.includes(note)) : []; + if (!first || missing.length === 0) return response; + return { content: [{ ...first, text: `${first.text}\n\n${missing.join("\n")}` }, ...rest] }; +} + +/** `.bicepparam` needs Bicep 0.14.85 or newer. */ +const usesBicepparam = (argv: string[]) => + argv.some((a) => /\.bicepparam$/i.test(a.replace(/^@/, ""))); + +/** + * Runs one `az` command against the LocalStack Azure emulator, in this order: the + * token, the config and the pure policy first, so a refusal needs no emulator; + * `version` after the CLI probe only; in Docker the loopback forwarder before the + * parallel group; Bicep checks before any spawn; the bootstrap; then the runner and + * the output. + */ +export default async function localstackAzureClient( + { command }: InferSchema, + extra?: ToolExtraArguments +) { + const config = azureConfig(); + const policy = evaluateAzCommand(command, policyOptions()); + const formatOptions = (argv: string[], notes: string[], isHelp: boolean): FormatOptions => ({ + argv, + notes, + isHelp, + guardOn: config.egressGuard, + port: config.port, + timeoutSeconds: Math.round(config.timeoutMs / 1000), + maxOutputChars: config.maxOutputChars, + maxHelpChars: config.maxHelpChars, + client: extra?.clientInfo, + envelope: config.testEnvelope, + inDocker: config.inDocker, + extensionFor: (tokens) => extensionFor(tokens, installedExtensionNames()), + }); + + // Only value-free fields reach analytics; exit code and class travel in the + // failure text's first line, which withToolAnalytics records as error_message. + return withToolAnalytics(TOOL, analyticsFields(command, policy), async () => { + try { + return await handle(); + } catch (error) { + // A thrown message would be recorded as error_message and could hold a path + // or value; the details go below the constant first line instead. + return withGuardNote( + ResponseBuilder.error( + "Azure Tool Error", + error instanceof Error ? error.message : String(error) + ) + ); + } + }); + + async function handle(): Promise { + const early = requireAuthToken() ?? requireAzureConfig(); + if (early) return early; + if (!policy.ok) return withGuardNote(ResponseBuilder.error(policy.title, policy.message)); + + if (policy.local === "version") { + // `az version` would call Microsoft: answered from the probe, no emulator. + const cli = await requireAzureCli(); + if (cli) return cli; + const az = await azCli(); + const answer = ResponseBuilder.markdown(localVersionText(az, installedExtensions())); + if (config.testEnvelope) { + // As if az had printed it, so the samples shim hands `az version -o json` callers + // (Terraform's provider) the JSON alone. + const envelope = { + exitCode: 0, + stdout: `${localVersionJson(az, installedExtensions())}\n`, + stderr: "", + notes: [localVersionNote(az)], + classId: null, + truncated: false, + stoppedByTool: false, + }; + answer.content.push({ type: "text", text: JSON.stringify(envelope) }); + } + return withGuardNote(answer); + } + + // Before the parallel group: requireStack would otherwise read 127.0.0.1 before + // the forwarder is bound. + if (config.inDocker) await ensureLoopbackForwarder(); + const preflightError = await runPreflights([ + requireStack("azure", TOOL, { baseUrl: config.healthBaseUrl }), + requireAzureEmulatorRunning(), + requireAzureCli(), + ]); + if (preflightError) return withGuardNote(preflightError); + + if (policy.needsBicep) { + let resolved; + try { + resolved = await bicep(); + } catch (error) { + if (error instanceof BicepPathError) { + return withGuardNote(ResponseBuilder.error("Bicep CLI Not Usable", error.message)); + } + throw error; + } + if (!resolved) return withGuardNote(bicepMissing({ inDocker: config.inDocker })); + if (usesBicepparam(policy.argv) && !resolved.supportsBicepparam) { + return withGuardNote( + ResponseBuilder.error( + "Bicep CLI Too Old", + `\`.bicepparam\` files need Bicep 0.14.85 or newer; ${resolved.path} is ${resolved.version}. Update it, or pass the parameters as JSON.` + ) + ); + } + // Answered before any spawn: on Windows a successful restore would write the + // real %USERPROFILE%\.bicep even with the private home. + const moduleRef = scanBicepModules(policy.argv, config.workdir); + if (moduleRef) return withGuardNote(bicepRegistryUnsupported(moduleRef)); + } + + const profileError = await requireAzureCliConfigured((error) => + formatAzResult( + error.result, + formatOptions( + error.step.split(" "), + [ + `This failed while the tool set up its own Azure CLI profile (\`az ${error.step}\`), before your command ran.`, + ], + false + ) + ) + ); + if (profileError) return withGuardNote(profileError); + + const result = await runAzCommand(policy.argv, { + signal: extra?.signal, + onProgress: progressSender(extra), + }); + return withEnvelopeExtras( + withGuardNote(formatAzResult(result, formatOptions(policy.argv, policy.notes, policy.isHelp))) + ); + } + + /** + * Tests only (LOCALSTACK_AZ_TEST_ENVELOPE=1): the envelope also carries the emulator + * session the profile was checked against, which shows a re-bootstrap after an emulator + * restart, and in Docker the loopback forwarder's connection count, which L5 reads to + * prove the traffic went through it. + */ + async function withEnvelopeExtras(response: ToolTextResponse): Promise { + if (!config.testEnvelope || response.content.length < 2) return response; + try { + const envelope = JSON.parse(response.content[1].text); + envelope.emulatorSession = emulatorSessionId() ?? null; + if (config.inDocker && (await activeForwarder())) { + envelope.forwarderConnections = await totalForwarderConnections(); + } + return { + content: [response.content[0], { type: "text", text: JSON.stringify(envelope) }], + }; + } catch { + return response; + } + } +} diff --git a/src/tools/localstack-chaos-injector.ts b/src/tools/localstack-chaos-injector.ts index 427a70d..3b7a105 100644 --- a/src/tools/localstack-chaos-injector.ts +++ b/src/tools/localstack-chaos-injector.ts @@ -8,6 +8,7 @@ import { requireAuthToken, requireLocalStackRunning, requireProFeature, + requireStack, } from "../core/preflight"; import { withToolAnalytics } from "../core/analytics"; @@ -137,6 +138,7 @@ export default async function localstackChaosInjector({ async () => { const preflightError = await runPreflights([ requireAuthToken(), + requireStack("aws", "localstack-chaos-injector"), requireLocalStackRunning(), requireProFeature(ProFeature.CHAOS_ENGINEERING), ]); diff --git a/src/tools/localstack-cloud-pods.ts b/src/tools/localstack-cloud-pods.ts index 7330f39..6dbf187 100644 --- a/src/tools/localstack-cloud-pods.ts +++ b/src/tools/localstack-cloud-pods.ts @@ -8,6 +8,7 @@ import { requireAuthToken, requireLocalStackRunning, requireProFeature, + requireStack, } from "../core/preflight"; import { withToolAnalytics } from "../core/analytics"; @@ -49,6 +50,7 @@ export default async function localstackCloudPods({ return withToolAnalytics("localstack-cloud-pods", { action, pod_name }, async () => { const preflightError = await runPreflights([ requireAuthToken(), + requireStack("aws", "localstack-cloud-pods"), requireLocalStackRunning(), requireProFeature(ProFeature.CLOUD_PODS), ]); diff --git a/src/tools/localstack-deployer.ts b/src/tools/localstack-deployer.ts index e96e4cc..9172674 100644 --- a/src/tools/localstack-deployer.ts +++ b/src/tools/localstack-deployer.ts @@ -3,7 +3,12 @@ import { type ToolMetadata, type InferSchema } from "xmcp"; import { runCommand, stripAnsiCodes } from "../core/command-runner"; import path from "path"; import fs from "fs"; -import { runPreflights, requireAuthToken, requireLocalStackRunning } from "../core/preflight"; +import { + runPreflights, + requireAuthToken, + requireLocalStackRunning, + requireStack, +} from "../core/preflight"; import { DockerApiClient } from "../lib/docker/docker.client"; import { checkDependencies, @@ -110,7 +115,11 @@ export default async function localstackDeployer({ saveParams, }, async () => { - const preflightError = await runPreflights([requireAuthToken(), requireLocalStackRunning()]); + const preflightError = await runPreflights([ + requireAuthToken(), + requireStack("aws", "localstack-deployer"), + requireLocalStackRunning(), + ]); if (preflightError) return preflightError; if (action === "create-stack") { diff --git a/src/tools/localstack-extensions.ts b/src/tools/localstack-extensions.ts index 2e1bbce..ce90e81 100644 --- a/src/tools/localstack-extensions.ts +++ b/src/tools/localstack-extensions.ts @@ -6,6 +6,7 @@ import { requireProFeature, requireAuthToken, requireDockerDaemon, + requireStack, } from "../core/preflight"; import { ResponseBuilder } from "../core/response-builder"; import { ProFeature } from "../lib/localstack/license-checker"; @@ -74,6 +75,7 @@ export default async function localstackExtensions({ ? [requireAuthToken()] : [ requireAuthToken(), + requireStack("aws", "localstack-extensions"), requireDockerDaemon(), requireLocalStackRunning(), requireProFeature(ProFeature.EXTENSIONS), diff --git a/src/tools/localstack-iam-policy-analyzer.ts b/src/tools/localstack-iam-policy-analyzer.ts index 4926acd..8c74579 100644 --- a/src/tools/localstack-iam-policy-analyzer.ts +++ b/src/tools/localstack-iam-policy-analyzer.ts @@ -15,6 +15,7 @@ import { requireAuthToken, requireLocalStackRunning, requireProFeature, + requireStack, } from "../core/preflight"; import { ResponseBuilder } from "../core/response-builder"; import { withToolAnalytics } from "../core/analytics"; @@ -57,6 +58,7 @@ export default async function localstackIamPolicyAnalyzer({ return withToolAnalytics("localstack-iam-policy-analyzer", { action, mode }, async () => { const preflightError = await runPreflights([ requireAuthToken(), + requireStack("aws", "localstack-iam-policy-analyzer"), requireLocalStackRunning(), requireProFeature(ProFeature.IAM_ENFORCEMENT), ]); diff --git a/src/tools/localstack-logs-analysis.ts b/src/tools/localstack-logs-analysis.ts index 4cd63d7..f116b70 100644 --- a/src/tools/localstack-logs-analysis.ts +++ b/src/tools/localstack-logs-analysis.ts @@ -1,7 +1,12 @@ import { z } from "zod"; import { type ToolMetadata, type InferSchema } from "xmcp"; import { LocalStackLogRetriever, type LogEntry } from "../lib/logs/log-retriever"; -import { runPreflights, requireAuthToken, requireLocalStackRunning } from "../core/preflight"; +import { + runPreflights, + requireAuthToken, + requireLocalStackRunning, + requireStack, +} from "../core/preflight"; import { ResponseBuilder } from "../core/response-builder"; import { withToolAnalytics } from "../core/analytics"; @@ -56,7 +61,15 @@ export default async function localstackLogsAnalysis({ "localstack-logs-analysis", { analysisType, lines, service, operation, filter }, async () => { - const preflightError = await runPreflights([requireAuthToken(), requireLocalStackRunning()]); + const preflightError = await runPreflights([ + requireAuthToken(), + analysisType === "logs" + ? null + : requireStack("aws", "localstack-logs-analysis", { + hint: "The summary, errors and requests analyses parse AWS request logs. For another emulator, use analysisType: logs for the raw log output.", + }), + requireLocalStackRunning(), + ]); if (preflightError) return preflightError; const retriever = new LocalStackLogRetriever(); diff --git a/src/tools/localstack-management.ts b/src/tools/localstack-management.ts index 6e33adc..6af0a2a 100644 --- a/src/tools/localstack-management.ts +++ b/src/tools/localstack-management.ts @@ -1,18 +1,28 @@ import { z } from "zod"; import { type ToolMetadata, type InferSchema } from "xmcp"; import { + CARRIED_ENV_KEY, + cannotRecreateResponse, + carriedRestartEnv, + pickCarriedEnv, deriveRecreateOverrides, getLocalStackStatus, getSnowflakeEmulatorStatus, launchRuntime, resolveContainerName, + unmountableBindSource, } from "../lib/localstack/localstack.utils"; import { DockerApiClient, isLocalStackContainerNotFoundError, type ContainerMetadata, } from "../lib/docker/docker.client"; -import { stackFromImage, type VolumeResolution } from "../lib/localstack/container-spec.logic"; +import { + forwardsHostEnvKey, + MCP_CLIENT_NAME, + stackFromImage, + type VolumeResolution, +} from "../lib/localstack/container-spec.logic"; import { runPreflights, requireProFeature, @@ -22,21 +32,33 @@ import { import { ResponseBuilder } from "../core/response-builder"; import { ProFeature } from "../lib/localstack/license-checker"; import { withToolAnalytics } from "../core/analytics"; +import { azureStartedCheck, getAzureRuntimeStatus } from "../lib/azure/runtime-status"; +import { ensureLoopbackForwarder } from "../lib/azure/loopback-forwarder"; +import { azureConfig } from "../lib/azure/services"; const AWS_ALREADY_RUNNING_MESSAGE = "⚠️ LocalStack is already running. Use 'restart' if you want to apply new configuration."; const SNOWFLAKE_ALREADY_RUNNING_MESSAGE = "⚠️ Snowflake emulator is already running. Use 'restart' if you want to apply new configuration."; +const AZURE_ALREADY_RUNNING_MESSAGE = + "⚠️ The LocalStack Azure emulator is already running. Use 'restart' if you want to apply new configuration."; + +type Service = "aws" | "snowflake" | "azure"; +const SERVICE_LABELS: Record = { + aws: "AWS", + snowflake: "Snowflake", + azure: "Azure", +}; export const schema = { action: z .enum(["start", "stop", "restart", "status"]) .describe("The LocalStack management action to perform"), service: z - .enum(["aws", "snowflake"]) + .enum(["aws", "snowflake", "azure"]) .default("aws") .describe( - "The LocalStack stack/service to manage. Use 'aws' for the default AWS emulator, or 'snowflake' for the Snowflake emulator." + "The LocalStack stack/service to manage. Use 'aws' for the default AWS emulator, 'snowflake' for the Snowflake emulator, or 'azure' for the Azure emulator." ), envVars: z .record(z.string(), z.string()) @@ -86,7 +108,7 @@ export default async function localstackManagement({ case "start": return await handleStart({ envVars, service }); case "stop": - return await handleStop(); + return await handleStop({ service }); case "restart": return await handleRestart({ envVars, service }); case "status": @@ -107,20 +129,35 @@ interface StartOverrides { } /** Best-effort look at the running LocalStack container (null when none/undetectable). */ -async function inspectRunningContainer(): Promise { +async function inspectRunningContainer( + stack?: "azure" | "aws" | "snowflake", + port?: string +): Promise { try { const dockerClient = new DockerApiClient(); - const containerId = await dockerClient.findLocalStackContainer(); + const containerId = await dockerClient.findLocalStackContainer(stack ? { stack, port } : {}); return await dockerClient.inspectContainer(containerId); } catch { return null; } } +/** + * The Azure tool's own port when the Azure emulator runs beside another stack's + * (LOCALSTACK_AZURE_PORT differs from LOCALSTACK_PORT), else undefined. There, the gateway + * the other tools use is not the Azure emulator, so status, stop and restart must not follow it. + */ +function azureSideBySidePort(service: Service): string | undefined { + if (service !== "azure") return undefined; + const gatewayPort = process.env.LOCALSTACK_PORT?.trim() || "4566"; + const azurePort = String(azureConfig().port ?? gatewayPort); + return azurePort !== gatewayPort ? azurePort : undefined; +} + /** Gate on the SNOWFLAKE pro feature only when the running container is the Snowflake stack. */ async function requireSnowflakeProIfSnowflakeRunning() { const metadata = await inspectRunningContainer(); - if (!metadata || stackFromImage(metadata.image) !== "snowflake") { + if (!metadata || stackFromImage(metadata.image, metadata.labels) !== "snowflake") { // Not running / different stack — the handlers report those states accurately. return null; } @@ -134,9 +171,35 @@ async function handleStart({ overrides, }: { envVars?: Record; - service: "aws" | "snowflake"; + service: Service; overrides?: StartOverrides; }) { + if (service === "azure") { + // The emulator is started on LOCALSTACK_PORT, and the readiness checks read the + // Azure tool's port: both must name the same gateway. + const azurePort = process.env.LOCALSTACK_AZURE_PORT?.trim(); + const gatewayPort = process.env.LOCALSTACK_PORT?.trim() || "4566"; + if (azurePort && azurePort !== gatewayPort) { + return ResponseBuilder.error( + "Conflicting Azure port settings", + `LOCALSTACK_AZURE_PORT (${azurePort}) differs from LOCALSTACK_PORT (${gatewayPort}), the port the start action publishes the emulator on. Set LOCALSTACK_PORT=${azurePort} for a port-shifted emulator, or unset LOCALSTACK_AZURE_PORT.` + ); + } + return await launchRuntime({ + stack: "azure", + envVars, + getStatus: getAzureRuntimeStatus, + processLabel: "LocalStack Azure emulator", + alreadyRunningMessage: AZURE_ALREADY_RUNNING_MESSAGE, + successTitle: "🚀 LocalStack Azure emulator started successfully!", + statusHeading: "Health check", + timeoutMessage: + "❌ LocalStack Azure emulator start timed out after 120 seconds. Its health check did not report the Azure edition. If this was the first start, the image pull may still be in progress — retry in a bit.", + onReady: azureStartedCheck, + ...overrides, + }); + } + if (service === "snowflake") { return await launchRuntime({ stack: "snowflake", @@ -170,11 +233,16 @@ async function handleStart({ // Handle stop action — stop the detected container via the Docker API. Also cleans up // stopped/stale containers occupying a LocalStack name, so start's conflict advice // ("stop it first") always has a working recovery path. -async function handleStop() { +async function handleStop({ service }: { service: Service }) { const dockerClient = new DockerApiClient(); + // Side by side, the Azure emulator is the Azure container on the Azure tool's own port; the + // gateway's container is another stack's emulator. + const azurePort = azureSideBySidePort(service); let containerId: string; try { - containerId = await dockerClient.findLocalStackContainer(); + containerId = await dockerClient.findLocalStackContainer( + azurePort ? { stack: "azure", port: azurePort } : {} + ); } catch (error) { if (!isLocalStackContainerNotFoundError(error)) { return ResponseBuilder.error( @@ -183,11 +251,12 @@ async function handleStop() { ); } - // No RUNNING container found — check for a stale stopped one holding the name. + // No RUNNING container found — check for a stale stopped one holding the name. The + // configured name is the gateway emulator's, so not side by side. try { - const stale = await dockerClient.findContainerByNameAnyState( - resolveContainerName(process.env) - ); + const stale = azurePort + ? undefined + : await dockerClient.findContainerByNameAnyState(resolveContainerName(process.env)); if (stale && !stale.running) { await dockerClient.removeContainer(stale.id); await dockerClient.waitForRemoval(stale.id); @@ -197,7 +266,7 @@ async function handleStop() { // fall through to the gateway-based reporting below } - const status = await getLocalStackStatus(); + const status = await getLocalStackStatus(azurePort ? azureConfig().healthBaseUrl : undefined); if (status.isRunning) { return ResponseBuilder.error( "LocalStack container not found", @@ -230,8 +299,17 @@ async function handleRestart({ service, }: { envVars?: Record; - service: "aws" | "snowflake"; + service: Service; }) { + // Side by side, the start action (which publishes the emulator on LOCALSTACK_PORT) refuses the + // Azure emulator, so a restart would stop it and leave it stopped: refuse before any stop. + const azurePort = azureSideBySidePort(service); + if (azurePort) { + return ResponseBuilder.error( + "Conflicting Azure port settings", + `LOCALSTACK_AZURE_PORT (${azurePort}) differs from LOCALSTACK_PORT (${process.env.LOCALSTACK_PORT?.trim() || "4566"}), the port the start action publishes the emulator on, so this server could not start the Azure emulator again after stopping it. Nothing was stopped. Restart it where it was started, or set LOCALSTACK_PORT=${azurePort}.` + ); + } const dockerClient = new DockerApiClient(); let containerId: string; try { @@ -263,6 +341,53 @@ async function handleRestart({ metadata = undefined; } + // Before stopping anything: a container whose state folder this machine cannot mount + // could not be recreated, and stopping it removes it (an lstk emulator started from + // WSL was deleted this way by a Windows server). + const overrides = deriveRecreateOverrides(metadata, service); + const unmountable = unmountableBindSource(overrides); + if (unmountable) return cannotRecreateResponse(unmountable, metadata?.name ?? containerId); + + // Carry an EXTERNALLY started container's own env flags over, so a restart does not silently + // drop settings this server never set and cannot reproduce — e.g. a shared Azure emulator's + // MSSQL_ACCEPT_EULA, DISABLE_EVENTS and portal flag, started by lstk or `docker run`. + // A container this server started (its launcher stamps LOCALSTACK_CLIENT_NAME) got its env from + // this server's config, so upstream's "restart applies new configuration" holds and nothing is + // carried: a restart can still drop a setting. The image's baked env is subtracted so only + // operator choices are carried; the caller's own `envVars` still win. Best-effort throughout. + const startedByThisServer = (metadata?.env ?? []).includes( + `LOCALSTACK_CLIENT_NAME=${MCP_CLIENT_NAME}` + ); + const image = metadata?.image; + // Promise.resolve().then: a synchronous throw from the client becomes a caught rejection too. + const imageEnv = + image && !startedByThisServer + ? await Promise.resolve() + .then(() => dockerClient.imageConfigEnv(image)) + .catch((): string[] => []) + : []; + // A server-started container that an earlier restart made from an external one lists what it + // carries (CARRIED_ENV_KEY): keep carrying exactly that, so a SECOND restart does not drop the + // external flags. Without the list, nothing is carried (upstream semantics). + const carriedList = (metadata?.env ?? []) + .find((entry) => entry.startsWith(`${CARRIED_ENV_KEY}=`)) + ?.slice(CARRIED_ENV_KEY.length + 1) + .split(",") + .filter(Boolean); + const carried = startedByThisServer + ? pickCarriedEnv(metadata?.env, carriedList ?? []) + : carriedRestartEnv(metadata?.env, imageEnv); + // This server's own env block wins, as on every start: drop what the start forwards itself. + for (const key of Object.keys(carried)) { + if (forwardsHostEnvKey(key, process.env)) delete carried[key]; + } + const carriedKeys = Object.keys(carried).filter((key) => !(envVars && key in envVars)); + const mergedEnv: Record = { ...carried, ...(envVars ?? {}) }; + if (Object.keys(carried).length > 0) { + mergedEnv[CARRIED_ENV_KEY] = Object.keys(carried).sort().join(","); + } + const startEnv = Object.keys(mergedEnv).length > 0 ? mergedEnv : undefined; + try { await dockerClient.stopContainer(containerId); await dockerClient.waitForRemoval(containerId); @@ -275,16 +400,56 @@ async function handleRestart({ ); } - return await handleStart({ - envVars, - service, - overrides: deriveRecreateOverrides(metadata, service), + const started = await handleStart({ envVars: startEnv, service, overrides }); + return carriedKeys.length > 0 ? appendCarriedEnvNote(started, carriedKeys) : started; +} + +/** Add a line to a restart response naming the settings carried from the old container. */ +function appendCarriedEnvNote( + response: ReturnType, + carriedKeys: string[] +): ReturnType { + const note = + `\n\n♻️ Carried ${carriedKeys.length} setting${carriedKeys.length === 1 ? "" : "s"} over from the ` + + `previous container: ${carriedKeys.sort().join(", ")}.`; + let appended = false; + const content = response.content?.map((part: { type: string; text?: string }) => { + if (appended || part.type !== "text" || typeof part.text !== "string") return part; + appended = true; + return { ...part, text: part.text + note }; }); + return { ...response, content } as ReturnType; +} + +/** `Container: "" (image , gateway :)`. */ +function describeContainer(running: ContainerMetadata): string { + const bindings = running.portBindings ?? {}; + const configured = process.env.LOCALSTACK_PORT?.trim() || "4566"; + const all = Object.values(bindings).flatMap((hosts) => hosts ?? []); + const gateway = + all.find((b) => b.HostPort === configured) ?? (bindings["4566/tcp"] ?? [])[0] ?? undefined; + const where = gateway + ? `${gateway.HostIp || "0.0.0.0"}:${gateway.HostPort}` + : "not published on the host"; + return `Container: "${running.name ?? running.id}" (image ${running.image ?? "unknown"}, gateway ${where})`; } // Handle status action -async function handleStatus({ service }: { service: "aws" | "snowflake" }) { - const statusResult = await getLocalStackStatus(); +async function handleStatus({ service }: { service: Service }) { + // In the Docker image the emulator is reached through the loopback forwarder, which the + // Azure tool starts on its first call: without it here, a status asked first probed an + // empty 127.0.0.1 and said "not running" for a running emulator. + if (service === "azure" && azureConfig().inDocker) { + await ensureLoopbackForwarder().catch(() => undefined); + } + // Side by side (LOCALSTACK_AZURE_PORT != LOCALSTACK_PORT): the Azure emulator sits on the + // Azure tool's own port, not on the gateway the other tools use, so status service: azure + // reads that one and its container (it used to report the AWS emulator on 4666). + const sideBySidePort = azureSideBySidePort(service); + const azureSideBySide = sideBySidePort !== undefined; + const statusResult = await getLocalStackStatus( + azureSideBySide ? azureConfig().healthBaseUrl : undefined + ); let result = "📊 LocalStack Status:\n\n"; result += statusResult.statusOutput || "LocalStack status is unavailable."; @@ -293,15 +458,47 @@ async function handleStatus({ service }: { service: "aws" | "snowflake" }) { return ResponseBuilder.markdown(result); } - if (service === "snowflake") { - const metadata = await inspectRunningContainer(); - if (metadata && stackFromImage(metadata.image) === "aws") { + const running = await inspectRunningContainer( + azureSideBySide ? "azure" : undefined, + sideBySidePort + ); + const runningStack = running ? stackFromImage(running.image, running.labels) : undefined; + // Always name the container, so a caller can check it before a stop or restart. + if (running) result += `\n\n${describeContainer(running)}`; + + // Another stack's container is running. A Snowflake image may report the AWS + // stack's `pro` edition, so for service: snowflake only a clear AWS or Azure image + // counts as foreign. + const foreign = + running && + runningStack && + runningStack !== service && + (service !== "snowflake" || runningStack !== "snowflake"); + if (running && runningStack && foreign) { + const next = + runningStack === "azure" + ? "Use localstack-azure-client with it, or stop it first" + : "Stop it first"; + result += + `\n\n⚠️ The running LocalStack container ("${running.name}", image: ${running.image}) is the ${SERVICE_LABELS[runningStack]} stack — ` + + `the ${SERVICE_LABELS[service]} emulator is not running. ${next}, then start with service: ${service}.`; + return ResponseBuilder.markdown(result); + } + + if (service === "azure") { + const azure = await getAzureRuntimeStatus(); + if (azure.isReady) { + result += "\n\n✅ LocalStack is running and the Azure emulator health check passed."; + } else { + const diagnostics = [azure.statusOutput, azure.status.message].filter(Boolean).join(" | "); result += - `\n\n⚠️ The running LocalStack container ("${metadata.name}", image: ${metadata.image}) is the AWS stack — ` + - "the Snowflake emulator is not running. Stop it first, then start with service: snowflake."; - return ResponseBuilder.markdown(result); + "\n\n⚠️ LocalStack is running, but the Azure emulator health check did not pass." + + (diagnostics ? ` (${diagnostics})` : ""); } + return ResponseBuilder.markdown(result); + } + if (service === "snowflake") { const snowflakeStatus = await getSnowflakeEmulatorStatus(); if (snowflakeStatus.isReady || snowflakeStatus.isRunning) { diff --git a/src/tools/localstack-snowflake-client.ts b/src/tools/localstack-snowflake-client.ts index febca3c..db6d2c6 100644 --- a/src/tools/localstack-snowflake-client.ts +++ b/src/tools/localstack-snowflake-client.ts @@ -7,6 +7,7 @@ import { requireAuthToken, requireSnowflakeCli, requireProFeature, + requireStack, } from "../core/preflight"; import { ResponseBuilder } from "../core/response-builder"; import { ProFeature } from "../lib/localstack/license-checker"; @@ -121,6 +122,7 @@ export default async function localstackSnowflakeClient({ return withToolAnalytics("localstack-snowflake-client", { action }, async () => { const preflightError = await runPreflights([ requireAuthToken(), + requireStack("snowflake", "localstack-snowflake-client"), requireSnowflakeCli(), requireProFeature(ProFeature.SNOWFLAKE), requireSnowflakeConnectionProfile(), diff --git a/src/tools/localstack-state-management.ts b/src/tools/localstack-state-management.ts index 24554fd..95e47a6 100644 --- a/src/tools/localstack-state-management.ts +++ b/src/tools/localstack-state-management.ts @@ -8,6 +8,7 @@ import { requireAuthToken, requireLocalStackRunning, requireProFeature, + requireStack, } from "../core/preflight"; import { withToolAnalytics } from "../core/analytics"; import { ProFeature } from "../lib/localstack/license-checker"; @@ -67,6 +68,7 @@ export default async function localstackStateManagement(args: StateManagementArg async () => { const preflightError = await runPreflights([ requireAuthToken(), + requireStack("aws", "localstack-state-management"), requireLocalStackRunning(), requireProFeature(ProFeature.STATE_MANAGEMENT), ]); diff --git a/tests/azure/README.md b/tests/azure/README.md new file mode 100644 index 0000000..5ade798 --- /dev/null +++ b/tests/azure/README.md @@ -0,0 +1,37 @@ +# Azure tests + +The tests of `localstack-azure-client` and of `localstack-management`'s Azure support, by layer. +The CI job names use these labels. + +| Layer | What it checks | Where | When | +| ------- | -------------------------------------------------------------------------------------------------------------- | ----------------------------------------------------------------------- | ------------------------------------------------------------------ | +| Unit | the policy, tokenizer, runner, egress guard, output classes, bootstrap, config, lifecycle and add-ons | `src/**/*.test.ts`, and `tests/azure/**/*.test.ts` without `.live` | `yarn test`, every PR (Linux, Windows, macOS) | +| L1 | a scenario through the MCP harness: start, readiness, commands, Bicep, stop; through npx and through the image | `tests/docker/validate-image.mjs` | `azure-live.yml`, `docker.yml` | +| L2 | the command matrix: one YAML file per provider, known gaps as expected failures | `matrix/`, `matrix.live.test.ts` | a PR subset in `azure-live.yml`; every case weekly | +| L3 | egress: the guard's records, the `~/.azure` home guard, the leak-command replay, a job with no route out | `egress.live.test.ts`, `home-guard.live.test.ts`, `egress-internal/` | `azure-live.yml`; in full weekly | +| L4 | the official samples, unmodified, with an `az` shim that routes them through the tool | `samples-shim/`, `samples-replay.live.test.ts` | three samples in `azure-live.yml`; all weekly | +| L5 | the Docker image: what it contains, what it must not, and its compressed size | `tests/docker/l5-image-assertions.sh`, `tests/docker/image-size.mjs` | `docker.yml` | +| E1 | Gemini tool-trigger evals | `tests/mcp/evals-gemini-azure.spec.mjs`, `data/evals/gemini-azure.json` | manual (`yarn test:mcp:evals:azure`) | +| E2 | Claude composition evals: a task, the agent loop, a verifier that reads the emulator | `evals/` | manual (`node tests/azure/evals/run.mjs`); offline tests with Unit | +| DR1-DR8 | drift gates, below | `drift/` | weekly | + +The drift gates catch changes outside this repository: + +- **DR1** the emulator's `/metadata/endpoints` suffixes still equal the snapshot; +- **DR2** its health still reports an Azure edition and a boolean `license`, as the preflight reads them; +- **DR3** the resolved `az` is supported; the weekly job runs L1 on `az` 2.85 (the minimum) and on the latest; +- **DR4** every curated extension installs at its pinned version and loads with the pinned `az`; +- **DR5** the tool's cloud config equals `lstk`'s; +- **DR6** every provider in the emulator's `/_localstack/coverage` has an L2 file; +- **DR7** (informational) changes to the emulator's certificate names and passthrough URLs; +- **DR8** the samples corpus, regenerated from the samples repo, still tokenizes to bash's argv and + passes the policy. + +The live suites need `AZURE_LIVE=1` and a running LocalStack Azure emulator: + +```bash +AZURE_LIVE=1 npx jest -c jest.azure-live.config.js --selectProjects matrix-subset --runInBand +``` + +The projects are `matrix-subset`, `matrix-full`, `egress`, `samples-subset`, `samples-all` and +`drift`; each folder's README has the details. diff --git a/tests/azure/drift/cli-and-extensions.live.test.ts b/tests/azure/drift/cli-and-extensions.live.test.ts new file mode 100644 index 0000000..77c667d --- /dev/null +++ b/tests/azure/drift/cli-and-extensions.live.test.ts @@ -0,0 +1,62 @@ +// Drift gates DR3 and DR4: the az version and the curated extensions. +import { readFileSync } from "fs"; +import path from "path"; +import { az, describeLive, setupLiveEnv } from "../live/harness"; +import { EXTENSION_COMMANDS, parsePinList } from "../../../src/lib/azure/extension-map"; +import { compareVersions, MIN_AZ_VERSION } from "../../../src/lib/azure/resolve-az"; +import { REPO, writeReport } from "./drift-helpers"; + +describeLive("DR3: the az version", () => { + test("the resolved az is supported, and the pinned version when the job pins one", async () => { + setupLiveEnv(); + // eslint-disable-next-line @typescript-eslint/no-require-imports + const services = + require("../../../src/lib/azure/services") as typeof import("../../../src/lib/azure/services"); + const exe = await services.azCli(); + writeReport("dr3-az-version", { + file: exe.file, + installer: exe.installer, + version: exe.version, + }); + expect(compareVersions(exe.version ?? "0", MIN_AZ_VERSION)).toBeGreaterThanOrEqual(0); + // The weekly matrix runs this with the minimum and the newest az (AZ_EXPECTED_VERSION). + if (process.env.AZ_EXPECTED_VERSION) expect(exe.version).toBe(process.env.AZ_EXPECTED_VERSION); + const listed = await az("group list --query length(@)"); + expect(listed.ok).toBe(true); + }); +}); + +// Runs after the job installed the pin list (scripts/install-azure-extensions.mjs), so it +// is opt-in: AZURE_DR4=1. +const describeDr4 = process.env.AZURE_DR4 === "1" ? describeLive : describe.skip; + +describeDr4("DR4: the curated extensions install and load with the pinned az", () => { + const pins = parsePinList(readFileSync(path.join(REPO, "docker/azure-extensions.txt"), "utf8")); + + test("every pinned extension is installed at its pinned version", async () => { + setupLiveEnv(); + // eslint-disable-next-line @typescript-eslint/no-require-imports + const services = + require("../../../src/lib/azure/services") as typeof import("../../../src/lib/azure/services"); + const installed = new Map(services.installedExtensions().map((e) => [e.name, e.version])); + const wrong = pins + .filter((p) => installed.get(p.name) !== p.version) + .map((p) => `${p.name}: pinned ${p.version}, installed ${installed.get(p.name) ?? "none"}`); + writeReport("dr4-extensions", { pins, installed: Object.fromEntries(installed), wrong }); + expect(wrong).toEqual([]); + }); + + test("one command group of each extension loads (--help)", async () => { + const failures: string[] = []; + for (const pin of pins) { + const group = EXTENSION_COMMANDS[pin.name]?.[0]; + if (!group) { + failures.push(`${pin.name}: no command group in extension-map.ts`); + continue; + } + const result = await az(`${group} --help`); + if (!result.ok) failures.push(`${pin.name} (${group}): ${result.text.split("\n")[0]}`); + } + expect(failures).toEqual([]); + }); +}); diff --git a/tests/azure/drift/coverage-and-samples.live.test.ts b/tests/azure/drift/coverage-and-samples.live.test.ts new file mode 100644 index 0000000..dc94659 --- /dev/null +++ b/tests/azure/drift/coverage-and-samples.live.test.ts @@ -0,0 +1,147 @@ +// Drift gates DR6 and DR8. +// DR6: every provider in the emulator's coverage list has an L2 matrix row (or a known gap). +// DR8: the samples corpus, regenerated from the samples repo, still tokenizes to bash's +// argv and passes the policy; new commands are reported. +import { spawnSync } from "child_process"; +import { mkdtempSync, readdirSync, readFileSync, rmSync } from "fs"; +import os from "os"; +import path from "path"; +import { describeLive } from "../live/harness"; +import { splitCliArgs } from "../../../src/lib/cli/argv"; +import { evaluateAzCommand } from "../../../src/lib/azure/policy"; +import { gatewayGet, REPO, writeReport } from "./drift-helpers"; + +/** Every `operations` entry anywhere in a parsed matrix file (cases and known gaps). */ +function collectOperations(node: unknown, out: string[]): void { + if (Array.isArray(node)) { + for (const item of node) collectOperations(item, out); + } else if (node && typeof node === "object") { + for (const [key, value] of Object.entries(node as Record)) { + if (key === "operations" && Array.isArray(value)) { + for (const op of value) if (typeof op === "string") out.push(op); + } else { + collectOperations(value, out); + } + } + } +} + +describeLive("DR6: coarse coverage", () => { + test("every provider in /_localstack/coverage has an L2 matrix row", async () => { + const coverage = JSON.parse((await gatewayGet("/_localstack/coverage")).body) as Array<{ + resource_provider: string; + implemented?: boolean; + }>; + const providers = [ + ...new Set(coverage.filter((e) => e.implemented).map((e) => e.resource_provider)), + ].sort(); + // eslint-disable-next-line @typescript-eslint/no-require-imports + const yaml = require("yaml") as { parse(text: string): unknown }; + const dir = path.join(REPO, "tests/azure/matrix"); + const operations: string[] = []; + for (const file of readdirSync(dir).filter((f) => /\.ya?ml$/.test(f))) { + collectOperations(yaml.parse(readFileSync(path.join(dir, file), "utf8")), operations); + } + const covered = new Set(operations.map((op) => op.split(" ")[0])); + const missing = providers.filter((p) => !covered.has(p)); + writeReport("dr6-coverage", { providers, covered: [...covered].sort(), missing }); + expect(missing).toEqual([]); + }); +}); + +const AZURE_TOKENIZER = { + quotedControlChars: true, + keepEmptyQuoted: true, + bashDoubleQuoteEscapes: true, +} as const; + +function python(): string { + for (const candidate of [process.env.PYTHON, "python3", "python"].filter(Boolean) as string[]) { + if (spawnSync(candidate, ["--version"], { timeout: 20_000 }).status === 0) return candidate; + } + throw new Error("DR8 needs Python (set PYTHON)"); +} + +// DR8 needs a samples checkout: the weekly job clones the samples repo's current commit. +const describeDr8 = process.env.AZURE_SAMPLES_DIR ? describeLive : describe.skip; + +describeDr8("DR8: the samples corpus", () => { + test("regenerated commands tokenize to bash's argv and pass the policy", () => { + const corpusDir = path.join(REPO, "tests/fixtures/azure/corpus"); + const work = mkdtempSync(path.join(os.tmpdir(), "lsaz-dr8-")); + try { + const env = { ...process.env, CORPUS_WORK_DIR: work }; + for (const script of ["extract.py", "bash_argv.py"]) { + const run = spawnSync(python(), [path.join(corpusDir, script)], { + env, + encoding: "utf8", + timeout: 600_000, + }); + if (run.status !== 0) throw new Error(`${script} failed:\n${run.stdout}\n${run.stderr}`); + } + const extracted = JSON.parse(readFileSync(path.join(work, "extracted.json"), "utf8")); + const bash = JSON.parse(readFileSync(path.join(work, "bash.json"), "utf8")); + const fixture = JSON.parse( + readFileSync(path.join(corpusDir, "samples-az-corpus.json"), "utf8") + ); + const known = new Map( + fixture.cases.map((c: { command: string; expect: "ok" | { ruleId: string } }) => [ + c.command, + c, + ]) + ); + const opts = { + workdir: "/work/samples", + homeDir: "/work/.mcp/azure/home", + platform: "linux" as const, + }; + + const unaccounted = extracted.reconciliation.flatMap( + (r: { file: string; unaccounted: string[] }) => r.unaccounted.map((u) => `${r.file}:${u}`) + ); + const argvMismatches: string[] = []; + const newlyRefused: string[] = []; + const newCommands = new Set(); + extracted.commands.forEach( + (c: { command: string; file: string; line: number }, i: number) => { + const origin = `${c.file}:${c.line}`; + const bashArgv: string[] | undefined = bash.argv[String(i)]; + if (!bashArgv) { + argvMismatches.push(`${origin}: bash oracle refused it (${bash.refused[String(i)]})`); + return; + } + let ours: string[] | undefined; + try { + ours = splitCliArgs(c.command, AZURE_TOKENIZER); + } catch (error) { + argvMismatches.push(`${origin}: tokenizer refused it (${(error as Error).message})`); + return; + } + if (JSON.stringify(ours) !== JSON.stringify(bashArgv)) + argvMismatches.push(`${origin}: argv differs from bash`); + const previous = known.get(c.command); + if (!previous) newCommands.add(c.command); + const verdict = evaluateAzCommand(c.command, opts); + const expectedRefusal = + previous && previous.expect !== "ok" ? previous.expect.ruleId : undefined; + if (!verdict.ok && verdict.ruleId !== expectedRefusal) + newlyRefused.push(`${origin}: ${verdict.ruleId}`); + } + ); + writeReport("dr8-samples-corpus", { + sourceCommit: extracted.source_commit, + fixtureCommit: fixture._metadata.source_commit, + commands: extracted.commands.length, + newCommands: [...newCommands], + unaccounted, + argvMismatches, + newlyRefused, + }); + expect(unaccounted).toEqual([]); + expect(argvMismatches).toEqual([]); + expect(newlyRefused).toEqual([]); + } finally { + rmSync(work, { recursive: true, force: true }); + } + }); +}); diff --git a/tests/azure/drift/drift-helpers.ts b/tests/azure/drift/drift-helpers.ts new file mode 100644 index 0000000..bb2a14a --- /dev/null +++ b/tests/azure/drift/drift-helpers.ts @@ -0,0 +1,88 @@ +// Helpers of the drift gates DR1-DR8. Every request goes to 127.0.0.1 +// with the ARM host as SNI, like the tool's own readiness check: no DNS is needed. +import { mkdirSync, writeFileSync } from "fs"; +import http from "http"; +import https from "https"; +import path from "path"; +import tls from "tls"; + +export const PORT = Number( + process.env.LOCALSTACK_AZURE_PORT || process.env.LOCALSTACK_PORT || 4566 +); +export const ARM_HOST = "azure.localhost.localstack.cloud"; +export const ENDPOINT = `https://${ARM_HOST}:${PORT}`; +export const REPO = path.resolve(__dirname, "../../.."); + +function collect(res: http.IncomingMessage): Promise { + return new Promise((resolve, reject) => { + const chunks: Buffer[] = []; + res.on("data", (c: Buffer) => chunks.push(c)); + res.on("end", () => resolve(Buffer.concat(chunks).toString("utf8"))); + res.on("error", reject); + }); +} + +/** GET over HTTPS from the emulator's ARM endpoint (no certificate check: drift only reads). */ +export function armGet(urlPath: string): Promise<{ status: number; body: string }> { + return new Promise((resolve, reject) => { + const req = https.request( + { + host: "127.0.0.1", + port: PORT, + servername: ARM_HOST, + path: urlPath, + headers: { Host: `${ARM_HOST}:${PORT}` }, + rejectUnauthorized: false, + timeout: 30_000, + }, + (res) => collect(res).then((body) => resolve({ status: res.statusCode ?? 0, body }), reject) + ); + req.on("timeout", () => req.destroy(new Error(`timeout: ${urlPath}`))); + req.on("error", reject); + req.end(); + }); +} + +/** GET over plain HTTP from the gateway (health, info, coverage). */ +export function gatewayGet(urlPath: string): Promise<{ status: number; body: string }> { + return new Promise((resolve, reject) => { + const req = http.get({ host: "127.0.0.1", port: PORT, path: urlPath, timeout: 60_000 }, (res) => + collect(res).then((body) => resolve({ status: res.statusCode ?? 0, body }), reject) + ); + req.on("timeout", () => req.destroy(new Error(`timeout: ${urlPath}`))); + req.on("error", reject); + }); +} + +/** The DNS names in the certificate the emulator serves for the ARM host. */ +export function certificateSans(): Promise { + return new Promise((resolve, reject) => { + const socket = tls.connect( + { host: "127.0.0.1", port: PORT, servername: ARM_HOST, rejectUnauthorized: false }, + () => { + const alt = socket.getPeerCertificate().subjectaltname ?? ""; + socket.end(); + resolve( + alt + .split(", ") + .filter(Boolean) + .map((s) => s.replace(/^DNS:/, "")) + .sort() + ); + } + ); + socket.on("error", reject); + }); +} + +/** Write a JSON report next to the other test artifacts (CI uploads test-results/). */ +export function writeReport(name: string, data: unknown): string { + const dir = path.join(REPO, "test-results"); + mkdirSync(dir, { recursive: true }); + const file = path.join(dir, `${name}.json`); + writeFileSync(file, JSON.stringify(data, null, 2) + "\n"); + return file; +} + +/** The local names az may reach; anything else is on L3's watch list. */ +export const isLocalSuffix = (value: string) => /localhost\.localstack\.cloud/i.test(value); diff --git a/tests/azure/drift/emulator-contract.live.test.ts b/tests/azure/drift/emulator-contract.live.test.ts new file mode 100644 index 0000000..db5d2b9 --- /dev/null +++ b/tests/azure/drift/emulator-contract.live.test.ts @@ -0,0 +1,111 @@ +// Drift gates DR1, DR2, DR5 and DR7: the +// emulator surfaces the tool depends on, checked weekly and on emulator releases. +import { existsSync, readFileSync } from "fs"; +import path from "path"; +import { describeLive } from "../live/harness"; +import { cloudConfigJson } from "../../../src/lib/azure/bootstrap"; +import { stackFromEdition } from "../../../src/lib/localstack/container-spec.logic"; +import { + armGet, + certificateSans, + ENDPOINT, + gatewayGet, + isLocalSuffix, + REPO, + writeReport, +} from "./drift-helpers"; + +const fixture = (name: string) => + JSON.parse(readFileSync(path.join(REPO, "tests/fixtures/azure/drift", name), "utf8")); + +describeLive("DR1: /metadata/endpoints suffixes", () => { + test("the suffixes equal the snapshot; the non-local ones are L3's watch list", async () => { + const res = await armGet("/metadata/endpoints?api-version=2022-09-01"); + expect(res.status).toBe(200); + const live = JSON.parse(res.body); + const snapshot = fixture("metadata-endpoints.json"); + const watchList = Object.entries(live.suffixes as Record) + .filter(([, value]) => !isLocalSuffix(value)) + .map(([key, value]) => `${key}=${value}`) + .sort(); + writeReport("dr1-metadata-endpoints", { suffixes: live.suffixes, watchList }); + // A change means: re-check the bootstrap's cloud registration, and put any new + // real-Azure suffix on L3's watch list. + expect(live.suffixes).toEqual(snapshot.suffixes); + expect(live.resourceManager.replace(/\/$/, "")).toBe(ENDPOINT); + }); +}); + +describeLive("DR2: health edition and license", () => { + test("health still reports an Azure edition and a boolean license, as the preflight expects", async () => { + const res = await gatewayGet("/_localstack/health"); + expect(res.status).toBe(200); + const health = JSON.parse(res.body); + writeReport("dr2-health", health); + expect(typeof health.edition).toBe("string"); + expect(stackFromEdition(health.edition)).toBe("azure"); + expect(typeof health.license).toBe("boolean"); + const info = JSON.parse((await gatewayGet("/_localstack/info")).body); + expect(typeof info.session_id).toBe("string"); + }); +}); + +describeLive("DR5: the cloud-config JSON equals lstk's BuildCloudConfig", () => { + // The weekly job fetches lstk's source; a local checkout works too. + const source = + process.env.LSTK_AZURECONFIG_GO || + "https://raw.githubusercontent.com/localstack/lstk/main/internal/azureconfig/azureconfig.go"; + + async function lstkSource(): Promise { + if (existsSync(source)) return readFileSync(source, "utf8"); + const res = await fetch(source); + if (!res.ok) throw new Error(`could not fetch ${source}: HTTP ${res.status}`); + return res.text(); + } + + test("same endpoint keys and values for the same endpoint", async () => { + const go = await lstkSource(); + const body = /func BuildCloudConfig\([\s\S]*?\n}/.exec(go)?.[0]; + expect(body).toBeDefined(); + // "activeDirectory": base, / "management": base + "/", + const lstk: Record = {}; + for (const m of body!.matchAll(/"(\w+)":\s*base(\s*\+\s*"\/")?,/g)) { + lstk[m[1]] = m[2] ? `${ENDPOINT}/` : ENDPOINT; + } + const ours = JSON.parse(cloudConfigJson(ENDPOINT)).endpoints; + writeReport("dr5-cloud-config", { lstk, ours }); + expect(Object.keys(lstk).length).toBeGreaterThanOrEqual(7); + expect(ours).toEqual(lstk); + }); +}); + +describeLive("DR7 (informational): certificate SANs and passthrough URLs", () => { + test("changes are reported, never a failure", async () => { + const sans = await certificateSans(); + const snapshot: string[] = fixture("certificate-sans.json").sans; + const added = sans.filter((s) => !snapshot.includes(s)); + const removed = snapshot.filter((s) => !sans.includes(s)); + // PASSTHROUGH_URLS is only visible in the CI emulator container's environment. + let passthrough: string | undefined; + const container = process.env.AZURE_CI_EMULATOR_CONTAINER; + if (container) { + try { + const { execFileSync } = await import("child_process"); + passthrough = execFileSync("docker", ["exec", container, "printenv", "PASSTHROUGH_URLS"], { + encoding: "utf8", + timeout: 30_000, + }).trim(); + } catch { + passthrough = "(not set)"; + } + } + const file = writeReport("dr7-emulator-surface", { added, removed, passthrough }); + if (added.length || removed.length) { + console.warn( + `DR7: certificate SANs changed (+${added.length} -${removed.length}); see ${file}` + ); + } + // The ARM host must stay covered, or az's certificate check would fail. + expect(sans.some((s) => s === "*.azure.localhost.localstack.cloud")).toBe(true); + }); +}); diff --git a/tests/azure/egress-internal/README.md b/tests/azure/egress-internal/README.md new file mode 100644 index 0000000..3139d7b --- /dev/null +++ b/tests/azure/egress-internal/README.md @@ -0,0 +1,125 @@ +# The internal-network egress job (L3) + +`run.sh` proves that the Azure tool needs nothing +outside the LocalStack emulator: the MCP server runs in a container that has **no route out**, +and a scenario through `localstack-azure-client` must still succeed. + +**CI only.** The script builds an image, creates a Docker network, and starts an emulator and +containers. It refuses to run unless `CI=true`. Never run it on a machine whose Docker engine +is shared: a second emulator on a shared engine has already destroyed another emulator's storage +side-car once. + +## Layout + +``` + default bridge (internet) --internal network (no route out) + | | + runner ── 127.0.0.1:4566 ─┤ | + ├── emulator (licence activation) ─────────────┤ :4566 + | | + | MCP server container (internal only) + | LOCALSTACK_AZURE_FORWARD_TARGET= + | az → egress guard → 127.0.0.1:4566 + | → loopback forwarder → :4566 +``` + +- The emulator is **dual-homed**: on the default bridge, so it can activate its licence online, + and on a network created with `docker network create --internal`. +- The MCP server's container is on the internal network **only**, with + `LOCALSTACK_AZURE_FORWARD_TARGET` set to the emulator's address there. It needs no DNS: the + egress guard maps `localhost.localstack.cloud` and every name under it to 127.0.0.1 itself, + and the loopback forwarder relays 127.0.0.1 to the emulator. +- The client (`tests/azure/tools/stdio-client.mjs`) runs on the runner and talks to the server + over `docker run -i` stdio. The server gets no Docker socket. + +## What the script does + +1. Checks `CI=true`, `LOCALSTACK_AUTH_TOKEN` (never printed), and `docker`, `node` and `curl`. +2. Builds the MCP server image from the repository, or uses `MCP_SERVER_IMAGE`. The image must + contain `az`; without it the script stops with that message. +3. Starts the emulator on the bridge with the CI flags of `scripts/ci/azure-emulator-up.sh`, published on `127.0.0.1` only, + and waits until health reports the Azure edition with `license: true` and HTTPS answers. +4. Connects the emulator to the internal network, then checks the layout: + - a container on the internal network reaches `http://:4566/_localstack/health`; + - **negative controls:** `https://management.azure.com/` and `https://1.1.1.1/` are _not_ + reachable from the internal network. If either is, the run fails, because a pass would + then prove nothing. +5. Runs one MCP session with the server on the internal network only: + - the first call (`group list`) bootstraps the tool's profile through the forwarder; + - group create and show; storage account create (a long-running operation, polled at the + apex `localhost.localstack.cloud`); a blob container, upload and list; a Key Vault with a + secret set and shown; `rest` with a relative URL; `rest` with an absolute + `management.azure.com` URL, which the tool rewrites; a `--help` page; a third-party URL, + which the policy refuses; then the cleanup (vault delete and purge, group delete); + - every answer must pass its check, no successful answer may carry the guard's "blocked a + connection" note, and no failure may be class `egress-refused`; + - the server's stderr must show the forwarder's target, and its connection count must be + above zero: the test envelope's `forwarderConnections`, or the `forwarder connections=` + line the server prints at exit. +6. Checks that the emulator's licence and health held through the scenario. + +On exit it saves the emulator log, removes the containers, the network and a built image, then +**scans every output file for the token** and replaces any occurrence, so an upload step cannot +publish it. + +### Feasibility first + +The first CI run is the feasibility check for this layout. The messages marked `FEASIBILITY:` +(licence activation while dual-homed, reachability on the internal network, the first bootstrap +through the forwarder) mean the layout itself does not work: redesign the job before building +more on it. + +## Inputs + +| Variable | Default | Use | +| ------------------------------ | ------------------------------------ | --------------------------------------------------------------------------------- | +| `CI` | none | must be `true` | +| `LOCALSTACK_AUTH_TOKEN` | none | an Azure-entitled token; passed to the emulator and (presence only) to the server | +| `MCP_SERVER_IMAGE` | built from the repository | an image that already contains `az` | +| `LOCALSTACK_AZURE_IMAGE_NAME` | `localstack/localstack-azure:latest` | the emulator image | +| `EGRESS_EMULATOR_HOST_PORT` | `4566` | the runner port the emulator is published on (`127.0.0.1` only) | +| `EGRESS_READY_TIMEOUT_SECONDS` | `300` | how long to wait for health, licence and HTTPS | +| `EGRESS_LS_LOG` | `debug` | the emulator's `LS_LOG` | +| `EGRESS_INTERNAL_OUT` | `$RUNNER_TEMP/egress-internal-` | where the outputs go | + +## Outputs + +In `EGRESS_INTERNAL_OUT`: `health.json` (the last health answer), `scenario.mjs` (the generated +client), `scenario.jsonl` (one line per step, written as each lands, plus the forwarder check), +`scenario.log`, and `emulator.log`. + +## Workflow + +`azure-weekly.yml`, job `egress-internal-network`: + +```yaml +jobs: + egress-internal-network: + runs-on: ubuntu-latest + timeout-minutes: 60 # an image build with az takes about 9 minutes cold + env: + LOCALSTACK_AUTH_TOKEN: "${{ secrets.LOCALSTACK_AUTH_TOKEN }}" + EGRESS_INTERNAL_OUT: "${{ runner.temp }}/egress-internal" + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: { node-version: 22.x } + - run: docker pull localstack/localstack-azure:latest + - run: bash tests/azure/egress-internal/run.sh + - uses: actions/upload-artifact@v4 # run.sh has already replaced any token in these files + if: failure() + with: + name: egress-internal + path: ${{ runner.temp }}/egress-internal + retention-days: 7 +``` + +`CI=true` comes from GitHub Actions. No `yarn install` is needed on the runner: the stdio client +has no dependencies, and the image build installs its own. + +## Not covered + +- Commands whose extensions call other Microsoft hosts, for example + `monitor app-insights query` (it calls `api.applicationinsights.io`): that is L3's weekly + run with the extensions installed. +- Bicep: L5 covers it in the image. diff --git a/tests/azure/egress-internal/run.sh b/tests/azure/egress-internal/run.sh new file mode 100644 index 0000000..7c3159c --- /dev/null +++ b/tests/azure/egress-internal/run.sh @@ -0,0 +1,331 @@ +#!/usr/bin/env bash +# L3 (b), the internal-network egress job. CI ONLY: see README.md. +# +# The emulator is dual-homed: on the default bridge, so it can activate its licence online, and +# on a network created with --internal. The MCP server's container is on the internal network +# only, with LOCALSTACK_AZURE_FORWARD_TARGET=. Inside it, the egress +# guard maps the emulator's names to 127.0.0.1 without DNS, and the loopback forwarder relays +# 127.0.0.1 to the emulator. A scenario through localstack-azure-client must then succeed: +# nothing on that network can reach the internet, so success proves that nothing leaked. +# +# CI=true bash tests/azure/egress-internal/run.sh (LOCALSTACK_AUTH_TOKEN in the env) +# +# It builds an image, creates a network and starts an emulator and containers, so it refuses to +# run unless CI=true. Never run it on a machine whose Docker engine is shared. +set -euo pipefail + +if [ "${CI:-}" != "true" ]; then + echo "egress-internal: refusing to run: CI is not \"true\". This job starts its own emulator," >&2 + echo "network and containers, and must only run on a CI runner (see README.md)." >&2 + exit 2 +fi +if [ -z "${LOCALSTACK_AUTH_TOKEN:-}" ]; then + echo "egress-internal: LOCALSTACK_AUTH_TOKEN is not set (an Azure-entitled token)." >&2 + exit 2 +fi +for tool in docker node curl; do + command -v "$tool" > /dev/null || { echo "egress-internal: $tool is missing" >&2; exit 2; } +done + +REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../.." && pwd)" +RUN_ID="${GITHUB_RUN_ID:-run}-$(date +%s)-$$" +NET="lsmcp-egress-internal-$RUN_ID" +EMU="lsmcp-egress-emulator-$RUN_ID" +MCP_NAME="lsmcp-egress-server-$RUN_ID" +EMU_IMAGE="${LOCALSTACK_AZURE_IMAGE_NAME:-localstack/localstack-azure:latest}" +EMU_PORT="${EGRESS_EMULATOR_HOST_PORT:-4566}" +READY_TIMEOUT="${EGRESS_READY_TIMEOUT_SECONDS:-300}" +OUT_DIR="${EGRESS_INTERNAL_OUT:-${RUNNER_TEMP:-/tmp}/egress-internal-$RUN_ID}" +MCP_IMAGE="${MCP_SERVER_IMAGE:-}" +WORKDIR="$(mktemp -d)" +mkdir -p "$OUT_DIR" + +log() { printf 'egress-internal: %s\n' "$*"; } +die() { + printf 'egress-internal: FAIL: %s\n' "$*" >&2 + exit 1 +} + +# Replace the token in every file this job wrote, so a later upload step cannot publish it. +# The value is read from the environment and never printed. +scan_outputs() { + node - "$OUT_DIR" << 'NODE' +const fs = require("fs"); +const path = require("path"); +const token = process.env.LOCALSTACK_AUTH_TOKEN || ""; +const found = []; +const walk = (dir) => { + for (const entry of fs.readdirSync(dir, { withFileTypes: true })) { + const file = path.join(dir, entry.name); + if (entry.isDirectory()) walk(file); + else if (token.length >= 8) { + const text = fs.readFileSync(file, "latin1"); + if (text.includes(token)) { + fs.writeFileSync(file, text.split(token).join("[redacted]"), "latin1"); + found.push(path.relative(process.argv[2], file)); + } + } + } +}; +walk(process.argv[2]); +if (found.length > 0) { + console.error(`egress-internal: FAIL: the auth token appeared in ${found.join(", ")} (redacted)`); + process.exit(1); +} +NODE +} + +cleanup() { + local code=$? + set +e + docker rm -f "$MCP_NAME" > /dev/null 2>&1 + if [ -n "${EMU_STARTED:-}" ]; then + docker logs "$EMU" > "$OUT_DIR/emulator.log" 2>&1 + docker rm -f "$EMU" > /dev/null 2>&1 + fi + if [ -n "${NET_CREATED:-}" ]; then + # Side-car containers the emulator attached to the network would keep it from being removed. + for c in $(docker network inspect -f '{{range .Containers}}{{.Name}} {{end}}' "$NET" 2> /dev/null); do + docker network disconnect -f "$NET" "$c" > /dev/null 2>&1 + done + docker network rm "$NET" > /dev/null 2>&1 || log "warning: could not remove network $NET" + fi + if [ -n "${MCP_IMAGE_BUILT:-}" ]; then docker rmi "$MCP_IMAGE" > /dev/null 2>&1; fi + rm -rf "$WORKDIR" + scan_outputs || code=1 + if [ "$code" -eq 0 ]; then log "PASSED (results in $OUT_DIR)"; else log "FAILED (see $OUT_DIR)"; fi + exit "$code" +} +trap cleanup EXIT + +# 1. The MCP server image, which must carry az. +if [ -z "$MCP_IMAGE" ]; then + MCP_IMAGE="lsmcp-egress-internal:$RUN_ID" + log "building $MCP_IMAGE from $REPO_ROOT" + docker build -q -t "$MCP_IMAGE" "$REPO_ROOT" > /dev/null + MCP_IMAGE_BUILT=1 +fi +docker run --rm --network none --entrypoint /bin/sh "$MCP_IMAGE" -c 'command -v az && command -v curl' > /dev/null \ + || die "$MCP_IMAGE has no az (or no curl); this job needs the MCP server image with az" + +# 2. The emulator on the default bridge (azure-emulator-up.sh's flags), published on 127.0.0.1 only, so +# this script can watch its health from the runner. +docker network create --internal "$NET" > /dev/null +NET_CREATED=1 +log "starting $EMU_IMAGE as $EMU" +# Its state folder as a bind mount, as azure-emulator-up.sh gives it: the containers the +# emulator starts for apps get their files from there. Not under $WORKDIR, whose cleanup could +# not remove the root-owned files the emulator writes. +EMU_VOLUME="${RUNNER_TEMP:-/tmp}/$EMU-volume" +mkdir -p "$EMU_VOLUME" +docker run -d --name "$EMU" \ + -p "127.0.0.1:$EMU_PORT:4566" \ + -e LOCALSTACK_AUTH_TOKEN \ + -e ACTIVATE_PRO=1 -e DNS_ADDRESS=0 -e DISABLE_EVENTS=1 -e "LS_LOG=${EGRESS_LS_LOG:-debug}" \ + -e MSSQL_ACCEPT_EULA=Y \ + -v "$EMU_VOLUME:/var/lib/localstack" \ + -v /var/run/docker.sock:/var/run/docker.sock \ + "$EMU_IMAGE" > /dev/null +EMU_STARTED=1 + +# Health says the edition and whether the licence activated; HTTPS comes up seconds later. +# The ARM name is pinned to 127.0.0.1, so no DNS is involved. +emulator_ready() { + local health + health="$(curl -fsS -m 5 "http://127.0.0.1:$EMU_PORT/_localstack/health" 2> /dev/null)" || return 1 + printf '%s' "$health" > "$OUT_DIR/health.json" + grep -Eq '"edition": *"azure' <<< "$health" || return 1 + grep -Eq '"license": *true' <<< "$health" || return 1 + curl -ksS -m 5 -o /dev/null \ + --resolve "azure.localhost.localstack.cloud:$EMU_PORT:127.0.0.1" \ + "https://azure.localhost.localstack.cloud:$EMU_PORT/_localstack/health" 2> /dev/null +} +deadline=$((SECONDS + READY_TIMEOUT)) +until emulator_ready; do + # An exited container never turns ready: stop at once, with its reason (exit code 55: the + # token's licence does not cover LocalStack for Azure). + state="$(docker inspect -f '{{.State.Status}} {{.State.ExitCode}}' "$EMU" 2> /dev/null || echo "missing -")" + case "$state" in + exited* | dead* | missing*) + docker logs "$EMU" > "$OUT_DIR/emulator-start.log" 2>&1 || true + grep -E '^(License activation failed|Reason: .)' "$OUT_DIR/emulator-start.log" >&2 || true + die "FEASIBILITY: the emulator stopped before it became ready (status and exit code: $state; logs in $OUT_DIR/emulator-start.log)" + ;; + esac + [ "$SECONDS" -lt "$deadline" ] \ + || die "FEASIBILITY: the emulator did not come up with an active licence within ${READY_TIMEOUT}s (last health in $OUT_DIR/health.json)" + sleep 5 +done +log "emulator ready: Azure edition, licence active, HTTPS up" + +# 3. Dual-home it: the internal network joins; the bridge (and its default route) stays. +docker network connect "$NET" "$EMU" +EMU_IP="$(docker inspect -f "{{with index .NetworkSettings.Networks \"$NET\"}}{{.IPAddress}}{{end}}" "$EMU")" +[ -n "$EMU_IP" ] || die "the emulator has no address on $NET" +log "emulator on $NET at $EMU_IP" +emulator_ready || die "FEASIBILITY: the emulator lost its licence or health after joining $NET" + +# 4. The internal network reaches the emulator and nothing else. +probe() { docker run --rm --network "$NET" --entrypoint curl "$MCP_IMAGE" "$@"; } +probe -fsS -m 15 -o /dev/null "http://$EMU_IP:4566/_localstack/health" \ + || die "FEASIBILITY: a container on $NET cannot reach the emulator at $EMU_IP:4566" +for outside in https://management.azure.com/ https://1.1.1.1/; do + if probe -sS -m 10 -o /dev/null "$outside" 2> /dev/null; then + die "$outside is reachable from $NET: the network is not internal, so a pass would prove nothing" + fi +done +log "negative controls: management.azure.com and 1.1.1.1 are unreachable from $NET" + +# 5. The scenario through the tool, with the server on the internal network only. +printf 'hello from the internal-network job\n' > "$WORKDIR/hello.txt" +cat > "$OUT_DIR/scenario.mjs" << 'NODE' +// Generated by tests/azure/egress-internal/run.sh: one MCP session with the server in a +// container on the internal network only. Each step's result is appended to scenario.jsonl. +import { appendFileSync } from "node:fs"; +import { join } from "node:path"; +import { pathToFileURL } from "node:url"; + +const e = process.env; +const { resultText, startServer } = await import( + pathToFileURL(join(e.REPO_ROOT, "tests", "azure", "tools", "stdio-client.mjs")).href +); +const out = join(e.EGRESS_OUT_DIR, "scenario.jsonl"); +const id = `${Date.now().toString(36)}${Math.random().toString(36).slice(2, 6)}`; +const rg = `mcp-${id}-rg`; +const account = `mcp${id}`.slice(0, 24); +const container = `mcp-${id}-c`; +const vault = `mcp${id}kv`.slice(0, 24); +const secret = `internal network ${id}`; + +const server = await startServer({ + command: "docker", + args: [ + "run", "-i", "--rm", "--name", e.EGRESS_MCP_NAME, "--network", e.EGRESS_NET, + // The value comes from this process's environment; it is never on a command line. + "-e", "LOCALSTACK_AUTH_TOKEN", + "-e", `LOCALSTACK_AZURE_FORWARD_TARGET=${e.EGRESS_EMU_IP}`, + "-e", "LOCALSTACK_AZ_TEST_ENVELOPE=1", + "-e", "MCP_ANALYTICS_DISABLED=1", + "-e", "LOCALSTACK_AZ_WORKDIR=/work", + "-v", `${e.EGRESS_WORKDIR}:/work`, + e.EGRESS_MCP_IMAGE, + ], +}); + +let failures = 0; +// In Docker the test envelope carries the forwarder's running connection count. +let envelopeConnections = 0; +async function step(name, command, check) { + const started = Date.now(); + let text = ""; + let envelope; + let ok = false; + let why = ""; + try { + const result = await server.callTool("localstack-azure-client", { command }, 900_000); + text = result.content?.[0]?.text ?? resultText(result); + try { + envelope = result.content[1] ? JSON.parse(result.content[1].text) : undefined; + } catch { + envelope = undefined; + } + envelopeConnections = Math.max(envelopeConnections, envelope?.forwarderConnections ?? 0); + // A command that succeeded while the guard refused a (non-housekeeping) host is a leak + // attempt, and a refusal class means az needed something outside the emulator. + if (/the egress guard blocked a connection to/.test(text)) throw new Error("egress blocked"); + if (envelope?.classId === "egress-refused") throw new Error("egress refused"); + ok = Boolean(check({ text, envelope, json: () => JSON.parse(envelope.stdout) })); + } catch (error) { + why = String(error?.message ?? error).slice(0, 300); + } + if (!ok) failures++; + const firstLine = text.split("\n")[0]; + appendFileSync(out, `${JSON.stringify({ name, command, ok, ms: Date.now() - started, firstLine, classId: envelope?.classId ?? null, why })}\n`); + console.log(`${ok ? "PASS" : "FAIL"} ${String(Date.now() - started).padStart(7)} ms ${name}: ${firstLine.slice(0, 100)}`); + return { ok, text, envelope }; +} + +const exit0 = ({ envelope }) => envelope?.exitCode === 0; +let subscription = ""; +try { + // The first call bootstraps the tool's profile through the forwarder: licence, name mapping + // and routing all have to work for this to pass (the layout's feasibility check). + await step("group list (bootstrap)", "group list -o json", exit0); + await step("group create", `group create --name ${rg} --location westeurope`, ({ json }) => { + subscription = /^\/subscriptions\/([^/]+)\//.exec(json().id)?.[1] ?? ""; + return json().name === rg && subscription !== ""; + }); + await step("group show", `group show --name ${rg} --query name -o tsv`, ({ envelope }) => envelope.stdout.trim() === rg); + await step( + "storage account create", + `storage account create --name ${account} --resource-group ${rg} --location westeurope --sku Standard_LRS`, + ({ json }) => json().name === account + ); + await step( + "storage container create", + `storage container create --name ${container} --account-name ${account} --auth-mode key`, + ({ json }) => json().created === true + ); + await step( + "storage blob upload", + `storage blob upload --container-name ${container} --name hello.txt --file hello.txt --account-name ${account} --auth-mode key --overwrite`, + exit0 + ); + await step( + "storage blob list", + `storage blob list --container-name ${container} --account-name ${account} --auth-mode key --query "[].name" -o tsv`, + ({ envelope }) => envelope.stdout.includes("hello.txt") + ); + await step("keyvault create", `keyvault create --name ${vault} --resource-group ${rg} --location westeurope`, ({ json }) => json().name === vault); + await step("keyvault secret set", `keyvault secret set --vault-name ${vault} --name internal --value "${secret}"`, ({ json }) => json().value === secret); + await step( + "keyvault secret show", + `keyvault secret show --vault-name ${vault} --name internal --query value -o tsv`, + ({ envelope }) => envelope.stdout.trim() === secret + ); + await step( + "rest (relative URL)", + `rest --method get --url "/subscriptions/${subscription}/resourceGroups/${rg}?api-version=2022-09-01"`, + ({ json }) => json().name === rg + ); + await step( + "rest (absolute management.azure.com URL, rewritten)", + `rest --method get --url "https://management.azure.com/subscriptions/${subscription}/resourceGroups/${rg}?api-version=2022-09-01"`, + ({ json, envelope }) => json().name === rg && envelope.notes.some((n) => n.startsWith("Rewrote the management.azure.com URL")) + ); + await step("group create --help", "group create --help", ({ text }) => text.includes("--location")); + await step( + "rest (third-party URL, a policy refusal)", + "rest --method get --url https://graph.microsoft.com/v1.0/me", + ({ text, envelope }) => text.startsWith("❌ **Address not allowed**") && envelope === undefined + ); +} finally { + await step("cleanup: keyvault delete", `keyvault delete --name ${vault}`, () => true); + await step("cleanup: keyvault purge", `keyvault purge --name ${vault}`, () => true); + await step("cleanup: group delete", `group delete --name ${rg} --yes --no-wait`, exit0); + await server.close(); +} + +// The forwarder logs its target when it starts, and its connection count at exit; the test +// envelope carries the running count too. Either count proves the traffic went through it. +const stderr = server.stderr(); +const target = new RegExp(`forwarder: 127\\.0\\.0\\.1:4566 -> ${e.EGRESS_EMU_IP.replace(/\./g, "\\.")}:4566`); +const atExit = Number(/forwarder connections=(\d+)/.exec(stderr)?.[1] ?? 0); +const connections = Math.max(atExit, envelopeConnections); +appendFileSync(out, `${JSON.stringify({ forwarderTarget: target.test(stderr), atExit, envelopeConnections })}\n`); +if (!target.test(stderr) || connections === 0) { + console.log(`FAIL forwarder: target seen ${target.test(stderr)}, connections ${connections}`); + failures++; +} else { + console.log(`PASS forwarder relayed ${connections} connections to ${e.EGRESS_EMU_IP}`); +} +process.exit(failures === 0 ? 0 : 1); +NODE + +log "running the scenario through $MCP_IMAGE on $NET" +REPO_ROOT="$REPO_ROOT" EGRESS_OUT_DIR="$OUT_DIR" EGRESS_NET="$NET" EGRESS_EMU_IP="$EMU_IP" \ + EGRESS_MCP_IMAGE="$MCP_IMAGE" EGRESS_MCP_NAME="$MCP_NAME" EGRESS_WORKDIR="$WORKDIR" \ + node "$OUT_DIR/scenario.mjs" | tee "$OUT_DIR/scenario.log" \ + || die "the scenario failed on the internal network (see $OUT_DIR/scenario.jsonl)" + +emulator_ready || die "the emulator's licence or health did not hold through the scenario" diff --git a/tests/azure/egress.live.test.ts b/tests/azure/egress.live.test.ts new file mode 100644 index 0000000..ad0e681 --- /dev/null +++ b/tests/azure/egress.live.test.ts @@ -0,0 +1,456 @@ +/** + * L3, egress: every command of a light scenario runs + * with the egress guard recording, and the suite fails if the guard refused any host that was + * not expected. Housekeeping refusals are reported and never fail the run, with + * one exception: app.aladdin.microsoft.com. The bootstrap turns az's command recommender off + * (`core.error_recommendation=off`), so any call there is a regression. + * + * Also here: + * - the first command after a fresh bootstrap makes no azcliprod CONNECT and prints no + * `WARNING:` line (the versionCheck.json seed); + * - CI only (AZURE_EGRESS_CI=1): the GET and DELETE leak commands + * (tests/fixtures/azure/leak-commands.json) replayed on the job's own emulator, each with no + * refusal and at least one relayed CONNECT; + * - CI only (AZURE_EGRESS_CI=1 and AZURE_CI_EMULATOR_CONTAINER): the job's own emulator is + * stopped between two calls, and the second gets the `conn-refused` answer (or the tool's "emulator not + * running" preflight answer) within about 5 s. The emulator is started again afterwards. This + * breaks any suite that runs at the same time, so CI runs this project on its own, last. + * + * AZURE_LIVE=1 npx jest -c jest.azure-live.config.js --selectProjects egress --runInBand + * + * On a shared emulator it only creates `mcp--*` resources, deletes its group with + * --no-wait, and deletes and purges its vault. It always uses a fresh config dir (it ignores + * LOCALSTACK_AZ_CONFIG_DIR). AZURE_EGRESS_REPORT= also appends one JSON line per command + * and the final summary there. + */ +import { execFileSync } from "child_process"; +import { appendFileSync, existsSync, readFileSync, writeFileSync } from "fs"; +import path from "path"; +import { MARKER_FILE, VERSION_CHECK_FILE } from "../../src/lib/azure/bootstrap"; +import { isAllowedEgressHost } from "../../src/lib/azure/egress-proxy"; +import { + az, + describeLive, + json, + names, + recordEgress, + runId, + setupLiveEnv, + type AzCall, + type EgressLog, + type LiveEnv, +} from "./live/harness"; + +// jest.azure-live.config.js sets testTimeout inside `projects`, where Jest ignores it (it is a +// global option), so without this every test and hook would get the 5 s default. +const LIVE_TIMEOUT_MS = 900_000; +jest.setTimeout(LIVE_TIMEOUT_MS); + +type Event = EgressLog["events"][number]; + +interface Step { + name: string; + call: AzCall; + /** The guard's events while this command ran, the bootstrap's calls included. */ + events: Event[]; +} + +interface LeakCommand { + id: string; + method: string; + command: string; +} + +const UPDATE_CHECK_HOST = "azcliprod.blob.core.windows.net"; +const RECOMMENDER_HOST = "app.aladdin.microsoft.com"; +const REWRITE_NOTE = "Rewrote the management.azure.com URL"; +/** The tool's health preflight; on a busy shared emulator it can miss its 3 s window. */ +const NOT_READY = "❌ **LocalStack Azure Emulator Not Ready**"; +const RETRY_DELAY_MS = 5_000; +/** The `conn-refused` answer must come within about 5 s of the stop. */ +const STOPPED_ANSWER_MS = 5_000; +/** The emulator must answer again within this long after `docker start`. */ +const RESTART_WAIT_MS = 300_000; +/** The shared emulators of a developer machine: never stopped outside CI, whatever the variables say. */ +const SHARED_EMULATOR_NAMES = new Set(["localstack-azure", "localstack-main", "localstack_main"]); + +const CI_EGRESS = process.env.AZURE_EGRESS_CI === "1"; +const CI_CONTAINER = process.env.AZURE_CI_EMULATOR_CONTAINER?.trim() ?? ""; +const describeCi = CI_EGRESS ? describe : describe.skip; +const describeStop = CI_EGRESS && CI_CONTAINER ? describe : describe.skip; + +const LEAK_COMMANDS: LeakCommand[] = ( + JSON.parse( + readFileSync(path.join(__dirname, "..", "fixtures", "azure", "leak-commands.json"), "utf8") + ) as { commands: LeakCommand[] } +).commands; +/** The others change state, so only these two methods are replayed. */ +const REPLAYED = LEAK_COMMANDS.filter((c) => c.method === "GET" || c.method === "DELETE"); + +const reportFile = process.env.AZURE_EGRESS_REPORT?.trim() || undefined; +function report(entry: Record): void { + if (!reportFile) return; + appendFileSync(reportFile, `${JSON.stringify({ at: new Date().toISOString(), ...entry })}\n`); +} + +const byKind = (events: Event[], kind: Event["kind"]) => events.filter((e) => e.kind === kind); +const hostsOf = (events: Event[]) => [...new Set(events.map((e) => e.host))].sort(); +function countHosts(events: Event[]): Record { + const counts: Record = {}; + for (const e of events) counts[e.host] = (counts[e.host] ?? 0) + 1; + return counts; +} + +describeLive("L3 egress: the guard's records over live commands", () => { + let env: LiveEnv; + let log: EgressLog; + let freshConfigDir = false; + const steps: Step[] = []; + /** Refusals a test provoked on purpose (the guard's positive control). */ + const expected = new Set(); + + /** + * Run one command and keep the guard events it caused (calls run one at a time). A "not ready" + * preflight answer is retried once, to allow for the readiness window: nothing + * was spawned, so the retry sees the same state. Every retry is logged. + */ + async function step( + name: string, + command: string, + opts: { retryNotReady?: boolean } = {} + ): Promise { + const from = log.events.length; + let call = await az(command); + if (opts.retryNotReady !== false && call.text.startsWith(NOT_READY)) { + console.log(`L3 retry (emulator not ready): ${name}\n${call.text.slice(0, 300)}`); + report({ retry: name, firstLine: call.text.split("\n")[0] }); + await new Promise((r) => setTimeout(r, RETRY_DELAY_MS)); + call = await az(command); + } + const events = log.events.slice(from); + const result: Step = { name, call, events }; + steps.push(result); + report({ + step: name, + command, + ms: call.ms, + exitCode: call.exitCode, + classId: call.classId, + ok: call.ok, + firstLine: call.text.split("\n")[0], + events: events.map((e) => `${e.kind} ${e.host}`), + }); + return result; + } + + beforeAll(async () => { + // The first-command check needs a config dir no bootstrap has touched: the harness makes a + // new one per process unless this variable points it elsewhere. + delete process.env.LOCALSTACK_AZ_CONFIG_DIR; + env = setupLiveEnv(); + freshConfigDir = !existsSync(path.join(env.configDir, MARKER_FILE)); + const recorder = await recordEgress(); + if (!recorder) throw new Error("L3 needs the egress guard: unset LOCALSTACK_AZ_EGRESS_GUARD"); + log = recorder; + if (reportFile) writeFileSync(reportFile, ""); + }); + + afterAll(async () => { + try { + if (log) verdict(); + } finally { + // The guard listens until it is closed, and Jest would never exit. + /* eslint-disable-next-line @typescript-eslint/no-require-imports */ + const services = + require("../../src/lib/azure/services") as typeof import("../../src/lib/azure/services"); + await services.resetAzureServices(); + } + }); + + /** Fails the suite on an unexpected refusal or any call to the command recommender. */ + function verdict(): void { + log.stop(); + const refused = byKind(log.events, "refused").filter((e) => !expected.has(e)); + const housekeeping = byKind(log.events, "housekeeping"); + const recommender = housekeeping.filter((e) => e.host === RECOMMENDER_HOST); + const summary = { + commands: steps.length, + allowed: countHosts(byKind(log.events, "allowed")), + unexpectedRefused: countHosts(refused), + expectedRefused: countHosts([...expected]), + housekeeping: countHosts(housekeeping), + upstream: countHosts(byKind(log.events, "upstream")), + }; + report({ summary }); + console.log(`L3 egress summary: ${JSON.stringify(summary, null, 2)}`); + const problems: string[] = []; + if (refused.length > 0) { + const by = steps + .filter((s) => s.events.some((e) => refused.includes(e))) + .map((s) => ` ${s.name}: ${hostsOf(s.events.filter((e) => refused.includes(e)))}`); + problems.push(`the guard refused hosts no test expected:\n${by.join("\n")}`); + } + if (recommender.length > 0) { + problems.push( + `az called ${RECOMMENDER_HOST} ${recommender.length} time(s): the bootstrap's ` + + "core.error_recommendation=off no longer holds" + ); + } + if (problems.length > 0) throw new Error(problems.join("\n")); + } + + describe("a light scenario on the emulator", () => { + const id = runId(); + const group = names.group(id); + const account = names.storage(id); + const container = names.container(id); + const vault = names.vault(id); + /** A storage account name that exists nowhere: its real-Azure host must be refused. */ + const outsider = names.storage(`${id}nx`); + const secretValue = `l3 dummy value ${id}`; + let subscription = ""; + const attempted = { group: false, vault: false }; + + const allowedHosts = (s: Step) => hostsOf(byKind(s.events, "allowed")); + const expectRelayed = (s: Step) => { + expect(byKind(s.events, "allowed").length).toBeGreaterThan(0); + for (const host of allowedHosts(s)) expect(isAllowedEgressHost(host)).toBe(true); + }; + + afterAll(async () => { + // The vault first: a group delete only soft-deletes it, and the emulator may be shared. + if (attempted.vault) { + await step("cleanup: keyvault delete", `keyvault delete --name ${vault}`); + await step("cleanup: keyvault purge", `keyvault purge --name ${vault}`); + } + if (attempted.group) { + await step("cleanup: group delete", `group delete --name ${group} --yes --no-wait`); + } + }); + + test("the first command after a fresh bootstrap: no update check, no WARNING", async () => { + attempted.group = true; + const s = await step( + "group create (first command, fresh bootstrap)", + `group create --name ${group} --location westeurope` + ); + expect(s.call.exitCode).toBe(0); + const created = json<{ name: string; id: string }>(s.call); + expect(created.name).toBe(group); + subscription = /^\/subscriptions\/([^/]+)\//.exec(created.id)?.[1] ?? ""; + expect(subscription).not.toBe(""); + + // The call bootstrapped a config dir that nothing had touched before, so its window holds + // the bootstrap's own az calls too: a missing seed shows up as an azcliprod CONNECT there. + expect(freshConfigDir).toBe(true); + expect(existsSync(path.join(env.configDir, MARKER_FILE))).toBe(true); + expect(existsSync(path.join(env.configDir, VERSION_CHECK_FILE))).toBe(true); + expect(s.events.filter((e) => e.host === UPDATE_CHECK_HOST)).toEqual([]); + // A malformed seed makes az log a WARNING on every command. + expect(s.call.envelope?.stderr ?? "").not.toMatch(/^WARNING:/m); + expectRelayed(s); + }); + + test("group show", async () => { + const s = await step("group show", `group show --name ${group} --query name -o tsv`); + expect(s.call.stdout.trim()).toBe(group); + expectRelayed(s); + }); + + test("storage account create (a long-running operation)", async () => { + const s = await step( + "storage account create", + `storage account create --name ${account} --resource-group ${group} --location westeurope --sku Standard_LRS` + ); + expect(json<{ name: string }>(s.call).name).toBe(account); + expectRelayed(s); + // Storage drops children created while the account is still being created. + let state = ""; + for (let i = 0; i < 60 && state !== "Succeeded"; i++) { + if (i > 0) await new Promise((r) => setTimeout(r, 5_000)); + const poll = await step( + "storage account show (provisioningState)", + `storage account show --name ${account} --resource-group ${group} --query provisioningState -o tsv` + ); + state = poll.call.stdout.trim(); + } + expect(state).toBe("Succeeded"); + }); + + test("storage container create (the blob data plane)", async () => { + const s = await step( + "storage container create", + `storage container create --name ${container} --account-name ${account} --auth-mode key` + ); + expect(json<{ created: boolean }>(s.call).created).toBe(true); + expectRelayed(s); + }); + + test("storage blob upload and list", async () => { + writeFileSync(path.join(env.workdir, "hello.txt"), `hello from L3 ${id}\n`); + const upload = await step( + "storage blob upload", + `storage blob upload --container-name ${container} --name hello.txt --file hello.txt --account-name ${account} --auth-mode key --overwrite` + ); + expect(upload.call.exitCode).toBe(0); + expectRelayed(upload); + const list = await step( + "storage blob list", + `storage blob list --container-name ${container} --account-name ${account} --auth-mode key --query "[].name" -o tsv` + ); + expect(list.call.stdout.split(/\r?\n/)).toContain("hello.txt"); + expectRelayed(list); + }); + + test("keyvault create, secret set and secret show (the vault data plane)", async () => { + attempted.vault = true; + const created = await step( + "keyvault create", + `keyvault create --name ${vault} --resource-group ${group} --location westeurope` + ); + expect(json<{ name: string }>(created.call).name).toBe(vault); + expectRelayed(created); + const set = await step( + "keyvault secret set", + `keyvault secret set --vault-name ${vault} --name l3-secret --value "${secretValue}"` + ); + expect(json<{ value: string }>(set.call).value).toBe(secretValue); + expectRelayed(set); + const shown = await step( + "keyvault secret show", + `keyvault secret show --vault-name ${vault} --name l3-secret --query value -o tsv` + ); + expect(shown.call.stdout.trim()).toBe(secretValue); + expectRelayed(shown); + }); + + test("rest with a relative URL", async () => { + const s = await step( + "rest (relative URL)", + `rest --method get --url "/subscriptions/${subscription}/resourceGroups/${group}?api-version=2022-09-01"` + ); + expect(json<{ name: string }>(s.call).name).toBe(group); + expectRelayed(s); + }); + + test("rest with an absolute management.azure.com URL is rewritten, not sent there", async () => { + const s = await step( + "rest (absolute management.azure.com URL)", + `rest --method get --url "https://management.azure.com/subscriptions/${subscription}/resourceGroups/${group}?api-version=2022-09-01"` + ); + expect(json<{ name: string }>(s.call).name).toBe(group); + expect(s.call.envelope?.notes.some((n) => n.startsWith(REWRITE_NOTE))).toBe(true); + expect(s.events.filter((e) => /management\.azure\.com$/i.test(e.host))).toEqual([]); + expectRelayed(s); + }); + + test("a --help page", async () => { + const s = await step("group create --help", "group create --help"); + expect(s.call.exitCode).toBe(0); + expect(s.call.text).toContain("--location"); + }); + + test("a third-party URL is a policy refusal: nothing is spawned, no CONNECT at all", async () => { + const s = await step( + "rest (third-party URL)", + "rest --method get --url https://graph.microsoft.com/v1.0/me" + ); + expect(s.call.text.startsWith("❌ **Address not allowed**")).toBe(true); + expect(s.call.text).toContain("graph.microsoft.com"); + expect(s.call.envelope).toBeUndefined(); + expect(s.events).toEqual([]); + }); + + test("positive control: a real-Azure data-plane host is refused by the guard and named", async () => { + const host = `${outsider}.blob.core.windows.net`; + const s = await step( + "storage container list (a real-Azure endpoint, refused)", + `storage container list --account-name ${outsider} --blob-endpoint https://${host} --account-key dGVzdA== -o json` + ); + const refusals = byKind(s.events, "refused"); + for (const e of refusals) if (e.host === host) expected.add(e); + // Only this host was refused, and the recorder saw it: an empty log would prove nothing. + expect(hostsOf(refusals)).toEqual([host]); + expect(s.call.classId).toBe("egress-refused"); + expect(s.call.text).toContain(host); + // Fail fast: without it the storage SDK retries for about 87 s. + expect(s.call.ms).toBeLessThan(30_000); + }); + + test("an unknown command (az 2.85 would report it to app.aladdin.microsoft.com)", async () => { + const s = await step("unknown command", "mcpl3nosuchgroup list"); + expect(s.call.ok).toBe(false); + expect(s.call.classId).toBe("unknown-command"); + expect(s.events.filter((e) => e.host === RECOMMENDER_HOST)).toEqual([]); + }); + }); + + describeCi("CI only: the GET and DELETE leak commands on the job's emulator", () => { + test("the fixture has GET and DELETE commands to replay", () => { + expect(REPLAYED.length).toBeGreaterThan(0); + expect(REPLAYED.every((c) => c.command.includes("https://management.azure.com"))).toBe(true); + }); + + test.each(REPLAYED)("$id ($method) reaches only the emulator", async (leak) => { + const s = await step(`leak ${leak.id} (${leak.method})`, leak.command); + // The exit code is not asserted: the targets are gone, and some answers are + // NotImplemented or validation errors. + expect(byKind(s.events, "refused")).toEqual([]); + expect(byKind(s.events, "allowed").length).toBeGreaterThan(0); + for (const host of hostsOf(byKind(s.events, "allowed"))) + expect(isAllowedEgressHost(host)).toBe(true); + expect(s.call.envelope?.notes.some((n) => n.startsWith(REWRITE_NOTE))).toBe(true); + }); + }); + + describeStop("CI only: the job's own emulator stops between two calls", () => { + let stopped = false; + + afterAll(async () => { + if (!stopped) return; + // Later files and steps expect a running emulator: start the same container again. + execFileSync("docker", ["start", CI_CONTAINER], { stdio: "pipe", timeout: 120_000 }); + /* eslint-disable @typescript-eslint/no-require-imports */ + const { getAzureEmulatorStatus } = + require("../../src/lib/azure/emulator") as typeof import("../../src/lib/azure/emulator"); + const { azureConfig } = + require("../../src/lib/azure/services") as typeof import("../../src/lib/azure/services"); + /* eslint-enable @typescript-eslint/no-require-imports */ + const deadline = Date.now() + RESTART_WAIT_MS; + let status = await getAzureEmulatorStatus(azureConfig()); + while (!status.ok && Date.now() < deadline) { + await new Promise((r) => setTimeout(r, 5_000)); + status = await getAzureEmulatorStatus(azureConfig()); + } + if (!status.ok) throw new Error(`the emulator did not come back: ${status.message}`); + }); + + test("the second call gets the emulator-down answer within about 5 s", async () => { + if (process.env.CI !== "true" && SHARED_EMULATOR_NAMES.has(CI_CONTAINER)) { + throw new Error(`refusing to stop ${CI_CONTAINER} outside CI: it is a shared emulator`); + } + const before = await step("group list (emulator up)", "group list -o json"); + expect(before.call.exitCode).toBe(0); + + execFileSync("docker", ["stop", CI_CONTAINER], { stdio: "pipe", timeout: 120_000 }); + stopped = true; + + const after = await step("group list (emulator stopped)", "group list -o json", { + retryNotReady: false, + }); + expect(after.call.ms).toBeLessThan(STOPPED_ANSWER_MS); + // With the guard on, `conn-refused` comes from the guard's upstream record; a call that + // starts after the stop is usually caught earlier, by the tool's own health preflight. + const connRefused = after.call.classId === "conn-refused"; + const preflight = + after.call.text.startsWith("❌ **LocalStack Azure Emulator Not Ready**") && + after.call.text.includes("is not running at"); + if (!connRefused && !preflight) { + throw new Error( + `expected conn-refused or the not-running preflight, got:\n${after.call.text.slice(0, 1000)}` + ); + } + expect(byKind(after.events, "refused")).toEqual([]); + }); + }); +}); diff --git a/tests/azure/evals/README.md b/tests/azure/evals/README.md new file mode 100644 index 0000000..5cc3b40 --- /dev/null +++ b/tests/azure/evals/README.md @@ -0,0 +1,209 @@ +# E2: composition evals for `localstack-azure-client` + +E2 measures whether Claude can do real Azure work through the server: it gets a task, calls +`localstack-azure-client` over MCP until it ends its turn, and then a verifier reads the outcome +from the emulator. + +- **No money is spent** by `--oracle`, `--negative`, `--dry-run` and `--list`, and by the unit + tests. Only a model run calls the Anthropic API, and it refuses to start without a key. +- **The gate:** at least 95% success at n=3, zero egress events, and the cost per completed task + recorded. + +## Quick start + +```bash +yarn build # dist/cli.js, the stdio server E2 drives + +node tests/azure/evals/run.mjs --list # the 50 tasks +node tests/azure/evals/run.mjs --dry-run --variant compact,long,help-tool,no-test-data + +# No model: each task's reference solution must pass its verifier, and setup alone must fail +# it (needs a running emulator and LOCALSTACK_AUTH_TOKEN). +node tests/azure/evals/run.mjs --oracle --out test-results/e2-oracle +node tests/azure/evals/run.mjs --negative --out test-results/e2-negative + +# The model run (needs an API key: see "What a run needs") +node tests/azure/evals/run.mjs --runs 3 --max-usd 20 --out test-results/e2 +``` + +**What a run needs** + +- Node.js 22.18 or newer: `run.mjs` loads the `.ts` modules with Node's built-in type + stripping (Node prints a `MODULE_TYPELESS_PACKAGE_JSON` warning once; `NODE_NO_WARNINGS=1` + hides it). +- A LocalStack Azure emulator on `LOCALSTACK_AZURE_PORT` (default 4566), started with + `CDN_CLASSIC_ALLOW_CREATE=1` (for `cdn-can-migrate`; `scripts/ci/azure-emulator-up.sh` sets it). +- `LOCALSTACK_AUTH_TOKEN` in the environment (the tool checks that it is set). +- An `az` the tool resolves (`LOCALSTACK_AZ_PATH`), with the curated extensions + (`node scripts/install-azure-extensions.mjs`). +- For a model run, an API key: `ANTHROPIC_API_KEY`, or `--key-file ` / + `ANTHROPIC_API_KEY_FILE` naming a file that holds only the key (mode 600 on POSIX). +- On a shared machine, give E2 its own CLI profile (`LOCALSTACK_AZ_CONFIG_DIR`) and keep + `--jobs 1`. + +## Options + +| Option | Default | Meaning | +| -------------------------------------------------- | ---------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `--runs N` | 1 | repetitions per task (and per variant) | +| `--max-usd X` | `$AZURE_EVALS_MAX_USD` or 20 | the total cap: no new run once the spend (pre-warms included) reaches X, and a running loop stops before its next request; every turn counts at once, for concurrent runs too | +| `--max-usd-per-run Y` | `$EVAL_MAX_USD_PER_RUN` or 1 | a run's agent loop stops (failure `spend_cap`) before a request once the run's own cost reaches Y | +| `--out DIR` | `test-results/e2` | results directory | +| `--tasks id,id` / `--tier T-A,T-B,T-APIM` | all | selection | +| `--model M` | `$EVAL_MODEL` or `claude-opus-5-5` | a model needs a price in `agent.ts` (`PRICES`), or the run refuses | +| `--effort E` | `$EVAL_EFFORT` or `high` | `output_config.effort` | +| `--variant v[,v]` | `compact` | see below; several variants interleave per task in a seeded order | +| `--all-tools` | off | offer every server tool, not only the Azure tool (`localstack-management` is withheld: it can stop the emulator) | +| `--key-file PATH` | | the key file (else `ANTHROPIC_API_KEY_FILE`, else `ANTHROPIC_API_KEY`) | +| `--phrasing alt\|0\|1` | `alt` | T-A has two phrasings: run _i_ uses phrasing _i_ mod 2, or a fixed one | +| `--verify-wait S` | 90 (10 with `--negative`) | how long a polling verifier waits for a state | +| `--jobs N` | `$EVAL_JOBS` or 1 | concurrent runs, each with its own agent and harness server | +| `--seed N` | 20260924 | the variant order | +| `--max-minutes M` | none | no new run after M minutes | +| `--no-prewarm` | | skip the `max_tokens: 0` cache pre-warm per variant | +| `--oracle` / `--negative` / `--dry-run` / `--list` | | the no-spend modes | + +Exit status: **0** passed (the gate for a model run; every expectation for `--oracle` and +`--negative`) and the final cleanup check was clean; **1** not passed, or stopped early (spend +cap, `--max-minutes`, a key that stopped working, Ctrl+C); **2** a usage or configuration error: +no API key for a model run, no price for the model, no `dist/cli.js`, no `LOCALSTACK_AUTH_TOKEN`, +Node.js too old. + +## How a run works + +``` +group create (harness session) +task setup (harness session: the task's fixtures, as az commands) +the agent loop | the oracle | nothing (--negative) (agent session) +the verifier (harness session; every answer recorded) +teardown, cleanup (harness session) +``` + +- **Two server sessions per worker**: the agent's (`node dist/cli.js`, what Claude or the + oracle calls) and the harness's own. Verification never uses the agent's session. Both get + the whole environment plus `LOCALSTACK_AZ_TEST_ENVELOPE=1`, minus every `ANTHROPIC_*` + variable (the key never reaches a server or its `az`). +- **The model never sees the test envelope.** The tool appends it as a second content item; + `agent.ts` (`splitEnvelope`) removes it before the `tool_result` is built and keeps it for + the record (exit code, class, egress refusals). +- **Egress events** are the tool's own reports: class `egress-refused`, or the note + "Note: the egress guard blocked a connection to ...". Housekeeping blocks (update checks the + tool always refuses) are recorded apart and are not events. +- **Cleanup** leaves nothing behind: it waits until nothing in the group is still provisioning, + removes the group's locks, deletes and purges its Key Vaults, deletes the group and waits, + purges soft-deleted vaults, APIM services and App Configuration stores the run named, and + checks the group is gone. At the end, an orphan sweep lists every resource type the tasks + create and deletes whatever still sits in one of the run's groups. Groups are named + `mcpe2---`. +- **Model settings** are fixed in `agent.ts`: `claude-opus-5-5`, `max_tokens` 16000, adaptive + thinking with summarized display, effort `high`, a cached system block plus top-level + automatic caching, and per-tier caps (T-A and T-APIM: 20 turns, 30 tool calls, 600 s; T-B: 40, + 60, 1800 s). The record keeps the billed cost and the cost re-priced as if the first turn + found the shared prefix cached ("warm") or not ("cold"); the gate uses the warm one. + +## Variants + +| Variant | What Claude gets | +| -------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `compact` | the server's own description, unchanged: the shipped default | +| `long` | a longer description in sections (Context, Input, Output, Rules, Examples) | +| `help-tool` | `compact` with "Run it with --help first" replaced by "Call `az_help` ...", plus a separate `az_help` tool that the runner runs as ` --help` through the tool | +| `no-test-data` | `compact` without ", so all data and secrets are local test data" | + +A variant whose anchor text is no longer in the server's description fails at start, rather than +silently testing the default. To compare a variant with the default, run both, for example +`--runs 3 --variant compact,long`, and read `summary.json`: `success_rate`, +`cost_per_completed_task_warm_usd`, `median_time_to_verified_ms`, `egress_events` and +`kv_crypto_refusals`. With two variants, `--max-usd` caps both together. + +## Output + +`--out DIR` holds: + +- `runs.jsonl`: one record per run, with the task, variant and mode, `success` (null for a setup + or infrastructure fault) and `failure` (`verify_failed`, `refusal`, `max_tokens`, + `cap_exceeded`, `spend_cap`, `setup_error`, `infra_error`, `account_error`, `api_error`, + `harness_error`; `verifier_passed_without_action` for `--negative`), the timings, turns, tool + calls, tokens and costs, the egress events, the verifier's result with every query it made, + `verify_input` (the slots and text the verifier read, so a verifier fixed later can re-score + the run offline), and the turn log and tool trace. +- `meta.json`: the settings (model, effort, thinking, prices, the tool definitions sent per + variant and their sizes, tasks, seed, jobs), the pre-warm usage, the spend, why it stopped. +- `summary.json`: per variant the success rate, costs, median time to verified state, egress + events, refusals, setup and infrastructure faults, the failing tasks, and the gate; for + `--oracle` / `--negative` the runs that met or missed their expectation. +- `cleanup.json`: the final orphan sweep and the leftover check (`clean`). +- `server-*.stderr.log`: each server's stderr. + +The API key never appears in any of them: it is read once, handed to the SDK client, removed from +the environment, and scrubbed from every error text and record before it is written. + +## The tasks + +50 tasks in three tiers; `node run.mjs --list` prints them. + +- **T-A** (28): single operations, each with two phrasings. +- **T-B** (8): multi-step goals, verified step by step (partial credit is recorded). +- **T-APIM** (14): API Management operations, each run on its own APIM service. + +| Tier | Tasks | +| ------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| T-A | `appconfig-kv-unlock`, `appconfig-check-keys`, `appconfig-regenerate-key`, `lock-delete`, `locks-list`, `role-definition-get`, `acr-login-server`, `acr-webhook-create`, `eventgrid-topic-regenerate-key`, `eventgrid-domain-disable-local-auth`, `eventhub-hub-auth-rule-rights`, `insights-activity-log-alert-create`, `kv-sign`, `kv-decrypt`, `kv-rotation-policy`, `kv-access-policy-grant`, `identity-create`, `identity-list`, `route-get`, `private-dns-zones-list`, `vnet-address-space`, `la-workspace-retention`, `deployment-sub-validate`, `sb-queue-regenerate-key`, `sb-dp-queue-max-delivery`, `storage-regenerate-key`, `disk-create`, `cdn-can-migrate` | +| T-B | `tb-storage-blob`, `tb-keyvault-secret-sign`, `tb-servicebus-topic`, `tb-resource-graph-tags`, `tb-tag-audit`, `tb-mysql-firewall`, `tb-eventgrid-to-eventhub`, `tb-webapp-settings` | +| T-APIM | `apim-group-user-check`, `apim-workspace-product-update`, `apim-tenant-access`, `apim-git-regenerate-primary`, `apim-api-create`, `apim-operation-delete`, `apim-operation-policy`, `apim-tag-assign-api`, `apim-tag-detach-product`, `apim-api-tag-descriptions`, `apim-graphql-resolver`, `apim-certificate-create`, `apim-oidc-secret`, `apim-migrate-stv2` | + +Every fixture runs as `az` commands through the harness's session (`az rest` where a resource has +no command), and the verifiers read state with `az rest` (relative URLs) and `az` data-plane +commands (`keyvault key verify`, `keyvault secret show`). With the oracle there is no answer, so +the claim verifiers read the oracle's transcript: one block per call, a mark line +(`ORACLE ✅ az `, ❌ on failure) and az's output fenced. + +## Tests + +`npx jest tests/azure/evals` (part of the default suite; no emulator, no network, no key): + +- `verifiers.test.ts`: each verifier against recorded answers, pass and fail; the query, record + and replay helpers; offline checks of the answer readers. +- `defects.test.ts`: seven answer-reading edge cases (a negation such as "doesn't", a table + instead of sentences, a host name with or without its port, and more): the naive reading gets + a correct answer wrong, the verifier reads it right, and a wrong answer still fails. +- `tasks.test.ts`: the catalogue (50 tasks, unique ids, the pitfall and crypto tasks present), + and every task's verifier replayed against answers recorded in live `--oracle` (must pass) + and `--negative` (must fail) runs: `fixtures/recorded-verifications.json`. +- `agent.test.ts`: the loop against a fake client (the request settings, the envelope never + sent, refusal, `pause_turn`, the caps, the per-run spend cap, error classes), the cost. +- `harness.test.ts`: a run in each mode against fake sessions, the setup-error path, the + cleanup order, the `az_help` dispatch, the spend cap, the orphan sweep, the gate. +- `keys.test.ts`: where the key is read from, how a bad file is refused, the scrubbing, the + child environment, and `run.mjs` without a key: exit 2 with the message, nothing written, + and not one socket opened (a preloaded hook fails on any connect). +- `variants.test.ts`: each variant changes only what it says, against the server's real + description. +- `run.test.ts`: Node loads every module through type stripping (`--list`), and bad options + exit 2 before anything starts. + +**Files.** `run.mjs` (the entry point), `harness.ts` (one run, cleanup, the orphan sweep, the +summary), `agent.ts` (the Messages API loop, envelope stripping, egress events, cost), +`verifiers.ts` (verifiers, fixture helpers, record and replay), `tasks-ta.ts`, `tasks-tb.ts`, +`tasks-tapim.ts` (the tasks), `variants.ts`, `keys.ts`, `types.ts`, `record-fixtures.mjs`. Node +loads the `.ts` modules unbundled, so none of them imports another local module at runtime +(only `import type`): `run.mjs` passes the verifier library to the task modules, and the other +modules to the harness. + +**Refreshing the recordings** after changing a task, or when the emulator's answers change: + +```bash +node tests/azure/evals/run.mjs --oracle --out test-results/e2-oracle +node tests/azure/evals/run.mjs --negative --out test-results/e2-negative +node tests/azure/evals/record-fixtures.mjs test-results/e2-oracle/runs.jsonl test-results/e2-negative/runs.jsonl +``` + +`record-fixtures.mjs` keeps, per task and mode, the latest record that met its expectation: its +`verify_input` (slots and text) and `verifier.queries` (JSON stdout minified; connection-string +key parts in the text, stdout and stderr, which no verifier reads, redacted). All values are +local emulator test data. + +## CI + +CI never calls a model: model runs are local only. CI runs the offline tests above, with the +unit suite. diff --git a/tests/azure/evals/agent.test.ts b/tests/azure/evals/agent.test.ts new file mode 100644 index 0000000..67f8897 --- /dev/null +++ b/tests/azure/evals/agent.test.ts @@ -0,0 +1,549 @@ +/** + * The E2 agent loop against a fake Messages client (no network, no key): the request + * settings are the fixed ones, the model never sees the test envelope, refusals and + * pause_turn are handled, the caps and the spend cap stop the loop, errors are classified, + * and the cost is computed from the fixed prices. + */ +import type Anthropic from "@anthropic-ai/sdk"; +import * as A from "./agent"; +import type { McpResult, Price } from "./types"; + +const PRICE: Price = A.PRICES["claude-opus-5-5"]; +const ENVELOPE = { + exitCode: 0, + stdout: '{\r\n "name": "ENVELOPE-ONLY-STDOUT-MARKER"\r\n}\r\n', + stderr: "", + notes: [], + classId: null, + truncated: false, + stoppedByTool: false, + emulatorSession: "s-1", +}; + +function mcp(text: string, envelope: unknown = ENVELOPE): McpResult { + const content = [{ type: "text", text }]; + if (envelope) content.push({ type: "text", text: JSON.stringify(envelope) }); + return { content }; +} + +type Resp = Partial | Error; + +function usage(input = 10, output = 5, cacheWrite = 0, cacheRead = 0): Anthropic.Usage { + const u = { + input_tokens: input, + output_tokens: output, + cache_creation_input_tokens: cacheWrite, + cache_read_input_tokens: cacheRead, + }; + return u as unknown as Anthropic.Usage; +} + +function fakeClient(responses: Resp[]) { + const calls: Anthropic.MessageCreateParamsNonStreaming[] = []; + const client: A.MessagesClient = { + messages: { + create: async (params) => { + calls.push(JSON.parse(JSON.stringify(params))); + const r = responses.shift(); + if (!r) throw new Error("no scripted response left"); + if (r instanceof Error) throw r; + return { + id: `msg_${calls.length}`, + type: "message", + role: "assistant", + model: params.model, + content: [], + stop_reason: "end_turn", + stop_sequence: null, + stop_details: null, + usage: usage(), + ...r, + } as unknown as Anthropic.Message; + }, + }, + }; + return { client, calls }; +} + +const toolUse = (id: string, command: string) => + ({ + type: "tool_use", + id, + name: "localstack-azure-client", + input: { command }, + }) as unknown as Anthropic.ContentBlock; +const text = (t: string) => + ({ type: "text", text: t, citations: null }) as unknown as Anthropic.ContentBlock; + +function options(client: A.MessagesClient, over: Partial = {}): A.AgentOptions { + return { + client, + model: "claude-opus-5-5", + effort: "high", + tools: [ + { + name: "localstack-azure-client", + description: "Run az.", + input_schema: { type: "object", properties: { command: { type: "string" } } }, + }, + ], + prompt: "List the groups.", + caps: A.CAPS["T-A"], + price: PRICE, + perRunUsd: 1, + remainingUsd: () => 20, + callTool: async () => { + const s = A.splitEnvelope(mcp('{\n "name": "g1"\n}')); + return { + text: s.text, + isError: s.isError, + ms: 3, + envelope: s.envelope, + command: "group list", + }; + }, + classifyError: () => "api_error", + scrub: (s) => s, + ...over, + }; +} + +describe("splitEnvelope: the model never sees the test envelope", () => { + test("the envelope (last text item) is split off", () => { + const s = A.splitEnvelope(mcp("visible")); + expect(s.text).toBe("visible"); + expect(s.envelope?.stdout).toContain("ENVELOPE-ONLY-STDOUT-MARKER"); + }); + + test("a tool refusal has no envelope: every item stays visible", () => { + const s = A.splitEnvelope({ content: [{ type: "text", text: "❌ **Command Not Allowed**" }] }); + expect(s).toEqual({ text: "❌ **Command Not Allowed**", envelope: null, isError: false }); + }); + + test("a last item that is JSON but not an envelope stays visible", () => { + const s = A.splitEnvelope({ + content: [ + { type: "text", text: "a" }, + { type: "text", text: '{"x": 1}' }, + ], + }); + expect(s.text).toBe('a\n{"x": 1}'); + expect(s.envelope).toBeNull(); + }); + + test("isError passes through", () => { + expect(A.splitEnvelope({ content: [{ type: "text", text: "x" }], isError: true }).isError).toBe( + true + ); + }); +}); + +describe("egress events", () => { + test("an egress-refused failure names the host", () => { + const env = { ...ENVELOPE, exitCode: 1, classId: "egress-refused", notes: [] }; + const t = + "❌ **Command Failed** (exit 1, egress-refused)\n\nBlocked a connection to `management.azure.com:443`: this tool only lets `az` talk to the local emulator."; + expect(A.egressOf(2, "rest --url https://x", t, env).events).toEqual([ + { + call: 2, + command: "rest --url https://x", + classId: "egress-refused", + hosts: ["management.azure.com:443"], + source: "classId", + }, + ]); + }); + + test("a blocked host on a successful command is an event; housekeeping blocks are not", () => { + const env = { + ...ENVELOPE, + notes: [ + "Note: the egress guard blocked a connection to `login.microsoftonline.com`; the command still succeeded.", + "Note: the egress guard also blocked `azcliprod.blob.core.windows.net`, which the tool always refuses (housekeeping calls such as update checks); these blocks are expected.", + ], + }; + const r = A.egressOf(0, "c", "ok", env); + expect(r.events.map((e) => e.hosts)).toEqual([["login.microsoftonline.com"]]); + expect(r.housekeeping).toEqual(["azcliprod.blob.core.windows.net"]); + }); + + test("no refusal, no event", () => { + expect(A.egressOf(0, "c", "ok", ENVELOPE)).toEqual({ events: [], housekeeping: [] }); + }); +}); + +describe("runAgent", () => { + test("the request carries the fixed settings", async () => { + const { client, calls } = fakeClient([{ content: [text("done")], stop_reason: "end_turn" }]); + const run = await A.runAgent(options(client)); + expect(run.final_text).toBe("done"); + expect(run.failure).toBeNull(); + const p = calls[0] as unknown as Record; + expect(p.model).toBe("claude-opus-5-5"); + expect(p.max_tokens).toBe(16000); + expect(p.thinking).toEqual({ type: "adaptive", display: "summarized" }); + expect(p.output_config).toEqual({ effort: "high" }); + expect(p.cache_control).toEqual({ type: "ephemeral" }); + expect(p.system).toEqual([ + { type: "text", text: A.SYSTEM_PROMPT, cache_control: { type: "ephemeral" } }, + ]); + expect(p.tool_choice).toBeUndefined(); + expect(p.messages).toEqual([{ role: "user", content: "List the groups." }]); + }); + + test("tool_use -> tool_result without the envelope, and the assistant turn replayed unmodified", async () => { + const first = { + content: [text("Checking."), toolUse("tu_1", "group list")], + stop_reason: "tool_use" as const, + usage: usage(100, 20), + }; + const { client, calls } = fakeClient([ + first, + { content: [text("Found g1.")], stop_reason: "end_turn", usage: usage(50, 10, 0, 100) }, + ]); + const run = await A.runAgent(options(client)); + expect(run.final_text).toBe("Found g1."); + expect(run.tool_calls).toHaveLength(1); + expect(run.tool_calls[0]).toMatchObject({ + name: "localstack-azure-client", + args: { command: "group list" }, + exit_code: 0, + ok: true, + }); + const second = calls[1].messages; + expect(second[1]).toEqual({ + role: "assistant", + content: JSON.parse(JSON.stringify(first.content)), + }); + expect(second[2]).toEqual({ + role: "user", + content: [ + { + type: "tool_result", + tool_use_id: "tu_1", + content: '{\n "name": "g1"\n}', + is_error: false, + }, + ], + }); + // Nothing of the envelope reaches the API, in any request. + const sent = JSON.stringify(calls); + expect(sent).not.toContain("ENVELOPE-ONLY-STDOUT-MARKER"); + expect(sent).not.toContain("emulatorSession"); + expect(sent).not.toContain('\\"exitCode\\"'); + expect(run.turns.map((t) => t.stop_reason)).toEqual(["tool_use", "end_turn"]); + }); + + test("every tool_use of a turn goes back in one user message", async () => { + const { client, calls } = fakeClient([ + { + content: [toolUse("a", "group list"), toolUse("b", "account show")], + stop_reason: "tool_use", + }, + { content: [text("ok")], stop_reason: "end_turn" }, + ]); + const run = await A.runAgent(options(client)); + expect(run.tool_calls).toHaveLength(2); + const results = calls[1].messages[2].content as Anthropic.ToolResultBlockParam[]; + expect(results.map((r) => r.tool_use_id)).toEqual(["a", "b"]); + }); + + test("egress events of a tool call are collected", async () => { + const env = { ...ENVELOPE, exitCode: 1, classId: "egress-refused" }; + const { client } = fakeClient([ + { + content: [toolUse("a", "rest --url https://management.azure.com/x")], + stop_reason: "tool_use", + }, + { content: [text("blocked")], stop_reason: "end_turn" }, + ]); + const run = await A.runAgent( + options(client, { + callTool: async () => { + const s = A.splitEnvelope( + mcp( + "❌ x\n\nBlocked a connection to `management.azure.com`: this tool only lets `az` talk", + env + ) + ); + return { + text: s.text, + isError: false, + ms: 1, + envelope: s.envelope, + command: "rest --url https://management.azure.com/x", + }; + }, + }) + ); + expect(run.egress).toHaveLength(1); + expect(run.egress[0].hosts).toEqual(["management.azure.com"]); + }); + + test("a refusal is recorded with its stop details", async () => { + const details = { type: "refusal", category: "cyber", explanation: "declined" }; + const { client } = fakeClient([ + { + content: [], + stop_reason: "refusal" as Anthropic.StopReason, + stop_details: details as never, + }, + ]); + const run = await A.runAgent(options(client)); + expect(run.refusal).toBe(true); + expect(run.failure).toBe("refusal"); + expect(run.stop_details).toEqual(details); + expect(run.turns[0].stop_details).toEqual(details); + }); + + test("pause_turn continues with the paused turn in the history", async () => { + const paused = { content: [text("thinking...")], stop_reason: "pause_turn" as const }; + const { client, calls } = fakeClient([ + paused, + { content: [text("done")], stop_reason: "end_turn" }, + ]); + const run = await A.runAgent(options(client)); + expect(run.final_text).toBe("done"); + expect(calls[1].messages).toHaveLength(2); + expect(calls[1].messages[1].role).toBe("assistant"); + }); + + test("max_tokens is a failure", async () => { + const { client } = fakeClient([{ content: [text("cut")], stop_reason: "max_tokens" }]); + expect((await A.runAgent(options(client))).failure).toBe("max_tokens"); + }); + + test("the turn cap stops a loop that never ends", async () => { + const loop = () => ({ + content: [toolUse(`t${Math.random()}`, "group list")], + stop_reason: "tool_use" as const, + }); + const { client, calls } = fakeClient([loop(), loop(), loop(), loop()]); + const run = await A.runAgent( + options(client, { caps: { turns: 3, toolCalls: 30, seconds: 600 } }) + ); + expect(run.failure).toBe("cap_exceeded"); + expect(calls).toHaveLength(3); + }); + + test("the per-run spend cap stops the loop before the next request", async () => { + // 100k output tokens at $20/M = $2 on the first turn, over a $1 per-run cap. + const big = { + content: [toolUse("a", "group list")], + stop_reason: "tool_use" as const, + usage: usage(10, 100_000), + }; + const { client, calls } = fakeClient([ + big, + { content: [text("never sent")], stop_reason: "end_turn" }, + ]); + const run = await A.runAgent(options(client, { perRunUsd: 1 })); + expect(run.failure).toBe("spend_cap"); + expect(calls).toHaveLength(1); + }); + + test("the campaign's remaining budget stops the loop too, and a spent cap sends nothing", async () => { + const { client, calls } = fakeClient([{ content: [text("x")], stop_reason: "end_turn" }]); + const run = await A.runAgent(options(client, { remainingUsd: () => 0 })); + expect(run.failure).toBe("spend_cap"); + expect(calls).toHaveLength(0); + }); + + test("each turn's cost is reported as it lands, so a concurrent run's spend stops this one", async () => { + // Each turn: (1000 * $4 + 100 * $20) / 1M = $0.006. A concurrent run spends $0.01 per + // tool call of this one; the campaign cap is $0.02. + const turn = () => ({ + content: [toolUse(`t${Math.random()}`, "group list")], + stop_reason: "tool_use" as const, + usage: usage(1000, 100), + }); + const { client, calls } = fakeClient([turn(), turn(), turn()]); + const budget = { max: 0.02, spent: 0 }; + const costs: number[] = []; + const run = await A.runAgent( + options(client, { + remainingUsd: () => budget.max - budget.spent, + onCost: (usd) => { + costs.push(usd); + budget.spent += usd; + }, + callTool: async () => { + budget.spent += 0.01; // the other run + return { text: "[]", isError: false, ms: 1, envelope: null }; + }, + }) + ); + expect(costs).toEqual([0.006, 0.006]); + expect(calls).toHaveLength(2); + expect(run.failure).toBe("spend_cap"); + }); + + test("an API error is classified and scrubbed", async () => { + const secret = "sk-ant-api03-SECRET"; + const { client } = fakeClient([new Error(`bad key ${secret}`)]); + const run = await A.runAgent( + options(client, { + classifyError: () => "account_error", + scrub: (s) => s.split(secret).join("[redacted]"), + }) + ); + expect(run.failure).toBe("account_error"); + expect(run.stop).toBe("Error: bad key [redacted]"); + }); + + test("an unknown tool's call reaches the dispatcher (which answers the error)", async () => { + const { client } = fakeClient([ + { + content: [ + { + type: "tool_use", + id: "x", + name: "nope", + input: {}, + } as unknown as Anthropic.ContentBlock, + ], + stop_reason: "tool_use", + }, + { content: [text("ok")], stop_reason: "end_turn" }, + ]); + const seen: string[] = []; + await A.runAgent( + options(client, { + callTool: async (name) => { + seen.push(name); + return { + text: "❌ Unknown tool 'nope' for this run", + isError: true, + ms: 0, + envelope: null, + }; + }, + }) + ); + expect(seen).toEqual(["nope"]); + }); +}); + +describe("prewarm", () => { + test("max_tokens 0, the same thinking and effort, no top-level cache_control", async () => { + const { client, calls } = fakeClient([ + { content: [], stop_reason: "max_tokens", usage: usage(3, 0, 1200, 0) }, + ]); + const pw = await A.prewarm(client, "claude-opus-5-5", "high", options(client).tools); + expect(pw.prefix_tokens).toBe(1200); + const p = calls[0] as unknown as Record; + expect(p.max_tokens).toBe(0); + expect(p.thinking).toEqual({ type: "adaptive", display: "summarized" }); + expect(p.output_config).toEqual({ effort: "high" }); + expect(p.cache_control).toBeUndefined(); + }); +}); + +describe("cost", () => { + const turn = (input: number, output: number, cache_write: number, cache_read: number) => ({ + usage: { input, output, cache_write, cache_read }, + }); + + test("prices: claude-opus-5-5 is $4 / $20, cache $5 / $0.20", () => { + expect(A.PRICES["claude-opus-5-5"]).toEqual({ + input: 4, + output: 20, + cache_write: 5, + cache_read: 0.2, + }); + expect(A.priceFor("claude-unknown")).toBeUndefined(); + }); + + test("totals and the initial tokens of the first request", () => { + expect(A.totals([turn(10, 5, 100, 0), turn(20, 7, 0, 100)])).toEqual({ + input: 30, + output: 12, + cache_write: 100, + cache_read: 100, + initial: 110, + }); + expect(A.totals([])).toEqual({ + input: 0, + output: 0, + cache_write: 0, + cache_read: 0, + initial: 0, + }); + }); + + test("billed cost", () => { + // (1000*4 + 500*20 + 2000*5 + 10000*0.2) / 1e6 = 0.026 + expect(A.costUsd([turn(1000, 500, 2000, 10000)], PRICE)).toBe(0.026); + }); + + test("warm and cold re-pricing of the first turn only", () => { + const turns = [turn(100, 50, 2000, 0), turn(10, 10, 0, 2100)]; + // warm: the 2,000-token prefix read instead of written + expect(A.pricedCostUsd(turns, 2000, "warm", PRICE)).toBe( + A.costUsd([turn(100, 50, 0, 2000), turns[1]], PRICE) + ); + // cold: a prefix that was read is priced as written + const read = [turn(100, 50, 0, 2000)]; + expect(A.pricedCostUsd(read, 2000, "cold", PRICE)).toBe( + A.costUsd([turn(100, 50, 2000, 0)], PRICE) + ); + expect(A.pricedCostUsd([], 2000, "warm", PRICE)).toBe(0); + expect(A.pricedCostUsd(turns, null, "warm", PRICE)).toBeNull(); + }); + + test("usageOf reads the SDK's usage fields", () => { + expect( + A.usageOf({ + input_tokens: 1, + output_tokens: 2, + cache_creation_input_tokens: 3, + cache_read_input_tokens: 4, + }) + ).toEqual({ + input: 1, + output: 2, + cache_write: 3, + cache_read: 4, + }); + expect(A.usageOf(null)).toEqual({ input: 0, output: 0, cache_write: 0, cache_read: 0 }); + }); +}); + +describe("error classification (account and transient errors)", () => { + class APIError extends Error { + constructor( + public status?: number, + message = "x", + public type?: string + ) { + super(message); + } + } + class AuthenticationError extends APIError {} + class PermissionDeniedError extends APIError {} + class APIConnectionError extends APIError {} + class RateLimitError extends APIError {} + class InternalServerError extends APIError {} + const classify = A.makeErrorClassifier({ + AuthenticationError, + PermissionDeniedError, + APIConnectionError, + RateLimitError, + InternalServerError, + }); + + test.each([ + [new AuthenticationError(401), "account_error"], + [new PermissionDeniedError(403), "account_error"], + [new APIError(402), "account_error"], + [new APIError(400, "Your credit balance is too low"), "account_error"], + [new APIError(400, "x", "billing_error"), "account_error"], + [new APIConnectionError(undefined), "infra_error"], + [new RateLimitError(429), "infra_error"], + [new InternalServerError(529), "infra_error"], + [new APIError(503), "infra_error"], + [new APIError(400, "tools.0: invalid"), "api_error"], + ] as Array<[Error, A.ErrorClass]>)("%s -> %s", (e, want) => { + expect(classify(e)).toBe(want); + }); +}); diff --git a/tests/azure/evals/agent.ts b/tests/azure/evals/agent.ts new file mode 100644 index 0000000..fe31e55 --- /dev/null +++ b/tests/azure/evals/agent.ts @@ -0,0 +1,518 @@ +/** + * The E2 agent loop and its bookkeeping: the loop, the cache pre-warm, the cost, and the + * fixed settings. + * + * The loop is hand-rolled on the Messages API, with fixed settings (model + * claude-opus-5-5, max_tokens 16000, adaptive thinking with summarized + * display, effort high, a cached system block plus top-level automatic caching, the + * assistant turn replayed unmodified). The client is injected: run.mjs passes the official + * SDK's client, the unit tests a fake. This module imports no local module at runtime and + * only types from the SDK, so run.mjs can load it with Node's type stripping. + * + * The model never sees the tool's test envelope (LOCALSTACK_AZ_TEST_ENVELOPE=1): + * `splitEnvelope` removes it from every tool result before the tool_result is built, and + * the runner keeps it for the record (exit code, class, egress refusals). + */ +import type Anthropic from "@anthropic-ai/sdk"; +import type { + Caps, + EgressEvent, + Envelope, + McpResult, + Price, + ToolDef, + TurnLog, + Usage, +} from "./types"; + +// ── fixed settings ──────────────────────────────────────────────────────────── + +export const DEFAULT_MODEL = "claude-opus-5-5"; +export const DEFAULT_EFFORT = "high"; +export const MAX_TOKENS = 16_000; +export const THINKING = { type: "adaptive", display: "summarized" } as const; +export const TOOL_CALL_TIMEOUT_S = 300; + +/** The system prompt, identical for every variant. */ +export const SYSTEM_PROMPT = + "You operate a local Azure emulator (LocalStack for Azure) on behalf of a developer. " + + "Everything you create lives in subscription 00000000-0000-0000-0000-000000000000; the " + + "default location is westeurope unless the request says otherwise. Use the tools you have " + + "been given to carry out the request. Do not ask the developer questions: if something is " + + "ambiguous, make a reasonable choice and say what you chose. When you are finished, reply " + + "with a short summary that states the outcome and names every resource you created, " + + "changed, read or deleted."; + +export const CAPS: Record<"T-A" | "T-B", Caps> = { + "T-A": { turns: 20, toolCalls: 30, seconds: 600 }, + "T-B": { turns: 40, toolCalls: 60, seconds: 1800 }, +}; + +/** + * USD per million tokens, fixed so that runs stay comparable; the cache write is the + * standard 1.25x (5-minute) one, except where a model's row says otherwise. + * A model without a price cannot run: the spend cap needs one. + */ +export const PRICES: Record = { + "claude-opus-5-5": { input: 4.0, output: 20.0, cache_write: 5.0, cache_read: 0.2 }, + "claude-opus-5": { input: 5.0, output: 25.0, cache_write: 6.25, cache_read: 0.5 }, + "claude-fable-5-1": { input: 10.0, output: 50.0, cache_write: 12.5, cache_read: 0.25 }, + "claude-sonnet-5": { input: 2.0, output: 10.0, cache_write: 2.5, cache_read: 0.2 }, + "claude-haiku-4-5": { input: 1.0, output: 5.0, cache_write: 1.25, cache_read: 0.1 }, +}; + +export function priceFor(model: string): Price | undefined { + return PRICES[model]; +} + +// ── tokens and cost ─────────────────────────────────────────────────────────── + +interface UsageLike { + input_tokens?: number | null; + output_tokens?: number | null; + cache_creation_input_tokens?: number | null; + cache_read_input_tokens?: number | null; +} + +export function usageOf(u: UsageLike | null | undefined): Usage { + return { + input: u?.input_tokens ?? 0, + output: u?.output_tokens ?? 0, + cache_write: u?.cache_creation_input_tokens ?? 0, + cache_read: u?.cache_read_input_tokens ?? 0, + }; +} + +export interface Totals extends Usage { + /** "Initial tokens": the whole first request (system + tool catalogue + prompt). */ + initial: number; +} + +export function totals(turns: ReadonlyArray<{ usage: Usage }>): Totals { + const t = { input: 0, output: 0, cache_write: 0, cache_read: 0 }; + for (const turn of turns) { + t.input += turn.usage.input; + t.output += turn.usage.output; + t.cache_write += turn.usage.cache_write; + t.cache_read += turn.usage.cache_read; + } + const first = turns[0]?.usage; + return { ...t, initial: first ? first.input + first.cache_write + first.cache_read : 0 }; +} + +function round6(x: number): number { + return Math.round(x * 1e6) / 1e6; +} + +/** Billed cost: each turn priced exactly as the API reported it. */ +export function costUsd(turns: ReadonlyArray<{ usage: Usage }>, price: Price): number { + const t = totals(turns); + return round6( + (t.input * price.input + + t.output * price.output + + t.cache_write * price.cache_write + + t.cache_read * price.cache_read) / + 1_000_000 + ); +} + +/** + * Cost as if the first turn found the shared prefix (tool catalogue + system prompt) + * already cached ("warm") or not cached ("cold"); later turns are priced as billed. The + * gate's cost per completed task uses the warm price. + */ +export function pricedCostUsd( + turns: ReadonlyArray<{ usage: Usage }>, + prefixTokens: number | null | undefined, + state: "warm" | "cold", + price: Price +): number | null { + if (turns.length === 0) return 0; + if (prefixTokens === null || prefixTokens === undefined) return null; + const first = { ...turns[0].usage }; + if (state === "warm") { + let need = Math.max(0, prefixTokens - first.cache_read); + for (const src of ["cache_write", "input"] as const) { + const moved = Math.min(need, first[src]); + first[src] -= moved; + first.cache_read += moved; + need -= moved; + } + } else { + const moved = Math.min(prefixTokens, first.cache_read); + first.cache_read -= moved; + first.cache_write += moved; + } + return costUsd([{ usage: first }, ...turns.slice(1)], price); +} + +// ── the test envelope and egress events ─────────────────────────────────────── + +function isEnvelope(v: unknown): v is Envelope { + return ( + typeof v === "object" && + v !== null && + "exitCode" in v && + "stdout" in v && + "stderr" in v && + "classId" in v + ); +} + +export interface SplitResult { + /** What the model sees: the tool's text items without the envelope. */ + text: string; + envelope: Envelope | null; + isError: boolean; +} + +/** + * Splits an MCP result into the model-visible text and the test envelope (the last text + * item, when it is the envelope JSON). Only the text goes into a tool_result. + */ +export function splitEnvelope(result: McpResult | null | undefined): SplitResult { + const items = (result?.content ?? []).filter((c) => c.type === "text").map((c) => c.text ?? ""); + let envelope: Envelope | null = null; + if (items.length >= 2) { + try { + const parsed: unknown = JSON.parse(items[items.length - 1]); + if (isEnvelope(parsed)) { + envelope = parsed; + items.pop(); + } + } catch { + // not an envelope: every item stays visible + } + } + return { text: items.join("\n"), envelope, isError: Boolean(result?.isError) }; +} + +const BLOCKED_NOTE = "Note: the egress guard blocked a connection to"; +const HOUSEKEEPING_NOTE = "Note: the egress guard also blocked"; + +function hostsIn(text: string): string[] { + return [...text.matchAll(/`([^`]+)`/g)].map((m) => m[1]); +} + +/** + * The egress refusals of one tool call: an `egress-refused` failure class, or a note that + * the guard blocked a host while the command still succeeded. Housekeeping blocks (update + * checks the tool always refuses) are not egress events; they are returned apart. + */ +export function egressOf( + call: number, + command: string, + text: string, + envelope: Envelope | null +): { events: EgressEvent[]; housekeeping: string[] } { + const events: EgressEvent[] = []; + const housekeeping: string[] = []; + const notes = envelope?.notes ?? []; + if (envelope?.classId === "egress-refused") { + const line = /Blocked (.*?): this tool only lets/.exec(text); + events.push({ + call, + command, + classId: envelope.classId, + hosts: line ? hostsIn(line[1]) : [], + source: "classId", + }); + } + for (const note of notes) { + if (note.startsWith(BLOCKED_NOTE)) { + events.push({ + call, + command, + classId: envelope?.classId ?? null, + hosts: hostsIn(note), + source: "note", + }); + } else if (note.startsWith(HOUSEKEEPING_NOTE)) { + housekeeping.push(...hostsIn(note)); + } + } + if (!envelope && text.includes(BLOCKED_NOTE)) { + const note = text.slice(text.indexOf(BLOCKED_NOTE)).split("\n")[0]; + events.push({ call, command, classId: null, hosts: hostsIn(note), source: "note" }); + } + return { events, housekeeping }; +} + +// ── the agent loop ──────────────────────────────────────────────────────────── + +/** What a tool call returns to the loop: the model-visible text plus the record. */ +export interface ToolOutcome { + text: string; + isError: boolean; + ms: number; + envelope: Envelope | null; + /** The command the Azure tool ran, when the call went to it. */ + command?: string; +} + +export interface ToolTrace { + name: string; + args: unknown; + ms: number; + is_error: boolean; + ok: boolean; + head: string; + exit_code?: number | null; + class_id?: string | null; + truncated?: boolean; + stopped_by_tool?: boolean; + stdout_chars?: number; +} + +export interface AgentRun { + final_text: string; + stop: string; + /** cap_exceeded | spend_cap | account_error | infra_error | api_error | any stop_reason but end_turn. */ + failure: string | null; + refusal: boolean; + stop_details: unknown; + turns: TurnLog[]; + tool_calls: ToolTrace[]; + egress: EgressEvent[]; + housekeeping: string[]; + wall_ms: number; +} + +/** The part of the SDK client the loop uses (the tests pass a fake). */ +export interface MessagesClient { + messages: { + create(params: Anthropic.MessageCreateParamsNonStreaming): Promise; + }; +} + +export type ErrorClass = "account_error" | "infra_error" | "api_error"; + +type ErrorCtor = abstract new (...args: never[]) => unknown; + +/** The SDK's typed error classes the classifier needs (run.mjs passes the real ones). */ +export interface ErrorClasses { + AuthenticationError: ErrorCtor; + PermissionDeniedError: ErrorCtor; + APIConnectionError: ErrorCtor; + RateLimitError: ErrorCtor; + InternalServerError: ErrorCtor; +} + +/** + * Error classes, on the SDK's typed errors. An + * account error (invalid or revoked key, no credit, a spend limit) stops the campaign + * unscored; an infrastructure error (connection, 429, 5xx after the SDK's retries) is not + * scored either; anything else (a 400) is the run's own failure. + */ +export function makeErrorClassifier(c: ErrorClasses): (e: unknown) => ErrorClass { + return (e) => { + const err = e as { + status?: unknown; + type?: unknown; + error?: { error?: { type?: unknown } }; + message?: unknown; + }; + if (e instanceof c.AuthenticationError || e instanceof c.PermissionDeniedError) + return "account_error"; + const type = err?.type ?? err?.error?.error?.type; + const text = String(err?.message ?? "").toLowerCase(); + if (err?.status === 402 || type === "billing_error") return "account_error"; + if ( + ["credit balance", "billing", "spend limit", "usage limit", "quota"].some((w) => + text.includes(w) + ) + ) + return "account_error"; + if ( + e instanceof c.APIConnectionError || + e instanceof c.RateLimitError || + e instanceof c.InternalServerError + ) + return "infra_error"; + const status = err?.status; + if (typeof status === "number" && (status === 429 || status >= 500)) return "infra_error"; + return "api_error"; + }; +} + +export interface AgentOptions { + client: MessagesClient; + model: string; + effort: string; + maxTokens?: number; + system?: string; + tools: ToolDef[]; + prompt: string; + caps: Caps; + price: Price; + /** Stops the loop before a request once the run's cost reaches it. */ + perRunUsd: number; + /** + * What is left of the campaign's cap, net of every turn already billed, this run's and + * any concurrent run's (`onCost` reports each turn as it lands). + */ + remainingUsd: () => number; + /** Called with each turn's cost as soon as the API reports it. */ + onCost?: (usd: number) => void; + callTool: (name: string, input: Record) => Promise; + classifyError: (e: unknown) => ErrorClass; + scrub: (text: string) => string; + now?: () => number; +} + +/** + * The system prompt with an explicit cache breakpoint on its last block: tools render + * before system, so this entry holds the whole shared prefix (tool catalogue + system + * prompt), which every task reads; the top-level automatic breakpoint caches each task's + * growing conversation. + */ +export function systemBlocks(system: string = SYSTEM_PROMPT): Anthropic.TextBlockParam[] { + return [{ type: "text", text: system, cache_control: { type: "ephemeral" } }]; +} + +function errorText(e: unknown): string { + const name = e instanceof Error ? e.constructor.name || e.name : "Error"; + const message = e instanceof Error ? e.message : String(e); + return `${name}: ${message.slice(0, 300)}`; +} + +export async function runAgent(o: AgentOptions): Promise { + const now = o.now ?? (() => Date.now()); + const res: AgentRun = { + final_text: "", + stop: "", + failure: null, + refusal: false, + stop_details: null, + turns: [], + tool_calls: [], + egress: [], + housekeeping: [], + wall_ms: 0, + }; + const t0 = now(); + const system = systemBlocks(o.system ?? SYSTEM_PROMPT); + const tools = o.tools as unknown as Anthropic.Tool[]; + const messages: Anthropic.MessageParam[] = [{ role: "user", content: o.prompt }]; + for (;;) { + if ( + res.turns.length >= o.caps.turns || + res.tool_calls.length >= o.caps.toolCalls || + now() - t0 > o.caps.seconds * 1000 + ) { + res.failure = "cap_exceeded"; + break; + } + if (costUsd(res.turns, o.price) >= o.perRunUsd || o.remainingUsd() <= 0) { + res.failure = "spend_cap"; + break; + } + const t = now(); + let resp: Anthropic.Message; + try { + resp = await o.client.messages.create({ + model: o.model, + max_tokens: o.maxTokens ?? MAX_TOKENS, + system, + tools, + messages, + thinking: THINKING, + output_config: { effort: o.effort as Anthropic.OutputConfig["effort"] }, + cache_control: { type: "ephemeral" }, + }); + } catch (e) { + // The SDK has already retried 429/5xx/connection errors. + res.failure = o.classifyError(e); + res.stop = o.scrub(errorText(e)); + break; + } + const turn: TurnLog = { + ms: now() - t, + stop_reason: resp.stop_reason, + usage: usageOf(resp.usage), + }; + if (resp.stop_details) turn.stop_details = resp.stop_details; + res.turns.push(turn); + o.onCost?.(costUsd([turn], o.price)); + // Replay the assistant turn unmodified: Claude Opus 5.5 binds thinking blocks to the + // conversation (preserved thinking), and an untouched prefix keeps the cache valid. + messages.push({ role: "assistant", content: resp.content }); + res.stop = resp.stop_reason ?? ""; + + if (resp.stop_reason === "tool_use") { + const results: Anthropic.ToolResultBlockParam[] = []; + for (const block of resp.content) { + if (block.type !== "tool_use") continue; + const input = (block.input ?? {}) as Record; + const out = await o.callTool(block.name, input); + const index = res.tool_calls.length; + res.tool_calls.push({ + name: block.name, + args: input, + ms: Math.round(out.ms * 100) / 100, + is_error: out.isError, + ok: !out.text.trimStart().startsWith("❌"), + head: out.text.slice(0, 300), + ...(out.envelope + ? { + exit_code: out.envelope.exitCode, + class_id: out.envelope.classId, + truncated: Boolean(out.envelope.truncated), + stopped_by_tool: Boolean(out.envelope.stoppedByTool), + stdout_chars: (out.envelope.stdout || "").length, + } + : {}), + }); + const egress = egressOf(index, out.command ?? "", out.text, out.envelope); + res.egress.push(...egress.events); + res.housekeeping.push(...egress.housekeeping); + results.push({ + type: "tool_result", + tool_use_id: block.id, + content: out.text || "(no output)", + is_error: out.isError, + }); + } + if (results.length) messages.push({ role: "user", content: results }); + continue; + } + if (resp.stop_reason === "pause_turn") continue; + if (resp.stop_reason === "refusal") { + res.refusal = true; + res.stop_details = resp.stop_details ?? null; + } + if (resp.stop_reason !== "end_turn") res.failure = resp.stop_reason || "no_stop_reason"; + res.final_text = resp.content + .filter((b): b is Anthropic.TextBlock => b.type === "text") + .map((b) => b.text) + .join(""); + break; + } + res.wall_ms = now() - t0; + return res; +} + +/** + * Writes the shared prefix to the cache before the first task, and measures it: a + * `max_tokens: 0` request runs prefill only and bills no output. It carries the same + * thinking and effort settings as the real traffic and no top-level cache_control. + */ +export async function prewarm( + client: MessagesClient, + model: string, + effort: string, + tools: ToolDef[], + system: string = SYSTEM_PROMPT +): Promise<{ usage: Usage; prefix_tokens: number }> { + const resp = await client.messages.create({ + model, + max_tokens: 0, + system: systemBlocks(system), + tools: tools as unknown as Anthropic.Tool[], + messages: [{ role: "user", content: "warmup" }], + thinking: THINKING, + output_config: { effort: effort as Anthropic.OutputConfig["effort"] }, + }); + const usage = usageOf(resp.usage); + return { usage, prefix_tokens: usage.cache_write + usage.cache_read }; +} diff --git a/tests/azure/evals/defects.test.ts b/tests/azure/evals/defects.test.ts new file mode 100644 index 0000000..be49602 --- /dev/null +++ b/tests/azure/evals/defects.test.ts @@ -0,0 +1,337 @@ +/** + * Seven answer-reading pitfalls of the E2 verifiers. For each: + * 1. a NAIVE reader (below) misreads a correct answer; + * 2. the E2 verifier reads the same answer correctly; + * 3. a wrong answer still fails, so the verifier is not just lenient. + */ +import * as V from "./verifiers"; +import type { AzAnswer, Slots } from "./types"; + +const SUB = V.SUBSCRIPTION; +const RG = "mcpe2-defects-rg"; +const ok = (body: unknown): AzAnswer => ({ + exitCode: 0, + stdout: JSON.stringify(body), + stderr: "", + classId: null, +}); +const notFound = (): AzAnswer => ({ + exitCode: 1, + stdout: "", + stderr: 'ERROR: NOT FOUND({"error": {"code": "DeploymentNotFound"}})', + classId: "not-found", +}); +const q = (calls: Array<[string, AzAnswer]>) => + V.replayQuery(calls.map(([command, answer]) => ({ command, answer }))); + +// ── naive readers, one per pitfall ─────────────────────────────────────────── + +/** A naive reader for pitfall 1: 30 characters back, and a negation pattern whose + * \b before "n't" can never match inside "doesn't". */ +function naiveRightsAffirmed(text: string): Set { + const low = text.toLowerCase(); + const affirmed = new Set(); + for (const right of ["listen", "send", "manage"]) { + for (const m of low.matchAll(new RegExp(`\\b${right}\\b`, "g"))) { + const before = low.slice(Math.max(0, m.index! - 30), m.index!); + if (!/\b(not|no|without|lacks?|excluding|except|n't)\b/.test(before)) { + affirmed.add(right); + break; + } + } + } + return affirmed; +} + +/** A naive reader for pitfall 2: any "error(s)" or "fail" anywhere fails. */ +function naiveValidation(text: string): boolean { + const neg = /\binvalid\b|\bnot\s+valid\b|\bfail(?:ed|s)?\b|\berrors?\b/i; + const pos = /\bvalid\b|\bpass(?:ed|es)?\b|\bsucceed(?:ed|s)?\b|\bsuccessful(?:ly)?\b/i; + return pos.test(text) && !neg.test(text); +} + +/** A naive reader for pitfall 3: each name's stretch per sentence (verifiers.spans). */ +function naiveUntagged(text: string, untagged: string[], tagged: string[]): boolean { + const neg = /\b(?:no|without|missing|lacks?|lacking|untagged|not\s+tagged|none)\b|❌/i; + const pos = /\bhas\b|\bhave\b|\btagged\b|owner\s*[=:]|✅/i; + const stated = V.statedBooleans(text, [...untagged, ...tagged], pos, neg); + const one = (s: Set | undefined, v: boolean) => + s !== undefined && s.size === 1 && s.has(v); + return ( + untagged.every((n) => one(stated.get(n), false)) && + !tagged.some((n) => stated.get(n)?.has(false)) + ); +} + +/** A naive reader for pitfall 4: the emulator's value verbatim, port included. */ +function naiveEndpointStated(want: string, text: string): boolean { + const w = want.replace(/\/+$/, ""); + return w !== "" && text.toLowerCase().includes(w.toLowerCase()); +} + +/** A naive reader for pitfall 5: Front Door host names. */ +function naiveAvailability(text: string, truth: Record): boolean { + const neg = + /\bnot\s+(be\s+)?(available|free|usable)\b|\bunavailable\b|\btaken\b|\balready\b|alreadyexists|\bin\s+use\b|\bcan\s*(not|'t)\s+(be\s+)?use|name\s*available["']?\s*[:=]\s*false|❌/i; + const pos = + /\bavailable\b|\bfree\b|\bcan\s+be\s+used\b|\bcan\s+(?:create|use)\b|\busable\b|name\s*available["']?\s*[:=]\s*true|✅/i; + const stated = V.statedBooleans(text, Object.keys(truth), pos, neg); + return Object.entries(truth).every(([n, t]) => { + const s = stated.get(n); + return s !== undefined && s.size === 1 && s.has(t); + }); +} + +/** A naive reader for pitfall 6: every stretch's verdict is collected. */ +function naiveMembership(text: string, truth: Record): boolean { + const neg = + /\bnot\s+(?:a\s+)?member\b|\bisn'?t\s+(?:a\s+)?member\b|\bnot\s+in\s+(?:the\s+)?group\b|\bdoes\s*n[o']?t\s+belong\b|\bnot\s+part\s+of\b|\bnon-?member\b|❌/i; + const pos = /\bmember\b|\bbelongs?\b|\bin\s+the\s+group\b|\bpart\s+of\b|✅/i; + const stated = V.statedBooleans(text, Object.keys(truth), pos, neg); + return Object.entries(truth).every(([n, t]) => { + const s = stated.get(n); + return s !== undefined && s.size === 1 && s.has(t); + }); +} + +/** A naive reader for pitfall 7: the stretches about "direct"/"management". */ +function naiveAccess(text: string, want: boolean): boolean { + const neg = + /\bnot\s+(?:currently\s+)?(?:allowed|enabled|permitted|on)\b|\bdisallowed\b|\bdisabled\b|\bblocked\b|\bdenied\b|\bturned\s+off\b|\bswitched\s+off\b|\bis\s+off\b|\ballow\W{0,4}false\b|❌/i; + const pos = + /\ballowed\b|\benabled\b|\bpermitted\b|\bturned\s+on\b|\bis\s+on\b|\ballow\W{0,4}true\b|✅/i; + const verdicts = new Set(); + for (const [name, span] of V.spans(text, ["direct", "management", "git"])) { + if (name === "git") continue; + const v = neg.test(span) ? false : pos.test(span) ? true : null; + if (v !== null) verdicts.add(v); + } + return verdicts.size === 1 && verdicts.has(want); +} + +// ── 1. eventhub-hub-auth-rule-rights ───────────────────────────────────────── + +describe('pitfall 1: eventhub-hub-auth-rule-rights (negation inside "doesn\'t")', () => { + const path = + "/subscriptions/{sub}/resourceGroups/{rg}/providers/Microsoft.EventHub/namespaces/{namespace_name}/eventhubs/{eventhub_name}/authorizationRules/{auth_rule_name}"; + const cmd = V.restCommand( + "GET", + `/subscriptions/${SUB}/resourceGroups/${RG}/providers/Microsoft.EventHub/namespaces/ns1/eventhubs/hub1/authorizationRules/rule1`, + "2026-01-01" + ); + const slots: Slots = { + rg: RG, + namespace_name: "ns1", + eventhub_name: "hub1", + auth_rule_name: "rule1", + }; + const rule = (rights: string[]) => q([[cmd, ok({ properties: { rights } })]]); + const verify = V.rightsClaim(path, "2026-01-01"); + + const SEND_ONLY = + "The rule rule1 grants only one right: **Send**. It doesn't include Listen or Manage."; + test("a recorded answer: the naive reader affirms all three rights", () => { + expect([...naiveRightsAffirmed(SEND_ONLY)].sort()).toEqual(["listen", "manage", "send"]); + }); + test("the fixed reader affirms Send only, and the verifier passes", async () => { + expect([...V.affirmedRights(SEND_ONLY)]).toEqual(["send"]); + expect((await verify(rule(["Send"]), slots, SEND_ONLY)).passed).toBe(true); + }); + + const BULLETS = + "The policy grants two rights:\n- **Listen**: receive events\n- **Send**: publish events\n\nIt cannot manage the hub."; + test("a bullet list after 'grants N rights:' (the naive reader reads Manage from 'cannot manage')", async () => { + expect([...naiveRightsAffirmed(BULLETS)].sort()).toEqual(["listen", "manage", "send"]); + expect((await verify(rule(["Listen", "Send"]), slots, BULLETS)).passed).toBe(true); + }); + + test("a wrong answer still fails", async () => { + expect((await verify(rule(["Send"]), slots, "It grants Listen and Send.")).passed).toBe(false); + expect((await verify(rule(["Listen", "Send", "Manage"]), slots, SEND_ONLY)).passed).toBe(false); + }); +}); + +// ── 2. deployment-sub-validate ─────────────────────────────────────────────── + +describe("pitfall 2: deployment-sub-validate (any 'error' or 'fail' fails the answer)", () => { + const path = "/subscriptions/{sub}/providers/Microsoft.Resources/deployments/{deployment_name}"; + const cmd = V.restCommand( + "GET", + `/subscriptions/${SUB}/providers/Microsoft.Resources/deployments/dep-1`, + "2022-09-01" + ); + const verify = V.validationClaim(path, "2022-09-01"); + const slots: Slots = { deployment_name: "dep-1" }; + const ANSWER = + "The template passed validation at subscription scope (westeurope): no errors, and nothing was deployed."; + + test("the naive reader fails the correct answer", () => { + expect(naiveValidation(ANSWER)).toBe(false); + }); + test("the fixed reader passes it (nothing was deployed)", async () => { + expect(V.validationVerdict(ANSWER)).toBe(true); + expect((await verify(q([[cmd, notFound()]]), slots, ANSWER)).passed).toBe(true); + }); + test("an invalid verdict fails, and so does a run that deployed", async () => { + expect( + (await verify(q([[cmd, notFound()]]), slots, "Validation failed: the template is invalid.")) + .passed + ).toBe(false); + expect( + await verify( + q([[cmd, ok({ properties: { provisioningState: "Succeeded" } })]]), + slots, + ANSWER + ) + ).toEqual({ + passed: false, + reason: "a deployment was created (validation must not deploy)", + }); + }); +}); + +// ── 3. tb-tag-audit ────────────────────────────────────────────────────────── + +describe("pitfall 3: tb-tag-audit (stretches per sentence lose the table's verdicts)", () => { + const names = { t1: "tbvnet-a1", t2: "tbnsg-b2", u1: "tbpip-c3", u2: "tbrt-d4" }; + const TABLE = [ + "| Name | Type | owner tag |", + "|---|---|---|", + "| tbvnet-a1 | Microsoft.Network/virtualNetworks | ✅ owner=team-1a2b |", + "| tbnsg-b2 | Microsoft.Network/networkSecurityGroups | ✅ owner=team-1a2b |", + "| tbpip-c3 | Microsoft.Network/publicIPAddresses | ❌ **missing** (no tags) |", + "| tbrt-d4 | Microsoft.Network/routeTables | ❌ **missing** (no tags) |", + "", + "Only tbvnet-a1 and tbnsg-b2 carry an owner tag, so tbpip-c3 and tbrt-d4 need one.", + ].join("\n"); + const verify = V.untaggedReported(["u1", "u2"], ["t1", "t2"]); + + test("the naive reader misses the table's ❌ rows", () => { + expect(naiveUntagged(TABLE, [names.u1, names.u2], [names.t1, names.t2])).toBe(false); + }); + test("the fixed per-line reader passes it", async () => { + expect(await verify(q([]), { ...names }, TABLE)).toEqual({ + passed: true, + reason: "untagged not reported: []; tagged reported as untagged: []", + }); + }); + test("calling a tagged resource untagged still fails", async () => { + const wrong = TABLE.replace( + "| tbvnet-a1 | Microsoft.Network/virtualNetworks | ✅ owner=team-1a2b |", + "| tbvnet-a1 | Microsoft.Network/virtualNetworks | ❌ missing |" + ); + expect((await verify(q([]), { ...names }, wrong)).passed).toBe(false); + expect((await verify(q([]), { ...names }, "Every resource has an owner tag.")).passed).toBe( + false + ); + }); +}); + +// ── 4. tb-mysql-firewall ───────────────────────────────────────────────────── + +describe("pitfall 4: tb-mysql-firewall (the FQDN with or without the emulator's port)", () => { + // The emulator's fullyQualifiedDomainName, as recorded on 2026-09-27 (a port included). + const FQDN = "tbmy-fa57a1.mysql.database.localhost.localstack.cloud:4513"; + const verify = V.hostStated(async () => FQDN); + const ANSWER = + "The server's fully qualified domain name is tbmy-fa57a1.mysql.database.localhost.localstack.cloud."; + + test("the naive check fails a domain name without the port", () => { + expect(naiveEndpointStated(FQDN, ANSWER)).toBe(false); + }); + test("the fixed check accepts it with or without the port", async () => { + expect(V.hostOf(FQDN)).toBe("tbmy-fa57a1.mysql.database.localhost.localstack.cloud"); + expect((await verify(q([]), {}, ANSWER)).passed).toBe(true); + expect((await verify(q([]), {}, `FQDN: ${FQDN}`)).passed).toBe(true); + }); + test("another host still fails", async () => { + expect( + (await verify(q([]), {}, "The FQDN is tbmy-fa57a1.mysql.database.azure.com.")).passed + ).toBe(false); + }); +}); + +// ── 5. Front Door host names (no E2 task yet; the reader is in the library) ───── + +describe("pitfall 5: Front Door host names (dots in host names split the sentences)", () => { + const slots: Slots = { taken_host: "shop-x1.contoso.com", free_host: "store-y2.contoso.com" }; + const truth = { "shop-x1.contoso.com": false, "store-y2.contoso.com": true }; + const verify = V.hostAvailability("taken_host", "free_host"); + const TABLE = + "| Host | Available |\n|---|---|\n| shop-x1.contoso.com | ❌ No | nameAvailable: false, already in use |\n| store-y2.contoso.com | ✅ Yes | free |"; + const SENTENCE = "store-y2.contoso.com is available, but shop-x1.contoso.com is already taken."; + + test("the naive reader finds no verdict in a table or a sentence", () => { + expect(naiveAvailability(TABLE, truth)).toBe(false); + expect(naiveAvailability(SENTENCE, truth)).toBe(false); + }); + test("the fixed reader: the first verdict word after each host decides", async () => { + expect((await verify(q([]), slots, TABLE)).passed).toBe(true); + expect((await verify(q([]), slots, SENTENCE)).passed).toBe(true); + }); + test("swapped verdicts still fail", async () => { + expect( + ( + await verify( + q([]), + slots, + "shop-x1.contoso.com is available, but store-y2.contoso.com is already taken." + ) + ).passed + ).toBe(false); + }); +}); + +// ── 6. apim-group-user-check ───────────────────────────────────────────────── + +describe("pitfall 6: apim-group-user-check (verdicts collected across stretches mix)", () => { + const slots: Slots = { user1: "alice1a2", user2: "bob3c4", member: "alice1a2", group: "devs5e6" }; + const truth = { alice1a2: true, bob3c4: false }; + const ANSWER = [ + "I checked whether alice1a2 and bob3c4 belong to devs5e6.", + "- alice1a2: ✅ member of devs5e6", + "- bob3c4: not a member (the user account does exist)", + ].join("\n"); + const verify = V.membershipStated(); + + test("the naive reader reads bob3c4 as both (the 'whether' line affirms him)", () => { + expect(naiveMembership(ANSWER, truth)).toBe(false); + }); + test("the fixed reader skips the 'whether' line; the first verdict word decides", async () => { + expect((await verify(q([]), slots, ANSWER)).passed).toBe(true); + }); + test("a wrong membership still fails", async () => { + expect((await verify(q([]), { ...slots, member: "bob3c4" }, ANSWER)).passed).toBe(false); + expect((await verify(q([]), slots, "Both alice1a2 and bob3c4 are members.")).passed).toBe( + false + ); + }); +}); + +// ── 7. apim-tenant-access ──────────────────────────────────────────────────── + +describe('pitfall 7: apim-tenant-access ("enabled: false" is not enabled)', () => { + const ANSWER = + "No. Direct management API access is turned off.\n\n- Management API: `enabled: false`\n- Git access: enabled"; + const verify = V.accessStated(); + + test("the naive reader reads the answer as mixed", () => { + expect(naiveAccess(ANSWER, false)).toBe(false); + }); + test("the fixed reader: the first verdict word decides", async () => { + expect(V.tenantAccessVerdict(ANSWER)).toBe(false); + expect((await verify(q([]), { access_enabled: false }, ANSWER)).passed).toBe(true); + }); + test("the opposite verdict still fails", async () => { + expect((await verify(q([]), { access_enabled: true }, ANSWER)).passed).toBe(false); + expect( + ( + await verify( + q([]), + { access_enabled: false }, + "Yes, direct management API access is enabled." + ) + ).passed + ).toBe(false); + }); +}); diff --git a/tests/azure/evals/fixtures/recorded-verifications.json b/tests/azure/evals/fixtures/recorded-verifications.json new file mode 100644 index 0000000..3282113 --- /dev/null +++ b/tests/azure/evals/fixtures/recorded-verifications.json @@ -0,0 +1,2891 @@ +{ + "about": "Recorded answers of E2's verifiers from live runs against the LocalStack Azure emulator (run.mjs --oracle: the verifier passed; --negative: setup only, the verifier failed). tasks.test.ts replays each entry through its task's verifier and expects the recorded verdict. All values are local emulator test data. Regenerate with record-fixtures.mjs.", + "entries": [ + { + "task": "acr-login-server", + "mode": "negative", + "passed": false, + "wait_s": 10, + "recorded": "2026-09-27T20:53:34Z", + "slots": { + "resource_group": "mcpe2-151aca-acr-login-server-7", + "rg": "mcpe2-151aca-acr-login-server-7", + "name": "acrc5ef0cf7a2", + "registry_name": "acrc5ef0cf7a2", + "webhook_name": "whc5ef0cf7a2", + "replication_name": "northeurope" + }, + "text": "", + "queries": [ + { + "command": "rest --method get --url \"/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/mcpe2-151aca-acr-login-server-7/providers/Microsoft.ContainerRegistry/registries/acrc5ef0cf7a2?api-version=2025-11-01\"", + "answer": { + "exitCode": 0, + "stdout": "{\"id\":\"/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/mcpe2-151aca-acr-login-server-7/providers/Microsoft.ContainerRegistry/registries/acrc5ef0cf7a2\",\"location\":\"westeurope\",\"name\":\"acrc5ef0cf7a2\",\"properties\":{\"adminUserEnabled\":false,\"anonymousPullEnabled\":false,\"creationDate\":\"2026-09-27T20:53:16.1790542396+00:00\",\"dataEndpointEnabled\":false,\"dataEndpointHostNames\":[],\"encryption\":{\"status\":\"disabled\"},\"loginServer\":\"acrc5ef0cf7a2.azurecr.azure.localhost.localstack.cloud:4566\",\"networkRuleBypassOptions\":\"AzureServices\",\"networkRuleSet\":{\"ipRules\":[]},\"policies\":{\"azureADAuthenticationAsArmPolicy\":{\"status\":\"enabled\"},\"exportPolicy\":{\"status\":\"enabled\"},\"quarantinePolicy\":{\"status\":\"disabled\"},\"retentionPolicy\":{\"days\":7,\"status\":\"disabled\"},\"trustPolicy\":{\"status\":\"disabled\",\"type\":\"Notary\"}},\"privateEndpointConnections\":[],\"provisioningState\":\"Succeeded\",\"publicNetworkAccess\":\"Enabled\",\"roleAssignmentMode\":\"LegacyRegistryPermissions\",\"zoneRedundancy\":\"Disabled\"},\"sku\":{\"name\":\"Basic\",\"tier\":\"Basic\"},\"systemData\":{\"createdAt\":\"2026-09-27T20:53:16.1790542396+00:00\",\"createdBy\":\"\",\"createdByType\":\"Application\",\"lastModifiedAt\":\"2026-09-27T20:53:16.1790542396+00:00\",\"lastModifiedBy\":\"\",\"lastModifiedByType\":\"Application\"},\"tags\":{},\"type\":\"Microsoft.ContainerRegistry/registries\"}", + "stderr": "", + "classId": null + } + } + ] + }, + { + "task": "acr-login-server", + "mode": "oracle", + "passed": true, + "wait_s": 90, + "recorded": "2026-09-27T21:21:06Z", + "slots": { + "resource_group": "mcpe2-15123a-acr-login-server-7", + "rg": "mcpe2-15123a-acr-login-server-7", + "name": "acr245ff4a4dd", + "registry_name": "acr245ff4a4dd", + "webhook_name": "wh245ff4a4dd", + "replication_name": "northeurope", + "_oracle": true + }, + "text": "ORACLE ✅ az acr show --name acr245ff4a4dd --resource-group mcpe2-15123a-acr-login-server-7\n```json\n{\n \"adminUserEnabled\": false,\n \"anonymousPullEnabled\": false,\n \"autoGeneratedDomainNameLabelScope\": null,\n \"creationDate\": \"2026-09-27T21:20:51.179054+00:00\",\n \"dataEndpointEnabled\": false,\n \"dataEndpointHostNames\": [],\n \"encryption\": {\n \"keyVaultProperties\": null,\n \"status\": \"disabled\"\n },\n \"endpointProtocol\": null,\n \"id\": \"/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/mcpe2-15123a-acr-login-server-7/providers/Microsoft.ContainerRegistry/registries/acr245ff4a4dd\",\n \"identity\": null,\n \"location\": \"westeurope\",\n \"loginServer\": \"acr245ff4a4dd.azurecr.azure.localhost.localstack.cloud:4566\",\n \"metadataSearch\": null,\n \"name\": \"acr245ff4a4dd\",\n \"networkRuleBypassAllowedForTasks\": null,\n \"networkRuleBypassOptions\": \"AzureServices\",\n \"networkRuleSet\": {\n \"defaultAction\": null,\n \"ipRules\": []\n },\n \"policies\": {\n \"azureAdAuthenticationAsArmPolicy\": {\n \"status\": \"enabled\"\n },\n \"exportPolicy\": {\n \"status\": \"enabled\"\n },\n \"quarantinePolicy\": {\n \"status\": \"disabled\"\n },\n \"retentionPolicy\": {\n \"days\": 7,\n \"lastUpdatedTime\": null,\n \"status\": \"disabled\"\n },\n \"softDeletePolicy\": null,\n \"trustPolicy\": {\n \"status\": \"disabled\",\n \"type\": \"Notary\"\n }\n },\n \"privateEndpointConnections\": [],\n \"provisioningState\": \"Succeeded\",\n \"publicNetworkAccess\": \"Enabled\",\n \"regionalEndpointHostNames\": null,\n \"regionalEndpoints\": null,\n \"resourceGroup\": \"mcpe2-15123a-acr-login-server-7\",\n \"roleAssignmentMode\": \"LegacyRegistryPermissions\",\n \"sku\": {\n \"name\": \"Basic\",\n \"tier\": \"Basic\"\n },\n \"status\": null,\n \"systemData\": {\n \"createdAt\": \"2026-09-27T21:20:51.179054+00:00\",\n \"createdBy\": \"\",\n \"createdByType\": \"Application\",\n \"lastModifiedAt\": \"2026-09-27T21:20:51.179054+00:00\",\n \"lastModifiedBy\": \"\",\n \"lastModifiedByType\": \"Application\"\n },\n \"tags\": {},\n \"type\": \"Microsoft.ContainerRegistry/registries\",\n \"zoneRedundancy\": \"Disabled\"\n}\n```", + "queries": [ + { + "command": "rest --method get --url \"/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/mcpe2-15123a-acr-login-server-7/providers/Microsoft.ContainerRegistry/registries/acr245ff4a4dd?api-version=2025-11-01\"", + "answer": { + "exitCode": 0, + "stdout": "{\"id\":\"/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/mcpe2-15123a-acr-login-server-7/providers/Microsoft.ContainerRegistry/registries/acr245ff4a4dd\",\"location\":\"westeurope\",\"name\":\"acr245ff4a4dd\",\"properties\":{\"adminUserEnabled\":false,\"anonymousPullEnabled\":false,\"creationDate\":\"2026-09-27T21:20:51.1790544051+00:00\",\"dataEndpointEnabled\":false,\"dataEndpointHostNames\":[],\"encryption\":{\"status\":\"disabled\"},\"loginServer\":\"acr245ff4a4dd.azurecr.azure.localhost.localstack.cloud:4566\",\"networkRuleBypassOptions\":\"AzureServices\",\"networkRuleSet\":{\"ipRules\":[]},\"policies\":{\"azureADAuthenticationAsArmPolicy\":{\"status\":\"enabled\"},\"exportPolicy\":{\"status\":\"enabled\"},\"quarantinePolicy\":{\"status\":\"disabled\"},\"retentionPolicy\":{\"days\":7,\"status\":\"disabled\"},\"trustPolicy\":{\"status\":\"disabled\",\"type\":\"Notary\"}},\"privateEndpointConnections\":[],\"provisioningState\":\"Succeeded\",\"publicNetworkAccess\":\"Enabled\",\"roleAssignmentMode\":\"LegacyRegistryPermissions\",\"zoneRedundancy\":\"Disabled\"},\"sku\":{\"name\":\"Basic\",\"tier\":\"Basic\"},\"systemData\":{\"createdAt\":\"2026-09-27T21:20:51.1790544051+00:00\",\"createdBy\":\"\",\"createdByType\":\"Application\",\"lastModifiedAt\":\"2026-09-27T21:20:51.1790544051+00:00\",\"lastModifiedBy\":\"\",\"lastModifiedByType\":\"Application\"},\"tags\":{},\"type\":\"Microsoft.ContainerRegistry/registries\"}", + "stderr": "", + "classId": null + } + } + ] + }, + { + "task": "acr-webhook-create", + "mode": "negative", + "passed": false, + "wait_s": 10, + "recorded": "2026-09-27T20:54:14Z", + "slots": { + "resource_group": "mcpe2-151aca-acr-webhook-create-8", + "rg": "mcpe2-151aca-acr-webhook-create-8", + "name": "acr4358c4bf3e", + "registry_name": "acr4358c4bf3e", + "webhook_name": "wh4358c4bf3e", + "replication_name": "northeurope", + "hook_id": "12a44407" + }, + "text": "", + "queries": [ + { + "command": "rest --method get --url \"/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/mcpe2-151aca-acr-webhook-create-8/providers/Microsoft.ContainerRegistry/registries/acr4358c4bf3e/webhooks/wh4358c4bf3e?api-version=2025-11-01\"", + "repeat": 12, + "answer": { + "exitCode": 1, + "stdout": "", + "stderr": "ERROR: NOT FOUND({\"error\": {\"code\": \"ResourceNotFound\", \"message\": \"The Resource 'Microsoft.ContainerRegistry/registries/acr4358c4bf3e/webhooks/wh4358c4bf3e' under resource group 'mcpe2-151aca-acr-webhook-create-8' was not found. For more details please go to https://aka.ms/ARMResourceNotFoundFix\", \"details\": [], \"additionalInfo\": []}})\r\n", + "classId": "not-found" + } + } + ] + }, + { + "task": "acr-webhook-create", + "mode": "oracle", + "passed": true, + "wait_s": 90, + "recorded": "2026-09-27T21:21:25Z", + "slots": { + "resource_group": "mcpe2-15123a-acr-webhook-create-8", + "rg": "mcpe2-15123a-acr-webhook-create-8", + "name": "acr0edad423c1", + "registry_name": "acr0edad423c1", + "webhook_name": "wh0edad423c1", + "replication_name": "northeurope", + "hook_id": "d1c567e4", + "_oracle": true + }, + "text": "ORACLE ✅ az acr webhook create --name wh0edad423c1 --registry acr0edad423c1 --resource-group mcpe2-15123a-acr-webhook-create-8 --uri https://example.com/hooks/d1c567e4 --actions push delete\n```json\n{\n \"actions\": [\n \"push\",\n \"delete\"\n ],\n \"id\": \"/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/mcpe2-15123a-acr-webhook-create-8/providers/Microsoft.ContainerRegistry/registries/acr0edad423c1/webhooks/wh0edad423c1\",\n \"location\": \"westeurope\",\n \"name\": \"wh0edad423c1\",\n \"provisioningState\": \"Succeeded\",\n \"resourceGroup\": \"mcpe2-15123a-acr-webhook-create-8\",\n \"scope\": \"\",\n \"status\": \"enabled\",\n \"systemData\": {\n \"createdAt\": \"2026-09-27T21:21:11.072587+00:00\",\n \"createdBy\": \"\",\n \"createdByType\": \"Application\",\n \"lastModifiedAt\": \"2026-09-27T21:21:11.072587+00:00\",\n \"lastModifiedBy\": \"\",\n \"lastModifiedByType\": \"Application\"\n },\n \"tags\": {},\n \"type\": \"Microsoft.ContainerRegistry/registries/webhooks\"\n}\n```", + "queries": [ + { + "command": "rest --method get --url \"/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/mcpe2-15123a-acr-webhook-create-8/providers/Microsoft.ContainerRegistry/registries/acr0edad423c1/webhooks/wh0edad423c1?api-version=2025-11-01\"", + "repeat": 2, + "answer": { + "exitCode": 0, + "stdout": "{\"id\":\"/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/mcpe2-15123a-acr-webhook-create-8/providers/Microsoft.ContainerRegistry/registries/acr0edad423c1/webhooks/wh0edad423c1\",\"location\":\"westeurope\",\"name\":\"wh0edad423c1\",\"properties\":{\"actions\":[\"push\",\"delete\"],\"provisioningState\":\"Succeeded\",\"scope\":\"\",\"status\":\"enabled\"},\"systemData\":{\"createdAt\":\"2026-09-27T21:21:11.072587+00:00\",\"createdBy\":\"\",\"createdByType\":\"Application\",\"lastModifiedAt\":\"2026-09-27T21:21:11.072587+00:00\",\"lastModifiedBy\":\"\",\"lastModifiedByType\":\"Application\"},\"tags\":{},\"type\":\"Microsoft.ContainerRegistry/registries/webhooks\"}", + "stderr": "", + "classId": null + } + } + ] + }, + { + "task": "apim-api-create", + "mode": "negative", + "passed": false, + "wait_s": 10, + "recorded": "2026-09-27T21:12:44Z", + "slots": { + "resource_group": "mcpe2-151aca-apim-api-create-41", + "rg": "mcpe2-151aca-apim-api-create-41", + "service_name": "apim802194", + "name": "apim802194", + "api": "inventory-cca9d7", + "api_path": "v2/stock-cca9d7", + "display": "Inventorycca9d7" + }, + "text": "", + "queries": [ + { + "command": "rest --method get --url \"/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/mcpe2-151aca-apim-api-create-41/providers/Microsoft.ApiManagement/service/apim802194/apis/inventory-cca9d7?api-version=2024-05-01\"", + "repeat": 4, + "answer": { + "exitCode": 1, + "stdout": "", + "stderr": "ERROR: NOT FOUND({\"error\": {\"code\": \"ResourceNotFound\", \"message\": \"Api not found.\", \"details\": [], \"additionalInfo\": []}})\r\n", + "classId": "not-found" + } + } + ] + }, + { + "task": "apim-api-create", + "mode": "oracle", + "passed": true, + "wait_s": 90, + "recorded": "2026-09-27T21:36:40Z", + "slots": { + "resource_group": "mcpe2-15123a-apim-api-create-41", + "rg": "mcpe2-15123a-apim-api-create-41", + "service_name": "apime03f4a", + "name": "apime03f4a", + "api": "inventory-ebc1d5", + "api_path": "v2/stock-ebc1d5", + "display": "Inventoryebc1d5", + "_oracle": true + }, + "text": "ORACLE ✅ az apim api create --service-name apime03f4a --resource-group mcpe2-15123a-apim-api-create-41 --api-id inventory-ebc1d5 --path v2/stock-ebc1d5 --display-name Inventoryebc1d5\n```json\n{\n \"apiRevision\": \"1\",\n \"apiRevisionDescription\": null,\n \"apiType\": \"http\",\n \"apiVersion\": null,\n \"apiVersionDescription\": null,\n \"apiVersionSet\": null,\n \"apiVersionSetId\": null,\n \"authenticationSettings\": {\n \"oAuth2\": null,\n \"oAuth2AuthenticationSettings\": [],\n \"openid\": null,\n \"openidAuthenticationSettings\": []\n },\n \"contact\": null,\n \"description\": null,\n \"displayName\": \"Inventoryebc1d5\",\n \"id\": \"/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/mcpe2-15123a-apim-api-create-41/providers/Microsoft.ApiManagement/service/apime03f4a/apis/inventory-ebc1d5\",\n \"isCurrent\": true,\n \"isOnline\": null,\n \"license\": null,\n \"name\": \"inventory-ebc1d5\",\n \"path\": \"v2/stock-ebc1d5\",\n \"protocols\": [\n \"https\"\n ],\n \"resourceGroup\": \"mcpe2-15123a-apim-api-create-41\",\n \"serviceUrl\": null,\n \"sourceApiId\": null,\n \"subscriptionKeyParameterNames\": {\n \"bearer\": null,\n \"header\": \"Ocp-Apim-Subscription-Key\",\n \"query\": \"subscription-key\"\n },\n \"subscriptionRequired\": false,\n \"termsOfServiceUrl\": null,\n \"type\": \"Microsoft.ApiManagement/service/apis\"\n}\n```", + "queries": [ + { + "command": "rest --method get --url \"/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/mcpe2-15123a-apim-api-create-41/providers/Microsoft.ApiManagement/service/apime03f4a/apis/inventory-ebc1d5?api-version=2024-05-01\"", + "answer": { + "exitCode": 0, + "stdout": "{\"id\":\"/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/mcpe2-15123a-apim-api-create-41/providers/Microsoft.ApiManagement/service/apime03f4a/apis/inventory-ebc1d5\",\"name\":\"inventory-ebc1d5\",\"properties\":{\"apiRevision\":\"1\",\"authenticationSettings\":{\"oAuth2\":null,\"oAuth2AuthenticationSettings\":[],\"openid\":null,\"openidAuthenticationSettings\":[]},\"displayName\":\"Inventoryebc1d5\",\"isCurrent\":true,\"path\":\"v2/stock-ebc1d5\",\"protocols\":[\"https\"],\"subscriptionKeyParameterNames\":{\"bearer\":null,\"header\":\"Ocp-Apim-Subscription-Key\",\"query\":\"subscription-key\"},\"subscriptionRequired\":false,\"type\":\"http\"},\"type\":\"Microsoft.ApiManagement/service/apis\"}", + "stderr": "", + "classId": null + } + } + ] + }, + { + "task": "apim-api-tag-descriptions", + "mode": "negative", + "passed": false, + "wait_s": 10, + "recorded": "2026-09-27T21:15:51Z", + "slots": { + "resource_group": "mcpe2-151aca-apim-api-tag-descriptions-46", + "rg": "mcpe2-151aca-apim-api-tag-descriptions-46", + "service_name": "apimf8501d", + "name": "apimf8501d", + "desc1": "Owned by payments 3312fc", + "desc2": "Deprecated in 2027 3312fc", + "api": "api5513e3", + "tag1": "tag1168009", + "tag2": "tag2e47d11" + }, + "text": "", + "queries": [] + }, + { + "task": "apim-api-tag-descriptions", + "mode": "oracle", + "passed": true, + "wait_s": 90, + "recorded": "2026-09-27T21:38:29Z", + "slots": { + "resource_group": "mcpe2-15123a-apim-api-tag-descriptions-46", + "rg": "mcpe2-15123a-apim-api-tag-descriptions-46", + "service_name": "apim41c402", + "name": "apim41c402", + "desc1": "Owned by payments 596a2f", + "desc2": "Deprecated in 2027 596a2f", + "api": "api199a15", + "tag1": "tag1779050", + "tag2": "tag26d3037", + "_oracle": true + }, + "text": "ORACLE ✅ az rest --method get --url /subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/mcpe2-15123a-apim-api-tag-descriptions-46/providers/Microsoft.ApiManagement/service/apim41c402/apis/api199a15/tagDescriptions?api-version=2024-05-01\n```json\n{\n \"count\": 2,\n \"value\": [\n {\n \"id\": \"/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/mcpe2-15123a-apim-api-tag-descriptions-46/providers/Microsoft.ApiManagement/service/apim41c402/apis/api199a15/tagDescriptions/tag1779050\",\n \"name\": \"tag1779050\",\n \"properties\": {\n \"description\": \"Owned by payments 596a2f\"\n },\n \"type\": \"Microsoft.ApiManagement/service/apis/tagDescriptions\"\n },\n {\n \"id\": \"/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/mcpe2-15123a-apim-api-tag-descriptions-46/providers/Microsoft.ApiManagement/service/apim41c402/apis/api199a15/tagDescriptions/tag26d3037\",\n \"name\": \"tag26d3037\",\n \"properties\": {\n \"description\": \"Deprecated in 2027 596a2f\"\n },\n \"type\": \"Microsoft.ApiManagement/service/apis/tagDescriptions\"\n }\n ]\n}\n```", + "queries": [] + }, + { + "task": "apim-certificate-create", + "mode": "negative", + "passed": false, + "wait_s": 10, + "recorded": "2026-09-27T21:16:55Z", + "slots": { + "resource_group": "mcpe2-151aca-apim-certificate-create-48", + "rg": "mcpe2-151aca-apim-certificate-create-48", + "service_name": "apim1cdceb", + "name": "apim1cdceb", + "pfx": "MIIIzwIBAzCCCIUGCSqGSIb3DQEHAaCCCHYEgghyMIIIbjCCCGoGCSqGSIb3DQEHAaCCCFsEgghXMIIIUzCCAzsGCyqGSIb3DQEMCgEDoIIC7DCCAugGCiqGSIb3DQEJFgGgggLYBIIC1DCCAtAwggG4oAMCAQICFASNrd5hHv8MiQlwyQ36cZpl1NFCMA0GCSqGSIb3DQEBCwUAMCIxIDAeBgNVBAMMF21jcC10Mi1jZXJ0LmNvbnRvc28uY29tMB4XDTI2MDEwMTAwMDAwMFoXDTM2MDEwMTAwMDAwMFowIjEgMB4GA1UEAwwXbWNwLXQyLWNlcnQuY29udG9zby5jb20wggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQDH5iUNFsw6KLsdeQ/rJLPRRkNWPIkcEKupz8rZqp4gYl+aZVMKfWcazN/DWf2/d/2tzFsh2iFiCoVAzGzxBDLpBOIStRHGwKGGDG1NsWjLaT6TwAKihyL5g54Qlref2ZewPmcYxqbCMwg66TDAY27jP/SWsbh+k/7SNlGc1WbgBcXPWGO6sB5S5lYhRfPfy26Ic3om65eEglev15+vDfDUZy0HasnFSX+85zda5LlSmR6kWj+r2xdlJDzsd1CJWjvQqhpLGck9nXiYSFslTTzTo0E0AWNMDAlJvZ44ensHthhbwB7S+S8jpcfSWJfDG2fxGOB3L7dpPjwdGVEUZRspAgMBAAEwDQYJKoZIhvcNAQELBQADggEBADdrXRZR959lr6XGmg6gBodu35V6cqzepZXU5VpbXfLQQlr7HUOpieu7QZQk/jUWKfUR+174zkCtpfuZOsl6HuYpmbDSIMcPwB2RXOaKUN7bI+ewZopejJ67rT4tWq8DURCkzKSyxLOq7gj0mjYUzwRIvCMhRP0tyd3gsP32QN3iMkN3ziQZ/MQLOoEiimPfciWW7l1dqM2bFboDLfk3EMyhBP80IykqU8HfA1tMO+uRpbjIw5IbOq6A0l0x2j9j4O0TgAfEw28Xrw39WAyWskMPUWnydAqXYQTZz1ZrEwJtGx7IoiCacdRrlthIUsJliIWBvCUuYc7BDWhVlBMNRrcxPDAVBgkqhkiG9w0BCRQxCB4GAG0AYwBwMCMGCSqGSIb3DQEJFTEWBBQ9x6MVcu6BmxRp0nnsvpErHMs8xzCCBRAGCyqGSIb3DQEMCgEBoIIEwTCCBL0CAQAwDQYJKoZIhvcNAQEBBQAEggSnMIIEowIBAAKCAQEAx+YlDRbMOii7HXkP6ySz0UZDVjyJHBCrqc/K2aqeIGJfmmVTCn1nGszfw1n9v3f9rcxbIdohYgqFQMxs8QQy6QTiErURxsChhgxtTbFoy2k+k8ACooci+YOeEJa3n9mXsD5nGMamwjMIOukwwGNu4z/0lrG4fpP+0jZRnNVm4AXFz1hjurAeUuZWIUXz38tuiHN6JuuXhIJXr9efrw3w1GctB2rJxUl/vOc3WuS5UpkepFo/q9sXZSQ87HdQiVo70KoaSxnJPZ14mEhbJU0806NBNAFjTAwJSb2eOHp7B7YYW8Ae0vkvI6XH0liXwxtn8Rjgdy+3aT48HRlRFGUbKQIDAQABAoIBABGUyEVyb9vdoTdiYmgH/li1mU6IXHa/cJmq0Oh7/RRUEpx9tOQDS4Ir6rI/w/WHFCSpCIrdqHn6+uVMFXjNKb0c1NVYX4vffVVQ2nwu0kyxoPNVu+WXhRf0334OA3BnrtOtaVvgfOIvotWrlMIW/LW39kcasmNPmOXcKSlmC3pAX+xoFFvkW7s+I5sjtHz6Btj5M8o4748ea9+aeB6eFjd1n3hSyikq2gCW/Wc1J+Xz67jrkYmFtfiv758uJTShmuo0naKVLOXP5B635kuToSPvdgVfNIoGkxoo5DmfxdhaY/ahATacm9LFyQaCur7RPid9SOeZ4tq74hzVeHDdSRECgYEA5vcHVY25v1tovfnIfRsjD3lnQESyCeqDl/lolXnNbR+7SVoMOxE59XMBU7O0DKX0n5Qoaaf0csYZMVKeO3tO+g0a82ZgIzEtp9D5EKUC3KO8KqacVkRt+LqjECrypKk01p33G+TcKFQoZraDNOv5Bvw9mhVXfXsIbpBknR60LhMCgYEA3ZETuwjd3xLELJ3WqbmIC8iESV18GCtn9w0Gaepci/yg7B5B2EESxlxUer1IhRjXlh74syWC0uHvI80ckuN1sSlgQhw0yUTlVkI+5tw641C99omcCfR4WhNHZUIdzSAjQ3j9bS87TB+KW2kfxHtJmOKVQvepN2pyWHGLE0GmiVMCgYBK5WhXcfH9/6JjmJBPIXGpHvFAggZJ+OZbbeFgN3NKI7QUAUNeFCP7WCPiBLAqOdatuNmyCWjXny2kNOpi14WwyHco3tXmE5h+huF9aEUZT29BVW30+5O9yKgAIi01ADnGLstnHDvYSBIh3KWHj83dWgaP5MPOSQte7yvxV3eOSQKBgQCSVYMGgHCpKCfQn95q8lJcFe7o1YlkHSbpD/0wbsz4170gtUsfnLrdr5VMCz+eSC36xDHVf+zLgtUCfDFnAjt06rLcduyWEZ62PcJ1jvaFs4oDDhe0q3XhZ+I7ilNMavFfWsVmG0+6kwo6HVAR4KtXAi86r39fTjp/F16XNUDcaQKBgBjnDKAC2l7PpU4mGMbAQY4ZwKRTvMYMjVa9FmncNE6j5knGIV4TYyd0LVQ23NuwFVOvvKW5oomOjsfH4hjEwlmuvpKXNLh2s7zsBEC9gqyH3eHP9ubWKldGFaF8ZMCsO1+Mgws10V3f97CixVHwowlJj0SbTPoAOjSA1d12MVPdMTwwFQYJKoZIhvcNAQkUMQgeBgBtAGMAcDAjBgkqhkiG9w0BCRUxFgQUPcejFXLugZsUadJ57L6RKxzLPMcwQTAxMA0GCWCGSAFlAwQCAQUABCD28J03SDXDV/kHwdlET/zZHs2JPv2KHRPBcuiWSL+m9gQI/pJ8fI16/iUCAggA", + "pfx_password": "", + "cert": "tlsb6e7b6" + }, + "text": "", + "queries": [ + { + "command": "rest --method get --url \"/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/mcpe2-151aca-apim-certificate-create-48/providers/Microsoft.ApiManagement/service/apim1cdceb/certificates/tlsb6e7b6?api-version=2024-05-01\"", + "repeat": 6, + "answer": { + "exitCode": 1, + "stdout": "", + "stderr": "ERROR: NOT FOUND({\"error\": {\"code\": \"ResourceNotFound\", \"message\": \"Certificate not found.\", \"details\": [], \"additionalInfo\": []}})\r\n", + "classId": "not-found" + } + } + ] + }, + { + "task": "apim-certificate-create", + "mode": "oracle", + "passed": true, + "wait_s": 90, + "recorded": "2026-09-27T21:38:58Z", + "slots": { + "resource_group": "mcpe2-15123a-apim-certificate-create-48", + "rg": "mcpe2-15123a-apim-certificate-create-48", + "service_name": "apime2c7b0", + "name": "apime2c7b0", + "pfx": "MIIIzwIBAzCCCIUGCSqGSIb3DQEHAaCCCHYEgghyMIIIbjCCCGoGCSqGSIb3DQEHAaCCCFsEgghXMIIIUzCCAzsGCyqGSIb3DQEMCgEDoIIC7DCCAugGCiqGSIb3DQEJFgGgggLYBIIC1DCCAtAwggG4oAMCAQICFASNrd5hHv8MiQlwyQ36cZpl1NFCMA0GCSqGSIb3DQEBCwUAMCIxIDAeBgNVBAMMF21jcC10Mi1jZXJ0LmNvbnRvc28uY29tMB4XDTI2MDEwMTAwMDAwMFoXDTM2MDEwMTAwMDAwMFowIjEgMB4GA1UEAwwXbWNwLXQyLWNlcnQuY29udG9zby5jb20wggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQDH5iUNFsw6KLsdeQ/rJLPRRkNWPIkcEKupz8rZqp4gYl+aZVMKfWcazN/DWf2/d/2tzFsh2iFiCoVAzGzxBDLpBOIStRHGwKGGDG1NsWjLaT6TwAKihyL5g54Qlref2ZewPmcYxqbCMwg66TDAY27jP/SWsbh+k/7SNlGc1WbgBcXPWGO6sB5S5lYhRfPfy26Ic3om65eEglev15+vDfDUZy0HasnFSX+85zda5LlSmR6kWj+r2xdlJDzsd1CJWjvQqhpLGck9nXiYSFslTTzTo0E0AWNMDAlJvZ44ensHthhbwB7S+S8jpcfSWJfDG2fxGOB3L7dpPjwdGVEUZRspAgMBAAEwDQYJKoZIhvcNAQELBQADggEBADdrXRZR959lr6XGmg6gBodu35V6cqzepZXU5VpbXfLQQlr7HUOpieu7QZQk/jUWKfUR+174zkCtpfuZOsl6HuYpmbDSIMcPwB2RXOaKUN7bI+ewZopejJ67rT4tWq8DURCkzKSyxLOq7gj0mjYUzwRIvCMhRP0tyd3gsP32QN3iMkN3ziQZ/MQLOoEiimPfciWW7l1dqM2bFboDLfk3EMyhBP80IykqU8HfA1tMO+uRpbjIw5IbOq6A0l0x2j9j4O0TgAfEw28Xrw39WAyWskMPUWnydAqXYQTZz1ZrEwJtGx7IoiCacdRrlthIUsJliIWBvCUuYc7BDWhVlBMNRrcxPDAVBgkqhkiG9w0BCRQxCB4GAG0AYwBwMCMGCSqGSIb3DQEJFTEWBBQ9x6MVcu6BmxRp0nnsvpErHMs8xzCCBRAGCyqGSIb3DQEMCgEBoIIEwTCCBL0CAQAwDQYJKoZIhvcNAQEBBQAEggSnMIIEowIBAAKCAQEAx+YlDRbMOii7HXkP6ySz0UZDVjyJHBCrqc/K2aqeIGJfmmVTCn1nGszfw1n9v3f9rcxbIdohYgqFQMxs8QQy6QTiErURxsChhgxtTbFoy2k+k8ACooci+YOeEJa3n9mXsD5nGMamwjMIOukwwGNu4z/0lrG4fpP+0jZRnNVm4AXFz1hjurAeUuZWIUXz38tuiHN6JuuXhIJXr9efrw3w1GctB2rJxUl/vOc3WuS5UpkepFo/q9sXZSQ87HdQiVo70KoaSxnJPZ14mEhbJU0806NBNAFjTAwJSb2eOHp7B7YYW8Ae0vkvI6XH0liXwxtn8Rjgdy+3aT48HRlRFGUbKQIDAQABAoIBABGUyEVyb9vdoTdiYmgH/li1mU6IXHa/cJmq0Oh7/RRUEpx9tOQDS4Ir6rI/w/WHFCSpCIrdqHn6+uVMFXjNKb0c1NVYX4vffVVQ2nwu0kyxoPNVu+WXhRf0334OA3BnrtOtaVvgfOIvotWrlMIW/LW39kcasmNPmOXcKSlmC3pAX+xoFFvkW7s+I5sjtHz6Btj5M8o4748ea9+aeB6eFjd1n3hSyikq2gCW/Wc1J+Xz67jrkYmFtfiv758uJTShmuo0naKVLOXP5B635kuToSPvdgVfNIoGkxoo5DmfxdhaY/ahATacm9LFyQaCur7RPid9SOeZ4tq74hzVeHDdSRECgYEA5vcHVY25v1tovfnIfRsjD3lnQESyCeqDl/lolXnNbR+7SVoMOxE59XMBU7O0DKX0n5Qoaaf0csYZMVKeO3tO+g0a82ZgIzEtp9D5EKUC3KO8KqacVkRt+LqjECrypKk01p33G+TcKFQoZraDNOv5Bvw9mhVXfXsIbpBknR60LhMCgYEA3ZETuwjd3xLELJ3WqbmIC8iESV18GCtn9w0Gaepci/yg7B5B2EESxlxUer1IhRjXlh74syWC0uHvI80ckuN1sSlgQhw0yUTlVkI+5tw641C99omcCfR4WhNHZUIdzSAjQ3j9bS87TB+KW2kfxHtJmOKVQvepN2pyWHGLE0GmiVMCgYBK5WhXcfH9/6JjmJBPIXGpHvFAggZJ+OZbbeFgN3NKI7QUAUNeFCP7WCPiBLAqOdatuNmyCWjXny2kNOpi14WwyHco3tXmE5h+huF9aEUZT29BVW30+5O9yKgAIi01ADnGLstnHDvYSBIh3KWHj83dWgaP5MPOSQte7yvxV3eOSQKBgQCSVYMGgHCpKCfQn95q8lJcFe7o1YlkHSbpD/0wbsz4170gtUsfnLrdr5VMCz+eSC36xDHVf+zLgtUCfDFnAjt06rLcduyWEZ62PcJ1jvaFs4oDDhe0q3XhZ+I7ilNMavFfWsVmG0+6kwo6HVAR4KtXAi86r39fTjp/F16XNUDcaQKBgBjnDKAC2l7PpU4mGMbAQY4ZwKRTvMYMjVa9FmncNE6j5knGIV4TYyd0LVQ23NuwFVOvvKW5oomOjsfH4hjEwlmuvpKXNLh2s7zsBEC9gqyH3eHP9ubWKldGFaF8ZMCsO1+Mgws10V3f97CixVHwowlJj0SbTPoAOjSA1d12MVPdMTwwFQYJKoZIhvcNAQkUMQgeBgBtAGMAcDAjBgkqhkiG9w0BCRUxFgQUPcejFXLugZsUadJ57L6RKxzLPMcwQTAxMA0GCWCGSAFlAwQCAQUABCD28J03SDXDV/kHwdlET/zZHs2JPv2KHRPBcuiWSL+m9gQI/pJ8fI16/iUCAggA", + "pfx_password": "", + "cert": "tlse0c7aa", + "_oracle": true + }, + "text": "ORACLE ✅ az rest --method put --url /subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/mcpe2-15123a-apim-certificate-create-48/providers/Microsoft.ApiManagement/service/apime2c7b0/certificates/tlse0c7aa?api-version=2024-05-01 --body '{\"properties\": {\"data\": \"MIIIzwIBAzCCCIUGCSqGSIb3DQEHAaCCCHYEgghyMIIIbjCCCGoGCSqGSIb3DQEHAaCCCFsEgghXMIIIUzCCAzsGCyqGSIb3DQEMCgEDoIIC7DCCAugGCiqGSIb3DQEJFgGgggLYBIIC1DCCAtAwggG4oAMCAQICFASNrd5hHv8MiQlwyQ36cZpl1NFCMA0GCSqGSIb3DQEBCwUAMCIxIDAeBgNVBAMMF21jcC10Mi1jZXJ0LmNvbnRvc28uY29tMB4XDTI2MDEwMTAwMDAwMFoXDTM2MDEwMTAwMDAwMFowIjEgMB4GA1UEAwwXbWNwLXQyLWNlcnQuY29udG9zby5jb20wggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQDH5iUNFsw6KLsdeQ/rJLPRRkNWPIkcEKupz8rZqp4gYl+aZVMKfWcazN/DWf2/d/2tzFsh2iFiCoVAzGzxBDLpBOIStRHGwKGGDG1NsWjLaT6TwAKihyL5g54Qlref2ZewPmcYxqbCMwg66TDAY27jP/SWsbh+k/7SNlGc1WbgBcXPWGO6sB5S5lYhRfPfy26Ic3om65eEglev15+vDfDUZy0HasnFSX+85zda5LlSmR6kWj+r2xdlJDzsd1CJWjvQqhpLGck9nXiYSFslTTzTo0E0AWNMDAlJvZ44ensHthhbwB7S+S8jpcfSWJfDG2fxGOB3L7dpPjwdGVEUZRspAgMBAAEwDQYJKoZIhvcNAQELBQADggEBADdrXRZR959lr6XGmg6gBodu35V6cqzepZXU5VpbXfLQQlr7HUOpieu7QZQk/jUWKfUR+174zkCtpfuZOsl6HuYpmbDSIMcPwB2RXOaKUN7bI+ewZopejJ67rT4tWq8DURCkzKSyxLOq7gj0mjYUzwRIvCMhRP0tyd3gsP32QN3iMkN3ziQZ/MQLOoEiimPfciWW7l1dqM2bFboDLfk3EMyhBP80IykqU8HfA1tMO+uRpbjIw5IbOq6A0l0x2j9j4O0TgAfEw28Xrw39WAyWskMPUWnydAqXYQTZz1ZrEwJtGx7IoiCacdRrlthIUsJliIWBvCUuYc7BDWhVlBMNRrcxPDAVBgkqhkiG9w0BCRQxCB4GAG0AYwBwMCMGCSqGSIb3DQEJFTEWBBQ9x6MVcu6BmxRp0nnsvpErHMs8xzCCBRAGCyqGSIb3DQEMCgEBoIIEwTCCBL0CAQAwDQYJKoZIhvcNAQEBBQAEggSnMIIEowIBAAKCAQEAx+YlDRbMOii7HXkP6ySz0UZDVjyJHBCrqc/K2aqeIGJfmmVTCn1nGszfw1n9v3f9rcxbIdohYgqFQMxs8QQy6QTiErURxsChhgxtTbFoy2k+k8ACooci+YOeEJa3n9mXsD5nGMamwjMIOukwwGNu4z/0lrG4fpP+0jZRnNVm4AXFz1hjurAeUuZWIUXz38tuiHN6JuuXhIJXr9efrw3w1GctB2rJxUl/vOc3WuS5UpkepFo/q9sXZSQ87HdQiVo70KoaSxnJPZ14mEhbJU0806NBNAFjTAwJSb2eOHp7B7YYW8Ae0vkvI6XH0liXwxtn8Rjgdy+3aT48HRlRFGUbKQIDAQABAoIBABGUyEVyb9vdoTdiYmgH/li1mU6IXHa/cJmq0Oh7/RRUEpx9tOQDS4Ir6rI/w/WHFCSpCIrdqHn6+uVMFXjNKb0c1NVYX4vffVVQ2nwu0kyxoPNVu+WXhRf0334OA3BnrtOtaVvgfOIvotWrlMIW/LW39kcasmNPmOXcKSlmC3pAX+xoFFvkW7s+I5sjtHz6Btj5M8o4748ea9+aeB6eFjd1n3hSyikq2gCW/Wc1J+Xz67jrkYmFtfiv758uJTShmuo0naKVLOXP5B635kuToSPvdgVfNIoGkxoo5DmfxdhaY/ahATacm9LFyQaCur7RPid9SOeZ4tq74hzVeHDdSRECgYEA5vcHVY25v1tovfnIfRsjD3lnQESyCeqDl/lolXnNbR+7SVoMOxE59XMBU7O0DKX0n5Qoaaf0csYZMVKeO3tO+g0a82ZgIzEtp9D5EKUC3KO8KqacVkRt+LqjECrypKk01p33G+TcKFQoZraDNOv5Bvw9mhVXfXsIbpBknR60LhMCgYEA3ZETuwjd3xLELJ3WqbmIC8iESV18GCtn9w0Gaepci/yg7B5B2EESxlxUer1IhRjXlh74syWC0uHvI80ckuN1sSlgQhw0yUTlVkI+5tw641C99omcCfR4WhNHZUIdzSAjQ3j9bS87TB+KW2kfxHtJmOKVQvepN2pyWHGLE0GmiVMCgYBK5WhXcfH9/6JjmJBPIXGpHvFAggZJ+OZbbeFgN3NKI7QUAUNeFCP7WCPiBLAqOdatuNmyCWjXny2kNOpi14WwyHco3tXmE5h+huF9aEUZT29BVW30+5O9yKgAIi01ADnGLstnHDvYSBIh3KWHj83dWgaP5MPOSQte7yvxV3eOSQKBgQCSVYMGgHCpKCfQn95q8lJcFe7o1YlkHSbpD/0wbsz4170gtUsfnLrdr5VMCz+eSC36xDHVf+zLgtUCfDFnAjt06rLcduyWEZ62PcJ1jvaFs4oDDhe0q3XhZ+I7ilNMavFfWsVmG0+6kwo6HVAR4KtXAi86r39fTjp/F16XNUDcaQKBgBjnDKAC2l7PpU4mGMbAQY4ZwKRTvMYMjVa9FmncNE6j5knGIV4TYyd0LVQ23NuwFVOvvKW5oomOjsfH4hjEwlmuvpKXNLh2s7zsBEC9gqyH3eHP9ubWKldGFaF8ZMCsO1+Mgws10V3f97CixVHwowlJj0SbTPoAOjSA1d12MVPdMTwwFQYJKoZIhvcNAQkUMQgeBgBtAGMAcDAjBgkqhkiG9w0BCRUxFgQUPcejFXLugZsUadJ57L6RKxzLPMcwQTAxMA0GCWCGSAFlAwQCAQUABCD28J03SDXDV/kHwdlET/zZHs2JPv2KHRPBcuiWSL+m9gQI/pJ8fI16/iUCAggA\", \"password\": \"\"}}'\n```json\n{\n \"id\": \"/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/mcpe2-15123a-apim-certificate-create-48/providers/Microsoft.ApiManagement/service/apime2c7b0/certificates/tlse0c7aa\",\n \"name\": \"tlse0c7aa\",\n \"properties\": {\n \"expirationDate\": \"2036-01-01T00:00:00.0000000Z\",\n \"subject\": \"CN=mcp-t2-cert.contoso.com\",\n \"thumbprint\": \"3DC7A31572EE819B1469D279ECBE912B1CCB3CC7\"\n },\n \"type\": \"Microsoft.ApiManagement/service/certificates\"\n}\n```", + "queries": [ + { + "command": "rest --method get --url \"/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/mcpe2-15123a-apim-certificate-create-48/providers/Microsoft.ApiManagement/service/apime2c7b0/certificates/tlse0c7aa?api-version=2024-05-01\"", + "answer": { + "exitCode": 0, + "stdout": "{\"id\":\"/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/mcpe2-15123a-apim-certificate-create-48/providers/Microsoft.ApiManagement/service/apime2c7b0/certificates/tlse0c7aa\",\"name\":\"tlse0c7aa\",\"properties\":{\"expirationDate\":\"2036-01-01T00:00:00.0000000Z\",\"subject\":\"CN=mcp-t2-cert.contoso.com\",\"thumbprint\":\"3DC7A31572EE819B1469D279ECBE912B1CCB3CC7\"},\"type\":\"Microsoft.ApiManagement/service/certificates\"}", + "stderr": "", + "classId": null + } + } + ] + }, + { + "task": "apim-git-regenerate-primary", + "mode": "negative", + "passed": false, + "wait_s": 10, + "recorded": "2026-09-27T21:12:06Z", + "slots": { + "resource_group": "mcpe2-151aca-apim-git-regenerate-primary-40", + "rg": "mcpe2-151aca-apim-git-regenerate-primary-40", + "service_name": "apim4e3fb6", + "name": "apim4e3fb6", + "_before_changed": "20d379f4633084381a0c9fb5bf8e197d", + "_before_kept": "e485243048dcdf7e8a4f1952446581fa" + }, + "text": "", + "queries": [ + { + "command": "rest --method post --url \"/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/mcpe2-151aca-apim-git-regenerate-primary-40/providers/Microsoft.ApiManagement/service/apim4e3fb6/tenant/gitAccess/listSecrets?api-version=2024-05-01\"", + "answer": { + "exitCode": 0, + "stdout": "{\"enabled\":true,\"primaryKey\":\"20d379f4633084381a0c9fb5bf8e197d\",\"principalId\":\"git\",\"secondaryKey\":\"e485243048dcdf7e8a4f1952446581fa\"}", + "stderr": "", + "classId": null + } + } + ] + }, + { + "task": "apim-git-regenerate-primary", + "mode": "oracle", + "passed": true, + "wait_s": 90, + "recorded": "2026-09-27T21:36:21Z", + "slots": { + "resource_group": "mcpe2-15123a-apim-git-regenerate-primary-40", + "rg": "mcpe2-15123a-apim-git-regenerate-primary-40", + "service_name": "apimbf2a40", + "name": "apimbf2a40", + "_before_changed": "06c6805eb78767acb7f91bf9244f73ea", + "_before_kept": "b5891b8031bb3e5c6213525aeb86e7bd", + "_oracle": true + }, + "text": "ORACLE ✅ az rest --method post --url /subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/mcpe2-15123a-apim-git-regenerate-primary-40/providers/Microsoft.ApiManagement/service/apimbf2a40/tenant/gitAccess/regeneratePrimaryKey?api-version=2024-05-01\n```\nThe command succeeded and printed no output.\n```", + "queries": [ + { + "command": "rest --method post --url \"/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/mcpe2-15123a-apim-git-regenerate-primary-40/providers/Microsoft.ApiManagement/service/apimbf2a40/tenant/gitAccess/listSecrets?api-version=2024-05-01\"", + "answer": { + "exitCode": 0, + "stdout": "{\"enabled\":true,\"primaryKey\":\"c64a642a3339e8f0b2e98480cf83d1b9\",\"principalId\":\"git\",\"secondaryKey\":\"b5891b8031bb3e5c6213525aeb86e7bd\"}", + "stderr": "", + "classId": null + } + } + ] + }, + { + "task": "apim-graphql-resolver", + "mode": "negative", + "passed": false, + "wait_s": 10, + "recorded": "2026-09-27T21:16:18Z", + "slots": { + "resource_group": "mcpe2-151aca-apim-graphql-resolver-47", + "rg": "mcpe2-151aca-apim-graphql-resolver-47", + "service_name": "apim411cc6", + "name": "apim411cc6", + "field": "orders616e5f", + "resolver": "orders-resolver-616e5f", + "api": "api8409af" + }, + "text": "", + "queries": [ + { + "command": "rest --method get --url \"/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/mcpe2-151aca-apim-graphql-resolver-47/providers/Microsoft.ApiManagement/service/apim411cc6/apis/api8409af/resolvers/orders-resolver-616e5f?api-version=2024-05-01\"", + "repeat": 7, + "answer": { + "exitCode": 1, + "stdout": "", + "stderr": "ERROR: NOT FOUND({\"error\": {\"code\": \"ResourceNotFound\", \"message\": \"GraphQLApiResolver not found.\", \"details\": [], \"additionalInfo\": []}})\r\n", + "classId": "not-found" + } + } + ] + }, + { + "task": "apim-graphql-resolver", + "mode": "oracle", + "passed": true, + "wait_s": 90, + "recorded": "2026-09-27T21:38:44Z", + "slots": { + "resource_group": "mcpe2-15123a-apim-graphql-resolver-47", + "rg": "mcpe2-15123a-apim-graphql-resolver-47", + "service_name": "apimf9382d", + "name": "apimf9382d", + "field": "ordersa3b775", + "resolver": "orders-resolver-a3b775", + "api": "apie01e63", + "_oracle": true + }, + "text": "ORACLE ✅ az apim graphql resolver create --service-name apimf9382d --resource-group mcpe2-15123a-apim-graphql-resolver-47 --api-id apie01e63 --resolver-id orders-resolver-a3b775 --display-name orders-resolver-a3b775 --path Query/ordersa3b775\n```json\n{\n \"description\": null,\n \"displayName\": \"orders-resolver-a3b775\",\n \"id\": \"/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/mcpe2-15123a-apim-graphql-resolver-47/providers/Microsoft.ApiManagement/service/apimf9382d/apis/apie01e63/resolvers/orders-resolver-a3b775\",\n \"name\": \"orders-resolver-a3b775\",\n \"path\": \"Query/ordersa3b775\",\n \"resourceGroup\": \"mcpe2-15123a-apim-graphql-resolver-47\",\n \"type\": \"Microsoft.ApiManagement/service/apis/resolvers\"\n}\n```", + "queries": [ + { + "command": "rest --method get --url \"/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/mcpe2-15123a-apim-graphql-resolver-47/providers/Microsoft.ApiManagement/service/apimf9382d/apis/apie01e63/resolvers/orders-resolver-a3b775?api-version=2024-05-01\"", + "answer": { + "exitCode": 0, + "stdout": "{\"id\":\"/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/mcpe2-15123a-apim-graphql-resolver-47/providers/Microsoft.ApiManagement/service/apimf9382d/apis/apie01e63/resolvers/orders-resolver-a3b775\",\"name\":\"orders-resolver-a3b775\",\"properties\":{\"displayName\":\"orders-resolver-a3b775\",\"path\":\"Query/ordersa3b775\"},\"type\":\"Microsoft.ApiManagement/service/apis/resolvers\"}", + "stderr": "", + "classId": null + } + } + ] + }, + { + "task": "apim-group-user-check", + "mode": "negative", + "passed": false, + "wait_s": 10, + "recorded": "2026-09-27T21:10:40Z", + "slots": { + "resource_group": "mcpe2-151aca-apim-group-user-check-37", + "rg": "mcpe2-151aca-apim-group-user-check-37", + "service_name": "apim4d29e8", + "name": "apim4d29e8", + "group": "devs2941ec", + "user1": "aliced62093", + "user2": "bob7cc433", + "member": "bob7cc433" + }, + "text": "", + "queries": [] + }, + { + "task": "apim-group-user-check", + "mode": "oracle", + "passed": true, + "wait_s": 90, + "recorded": "2026-09-27T21:35:29Z", + "slots": { + "resource_group": "mcpe2-15123a-apim-group-user-check-37", + "rg": "mcpe2-15123a-apim-group-user-check-37", + "service_name": "apim141fae", + "name": "apim141fae", + "group": "devse9b8c4", + "user1": "aliceeee20c", + "user2": "bobe65de8", + "member": "aliceeee20c", + "_oracle": true + }, + "text": "ORACLE ✅ az rest --method head --url /subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/mcpe2-15123a-apim-group-user-check-37/providers/Microsoft.ApiManagement/service/apim141fae/groups/devse9b8c4/users/aliceeee20c?api-version=2024-05-01\n```\nThe command succeeded and printed no output.\n```\nORACLE ❌ az rest --method head --url /subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/mcpe2-15123a-apim-group-user-check-37/providers/Microsoft.ApiManagement/service/apim141fae/groups/devse9b8c4/users/bobe65de8?api-version=2024-05-01\n```\n❌ **Command Failed** (exit 1, other)\n\nERROR: NOT FOUND\n```", + "queries": [] + }, + { + "task": "apim-migrate-stv2", + "mode": "negative", + "passed": false, + "wait_s": 10, + "recorded": "2026-09-27T21:17:45Z", + "slots": { + "resource_group": "mcpe2-151aca-apim-migrate-stv2-50", + "rg": "mcpe2-151aca-apim-migrate-stv2-50", + "service_name": "apim9b8a7c", + "name": "apim9b8a7c" + }, + "text": "", + "queries": [ + { + "command": "rest --method get --url \"/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/mcpe2-151aca-apim-migrate-stv2-50/providers/Microsoft.ApiManagement/service/apim9b8a7c?api-version=2024-05-01\"", + "repeat": 7, + "answer": { + "exitCode": 0, + "stdout": "{\"etag\":\"AAAAAAAAAAE=\",\"id\":\"/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/mcpe2-151aca-apim-migrate-stv2-50/providers/Microsoft.ApiManagement/service/apim9b8a7c\",\"location\":\"West Europe\",\"name\":\"apim9b8a7c\",\"properties\":{\"apiVersionConstraint\":{\"minApiVersion\":null},\"createdAtUtc\":\"2026-09-27T21:17:16.3084470Z\",\"customProperties\":{\"Microsoft.WindowsAzure.ApiManagement.Gateway.Protocols.Server.Http2\":\"False\",\"Microsoft.WindowsAzure.ApiManagement.Gateway.Security.Backend.Protocols.Ssl30\":\"False\",\"Microsoft.WindowsAzure.ApiManagement.Gateway.Security.Backend.Protocols.Tls10\":\"False\",\"Microsoft.WindowsAzure.ApiManagement.Gateway.Security.Backend.Protocols.Tls11\":\"False\",\"Microsoft.WindowsAzure.ApiManagement.Gateway.Security.Ciphers.TripleDes168\":\"False\",\"Microsoft.WindowsAzure.ApiManagement.Gateway.Security.Protocols.Ssl30\":\"False\",\"Microsoft.WindowsAzure.ApiManagement.Gateway.Security.Protocols.Tls10\":\"False\",\"Microsoft.WindowsAzure.ApiManagement.Gateway.Security.Protocols.Tls11\":\"False\"},\"developerPortalStatus\":\"Enabled\",\"developerPortalUrl\":\"https://apim9b8a7c.developer.azure-api.net\",\"disableGateway\":false,\"gatewayRegionalUrl\":\"https://apim9b8a7c-westeurope-01.regional.azure-api.net\",\"gatewayUrl\":\"https://apim9b8a7c.azure-api.net\",\"hostnameConfigurations\":[{\"certificateSource\":\"BuiltIn\",\"defaultSslBinding\":true,\"hostName\":\"apim9b8a7c.azure-api.net\",\"negotiateClientCertificate\":false,\"type\":\"Proxy\"}],\"legacyPortalStatus\":\"Disabled\",\"managementApiUrl\":\"https://apim9b8a7c.management.azure-api.net\",\"natGatewayState\":\"Unsupported\",\"notificationSenderEmail\":\"apimgmt-noreply@mail.windowsazure.com\",\"outboundPublicIPAddresses\":[\"203.0.113.6\"],\"platformVersion\":\"stv2.1\",\"portalUrl\":null,\"provisioningState\":\"Succeeded\",\"publicIPAddresses\":[\"203.0.113.6\"],\"publicNetworkAccess\":\"Enabled\",\"publisherEmail\":\"ops@contoso.com\",\"publisherName\":\"Contoso\",\"scmUrl\":\"https://apim9b8a7c.scm.azure-api.net\",\"targetProvisioningState\":\"\",\"virtualNetworkType\":\"None\"},\"sku\":{\"capacity\":1,\"name\":\"Developer\"},\"systemData\":{\"createdAt\":\"2026-09-27T21:17:16.3084470Z\",\"createdBy\":\"any-app\",\"createdByType\":\"Application\",\"lastModifiedAt\":\"2026-09-27T21:17:16.3084470Z\",\"lastModifiedBy\":\"any-app\",\"lastModifiedByType\":\"Application\"},\"tags\":{},\"type\":\"Microsoft.ApiManagement/service\"}", + "stderr": "", + "classId": null + } + } + ] + }, + { + "task": "apim-migrate-stv2", + "mode": "oracle", + "passed": true, + "wait_s": 90, + "recorded": "2026-09-27T21:39:26Z", + "slots": { + "resource_group": "mcpe2-15123a-apim-migrate-stv2-50", + "rg": "mcpe2-15123a-apim-migrate-stv2-50", + "service_name": "apim48f45b", + "name": "apim48f45b", + "_oracle": true + }, + "text": "ORACLE ✅ az rest --method post --url /subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/mcpe2-15123a-apim-migrate-stv2-50/providers/Microsoft.ApiManagement/service/apim48f45b/migrateToStv2?api-version=2024-05-01 --body '{\"mode\": \"PreserveIp\"}'\n```\nThe command succeeded and printed no output.\n```", + "queries": [ + { + "command": "rest --method get --url \"/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/mcpe2-15123a-apim-migrate-stv2-50/providers/Microsoft.ApiManagement/service/apim48f45b?api-version=2024-05-01\"", + "answer": { + "exitCode": 0, + "stdout": "{\"etag\":\"AAAAAAAAAAI=\",\"id\":\"/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/mcpe2-15123a-apim-migrate-stv2-50/providers/Microsoft.ApiManagement/service/apim48f45b\",\"location\":\"West Europe\",\"name\":\"apim48f45b\",\"properties\":{\"apiVersionConstraint\":{\"minApiVersion\":null},\"createdAtUtc\":\"2026-09-27T21:39:15.3426200Z\",\"customProperties\":{\"Microsoft.WindowsAzure.ApiManagement.Gateway.Protocols.Server.Http2\":\"False\",\"Microsoft.WindowsAzure.ApiManagement.Gateway.Security.Backend.Protocols.Ssl30\":\"False\",\"Microsoft.WindowsAzure.ApiManagement.Gateway.Security.Backend.Protocols.Tls10\":\"False\",\"Microsoft.WindowsAzure.ApiManagement.Gateway.Security.Backend.Protocols.Tls11\":\"False\",\"Microsoft.WindowsAzure.ApiManagement.Gateway.Security.Ciphers.TripleDes168\":\"False\",\"Microsoft.WindowsAzure.ApiManagement.Gateway.Security.Protocols.Ssl30\":\"False\",\"Microsoft.WindowsAzure.ApiManagement.Gateway.Security.Protocols.Tls10\":\"False\",\"Microsoft.WindowsAzure.ApiManagement.Gateway.Security.Protocols.Tls11\":\"False\"},\"developerPortalStatus\":\"Enabled\",\"developerPortalUrl\":\"https://apim48f45b.developer.azure-api.net\",\"disableGateway\":false,\"gatewayRegionalUrl\":\"https://apim48f45b-westeurope-01.regional.azure-api.net\",\"gatewayUrl\":\"https://apim48f45b.azure-api.net\",\"hostnameConfigurations\":[{\"certificateSource\":\"BuiltIn\",\"defaultSslBinding\":true,\"hostName\":\"apim48f45b.azure-api.net\",\"negotiateClientCertificate\":false,\"type\":\"Proxy\"}],\"legacyPortalStatus\":\"Disabled\",\"managementApiUrl\":\"https://apim48f45b.management.azure-api.net\",\"natGatewayState\":\"Unsupported\",\"notificationSenderEmail\":\"apimgmt-noreply@mail.windowsazure.com\",\"outboundPublicIPAddresses\":[\"203.0.113.234\"],\"platformVersion\":\"stv2\",\"portalUrl\":null,\"provisioningState\":\"Succeeded\",\"publicIPAddresses\":[\"203.0.113.234\"],\"publicNetworkAccess\":\"Enabled\",\"publisherEmail\":\"ops@contoso.com\",\"publisherName\":\"Contoso\",\"scmUrl\":\"https://apim48f45b.scm.azure-api.net\",\"targetProvisioningState\":\"\",\"virtualNetworkType\":\"None\"},\"sku\":{\"capacity\":1,\"name\":\"Developer\"},\"systemData\":{\"createdAt\":\"2026-09-27T21:39:15.3426200Z\",\"createdBy\":\"any-app\",\"createdByType\":\"Application\",\"lastModifiedAt\":\"2026-09-27T21:39:15.3426200Z\",\"lastModifiedBy\":\"any-app\",\"lastModifiedByType\":\"Application\"},\"tags\":{},\"type\":\"Microsoft.ApiManagement/service\"}", + "stderr": "", + "classId": null + } + } + ] + }, + { + "task": "apim-oidc-secret", + "mode": "negative", + "passed": false, + "wait_s": 10, + "recorded": "2026-09-27T21:17:14Z", + "slots": { + "resource_group": "mcpe2-151aca-apim-oidc-secret-49", + "rg": "mcpe2-151aca-apim-oidc-secret-49", + "service_name": "apimc5228b", + "name": "apimc5228b", + "secret": "oidc-97e929-s3cr3t", + "oidc": "oidcf330ad" + }, + "text": "", + "queries": [] + }, + { + "task": "apim-oidc-secret", + "mode": "oracle", + "passed": true, + "wait_s": 90, + "recorded": "2026-09-27T21:39:12Z", + "slots": { + "resource_group": "mcpe2-15123a-apim-oidc-secret-49", + "rg": "mcpe2-15123a-apim-oidc-secret-49", + "service_name": "apim49b37e", + "name": "apim49b37e", + "secret": "oidc-b944e1-s3cr3t", + "oidc": "oidca38a4f", + "_oracle": true + }, + "text": "ORACLE ✅ az rest --method post --url /subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/mcpe2-15123a-apim-oidc-secret-49/providers/Microsoft.ApiManagement/service/apim49b37e/openidConnectProviders/oidca38a4f/listSecrets?api-version=2024-05-01\n```json\n{\n \"clientSecret\": \"oidc-b944e1-s3cr3t\"\n}\n```", + "queries": [] + }, + { + "task": "apim-operation-delete", + "mode": "negative", + "passed": false, + "wait_s": 10, + "recorded": "2026-09-27T21:13:15Z", + "slots": { + "resource_group": "mcpe2-151aca-apim-operation-delete-42", + "rg": "mcpe2-151aca-apim-operation-delete-42", + "service_name": "apim719a60", + "name": "apim719a60", + "api": "ordersc34bb8", + "operation": "getorder0e0a18", + "calls": 17 + }, + "text": "", + "queries": [ + { + "command": "rest --method get --url \"/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/mcpe2-151aca-apim-operation-delete-42/providers/Microsoft.ApiManagement/service/apim719a60/apis/ordersc34bb8/operations/getorder0e0a18?api-version=2024-05-01\"", + "repeat": 6, + "answer": { + "exitCode": 0, + "stdout": "{\"id\":\"/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/mcpe2-151aca-apim-operation-delete-42/providers/Microsoft.ApiManagement/service/apim719a60/apis/ordersc34bb8/operations/getorder0e0a18\",\"name\":\"getorder0e0a18\",\"properties\":{\"displayName\":\"Get order\",\"method\":\"GET\",\"request\":{\"description\":null,\"headers\":[],\"queryParameters\":[],\"representations\":[]},\"responses\":[],\"templateParameters\":[{\"name\":\"id\",\"required\":true,\"type\":\"string\",\"values\":[]}],\"urlTemplate\":\"/orders/{id}\"},\"type\":\"Microsoft.ApiManagement/service/apis/operations\"}", + "stderr": "", + "classId": null + } + } + ] + }, + { + "task": "apim-operation-delete", + "mode": "oracle", + "passed": true, + "wait_s": 90, + "recorded": "2026-09-27T21:37:08Z", + "slots": { + "resource_group": "mcpe2-15123a-apim-operation-delete-42", + "rg": "mcpe2-15123a-apim-operation-delete-42", + "service_name": "apimc60c34", + "name": "apimc60c34", + "api": "ordersc600b5", + "operation": "getorder72f591", + "calls": 40, + "_oracle": true + }, + "text": "ORACLE ✅ az apim api operation delete --service-name apimc60c34 --resource-group mcpe2-15123a-apim-operation-delete-42 --api-id ordersc600b5 --operation-id getorder72f591\n```\nThe command succeeded and printed no output.\n```", + "queries": [ + { + "command": "rest --method get --url \"/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/mcpe2-15123a-apim-operation-delete-42/providers/Microsoft.ApiManagement/service/apimc60c34/apis/ordersc600b5/operations/getorder72f591?api-version=2024-05-01\"", + "answer": { + "exitCode": 1, + "stdout": "", + "stderr": "ERROR: NOT FOUND({\"error\": {\"code\": \"ResourceNotFound\", \"message\": \"Operation not found.\", \"details\": [], \"additionalInfo\": []}})\r\n", + "classId": "not-found" + } + } + ] + }, + { + "task": "apim-operation-policy", + "mode": "negative", + "passed": false, + "wait_s": 10, + "recorded": "2026-09-27T21:13:44Z", + "slots": { + "resource_group": "mcpe2-151aca-apim-operation-policy-43", + "rg": "mcpe2-151aca-apim-operation-policy-43", + "service_name": "apim651ac2", + "name": "apim651ac2", + "api": "orders531b06", + "operation": "getorder65c509", + "calls": 12 + }, + "text": "", + "queries": [ + { + "command": "rest --method get --url \"/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/mcpe2-151aca-apim-operation-policy-43/providers/Microsoft.ApiManagement/service/apim651ac2/apis/orders531b06/operations/getorder65c509/policies/policy?api-version=2024-05-01\"", + "repeat": 6, + "answer": { + "exitCode": 1, + "stdout": "", + "stderr": "ERROR: NOT FOUND({\"error\": {\"code\": \"ResourceNotFound\", \"message\": \"PoliciesConfiguration not found.\", \"details\": [], \"additionalInfo\": []}})\r\n", + "classId": "not-found" + } + } + ] + }, + { + "task": "apim-operation-policy", + "mode": "oracle", + "passed": true, + "wait_s": 90, + "recorded": "2026-09-27T21:37:33Z", + "slots": { + "resource_group": "mcpe2-15123a-apim-operation-policy-43", + "rg": "mcpe2-15123a-apim-operation-policy-43", + "service_name": "apim33bb95", + "name": "apim33bb95", + "api": "orders72b0e9", + "operation": "getorder8e5a65", + "calls": 40, + "_oracle": true + }, + "text": "ORACLE ✅ az rest --method put --url /subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/mcpe2-15123a-apim-operation-policy-43/providers/Microsoft.ApiManagement/service/apim33bb95/apis/orders72b0e9/operations/getorder8e5a65/policies/policy?api-version=2024-05-01 --body '{\"properties\": {\"format\": \"rawxml\", \"value\": \"\\u003cpolicies\\u003e\\u003cinbound\\u003e\\u003cbase /\\u003e\\u003crate-limit calls=\\\"40\\\" renewal-period=\\\"60\\\" /\\u003e\\u003c/inbound\\u003e\\u003cbackend\\u003e\\u003cbase /\\u003e\\u003c/backend\\u003e\\u003coutbound\\u003e\\u003cbase /\\u003e\\u003c/outbound\\u003e\\u003con-error\\u003e\\u003cbase /\\u003e\\u003c/on-error\\u003e\\u003c/policies\\u003e\"}}'\n```json\n{\n \"id\": \"/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/mcpe2-15123a-apim-operation-policy-43/providers/Microsoft.ApiManagement/service/apim33bb95/apis/orders72b0e9/operations/getorder8e5a65/policies/policy\",\n \"name\": \"policy\",\n \"properties\": {\n \"format\": \"rawxml\",\n \"value\": \"\\r\\n\\t\\r\\n\\t\\t\\r\\n\\t\\t\\r\\n\\t\\r\\n\\t\\r\\n\\t\\t\\r\\n\\t\\r\\n\\t\\r\\n\\t\\t\\r\\n\\t\\r\\n\\t\\r\\n\\t\\t\\r\\n\\t\\r\\n\"\n },\n \"type\": \"Microsoft.ApiManagement/service/apis/operations/policies\"\n}\n```", + "queries": [ + { + "command": "rest --method get --url \"/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/mcpe2-15123a-apim-operation-policy-43/providers/Microsoft.ApiManagement/service/apim33bb95/apis/orders72b0e9/operations/getorder8e5a65/policies/policy?api-version=2024-05-01\"", + "answer": { + "exitCode": 0, + "stdout": "{\"id\":\"/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/mcpe2-15123a-apim-operation-policy-43/providers/Microsoft.ApiManagement/service/apim33bb95/apis/orders72b0e9/operations/getorder8e5a65/policies/policy\",\"name\":\"policy\",\"properties\":{\"format\":\"rawxml\",\"value\":\"\\r\\n\\t\\r\\n\\t\\t\\r\\n\\t\\t\\r\\n\\t\\r\\n\\t\\r\\n\\t\\t\\r\\n\\t\\r\\n\\t\\r\\n\\t\\t\\r\\n\\t\\r\\n\\t\\r\\n\\t\\t\\r\\n\\t\\r\\n\"},\"type\":\"Microsoft.ApiManagement/service/apis/operations/policies\"}", + "stderr": "", + "classId": null + } + } + ] + }, + { + "task": "apim-tag-assign-api", + "mode": "negative", + "passed": false, + "wait_s": 10, + "recorded": "2026-09-27T21:14:18Z", + "slots": { + "resource_group": "mcpe2-151aca-apim-tag-assign-api-44", + "rg": "mcpe2-151aca-apim-tag-assign-api-44", + "service_name": "apima007d1", + "name": "apima007d1", + "api": "apia189a1", + "tag": "tagfb7aa6" + }, + "text": "", + "queries": [ + { + "command": "rest --method get --url \"/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/mcpe2-151aca-apim-tag-assign-api-44/providers/Microsoft.ApiManagement/service/apima007d1/apis/apia189a1/tags/tagfb7aa6?api-version=2024-05-01\"", + "repeat": 7, + "answer": { + "exitCode": 1, + "stdout": "", + "stderr": "ERROR: NOT FOUND({\"error\": {\"code\": \"ResourceNotFound\", \"message\": \"Tag not found.\", \"details\": [], \"additionalInfo\": []}})\r\n", + "classId": "not-found" + } + } + ] + }, + { + "task": "apim-tag-assign-api", + "mode": "oracle", + "passed": true, + "wait_s": 90, + "recorded": "2026-09-27T21:37:49Z", + "slots": { + "resource_group": "mcpe2-15123a-apim-tag-assign-api-44", + "rg": "mcpe2-15123a-apim-tag-assign-api-44", + "service_name": "apimecdfeb", + "name": "apimecdfeb", + "api": "apiffcc7c", + "tag": "tag537932", + "_oracle": true + }, + "text": "ORACLE ✅ az rest --method put --url /subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/mcpe2-15123a-apim-tag-assign-api-44/providers/Microsoft.ApiManagement/service/apimecdfeb/apis/apiffcc7c/tags/tag537932?api-version=2024-05-01\n```json\n{\n \"id\": \"/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/mcpe2-15123a-apim-tag-assign-api-44/providers/Microsoft.ApiManagement/service/apimecdfeb/apis/apiffcc7c/tags/tag537932\",\n \"name\": \"tag537932\",\n \"properties\": {\n \"displayName\": \"tag537932\"\n },\n \"type\": \"Microsoft.ApiManagement/service/apis/tags\"\n}\n```", + "queries": [ + { + "command": "rest --method get --url \"/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/mcpe2-15123a-apim-tag-assign-api-44/providers/Microsoft.ApiManagement/service/apimecdfeb/apis/apiffcc7c/tags/tag537932?api-version=2024-05-01\"", + "answer": { + "exitCode": 0, + "stdout": "{\"id\":\"/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/mcpe2-15123a-apim-tag-assign-api-44/providers/Microsoft.ApiManagement/service/apimecdfeb/apis/apiffcc7c/tags/tag537932\",\"name\":\"tag537932\",\"properties\":{\"displayName\":\"tag537932\"},\"type\":\"Microsoft.ApiManagement/service/apis/tags\"}", + "stderr": "", + "classId": null + } + } + ] + }, + { + "task": "apim-tag-detach-product", + "mode": "negative", + "passed": false, + "wait_s": 10, + "recorded": "2026-09-27T21:15:06Z", + "slots": { + "resource_group": "mcpe2-151aca-apim-tag-detach-product-45", + "rg": "mcpe2-151aca-apim-tag-detach-product-45", + "service_name": "apim11ba0f", + "name": "apim11ba0f", + "product": "produceb304a", + "tag": "taga98b6c" + }, + "text": "", + "queries": [ + { + "command": "rest --method get --url \"/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/mcpe2-151aca-apim-tag-detach-product-45/providers/Microsoft.ApiManagement/service/apim11ba0f/products/produceb304a/tags/taga98b6c?api-version=2024-05-01\"", + "repeat": 6, + "answer": { + "exitCode": 0, + "stdout": "{\"id\":\"/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/mcpe2-151aca-apim-tag-detach-product-45/providers/Microsoft.ApiManagement/service/apim11ba0f/products/produceb304a/tags/taga98b6c\",\"name\":\"taga98b6c\",\"properties\":{\"displayName\":\"taga98b6c\"},\"type\":\"Microsoft.ApiManagement/service/tags\"}", + "stderr": "", + "classId": null + } + }, + { + "command": "rest --method get --url \"/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/mcpe2-151aca-apim-tag-detach-product-45/providers/Microsoft.ApiManagement/service/apim11ba0f/tags/taga98b6c?api-version=2024-05-01\"", + "answer": { + "exitCode": 0, + "stdout": "{\"id\":\"/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/mcpe2-151aca-apim-tag-detach-product-45/providers/Microsoft.ApiManagement/service/apim11ba0f/tags/taga98b6c\",\"name\":\"taga98b6c\",\"properties\":{\"displayName\":\"taga98b6c\"},\"type\":\"Microsoft.ApiManagement/service/tags\"}", + "stderr": "", + "classId": null + } + } + ] + }, + { + "task": "apim-tag-detach-product", + "mode": "oracle", + "passed": true, + "wait_s": 90, + "recorded": "2026-09-27T21:38:11Z", + "slots": { + "resource_group": "mcpe2-15123a-apim-tag-detach-product-45", + "rg": "mcpe2-15123a-apim-tag-detach-product-45", + "service_name": "apim79229c", + "name": "apim79229c", + "product": "produc50eb22", + "tag": "tag22718c", + "_oracle": true + }, + "text": "ORACLE ✅ az rest --method delete --url /subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/mcpe2-15123a-apim-tag-detach-product-45/providers/Microsoft.ApiManagement/service/apim79229c/products/produc50eb22/tags/tag22718c?api-version=2024-05-01\n```\nThe command succeeded and printed no output.\n```", + "queries": [ + { + "command": "rest --method get --url \"/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/mcpe2-15123a-apim-tag-detach-product-45/providers/Microsoft.ApiManagement/service/apim79229c/products/produc50eb22/tags/tag22718c?api-version=2024-05-01\"", + "answer": { + "exitCode": 1, + "stdout": "", + "stderr": "ERROR: NOT FOUND({\"error\": {\"code\": \"ResourceNotFound\", \"message\": \"Tag not found.\", \"details\": [], \"additionalInfo\": []}})\r\n", + "classId": "not-found" + } + }, + { + "command": "rest --method get --url \"/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/mcpe2-15123a-apim-tag-detach-product-45/providers/Microsoft.ApiManagement/service/apim79229c/tags/tag22718c?api-version=2024-05-01\"", + "answer": { + "exitCode": 0, + "stdout": "{\"id\":\"/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/mcpe2-15123a-apim-tag-detach-product-45/providers/Microsoft.ApiManagement/service/apim79229c/tags/tag22718c\",\"name\":\"tag22718c\",\"properties\":{\"displayName\":\"tag22718c\"},\"type\":\"Microsoft.ApiManagement/service/tags\"}", + "stderr": "", + "classId": null + } + } + ] + }, + { + "task": "apim-tenant-access", + "mode": "negative", + "passed": false, + "wait_s": 10, + "recorded": "2026-09-27T21:11:40Z", + "slots": { + "resource_group": "mcpe2-151aca-apim-tenant-access-39", + "rg": "mcpe2-151aca-apim-tenant-access-39", + "service_name": "apim044e5c", + "name": "apim044e5c", + "access_enabled": false + }, + "text": "", + "queries": [] + }, + { + "task": "apim-tenant-access", + "mode": "oracle", + "passed": true, + "wait_s": 90, + "recorded": "2026-09-27T21:36:03Z", + "slots": { + "resource_group": "mcpe2-15123a-apim-tenant-access-39", + "rg": "mcpe2-15123a-apim-tenant-access-39", + "service_name": "apim789e6d", + "name": "apim789e6d", + "access_enabled": false, + "_oracle": true + }, + "text": "ORACLE ✅ az rest --method get --url /subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/mcpe2-15123a-apim-tenant-access-39/providers/Microsoft.ApiManagement/service/apim789e6d/tenant?api-version=2024-05-01\n```json\n{\n \"count\": 2,\n \"value\": [\n {\n \"id\": \"/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/mcpe2-15123a-apim-tenant-access-39/providers/Microsoft.ApiManagement/service/apim789e6d/tenant/access\",\n \"name\": \"access\",\n \"properties\": {\n \"enabled\": false,\n \"id\": null,\n \"principalId\": \"integration\"\n },\n \"type\": \"Microsoft.ApiManagement/service/tenant\"\n },\n {\n \"id\": \"/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/mcpe2-15123a-apim-tenant-access-39/providers/Microsoft.ApiManagement/service/apim789e6d/tenant/gitAccess\",\n \"name\": \"gitAccess\",\n \"properties\": {\n \"enabled\": true,\n \"id\": null,\n \"principalId\": \"git\"\n },\n \"type\": \"Microsoft.ApiManagement/service/tenant\"\n }\n ]\n}\n```", + "queries": [] + }, + { + "task": "apim-workspace-product-update", + "mode": "negative", + "passed": false, + "wait_s": 10, + "recorded": "2026-09-27T21:11:17Z", + "slots": { + "resource_group": "mcpe2-151aca-apim-workspace-product-update-38", + "rg": "mcpe2-151aca-apim-workspace-product-update-38", + "service_name": "apimc0d2f0", + "name": "apimc0d2f0", + "description": "Partner tier, reviewed 713906", + "workspace": "worksp43df5c", + "product": "producfa03ab" + }, + "text": "", + "queries": [ + { + "command": "rest --method get --url \"/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/mcpe2-151aca-apim-workspace-product-update-38/providers/Microsoft.ApiManagement/service/apimc0d2f0/workspaces/worksp43df5c/products/producfa03ab?api-version=2024-05-01\"", + "repeat": 6, + "answer": { + "exitCode": 0, + "stdout": "{\"id\":\"/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/mcpe2-151aca-apim-workspace-product-update-38/providers/Microsoft.ApiManagement/service/apimc0d2f0/workspaces/worksp43df5c/products/producfa03ab\",\"name\":\"producfa03ab\",\"properties\":{\"approvalRequired\":false,\"displayName\":\"producfa03ab\",\"state\":\"notPublished\",\"subscriptionRequired\":true},\"type\":\"Microsoft.ApiManagement/service/workspaces/products\"}", + "stderr": "", + "classId": null + } + } + ] + }, + { + "task": "apim-workspace-product-update", + "mode": "oracle", + "passed": true, + "wait_s": 90, + "recorded": "2026-09-27T21:35:47Z", + "slots": { + "resource_group": "mcpe2-15123a-apim-workspace-product-update-38", + "rg": "mcpe2-15123a-apim-workspace-product-update-38", + "service_name": "apimb2be75", + "name": "apimb2be75", + "description": "Partner tier, reviewed 85d6ee", + "workspace": "worksp51c7b3", + "product": "produc881971", + "_oracle": true + }, + "text": "ORACLE ✅ az rest --method patch --url /subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/mcpe2-15123a-apim-workspace-product-update-38/providers/Microsoft.ApiManagement/service/apimb2be75/workspaces/worksp51c7b3/products/produc881971?api-version=2024-05-01 --body '{\"properties\": {\"description\": \"Partner tier, reviewed 85d6ee\"}}'\n```json\n{\n \"id\": \"/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/mcpe2-15123a-apim-workspace-product-update-38/providers/Microsoft.ApiManagement/service/apimb2be75/workspaces/worksp51c7b3/products/produc881971\",\n \"name\": \"produc881971\",\n \"properties\": {\n \"approvalRequired\": false,\n \"description\": \"Partner tier, reviewed 85d6ee\",\n \"displayName\": \"produc881971\",\n \"state\": \"notPublished\",\n \"subscriptionRequired\": true\n },\n \"type\": \"Microsoft.ApiManagement/service/workspaces/products\"\n}\n```", + "queries": [ + { + "command": "rest --method get --url \"/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/mcpe2-15123a-apim-workspace-product-update-38/providers/Microsoft.ApiManagement/service/apimb2be75/workspaces/worksp51c7b3/products/produc881971?api-version=2024-05-01\"", + "answer": { + "exitCode": 0, + "stdout": "{\"id\":\"/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/mcpe2-15123a-apim-workspace-product-update-38/providers/Microsoft.ApiManagement/service/apimb2be75/workspaces/worksp51c7b3/products/produc881971\",\"name\":\"produc881971\",\"properties\":{\"approvalRequired\":false,\"description\":\"Partner tier, reviewed 85d6ee\",\"displayName\":\"produc881971\",\"state\":\"notPublished\",\"subscriptionRequired\":true},\"type\":\"Microsoft.ApiManagement/service/workspaces/products\"}", + "stderr": "", + "classId": null + } + } + ] + }, + { + "task": "appconfig-check-keys", + "mode": "negative", + "passed": false, + "wait_s": 10, + "recorded": "2026-09-27T20:52:03Z", + "slots": { + "resource_group": "mcpe2-151aca-appconfig-check-keys-2", + "rg": "mcpe2-151aca-appconfig-check-keys-2", + "name": "appcfg01bc3e34", + "store_name": "appcfg01bc3e34", + "key": "key01bc3e34", + "replica_name": "rep01bc3e34", + "snapshot_name": "snap01bc3e34", + "missing_key": "legacy-timeout-137343" + }, + "text": "", + "queries": [ + { + "command": "rest --method get --url \"/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/mcpe2-151aca-appconfig-check-keys-2/providers/Microsoft.AppConfiguration/configurationStores/appcfg01bc3e34/keyValues/key01bc3e34?api-version=2024-06-01\"", + "answer": { + "exitCode": 0, + "stdout": "{\"id\":\"/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/mcpe2-151aca-appconfig-check-keys-2/providers/Microsoft.AppConfiguration/configurationStores/appcfg01bc3e34/keyValues/key01bc3e34\",\"name\":\"key01bc3e34\",\"properties\":{\"contentType\":\"\",\"eTag\":\"0a0ed4a1c00f462dabbabfaac0ebdf9b\",\"key\":\"key01bc3e34\",\"lastModified\":\"2026-09-27T20:51:48.073009+00:00\",\"locked\":false,\"tags\":{},\"value\":\"v1\"},\"type\":\"Microsoft.AppConfiguration/configurationStores/keyValues\"}", + "stderr": "", + "classId": null + } + }, + { + "command": "rest --method get --url \"/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/mcpe2-151aca-appconfig-check-keys-2/providers/Microsoft.AppConfiguration/configurationStores/appcfg01bc3e34/keyValues/legacy-timeout-137343?api-version=2024-06-01\"", + "answer": { + "exitCode": 1, + "stdout": "", + "stderr": "ERROR: NOT FOUND({\"error\": {\"code\": \"EntityNotFound\", \"message\": \"The resource with id 'legacy-timeout-137343' was not found.\", \"details\": [], \"additionalInfo\": []}})\r\n", + "classId": "other" + } + } + ] + }, + { + "task": "appconfig-check-keys", + "mode": "oracle", + "passed": true, + "wait_s": 90, + "recorded": "2026-09-27T21:19:38Z", + "slots": { + "resource_group": "mcpe2-15123a-appconfig-check-keys-2", + "rg": "mcpe2-15123a-appconfig-check-keys-2", + "name": "appcfg0b6c257f", + "store_name": "appcfg0b6c257f", + "key": "key0b6c257f", + "replica_name": "rep0b6c257f", + "snapshot_name": "snap0b6c257f", + "missing_key": "legacy-timeout-49967b", + "_oracle": true + }, + "text": "ORACLE ✅ az appconfig kv list --name appcfg0b6c257f --key key0b6c257f\n```json\n[\n {\n \"contentType\": \"\",\n \"etag\": \"b5051d35ca9145c5b20f09fc4e1a45d3\",\n \"key\": \"key0b6c257f\",\n \"label\": null,\n \"lastModified\": \"2026-09-27T21:19:24.416836+00:00\",\n \"locked\": false,\n \"tags\": {},\n \"value\": \"v1\"\n }\n]\n```\nORACLE ✅ az appconfig kv list --name appcfg0b6c257f --key legacy-timeout-49967b\n```json\n[]\n```", + "queries": [ + { + "command": "rest --method get --url \"/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/mcpe2-15123a-appconfig-check-keys-2/providers/Microsoft.AppConfiguration/configurationStores/appcfg0b6c257f/keyValues/key0b6c257f?api-version=2024-06-01\"", + "answer": { + "exitCode": 0, + "stdout": "{\"id\":\"/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/mcpe2-15123a-appconfig-check-keys-2/providers/Microsoft.AppConfiguration/configurationStores/appcfg0b6c257f/keyValues/key0b6c257f\",\"name\":\"key0b6c257f\",\"properties\":{\"contentType\":\"\",\"eTag\":\"b5051d35ca9145c5b20f09fc4e1a45d3\",\"key\":\"key0b6c257f\",\"lastModified\":\"2026-09-27T21:19:24.416836+00:00\",\"locked\":false,\"tags\":{},\"value\":\"v1\"},\"type\":\"Microsoft.AppConfiguration/configurationStores/keyValues\"}", + "stderr": "", + "classId": null + } + }, + { + "command": "rest --method get --url \"/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/mcpe2-15123a-appconfig-check-keys-2/providers/Microsoft.AppConfiguration/configurationStores/appcfg0b6c257f/keyValues/legacy-timeout-49967b?api-version=2024-06-01\"", + "answer": { + "exitCode": 1, + "stdout": "", + "stderr": "ERROR: NOT FOUND({\"error\": {\"code\": \"EntityNotFound\", \"message\": \"The resource with id 'legacy-timeout-49967b' was not found.\", \"details\": [], \"additionalInfo\": []}})\r\n", + "classId": "other" + } + } + ] + }, + { + "task": "appconfig-kv-unlock", + "mode": "negative", + "passed": false, + "wait_s": 10, + "recorded": "2026-09-27T20:51:42Z", + "slots": { + "resource_group": "mcpe2-151aca-appconfig-kv-unlock-1", + "rg": "mcpe2-151aca-appconfig-kv-unlock-1", + "name": "appcfg080d4f71", + "store_name": "appcfg080d4f71", + "key": "key080d4f71", + "replica_name": "rep080d4f71", + "snapshot_name": "snap080d4f71" + }, + "text": "", + "queries": [ + { + "command": "rest --method get --url \"/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/mcpe2-151aca-appconfig-kv-unlock-1/providers/Microsoft.AppConfiguration/configurationStores/appcfg080d4f71/keyValues/key080d4f71?api-version=2024-06-01\"", + "repeat": 8, + "answer": { + "exitCode": 0, + "stdout": "{\"id\":\"/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/mcpe2-151aca-appconfig-kv-unlock-1/providers/Microsoft.AppConfiguration/configurationStores/appcfg080d4f71/keyValues/key080d4f71\",\"name\":\"key080d4f71\",\"properties\":{\"contentType\":\"\",\"eTag\":\"42c0d6b6625542cc88ae5822bd2c8b1f\",\"key\":\"key080d4f71\",\"lastModified\":\"2026-09-27T20:51:20.901979+00:00\",\"locked\":true,\"tags\":{},\"value\":\"v1\"},\"type\":\"Microsoft.AppConfiguration/configurationStores/keyValues\"}", + "stderr": "", + "classId": null + } + } + ] + }, + { + "task": "appconfig-kv-unlock", + "mode": "oracle", + "passed": true, + "wait_s": 90, + "recorded": "2026-09-27T21:19:18Z", + "slots": { + "resource_group": "mcpe2-15123a-appconfig-kv-unlock-1", + "rg": "mcpe2-15123a-appconfig-kv-unlock-1", + "name": "appcfg0e6901cb", + "store_name": "appcfg0e6901cb", + "key": "key0e6901cb", + "replica_name": "rep0e6901cb", + "snapshot_name": "snap0e6901cb", + "_oracle": true + }, + "text": "ORACLE ✅ az appconfig kv unlock --name appcfg0e6901cb --key key0e6901cb --yes\n```json\n{\n \"contentType\": \"\",\n \"etag\": \"955efdbb68fc4449baa426a8b21c7012\",\n \"key\": \"key0e6901cb\",\n \"label\": null,\n \"lastModified\": \"2026-09-27T21:18:55.755099+00:00\",\n \"locked\": false,\n \"tags\": {},\n \"value\": \"v1\"\n}\n```", + "queries": [ + { + "command": "rest --method get --url \"/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/mcpe2-15123a-appconfig-kv-unlock-1/providers/Microsoft.AppConfiguration/configurationStores/appcfg0e6901cb/keyValues/key0e6901cb?api-version=2024-06-01\"", + "answer": { + "exitCode": 0, + "stdout": "{\"id\":\"/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/mcpe2-15123a-appconfig-kv-unlock-1/providers/Microsoft.AppConfiguration/configurationStores/appcfg0e6901cb/keyValues/key0e6901cb\",\"name\":\"key0e6901cb\",\"properties\":{\"contentType\":\"\",\"eTag\":\"955efdbb68fc4449baa426a8b21c7012\",\"key\":\"key0e6901cb\",\"lastModified\":\"2026-09-27T21:18:55.755099+00:00\",\"locked\":false,\"tags\":{},\"value\":\"v1\"},\"type\":\"Microsoft.AppConfiguration/configurationStores/keyValues\"}", + "stderr": "", + "classId": null + } + } + ] + }, + { + "task": "appconfig-regenerate-key", + "mode": "negative", + "passed": false, + "wait_s": 10, + "recorded": "2026-09-27T20:52:22Z", + "slots": { + "resource_group": "mcpe2-151aca-appconfig-regenerate-key-3", + "rg": "mcpe2-151aca-appconfig-regenerate-key-3", + "name": "appcfga1be0d46", + "store_name": "appcfga1be0d46", + "key": "keya1be0d46", + "replica_name": "repa1be0d46", + "snapshot_name": "snapa1be0d46", + "key_id": "rjbKUWw6e92_M53gz85iFA", + "_primary_id": "rjbKUWw6e92_M53gz85iFA", + "_before_changed": "3799j/haA/KW0eszeyaf4hlBYpkYoDk5Fj30+UawpkI=", + "_before_kept": "4+BUjZ9GOlLcAny8tCQNXCmu/0ED01bod6aINkQWn8Y=" + }, + "text": "", + "queries": [ + { + "command": "rest --method post --url \"/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/mcpe2-151aca-appconfig-regenerate-key-3/providers/Microsoft.AppConfiguration/configurationStores/appcfga1be0d46/listKeys?api-version=2024-06-01\"", + "answer": { + "exitCode": 0, + "stdout": "{\"value\":[{\"connectionString\":\"Endpoint=https://appcfga1be0d46.azure.localhost.localstack.cloud:4566;Id=rjbKUWw6e92_M53gz85iFA;Secret=3799j/haA/KW0eszeyaf4hlBYpkYoDk5Fj30+UawpkI=\",\"id\":\"rjbKUWw6e92_M53gz85iFA\",\"lastModified\":\"2026-09-27T20:52:06.691883+00:00\",\"name\":\"Primary\",\"readOnly\":false,\"value\":\"3799j/haA/KW0eszeyaf4hlBYpkYoDk5Fj30+UawpkI=\"},{\"connectionString\":\"Endpoint=https://appcfga1be0d46.azure.localhost.localstack.cloud:4566;Id=95nlHhmiae3PkVWtPZsySQ;Secret=4+BUjZ9GOlLcAny8tCQNXCmu/0ED01bod6aINkQWn8Y=\",\"id\":\"95nlHhmiae3PkVWtPZsySQ\",\"lastModified\":\"2026-09-27T20:52:06.691890+00:00\",\"name\":\"Secondary\",\"readOnly\":false,\"value\":\"4+BUjZ9GOlLcAny8tCQNXCmu/0ED01bod6aINkQWn8Y=\"},{\"connectionString\":\"Endpoint=https://appcfga1be0d46.azure.localhost.localstack.cloud:4566;Id=JJDrhRaoKz22KmY7yNwxXA;Secret=OZ3gDo8doL81GumfK0VWrovkOuaqDypCSx9d/guM/yo=\",\"id\":\"JJDrhRaoKz22KmY7yNwxXA\",\"lastModified\":\"2026-09-27T20:52:06.691893+00:00\",\"name\":\"Primary Read Only\",\"readOnly\":true,\"value\":\"OZ3gDo8doL81GumfK0VWrovkOuaqDypCSx9d/guM/yo=\"},{\"connectionString\":\"Endpoint=https://appcfga1be0d46.azure.localhost.localstack.cloud:4566;Id=I6lD2X9ftjYHWpYBLcEotA;Secret=DXDbGUZRs+ewmiMVLWoBj5SU6GyjheNFjFNX/tMu/Kw=\",\"id\":\"I6lD2X9ftjYHWpYBLcEotA\",\"lastModified\":\"2026-09-27T20:52:06.691983+00:00\",\"name\":\"Secondary Read Only\",\"readOnly\":true,\"value\":\"DXDbGUZRs+ewmiMVLWoBj5SU6GyjheNFjFNX/tMu/Kw=\"}]}", + "stderr": "", + "classId": null + } + } + ] + }, + { + "task": "appconfig-regenerate-key", + "mode": "oracle", + "passed": true, + "wait_s": 90, + "recorded": "2026-09-27T21:19:55Z", + "slots": { + "resource_group": "mcpe2-15123a-appconfig-regenerate-key-3", + "rg": "mcpe2-15123a-appconfig-regenerate-key-3", + "name": "appcfga0204d03", + "store_name": "appcfga0204d03", + "key": "keya0204d03", + "replica_name": "repa0204d03", + "snapshot_name": "snapa0204d03", + "key_id": "3-3EkcvfxrLG-3iiEFe0gg", + "_primary_id": "3-3EkcvfxrLG-3iiEFe0gg", + "_before_changed": "dNRHwiLwJfCDsJ+kQawwo95sOhKZMWhOBtL7pwwjJ+0=", + "_before_kept": "sXWNx7Y60HYKl3AgwbEmrTs0Qm4MNU6pAtKgWqgBhxk=", + "_oracle": true + }, + "text": "ORACLE ✅ az appconfig credential regenerate --name appcfga0204d03 --resource-group mcpe2-15123a-appconfig-regenerate-key-3 --id 3-3EkcvfxrLG-3iiEFe0gg\n```json\n{\n \"connectionString\": \"Endpoint=https://appcfga0204d03.azure.localhost.localstack.cloud:4566;Id=9DJcUiyNKYbJUCWbSU6zJA;Secret=TJQNGUL+s3b5Kb3uT/Vjryv4k4Dq0FLSfN2Oj9NUYKc=\",\n \"id\": \"9DJcUiyNKYbJUCWbSU6zJA\",\n \"lastModified\": \"2026-09-27T21:19:46.159356+00:00\",\n \"name\": \"Primary\",\n \"readOnly\": false,\n \"value\": \"TJQNGUL+s3b5Kb3uT/Vjryv4k4Dq0FLSfN2Oj9NUYKc=\"\n}\n```", + "queries": [ + { + "command": "rest --method post --url \"/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/mcpe2-15123a-appconfig-regenerate-key-3/providers/Microsoft.AppConfiguration/configurationStores/appcfga0204d03/listKeys?api-version=2024-06-01\"", + "answer": { + "exitCode": 0, + "stdout": "{\"value\":[{\"connectionString\":\"Endpoint=https://appcfga0204d03.azure.localhost.localstack.cloud:4566;Id=vVdpxkBh_Vcslmm20rwRaw;Secret=sXWNx7Y60HYKl3AgwbEmrTs0Qm4MNU6pAtKgWqgBhxk=\",\"id\":\"vVdpxkBh_Vcslmm20rwRaw\",\"lastModified\":\"2026-09-27T21:19:42.780765+00:00\",\"name\":\"Secondary\",\"readOnly\":false,\"value\":\"sXWNx7Y60HYKl3AgwbEmrTs0Qm4MNU6pAtKgWqgBhxk=\"},{\"connectionString\":\"Endpoint=https://appcfga0204d03.azure.localhost.localstack.cloud:4566;Id=O90i9tQUen2F3aHrP04gPA;Secret=cGBPC7nK6R3ePIL8GEg+eCXqPje5mPxO1Wr3hM5Mzo8=\",\"id\":\"O90i9tQUen2F3aHrP04gPA\",\"lastModified\":\"2026-09-27T21:19:42.780768+00:00\",\"name\":\"Primary Read Only\",\"readOnly\":true,\"value\":\"cGBPC7nK6R3ePIL8GEg+eCXqPje5mPxO1Wr3hM5Mzo8=\"},{\"connectionString\":\"Endpoint=https://appcfga0204d03.azure.localhost.localstack.cloud:4566;Id=_O9JcZwzd2ldJca5WQl-4w;Secret=L0zy5I1pq29oH50k40dlM3x+Z6WL+/ihBeEn/6Cc7ec=\",\"id\":\"_O9JcZwzd2ldJca5WQl-4w\",\"lastModified\":\"2026-09-27T21:19:42.780781+00:00\",\"name\":\"Secondary Read Only\",\"readOnly\":true,\"value\":\"L0zy5I1pq29oH50k40dlM3x+Z6WL+/ihBeEn/6Cc7ec=\"},{\"connectionString\":\"Endpoint=https://appcfga0204d03.azure.localhost.localstack.cloud:4566;Id=9DJcUiyNKYbJUCWbSU6zJA;Secret=TJQNGUL+s3b5Kb3uT/Vjryv4k4Dq0FLSfN2Oj9NUYKc=\",\"id\":\"9DJcUiyNKYbJUCWbSU6zJA\",\"lastModified\":\"2026-09-27T21:19:46.159356+00:00\",\"name\":\"Primary\",\"readOnly\":false,\"value\":\"TJQNGUL+s3b5Kb3uT/Vjryv4k4Dq0FLSfN2Oj9NUYKc=\"}]}", + "stderr": "", + "classId": null + } + } + ] + }, + { + "task": "cdn-can-migrate", + "mode": "negative", + "passed": false, + "wait_s": 10, + "recorded": "2026-09-27T21:03:31Z", + "slots": { + "resource_group": "mcpe2-151aca-cdn-can-migrate-28", + "rg": "mcpe2-151aca-cdn-can-migrate-28", + "profile_name": "cdnp919df2878f", + "endpoint_name": "cdne919df2878f", + "origin_name": "or919df2878f", + "can_migrate": true + }, + "text": "", + "queries": [] + }, + { + "task": "cdn-can-migrate", + "mode": "oracle", + "passed": true, + "wait_s": 90, + "recorded": "2026-09-27T21:29:54Z", + "slots": { + "resource_group": "mcpe2-15123a-cdn-can-migrate-28", + "rg": "mcpe2-15123a-cdn-can-migrate-28", + "profile_name": "cdnp776c3d6e6b", + "endpoint_name": "cdne776c3d6e6b", + "origin_name": "or776c3d6e6b", + "can_migrate": false, + "_oracle": true + }, + "text": "ORACLE ✅ az cdn profile-migration check-compatibility --profile-name cdnp776c3d6e6b --resource-group mcpe2-15123a-cdn-can-migrate-28\n```json\n{\n \"canMigrate\": false,\n \"defaultSku\": \"Standard_AzureFrontDoor\",\n \"errors\": [\n {\n \"code\": \"NoEndpointsToMigrate\",\n \"errorMessage\": \"Profile 'cdnp776c3d6e6b' has no endpoints to migrate.\",\n \"nextSteps\": \"Create an endpoint before migrating.\",\n \"resourceName\": \"cdnp776c3d6e6b\"\n }\n ],\n \"id\": \"/subscriptions/00000000-0000-0000-0000-000000000000/resourcegroups/mcpe2-15123a-cdn-can-migrate-28/providers/Microsoft.Cdn/profiles/cdnp776c3d6e6b\",\n \"resourceGroup\": \"mcpe2-15123a-cdn-can-migrate-28\",\n \"type\": \"Microsoft.Cdn/profiles/canmigrate\"\n}\n```", + "queries": [] + }, + { + "task": "deployment-sub-validate", + "mode": "negative", + "passed": false, + "wait_s": 10, + "recorded": "2026-09-27T21:00:40Z", + "slots": { + "resource_group": "mcpe2-151aca-deployment-sub-validate-23", + "rg": "mcpe2-151aca-deployment-sub-validate-23", + "deployment_name": "dep-23b2870a", + "template_json": "{\"$schema\": \"https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#\", \"contentVersion\": \"1.0.0.0\", \"resources\": []}", + "deploy_body": "{\"location\": \"westeurope\", \"properties\": {\"mode\": \"Incremental\", \"template\": {\"$schema\": \"https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#\", \"contentVersion\": \"1.0.0.0\", \"resources\": []}}}" + }, + "text": "", + "queries": [ + { + "command": "rest --method get --url \"/subscriptions/00000000-0000-0000-0000-000000000000/providers/Microsoft.Resources/deployments/dep-23b2870a?api-version=2022-09-01\"", + "answer": { + "exitCode": 1, + "stdout": "", + "stderr": "ERROR: NOT FOUND({\"error\": {\"code\": \"DeploymentNotFound\", \"message\": \"Deployment 'unknown' could not be found.\", \"details\": [], \"additionalInfo\": []}})\r\n", + "classId": "other" + } + } + ] + }, + { + "task": "deployment-sub-validate", + "mode": "oracle", + "passed": true, + "wait_s": 90, + "recorded": "2026-09-27T21:27:57Z", + "slots": { + "resource_group": "mcpe2-15123a-deployment-sub-validate-23", + "rg": "mcpe2-15123a-deployment-sub-validate-23", + "deployment_name": "dep-bc198d2f", + "template_json": "{\"$schema\": \"https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#\", \"contentVersion\": \"1.0.0.0\", \"resources\": []}", + "deploy_body": "{\"location\": \"westeurope\", \"properties\": {\"mode\": \"Incremental\", \"template\": {\"$schema\": \"https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#\", \"contentVersion\": \"1.0.0.0\", \"resources\": []}}}", + "_oracle": true + }, + "text": "ORACLE ✅ az rest --method post --url /subscriptions/00000000-0000-0000-0000-000000000000/providers/Microsoft.Resources/deployments/dep-bc198d2f/validate?api-version=2022-09-01 --body '{\"location\": \"westeurope\", \"properties\": {\"mode\": \"Incremental\", \"template\": {\"$schema\": \"https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#\", \"contentVersion\": \"1.0.0.0\", \"resources\": []}}}'\n```json\n{}\n```", + "queries": [ + { + "command": "rest --method get --url \"/subscriptions/00000000-0000-0000-0000-000000000000/providers/Microsoft.Resources/deployments/dep-bc198d2f?api-version=2022-09-01\"", + "answer": { + "exitCode": 1, + "stdout": "", + "stderr": "ERROR: NOT FOUND({\"error\": {\"code\": \"DeploymentNotFound\", \"message\": \"Deployment 'unknown' could not be found.\", \"details\": [], \"additionalInfo\": []}})\r\n", + "classId": "other" + } + } + ] + }, + { + "task": "disk-create", + "mode": "negative", + "passed": false, + "wait_s": 10, + "recorded": "2026-09-27T21:03:03Z", + "slots": { + "resource_group": "mcpe2-151aca-disk-create-27", + "rg": "mcpe2-151aca-disk-create-27", + "disk_name": "disk-ea10211f8a", + "size_gb": 64 + }, + "text": "", + "queries": [ + { + "command": "rest --method get --url \"/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/mcpe2-151aca-disk-create-27/providers/Microsoft.Compute/disks/disk-ea10211f8a?api-version=2026-03-02\"", + "repeat": 5, + "answer": { + "exitCode": 1, + "stdout": "", + "stderr": "ERROR: NOT FOUND({\"error\": {\"code\": \"ResourceNotFound\", \"message\": \"The Resource 'Microsoft.Compute/disks/disk-ea10211f8a' under resource group 'mcpe2-151aca-disk-create-27' was not found. For more details please go to https://aka.ms/ARMResourceNotFoundFix\", \"details\": [], \"additionalInfo\": []}})\r\n", + "classId": "not-found" + } + } + ] + }, + { + "task": "disk-create", + "mode": "oracle", + "passed": true, + "wait_s": 90, + "recorded": "2026-09-27T21:29:40Z", + "slots": { + "resource_group": "mcpe2-15123a-disk-create-27", + "rg": "mcpe2-15123a-disk-create-27", + "disk_name": "disk-f3c7657784", + "size_gb": 16, + "_oracle": true + }, + "text": "ORACLE ✅ az disk create --name disk-f3c7657784 --resource-group mcpe2-15123a-disk-create-27 --size-gb 16 --sku StandardSSD_LRS\n```json\n{\n \"creationData\": {\n \"createOption\": \"Empty\"\n },\n \"diskIOPSReadWrite\": 500,\n \"diskMBpsReadWrite\": 100,\n \"diskSizeBytes\": 17179869184,\n \"diskSizeGB\": 16,\n \"diskState\": \"Unattached\",\n \"encryption\": {\n \"type\": \"EncryptionAtRestWithPlatformKey\"\n },\n \"id\": \"/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/mcpe2-15123a-disk-create-27/providers/Microsoft.Compute/disks/disk-f3c7657784\",\n \"location\": \"westeurope\",\n \"name\": \"disk-f3c7657784\",\n \"networkAccessPolicy\": \"AllowAll\",\n \"provisioningState\": \"Succeeded\",\n \"publicNetworkAccess\": \"Enabled\",\n \"resourceGroup\": \"mcpe2-15123a-disk-create-27\",\n \"sku\": {\n \"name\": \"StandardSSD_LRS\",\n \"tier\": \"Standard\"\n },\n \"timeCreated\": \"2026-09-27T21:29:28.869514Z\",\n \"type\": \"Microsoft.Compute/disks\",\n \"uniqueId\": \"7a2d385a-c81f-4b5c-9a4b-6827b16e0054\"\n}\n```", + "queries": [ + { + "command": "rest --method get --url \"/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/mcpe2-15123a-disk-create-27/providers/Microsoft.Compute/disks/disk-f3c7657784?api-version=2026-03-02\"", + "repeat": 2, + "answer": { + "exitCode": 0, + "stdout": "{\"id\":\"/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/mcpe2-15123a-disk-create-27/providers/Microsoft.Compute/disks/disk-f3c7657784\",\"location\":\"westeurope\",\"name\":\"disk-f3c7657784\",\"properties\":{\"creationData\":{\"createOption\":\"Empty\"},\"diskIOPSReadWrite\":500,\"diskMBpsReadWrite\":100,\"diskSizeBytes\":17179869184,\"diskSizeGB\":16,\"diskState\":\"Unattached\",\"encryption\":{\"type\":\"EncryptionAtRestWithPlatformKey\"},\"networkAccessPolicy\":\"AllowAll\",\"provisioningState\":\"Succeeded\",\"publicNetworkAccess\":\"Enabled\",\"timeCreated\":\"2026-09-27T21:29:28.869514Z\",\"uniqueId\":\"7a2d385a-c81f-4b5c-9a4b-6827b16e0054\"},\"sku\":{\"name\":\"StandardSSD_LRS\",\"tier\":\"Standard\"},\"type\":\"Microsoft.Compute/disks\"}", + "stderr": "", + "classId": null + } + } + ] + }, + { + "task": "eventgrid-domain-disable-local-auth", + "mode": "negative", + "passed": false, + "wait_s": 10, + "recorded": "2026-09-27T20:54:57Z", + "slots": { + "resource_group": "mcpe2-151aca-eventgrid-domain-disable-local-auth-10", + "rg": "mcpe2-151aca-eventgrid-domain-disable-local-auth-10", + "domain_name": "d652bea3cef", + "location": "westeurope" + }, + "text": "", + "queries": [ + { + "command": "rest --method get --url \"/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/mcpe2-151aca-eventgrid-domain-disable-local-auth-10/providers/Microsoft.EventGrid/domains/d652bea3cef?api-version=2025-02-15\"", + "repeat": 6, + "answer": { + "exitCode": 0, + "stdout": "{\"id\":\"/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/mcpe2-151aca-eventgrid-domain-disable-local-auth-10/providers/Microsoft.EventGrid/domains/d652bea3cef\",\"location\":\"westeurope\",\"name\":\"d652bea3cef\",\"properties\":{\"autoCreateTopicWithFirstSubscription\":true,\"autoDeleteTopicWithLastSubscription\":true,\"dataResidencyBoundary\":\"WithinGeopair\",\"disableLocalAuth\":false,\"endpoint\":\"https://d652bea3cef.azure.localhost.localstack.cloud:4566/api/events\",\"inputSchema\":\"EventGridSchema\",\"metricResourceId\":\"d97a99c8-1d1f-49bc-801b-9bd31af4f6b1\",\"minimumTlsVersionAllowed\":\"1.0\",\"provisioningState\":\"Succeeded\",\"publicNetworkAccess\":\"Enabled\"},\"type\":\"Microsoft.EventGrid/domains\"}", + "stderr": "", + "classId": null + } + } + ] + }, + { + "task": "eventgrid-domain-disable-local-auth", + "mode": "oracle", + "passed": true, + "wait_s": 90, + "recorded": "2026-09-27T21:22:10Z", + "slots": { + "resource_group": "mcpe2-15123a-eventgrid-domain-disable-local-auth-10", + "rg": "mcpe2-15123a-eventgrid-domain-disable-local-auth-10", + "domain_name": "d8e0a95215a", + "location": "westeurope", + "_oracle": true + }, + "text": "ORACLE ✅ az resource update --resource-group mcpe2-15123a-eventgrid-domain-disable-local-auth-10 --name d8e0a95215a --resource-type Microsoft.EventGrid/domains --set properties.disableLocalAuth=true\n```json\n{\n \"extendedLocation\": null,\n \"id\": \"/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/mcpe2-15123a-eventgrid-domain-disable-local-auth-10/providers/Microsoft.EventGrid/domains/d8e0a95215a\",\n \"identity\": null,\n \"kind\": null,\n \"location\": \"westeurope\",\n \"managedBy\": null,\n \"name\": \"d8e0a95215a\",\n \"plan\": null,\n \"properties\": {\n \"autoCreateTopicWithFirstSubscription\": true,\n \"autoDeleteTopicWithLastSubscription\": true,\n \"dataResidencyBoundary\": \"WithinGeopair\",\n \"disableLocalAuth\": true,\n \"endpoint\": \"https://d8e0a95215a.azure.localhost.localstack.cloud:4566/api/events\",\n \"inputSchema\": \"EventGridSchema\",\n \"metricResourceId\": \"8c8aef35-ac20-4922-96ab-f0ae202efea3\",\n \"minimumTlsVersionAllowed\": \"1.0\",\n \"provisioningState\": \"Succeeded\",\n \"publicNetworkAccess\": \"Enabled\"\n },\n \"resourceGroup\": \"mcpe2-15123a-eventgrid-domain-disable-local-auth-10\",\n \"sku\": null,\n \"tags\": null,\n \"type\": \"Microsoft.EventGrid/domains\"\n}\n```", + "queries": [ + { + "command": "rest --method get --url \"/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/mcpe2-15123a-eventgrid-domain-disable-local-auth-10/providers/Microsoft.EventGrid/domains/d8e0a95215a?api-version=2025-02-15\"", + "answer": { + "exitCode": 0, + "stdout": "{\"id\":\"/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/mcpe2-15123a-eventgrid-domain-disable-local-auth-10/providers/Microsoft.EventGrid/domains/d8e0a95215a\",\"location\":\"westeurope\",\"name\":\"d8e0a95215a\",\"properties\":{\"autoCreateTopicWithFirstSubscription\":true,\"autoDeleteTopicWithLastSubscription\":true,\"dataResidencyBoundary\":\"WithinGeopair\",\"disableLocalAuth\":true,\"endpoint\":\"https://d8e0a95215a.azure.localhost.localstack.cloud:4566/api/events\",\"inputSchema\":\"EventGridSchema\",\"metricResourceId\":\"8c8aef35-ac20-4922-96ab-f0ae202efea3\",\"minimumTlsVersionAllowed\":\"1.0\",\"provisioningState\":\"Succeeded\",\"publicNetworkAccess\":\"Enabled\"},\"type\":\"Microsoft.EventGrid/domains\"}", + "stderr": "", + "classId": null + } + } + ] + }, + { + "task": "eventgrid-topic-regenerate-key", + "mode": "negative", + "passed": false, + "wait_s": 10, + "recorded": "2026-09-27T20:54:30Z", + "slots": { + "resource_group": "mcpe2-151aca-eventgrid-topic-regenerate-key-9", + "rg": "mcpe2-151aca-eventgrid-topic-regenerate-key-9", + "topic_name": "t9bf3e42948", + "_before_changed": "NDMyMGE5Mzg0ODIxNDFhMjkzZWI2NDA0ZGYwYWM1ODk=", + "_before_kept": "MzdjZTU3NzIyNjY3NDE0N2FjMzc0NjU1ZjVmYWVhYWE=" + }, + "text": "", + "queries": [ + { + "command": "rest --method post --url \"/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/mcpe2-151aca-eventgrid-topic-regenerate-key-9/providers/Microsoft.EventGrid/topics/t9bf3e42948/listKeys?api-version=2025-02-15\"", + "answer": { + "exitCode": 0, + "stdout": "{\"key1\":\"NDMyMGE5Mzg0ODIxNDFhMjkzZWI2NDA0ZGYwYWM1ODk=\",\"key2\":\"MzdjZTU3NzIyNjY3NDE0N2FjMzc0NjU1ZjVmYWVhYWE=\"}", + "stderr": "", + "classId": null + } + } + ] + }, + { + "task": "eventgrid-topic-regenerate-key", + "mode": "oracle", + "passed": true, + "wait_s": 90, + "recorded": "2026-09-27T21:21:45Z", + "slots": { + "resource_group": "mcpe2-15123a-eventgrid-topic-regenerate-key-9", + "rg": "mcpe2-15123a-eventgrid-topic-regenerate-key-9", + "topic_name": "t06d103b196", + "_before_changed": "ZDhhZDViOWQ4OWYxNDg3ZGFmNmI5MTRmMzdmYTA3ODY=", + "_before_kept": "NmZhY2M2MjEzZjJjNDdlODk0ZjY1MDU3NmQzMzNhZjE=", + "_oracle": true + }, + "text": "ORACLE ✅ az eventgrid topic key regenerate --name t06d103b196 --resource-group mcpe2-15123a-eventgrid-topic-regenerate-key-9 --key-name key1\n```json\n{\n \"key1\": \"M2E0OTYzZWY4MzNmNGE4ODk0ZGY3MDEyYjkyM2E0ZmI=\",\n \"key2\": \"NmZhY2M2MjEzZjJjNDdlODk0ZjY1MDU3NmQzMzNhZjE=\"\n}\n```", + "queries": [ + { + "command": "rest --method post --url \"/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/mcpe2-15123a-eventgrid-topic-regenerate-key-9/providers/Microsoft.EventGrid/topics/t06d103b196/listKeys?api-version=2025-02-15\"", + "answer": { + "exitCode": 0, + "stdout": "{\"key1\":\"M2E0OTYzZWY4MzNmNGE4ODk0ZGY3MDEyYjkyM2E0ZmI=\",\"key2\":\"NmZhY2M2MjEzZjJjNDdlODk0ZjY1MDU3NmQzMzNhZjE=\"}", + "stderr": "", + "classId": null + } + } + ] + }, + { + "task": "eventhub-hub-auth-rule-rights", + "mode": "negative", + "passed": false, + "wait_s": 10, + "recorded": "2026-09-27T20:55:23Z", + "slots": { + "resource_group": "mcpe2-151aca-eventhub-hub-auth-rule-rights-11", + "rg": "mcpe2-151aca-eventhub-hub-auth-rule-rights-11", + "name": "ehns-a7c33381", + "namespace_name": "ehns-a7c33381", + "eventhub_name": "hub-a7c33381", + "auth_rule_name": "rule-a7c33381", + "_rights": "Send" + }, + "text": "", + "queries": [ + { + "command": "rest --method get --url \"/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/mcpe2-151aca-eventhub-hub-auth-rule-rights-11/providers/Microsoft.EventHub/namespaces/ehns-a7c33381/eventhubs/hub-a7c33381/authorizationRules/rule-a7c33381?api-version=2026-01-01\"", + "answer": { + "exitCode": 0, + "stdout": "{\"id\":\"/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/mcpe2-151aca-eventhub-hub-auth-rule-rights-11/providers/Microsoft.EventHub/namespaces/ehns-a7c33381/eventhubs/hub-a7c33381/authorizationrules/rule-a7c33381\",\"location\":\"westeurope\",\"name\":\"rule-a7c33381\",\"properties\":{\"rights\":[\"Send\"]},\"type\":\"Microsoft.EventHub/namespaces/eventhubs/authorizationrules\"}", + "stderr": "", + "classId": null + } + } + ] + }, + { + "task": "eventhub-hub-auth-rule-rights", + "mode": "oracle", + "passed": true, + "wait_s": 90, + "recorded": "2026-09-27T21:23:48Z", + "slots": { + "resource_group": "mcpe2-15123a-eventhub-hub-auth-rule-rights-11", + "rg": "mcpe2-15123a-eventhub-hub-auth-rule-rights-11", + "name": "ehns-c421ffd4", + "namespace_name": "ehns-c421ffd4", + "eventhub_name": "hub-c421ffd4", + "auth_rule_name": "rule-c421ffd4", + "_rights": "Listen,Send,Manage", + "_oracle": true + }, + "text": "ORACLE ✅ az eventhubs eventhub authorization-rule show --name rule-c421ffd4 --eventhub-name hub-c421ffd4 --namespace-name ehns-c421ffd4 --resource-group mcpe2-15123a-eventhub-hub-auth-rule-rights-11\n```json\n{\n \"id\": \"/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/mcpe2-15123a-eventhub-hub-auth-rule-rights-11/providers/Microsoft.EventHub/namespaces/ehns-c421ffd4/eventhubs/hub-c421ffd4/authorizationrules/rule-c421ffd4\",\n \"location\": \"westeurope\",\n \"name\": \"rule-c421ffd4\",\n \"resourceGroup\": \"mcpe2-15123a-eventhub-hub-auth-rule-rights-11\",\n \"rights\": [\n \"Listen\",\n \"Send\",\n \"Manage\"\n ],\n \"type\": \"Microsoft.EventHub/namespaces/eventhubs/authorizationrules\"\n}\n```", + "queries": [ + { + "command": "rest --method get --url \"/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/mcpe2-15123a-eventhub-hub-auth-rule-rights-11/providers/Microsoft.EventHub/namespaces/ehns-c421ffd4/eventhubs/hub-c421ffd4/authorizationRules/rule-c421ffd4?api-version=2026-01-01\"", + "answer": { + "exitCode": 0, + "stdout": "{\"id\":\"/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/mcpe2-15123a-eventhub-hub-auth-rule-rights-11/providers/Microsoft.EventHub/namespaces/ehns-c421ffd4/eventhubs/hub-c421ffd4/authorizationrules/rule-c421ffd4\",\"location\":\"westeurope\",\"name\":\"rule-c421ffd4\",\"properties\":{\"rights\":[\"Listen\",\"Send\",\"Manage\"]},\"type\":\"Microsoft.EventHub/namespaces/eventhubs/authorizationrules\"}", + "stderr": "", + "classId": null + } + } + ] + }, + { + "task": "identity-create", + "mode": "negative", + "passed": false, + "wait_s": 10, + "recorded": "2026-09-27T20:58:21Z", + "slots": { + "resource_group": "mcpe2-151aca-identity-create-17", + "rg": "mcpe2-151aca-identity-create-17", + "name": "id-4816c001" + }, + "text": "", + "queries": [ + { + "command": "rest --method get --url \"/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/mcpe2-151aca-identity-create-17/providers/Microsoft.ManagedIdentity/userAssignedIdentities/id-4816c001?api-version=2024-11-30\"", + "repeat": 7, + "answer": { + "exitCode": 1, + "stdout": "", + "stderr": "ERROR: NOT FOUND({\"error\": {\"code\": \"ResourceNotFound\", \"message\": \"The Resource 'Microsoft.ManagedIdentity/userAssignedIdentities/id-4816c001' under resource group 'mcpe2-151aca-identity-create-17' was not found. For more details please go to https://aka.ms/ARMResourceNotFoundFix\", \"details\": [], \"additionalInfo\": []}})\r\n", + "classId": "not-found" + } + } + ] + }, + { + "task": "identity-create", + "mode": "oracle", + "passed": true, + "wait_s": 90, + "recorded": "2026-09-27T21:26:19Z", + "slots": { + "resource_group": "mcpe2-15123a-identity-create-17", + "rg": "mcpe2-15123a-identity-create-17", + "name": "id-bf316c58", + "_oracle": true + }, + "text": "ORACLE ✅ az identity create --name id-bf316c58 --resource-group mcpe2-15123a-identity-create-17\n```json\n{\n \"clientId\": \"95760d73-e77a-459c-9e35-1e1312794107\",\n \"id\": \"/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/mcpe2-15123a-identity-create-17/providers/Microsoft.ManagedIdentity/userAssignedIdentities/id-bf316c58\",\n \"isolationScope\": \"None\",\n \"location\": \"westeurope\",\n \"name\": \"id-bf316c58\",\n \"principalId\": \"2d651af2-8b5b-45f5-89db-9a8997668f73\",\n \"resourceGroup\": \"mcpe2-15123a-identity-create-17\",\n \"systemData\": null,\n \"tags\": {},\n \"tenantId\": \"00000000-0000-0000-0000-000000000000\",\n \"type\": \"Microsoft.ManagedIdentity/userAssignedIdentities\"\n}\n```", + "queries": [ + { + "command": "rest --method get --url \"/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/mcpe2-15123a-identity-create-17/providers/Microsoft.ManagedIdentity/userAssignedIdentities/id-bf316c58?api-version=2024-11-30\"", + "repeat": 2, + "answer": { + "exitCode": 0, + "stdout": "{\"id\":\"/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/mcpe2-15123a-identity-create-17/providers/Microsoft.ManagedIdentity/userAssignedIdentities/id-bf316c58\",\"location\":\"westeurope\",\"name\":\"id-bf316c58\",\"properties\":{\"clientId\":\"95760d73-e77a-459c-9e35-1e1312794107\",\"isolationScope\":\"None\",\"principalId\":\"2d651af2-8b5b-45f5-89db-9a8997668f73\",\"tenantId\":\"00000000-0000-0000-0000-000000000000\"},\"tags\":{},\"type\":\"Microsoft.ManagedIdentity/userAssignedIdentities\"}", + "stderr": "", + "classId": null + } + } + ] + }, + { + "task": "identity-list", + "mode": "negative", + "passed": false, + "wait_s": 10, + "recorded": "2026-09-27T20:58:39Z", + "slots": { + "resource_group": "mcpe2-151aca-identity-list-18", + "rg": "mcpe2-151aca-identity-list-18", + "id1": "id-ci-ecd1d7", + "id2": "id-backup-fa1ec4" + }, + "text": "", + "queries": [] + }, + { + "task": "identity-list", + "mode": "oracle", + "passed": true, + "wait_s": 90, + "recorded": "2026-09-27T21:26:32Z", + "slots": { + "resource_group": "mcpe2-15123a-identity-list-18", + "rg": "mcpe2-15123a-identity-list-18", + "id1": "id-ci-ca3e70", + "id2": "id-backup-8a1d06", + "_oracle": true + }, + "text": "ORACLE ✅ az identity list\n```json\n[\n {\n \"clientId\": \"2d065bc8-e24d-49ab-861a-113c0ee3d7d2\",\n \"id\": \"/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/mcpe2-15123a-identity-list-18/providers/Microsoft.ManagedIdentity/userAssignedIdentities/id-ci-ca3e70\",\n \"isolationScope\": \"None\",\n \"location\": \"westeurope\",\n \"name\": \"id-ci-ca3e70\",\n \"principalId\": \"c9ebd571-9591-4f3f-a6a5-df01ead4633b\",\n \"resourceGroup\": \"mcpe2-15123a-identity-list-18\",\n \"systemData\": null,\n \"tags\": {},\n \"tenantId\": \"00000000-0000-0000-0000-000000000000\",\n \"type\": \"Microsoft.ManagedIdentity/userAssignedIdentities\"\n },\n {\n \"clientId\": \"b18fa735-c94b-4ff5-a75b-8062bd5802a6\",\n \"id\": \"/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/mcpe2-15123a-identity-list-18/providers/Microsoft.ManagedIdentity/userAssignedIdentities/id-backup-8a1d06\",\n \"isolationScope\": \"None\",\n \"location\": \"westeurope\",\n \"name\": \"id-backup-8a1d06\",\n \"principalId\": \"61d8f599-ab9a-490a-8aa0-038a46a59efc\",\n \"resourceGroup\": \"mcpe2-15123a-identity-list-18\",\n \"systemData\": null,\n \"tags\": {},\n \"tenantId\": \"00000000-0000-0000-0000-000000000000\",\n \"type\": \"Microsoft.ManagedIdentity/userAssignedIdentities\"\n }\n]\n```", + "queries": [] + }, + { + "task": "insights-activity-log-alert-create", + "mode": "negative", + "passed": false, + "wait_s": 10, + "recorded": "2026-09-27T20:56:10Z", + "slots": { + "resource_group": "mcpe2-151aca-insights-activity-log-alert-create-12", + "rg": "mcpe2-151aca-insights-activity-log-alert-create-12", + "name": "activitye57773512a", + "scope": "/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/mcpe2-151aca-insights-activity-log-alert-create-12" + }, + "text": "", + "queries": [ + { + "command": "rest --method get --url \"/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/mcpe2-151aca-insights-activity-log-alert-create-12/providers/Microsoft.Insights/activityLogAlerts/activitye57773512a?api-version=2026-01-01\"", + "repeat": 20, + "answer": { + "exitCode": 1, + "stdout": "", + "stderr": "ERROR: NOT FOUND({\"error\": {\"code\": \"ResourceNotFound\", \"message\": \"The Resource 'Microsoft.Insights/activityLogAlerts/activitye57773512a' under resource group 'mcpe2-151aca-insights-activity-log-alert-create-12' was not found.\", \"details\": [], \"additionalInfo\": []}})\r\n", + "classId": "not-found" + } + } + ] + }, + { + "task": "insights-activity-log-alert-create", + "mode": "oracle", + "passed": true, + "wait_s": 90, + "recorded": "2026-09-27T21:24:07Z", + "slots": { + "resource_group": "mcpe2-15123a-insights-activity-log-alert-create-12", + "rg": "mcpe2-15123a-insights-activity-log-alert-create-12", + "name": "activity19be28d680", + "scope": "/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/mcpe2-15123a-insights-activity-log-alert-create-12", + "_oracle": true + }, + "text": "ORACLE ✅ az monitor activity-log alert create --name activity19be28d680 --resource-group mcpe2-15123a-insights-activity-log-alert-create-12 --scope /subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/mcpe2-15123a-insights-activity-log-alert-create-12 --condition category=Administrative\n```json\n{\n \"actions\": {\n \"actionGroups\": []\n },\n \"condition\": {\n \"allOf\": [\n {\n \"equals\": \"Administrative\",\n \"field\": \"category\"\n }\n ]\n },\n \"enabled\": true,\n \"id\": \"/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/mcpe2-15123a-insights-activity-log-alert-create-12/providers/Microsoft.Insights/activityLogAlerts/activity19be28d680\",\n \"location\": \"Global\",\n \"name\": \"activity19be28d680\",\n \"resourceGroup\": \"mcpe2-15123a-insights-activity-log-alert-create-12\",\n \"scopes\": [\n \"/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/mcpe2-15123a-insights-activity-log-alert-create-12\"\n ],\n \"type\": \"Microsoft.Insights/ActivityLogAlerts\"\n}\n```", + "queries": [ + { + "command": "rest --method get --url \"/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/mcpe2-15123a-insights-activity-log-alert-create-12/providers/Microsoft.Insights/activityLogAlerts/activity19be28d680?api-version=2026-01-01\"", + "repeat": 3, + "answer": { + "exitCode": 0, + "stdout": "{\"id\":\"/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/mcpe2-15123a-insights-activity-log-alert-create-12/providers/Microsoft.Insights/activityLogAlerts/activity19be28d680\",\"location\":\"Global\",\"name\":\"activity19be28d680\",\"properties\":{\"actions\":{\"actionGroups\":[]},\"condition\":{\"allOf\":[{\"equals\":\"Administrative\",\"field\":\"category\"}]},\"enabled\":true,\"scopes\":[\"/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/mcpe2-15123a-insights-activity-log-alert-create-12\"]},\"type\":\"Microsoft.Insights/ActivityLogAlerts\"}", + "stderr": "", + "classId": null + } + } + ] + }, + { + "task": "kv-access-policy-grant", + "mode": "negative", + "passed": false, + "wait_s": 10, + "recorded": "2026-09-27T20:57:57Z", + "slots": { + "resource_group": "mcpe2-151aca-kv-access-policy-grant-16", + "rg": "mcpe2-151aca-kv-access-policy-grant-16", + "vault_name": "kv62c1bdb510", + "object_id": "70d3066b-41c4-4f41-9906-3b3bf6e8f3b3" + }, + "text": "", + "queries": [ + { + "command": "rest --method get --url \"/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/mcpe2-151aca-kv-access-policy-grant-16/providers/Microsoft.KeyVault/vaults/kv62c1bdb510?api-version=2023-07-01\"", + "answer": { + "exitCode": 0, + "stdout": "{\"id\":\"/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/mcpe2-151aca-kv-access-policy-grant-16/providers/Microsoft.KeyVault/vaults/kv62c1bdb510\",\"location\":\"westeurope\",\"name\":\"kv62c1bdb510\",\"properties\":{\"accessPolicies\":[],\"enableRbacAuthorization\":false,\"enableSoftDelete\":true,\"enabledForDeployment\":false,\"enabledForDiskEncryption\":false,\"enabledForTemplateDeployment\":false,\"provisioningState\":\"Succeeded\",\"publicNetworkAccess\":\"Enabled\",\"sku\":{\"family\":\"A\",\"name\":\"standard\"},\"softDeleteRetentionInDays\":90,\"tenantId\":\"00000000-0000-0000-0000-000000000000\",\"vaultUri\":\"https://kv62c1bdb510.vault.azure.localhost.localstack.cloud:4566\"},\"systemData\":{\"createdAt\":\"2026-09-27T20:57:38.1790542658+00:00\",\"createdBy\":\"any-app\",\"createdByType\":\"Application\",\"lastModifiedAt\":\"2026-09-27T20:57:38.1790542658+00:00\",\"lastModifiedBy\":\"any-app\",\"lastModifiedByType\":\"Application\"},\"tags\":{},\"type\":\"Microsoft.KeyVault/vaults\"}", + "stderr": "", + "classId": null + } + } + ] + }, + { + "task": "kv-access-policy-grant", + "mode": "oracle", + "passed": true, + "wait_s": 90, + "recorded": "2026-09-27T21:26:03Z", + "slots": { + "resource_group": "mcpe2-15123a-kv-access-policy-grant-16", + "rg": "mcpe2-15123a-kv-access-policy-grant-16", + "vault_name": "kvb3754a4418", + "object_id": "43cdbfac-0a96-4f86-827e-a27fc32aae87", + "_oracle": true + }, + "text": "ORACLE ✅ az keyvault set-policy --name kvb3754a4418 --resource-group mcpe2-15123a-kv-access-policy-grant-16 --object-id 43cdbfac-0a96-4f86-827e-a27fc32aae87 --secret-permissions get list\n```json\n{\n \"id\": \"/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/mcpe2-15123a-kv-access-policy-grant-16/providers/Microsoft.KeyVault/vaults/kvb3754a4418\",\n \"location\": \"westeurope\",\n \"name\": \"kvb3754a4418\",\n \"properties\": {\n \"accessPolicies\": [\n {\n \"applicationId\": null,\n \"objectId\": \"43cdbfac-0a96-4f86-827e-a27fc32aae87\",\n \"permissions\": {\n \"certificates\": null,\n \"keys\": null,\n \"secrets\": [\n \"list\",\n \"get\"\n ],\n \"storage\": null\n },\n \"tenantId\": \"00000000-0000-0000-0000-000000000000\"\n }\n ],\n \"createMode\": null,\n \"enablePurgeProtection\": null,\n \"enableRbacAuthorization\": false,\n \"enableSoftDelete\": true,\n \"enabledForDeployment\": false,\n \"enabledForDiskEncryption\": false,\n \"enabledForTemplateDeployment\": false,\n \"hsmPoolResourceId\": null,\n \"networkAcls\": null,\n \"privateEndpointConnections\": null,\n \"provisioningState\": \"Succeeded\",\n \"publicNetworkAccess\": \"Enabled\",\n \"sku\": {\n \"family\": \"A\",\n \"name\": \"standard\"\n },\n \"softDeleteRetentionInDays\": 90,\n \"tenantId\": \"00000000-0000-0000-0000-000000000000\",\n \"vaultUri\": \"https://kvb3754a4418.vault.azure.localhost.localstack.cloud:4566\"\n },\n \"resourceGroup\": \"mcpe2-15123a-kv-access-policy-grant-16\",\n \"systemData\": {\n \"createdAt\": \"2026-09-27T21:25:40.179054+00:00\",\n \"createdBy\": \"any-app\",\n \"createdByType\": \"Application\",\n \"lastModifiedAt\": \"2026-09-27T21:25:40.179054+00:00\",\n \"lastModifiedBy\": \"any-app\",\n \"lastModifiedByType\": \"Application\"\n },\n \"tags\": {},\n \"type\": \"Microsoft.KeyVault/vaults\"\n}\n```", + "queries": [ + { + "command": "rest --method get --url \"/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/mcpe2-15123a-kv-access-policy-grant-16/providers/Microsoft.KeyVault/vaults/kvb3754a4418?api-version=2023-07-01\"", + "answer": { + "exitCode": 0, + "stdout": "{\"id\":\"/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/mcpe2-15123a-kv-access-policy-grant-16/providers/Microsoft.KeyVault/vaults/kvb3754a4418\",\"location\":\"westeurope\",\"name\":\"kvb3754a4418\",\"properties\":{\"accessPolicies\":[{\"objectId\":\"43cdbfac-0a96-4f86-827e-a27fc32aae87\",\"permissions\":{\"secrets\":[\"list\",\"get\"]},\"tenantId\":\"00000000-0000-0000-0000-000000000000\"}],\"enableRbacAuthorization\":false,\"enableSoftDelete\":true,\"enabledForDeployment\":false,\"enabledForDiskEncryption\":false,\"enabledForTemplateDeployment\":false,\"provisioningState\":\"Succeeded\",\"publicNetworkAccess\":\"Enabled\",\"sku\":{\"family\":\"A\",\"name\":\"standard\"},\"softDeleteRetentionInDays\":90,\"tenantId\":\"00000000-0000-0000-0000-000000000000\",\"vaultUri\":\"https://kvb3754a4418.vault.azure.localhost.localstack.cloud:4566\"},\"systemData\":{\"createdAt\":\"2026-09-27T21:25:40.1790544340+00:00\",\"createdBy\":\"any-app\",\"createdByType\":\"Application\",\"lastModifiedAt\":\"2026-09-27T21:25:40.1790544340+00:00\",\"lastModifiedBy\":\"any-app\",\"lastModifiedByType\":\"Application\"},\"tags\":{},\"type\":\"Microsoft.KeyVault/vaults\"}", + "stderr": "", + "classId": null + } + } + ] + }, + { + "task": "kv-decrypt", + "mode": "negative", + "passed": false, + "wait_s": 10, + "recorded": "2026-09-27T20:57:10Z", + "slots": { + "resource_group": "mcpe2-151aca-kv-decrypt-14", + "rg": "mcpe2-151aca-kv-decrypt-14", + "vault_name": "kv5650318a38", + "key_name": "k5650318a38", + "key_version": "22863c0fa78a46f5a5bee69263b91b8a", + "crypto_value": "kPDIofq_SlJfwRhmdJR5AkBcc0K1gn6O5CgHL0wcInE-AnzgK9vlMAeP9KwR7MQ2m_i4XGanAJDVMc6GOe1vzdgFRGD0Cn-OgwJX8Jr_zLL_LyVMhTkHkWtdNxaORGYs3zBdkyp4eXCUXaZuigxl2Y3gPCnZL_UxX7KeU-8r7bp74XPLpivVJ8t9XHbk_Ry1EGlWt7I1Gh7wVZRRiPJhBwGsqNCI-ZkFawEEY_vFQUe2iHgYMPspZ7dOfGMuHaT534eIPherqvkLwf1BhLTT_g3s0Xuy1LxRvXwudA05fSXE8y6r_nSA1f16Ug6hKDtCZHVFuMkNqW7fhZ-k89gufw", + "ciphertext_b64": "kPDIofq/SlJfwRhmdJR5AkBcc0K1gn6O5CgHL0wcInE+AnzgK9vlMAeP9KwR7MQ2m/i4XGanAJDVMc6GOe1vzdgFRGD0Cn+OgwJX8Jr/zLL/LyVMhTkHkWtdNxaORGYs3zBdkyp4eXCUXaZuigxl2Y3gPCnZL/UxX7KeU+8r7bp74XPLpivVJ8t9XHbk/Ry1EGlWt7I1Gh7wVZRRiPJhBwGsqNCI+ZkFawEEY/vFQUe2iHgYMPspZ7dOfGMuHaT534eIPherqvkLwf1BhLTT/g3s0Xuy1LxRvXwudA05fSXE8y6r/nSA1f16Ug6hKDtCZHVFuMkNqW7fhZ+k89gufw==", + "plaintext": "order-8efb37ccc2" + }, + "text": "", + "queries": [] + }, + { + "task": "kv-decrypt", + "mode": "oracle", + "passed": true, + "wait_s": 90, + "recorded": "2026-09-27T21:25:06Z", + "slots": { + "resource_group": "mcpe2-15123a-kv-decrypt-14", + "rg": "mcpe2-15123a-kv-decrypt-14", + "vault_name": "kv9cefe95e09", + "key_name": "k9cefe95e09", + "key_version": "56b4674b5abd4cebab374511c6080c7f", + "crypto_value": "qESCwfEGpKQoWm0WAupwkTYHF74wHQ17x3GQzapZ2axspm4wwAtXASRthTEtRZ9sNMLAMuwbWY4fLzQYWh3xneA3V87xzPq-z37rlP7l-R08UpJV5erA115_aVigg5HPAFP1li8F42EY2WSS6ZEPwVhsjuuXT6L2SffYBeLi6FVnojWZgk1Kqd_d8oupJZ8eBrjvRNxCvEw0ZI8oWfrGaUt12AxUTISfkjcA2WJxpLJzSMhHmTTW_R9lpFb5W3AYAeqbb6RxmIaG1FJdujib1REmhNDs6A8vbUNF3BsNEEqlWZTwK1kQ7nk4Vmi3JCwgWMgOztMolw5hmaRmXX89lA", + "ciphertext_b64": "qESCwfEGpKQoWm0WAupwkTYHF74wHQ17x3GQzapZ2axspm4wwAtXASRthTEtRZ9sNMLAMuwbWY4fLzQYWh3xneA3V87xzPq+z37rlP7l+R08UpJV5erA115/aVigg5HPAFP1li8F42EY2WSS6ZEPwVhsjuuXT6L2SffYBeLi6FVnojWZgk1Kqd/d8oupJZ8eBrjvRNxCvEw0ZI8oWfrGaUt12AxUTISfkjcA2WJxpLJzSMhHmTTW/R9lpFb5W3AYAeqbb6RxmIaG1FJdujib1REmhNDs6A8vbUNF3BsNEEqlWZTwK1kQ7nk4Vmi3JCwgWMgOztMolw5hmaRmXX89lA==", + "plaintext": "order-d18621ca59", + "_oracle": true + }, + "text": "ORACLE ✅ az keyvault key decrypt --vault-name kv9cefe95e09 --name k9cefe95e09 --version 56b4674b5abd4cebab374511c6080c7f --algorithm RSA-OAEP --value qESCwfEGpKQoWm0WAupwkTYHF74wHQ17x3GQzapZ2axspm4wwAtXASRthTEtRZ9sNMLAMuwbWY4fLzQYWh3xneA3V87xzPq+z37rlP7l+R08UpJV5erA115/aVigg5HPAFP1li8F42EY2WSS6ZEPwVhsjuuXT6L2SffYBeLi6FVnojWZgk1Kqd/d8oupJZ8eBrjvRNxCvEw0ZI8oWfrGaUt12AxUTISfkjcA2WJxpLJzSMhHmTTW/R9lpFb5W3AYAeqbb6RxmIaG1FJdujib1REmhNDs6A8vbUNF3BsNEEqlWZTwK1kQ7nk4Vmi3JCwgWMgOztMolw5hmaRmXX89lA== --data-type plaintext\n```json\n{\n \"algorithm\": \"RSA-OAEP\",\n \"kid\": \"https://kv9cefe95e09.vault.azure.localhost.localstack.cloud:4566/keys/k9cefe95e09/56b4674b5abd4cebab374511c6080c7f\",\n \"result\": \"order-d18621ca59\"\n}\n```", + "queries": [] + }, + { + "task": "kv-rotation-policy", + "mode": "negative", + "passed": false, + "wait_s": 10, + "recorded": "2026-09-27T20:57:35Z", + "slots": { + "resource_group": "mcpe2-151aca-kv-rotation-policy-15", + "rg": "mcpe2-151aca-kv-rotation-policy-15", + "vault_name": "kvb6789033fd", + "key_name": "kb6789033fd", + "key_version": "6bbacbe341a44bcf84e78819a86b0036", + "rotate_days": 120, + "expiry_days": 270 + }, + "text": "", + "queries": [] + }, + { + "task": "kv-rotation-policy", + "mode": "oracle", + "passed": true, + "wait_s": 90, + "recorded": "2026-09-27T21:25:37Z", + "slots": { + "resource_group": "mcpe2-15123a-kv-rotation-policy-15", + "rg": "mcpe2-15123a-kv-rotation-policy-15", + "vault_name": "kvf47800e1de", + "key_name": "kf47800e1de", + "key_version": "4b5a579c024f42e389d2e142dd24a3ee", + "rotate_days": 75, + "expiry_days": 270, + "_oracle": true + }, + "text": "ORACLE ✅ az keyvault key rotation-policy show --vault-name kvf47800e1de --name kf47800e1de\n```json\n{\n \"createdOn\": \"2026-09-27T21:25:15+00:00\",\n \"expiresIn\": \"P270D\",\n \"id\": \"https://kvf47800e1de.vault.azure.localhost.localstack.cloud:4566/keys/kf47800e1de/rotationpolicy\",\n \"lifetimeActions\": [\n {\n \"action\": \"Rotate\",\n \"timeAfterCreate\": \"P75D\",\n \"timeBeforeExpiry\": null\n },\n {\n \"action\": \"Notify\",\n \"timeAfterCreate\": null,\n \"timeBeforeExpiry\": \"P30D\"\n }\n ],\n \"updatedOn\": \"2026-09-27T21:25:15+00:00\"\n}\n```", + "queries": [] + }, + { + "task": "kv-sign", + "mode": "negative", + "passed": false, + "wait_s": 10, + "recorded": "2026-09-27T20:56:36Z", + "slots": { + "resource_group": "mcpe2-151aca-kv-sign-13", + "rg": "mcpe2-151aca-kv-sign-13", + "vault_name": "kv4c794fdee2", + "key_name": "k4c794fdee2", + "key_version": "4dfcbe561f4849eeb7289965efb2d204", + "crypto_value": "Uk5RS8khj8InWhafHksJqekD-Fx8UfPVPAuEoXUElg0", + "digest": "Uk5RS8khj8InWhafHksJqekD-Fx8UfPVPAuEoXUElg0", + "digest_b64": "Uk5RS8khj8InWhafHksJqekD+Fx8UfPVPAuEoXUElg0=" + }, + "text": "", + "queries": [] + }, + { + "task": "kv-sign", + "mode": "oracle", + "passed": true, + "wait_s": 90, + "recorded": "2026-09-27T21:24:33Z", + "slots": { + "resource_group": "mcpe2-15123a-kv-sign-13", + "rg": "mcpe2-15123a-kv-sign-13", + "vault_name": "kv561194439a", + "key_name": "k561194439a", + "key_version": "e9c7812bc1c74a5f9c51e5fe3c5a5b0c", + "crypto_value": "RNs7ajJdI1R7Z05Gjky0NeMZlPVs4TziT0RmtdF2jXo", + "digest": "RNs7ajJdI1R7Z05Gjky0NeMZlPVs4TziT0RmtdF2jXo", + "digest_b64": "RNs7ajJdI1R7Z05Gjky0NeMZlPVs4TziT0RmtdF2jXo=", + "_oracle": true + }, + "text": "ORACLE ✅ az keyvault key sign --vault-name kv561194439a --name k561194439a --version e9c7812bc1c74a5f9c51e5fe3c5a5b0c --algorithm RS256 --digest RNs7ajJdI1R7Z05Gjky0NeMZlPVs4TziT0RmtdF2jXo=\n```json\n{\n \"algorithm\": \"RS256\",\n \"keyId\": \"https://kv561194439a.vault.azure.localhost.localstack.cloud:4566/keys/k561194439a/e9c7812bc1c74a5f9c51e5fe3c5a5b0c\",\n \"signature\": \"NhIDcC/NXtnoYHcfSvC8WZK0zHioakmi6hs8C4KK5aHrx7TPrOlGqMAis0PjtAQgMO1jJC/lkOdtvi7Gm+gloHA1G8SCmET/xIEOEFJbICKWXY+zol2qTLFr8QWr0kTyA6NEFzl4kdofo/yavxKSed60R4DKbf5eSarawexkziAOsXUNdmPk4BBNw5orhTiaS3P1sclHaCeT1CltcOdsdxMYaMjRw8nOEbW+gOyPjsg/xPGCRIhd/1Aa6iVZ1BYMktl/xoJMLQAoddGEcJc00+4T6L6XwN96v8ho/SJP9sajqwV4JyjXKffi5Jks1bVj52ZEQ8nX7uM1qV05F+jibQ==\"\n}\n```", + "queries": [ + { + "command": "keyvault key verify --vault-name kv561194439a --name k561194439a --version e9c7812bc1c74a5f9c51e5fe3c5a5b0c --algorithm RS256 --digest RNs7ajJdI1R7Z05Gjky0NeMZlPVs4TziT0RmtdF2jXo= --signature NhIDcC/NXtnoYHcfSvC8WZK0zHioakmi6hs8C4KK5aHrx7TPrOlGqMAis0PjtAQgMO1jJC/lkOdtvi7Gm+gloHA1G8SCmET/xIEOEFJbICKWXY+zol2qTLFr8QWr0kTyA6NEFzl4kdofo/yavxKSed60R4DKbf5eSarawexkziAOsXUNdmPk4BBNw5orhTiaS3P1sclHaCeT1CltcOdsdxMYaMjRw8nOEbW+gOyPjsg/xPGCRIhd/1Aa6iVZ1BYMktl/xoJMLQAoddGEcJc00+4T6L6XwN96v8ho/SJP9sajqwV4JyjXKffi5Jks1bVj52ZEQ8nX7uM1qV05F+jibQ==", + "answer": { + "exitCode": 0, + "stdout": "{\"algorithm\":\"RS256\",\"isValid\":true,\"keyId\":\"https://kv561194439a.vault.azure.localhost.localstack.cloud:4566/keys/k561194439a/e9c7812bc1c74a5f9c51e5fe3c5a5b0c\"}", + "stderr": "", + "classId": null + } + } + ] + }, + { + "task": "la-workspace-retention", + "mode": "negative", + "passed": false, + "wait_s": 10, + "recorded": "2026-09-27T21:00:13Z", + "slots": { + "resource_group": "mcpe2-151aca-la-workspace-retention-22", + "rg": "mcpe2-151aca-la-workspace-retention-22", + "name": "la-6ec38d10", + "days": 60 + }, + "text": "", + "queries": [ + { + "command": "rest --method get --url \"/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/mcpe2-151aca-la-workspace-retention-22/providers/Microsoft.OperationalInsights/workspaces/la-6ec38d10?api-version=2026-03-01\"", + "answer": { + "exitCode": 0, + "stdout": "{\"etag\":\"\\\"a2d3350e-58ec-4c7e-a3ee-43af00a0df34\\\"\",\"id\":\"/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/mcpe2-151aca-la-workspace-retention-22/providers/Microsoft.OperationalInsights/workspaces/la-6ec38d10\",\"location\":\"westeurope\",\"name\":\"la-6ec38d10\",\"properties\":{\"createdDate\":\"2026-09-27T20:59:46.1790542786Z\",\"customerId\":\"9447d0af-c38a-4ef4-a9f9-f3bc19d93e18\",\"features\":{\"enableLogAccessUsingOnlyResourcePermissions\":true},\"modifiedDate\":\"2026-09-27T20:59:46.1790542786Z\",\"provisioningState\":\"Succeeded\",\"publicNetworkAccessForIngestion\":\"Enabled\",\"publicNetworkAccessForQuery\":\"Enabled\",\"retentionInDays\":30,\"sku\":{\"lastSkuUpdate\":\"2026-09-27T20:59:46.1790542786Z\",\"name\":\"PerGB2018\"},\"workspaceCapping\":{\"dailyQuotaGb\":-1,\"dataIngestionStatus\":\"RespectQuota\",\"quotaNextResetTime\":\"2026-09-27T20:59:47Z\"}},\"type\":\"Microsoft.OperationalInsights/workspaces\"}", + "stderr": "", + "classId": null + } + }, + { + "command": "rest --method get --url \"/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/mcpe2-151aca-la-workspace-retention-22/providers/Microsoft.OperationalInsights/workspaces/la-6ec38d10?api-version=2026-03-01\"", + "answer": { + "exitCode": 0, + "stdout": "{\"etag\":\"\\\"a2d3350e-58ec-4c7e-a3ee-43af00a0df34\\\"\",\"id\":\"/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/mcpe2-151aca-la-workspace-retention-22/providers/Microsoft.OperationalInsights/workspaces/la-6ec38d10\",\"location\":\"westeurope\",\"name\":\"la-6ec38d10\",\"properties\":{\"createdDate\":\"2026-09-27T20:59:46.1790542786Z\",\"customerId\":\"9447d0af-c38a-4ef4-a9f9-f3bc19d93e18\",\"features\":{\"enableLogAccessUsingOnlyResourcePermissions\":true},\"modifiedDate\":\"2026-09-27T20:59:46.1790542786Z\",\"provisioningState\":\"Succeeded\",\"publicNetworkAccessForIngestion\":\"Enabled\",\"publicNetworkAccessForQuery\":\"Enabled\",\"retentionInDays\":30,\"sku\":{\"lastSkuUpdate\":\"2026-09-27T20:59:46.1790542786Z\",\"name\":\"PerGB2018\"},\"workspaceCapping\":{\"dailyQuotaGb\":-1,\"dataIngestionStatus\":\"RespectQuota\",\"quotaNextResetTime\":\"2026-09-27T20:59:49Z\"}},\"type\":\"Microsoft.OperationalInsights/workspaces\"}", + "stderr": "", + "classId": null + } + }, + { + "command": "rest --method get --url \"/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/mcpe2-151aca-la-workspace-retention-22/providers/Microsoft.OperationalInsights/workspaces/la-6ec38d10?api-version=2026-03-01\"", + "answer": { + "exitCode": 0, + "stdout": "{\"etag\":\"\\\"a2d3350e-58ec-4c7e-a3ee-43af00a0df34\\\"\",\"id\":\"/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/mcpe2-151aca-la-workspace-retention-22/providers/Microsoft.OperationalInsights/workspaces/la-6ec38d10\",\"location\":\"westeurope\",\"name\":\"la-6ec38d10\",\"properties\":{\"createdDate\":\"2026-09-27T20:59:46.1790542786Z\",\"customerId\":\"9447d0af-c38a-4ef4-a9f9-f3bc19d93e18\",\"features\":{\"enableLogAccessUsingOnlyResourcePermissions\":true},\"modifiedDate\":\"2026-09-27T20:59:46.1790542786Z\",\"provisioningState\":\"Succeeded\",\"publicNetworkAccessForIngestion\":\"Enabled\",\"publicNetworkAccessForQuery\":\"Enabled\",\"retentionInDays\":30,\"sku\":{\"lastSkuUpdate\":\"2026-09-27T20:59:46.1790542786Z\",\"name\":\"PerGB2018\"},\"workspaceCapping\":{\"dailyQuotaGb\":-1,\"dataIngestionStatus\":\"RespectQuota\",\"quotaNextResetTime\":\"2026-09-27T20:59:51Z\"}},\"type\":\"Microsoft.OperationalInsights/workspaces\"}", + "stderr": "", + "classId": null + } + }, + { + "command": "rest --method get --url \"/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/mcpe2-151aca-la-workspace-retention-22/providers/Microsoft.OperationalInsights/workspaces/la-6ec38d10?api-version=2026-03-01\"", + "answer": { + "exitCode": 0, + "stdout": "{\"etag\":\"\\\"a2d3350e-58ec-4c7e-a3ee-43af00a0df34\\\"\",\"id\":\"/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/mcpe2-151aca-la-workspace-retention-22/providers/Microsoft.OperationalInsights/workspaces/la-6ec38d10\",\"location\":\"westeurope\",\"name\":\"la-6ec38d10\",\"properties\":{\"createdDate\":\"2026-09-27T20:59:46.1790542786Z\",\"customerId\":\"9447d0af-c38a-4ef4-a9f9-f3bc19d93e18\",\"features\":{\"enableLogAccessUsingOnlyResourcePermissions\":true},\"modifiedDate\":\"2026-09-27T20:59:46.1790542786Z\",\"provisioningState\":\"Succeeded\",\"publicNetworkAccessForIngestion\":\"Enabled\",\"publicNetworkAccessForQuery\":\"Enabled\",\"retentionInDays\":30,\"sku\":{\"lastSkuUpdate\":\"2026-09-27T20:59:46.1790542786Z\",\"name\":\"PerGB2018\"},\"workspaceCapping\":{\"dailyQuotaGb\":-1,\"dataIngestionStatus\":\"RespectQuota\",\"quotaNextResetTime\":\"2026-09-27T20:59:53Z\"}},\"type\":\"Microsoft.OperationalInsights/workspaces\"}", + "stderr": "", + "classId": null + } + }, + { + "command": "rest --method get --url \"/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/mcpe2-151aca-la-workspace-retention-22/providers/Microsoft.OperationalInsights/workspaces/la-6ec38d10?api-version=2026-03-01\"", + "answer": { + "exitCode": 0, + "stdout": "{\"etag\":\"\\\"a2d3350e-58ec-4c7e-a3ee-43af00a0df34\\\"\",\"id\":\"/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/mcpe2-151aca-la-workspace-retention-22/providers/Microsoft.OperationalInsights/workspaces/la-6ec38d10\",\"location\":\"westeurope\",\"name\":\"la-6ec38d10\",\"properties\":{\"createdDate\":\"2026-09-27T20:59:46.1790542786Z\",\"customerId\":\"9447d0af-c38a-4ef4-a9f9-f3bc19d93e18\",\"features\":{\"enableLogAccessUsingOnlyResourcePermissions\":true},\"modifiedDate\":\"2026-09-27T20:59:46.1790542786Z\",\"provisioningState\":\"Succeeded\",\"publicNetworkAccessForIngestion\":\"Enabled\",\"publicNetworkAccessForQuery\":\"Enabled\",\"retentionInDays\":30,\"sku\":{\"lastSkuUpdate\":\"2026-09-27T20:59:46.1790542786Z\",\"name\":\"PerGB2018\"},\"workspaceCapping\":{\"dailyQuotaGb\":-1,\"dataIngestionStatus\":\"RespectQuota\",\"quotaNextResetTime\":\"2026-09-27T20:59:54Z\"}},\"type\":\"Microsoft.OperationalInsights/workspaces\"}", + "stderr": "", + "classId": null + } + }, + { + "command": "rest --method get --url \"/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/mcpe2-151aca-la-workspace-retention-22/providers/Microsoft.OperationalInsights/workspaces/la-6ec38d10?api-version=2026-03-01\"", + "answer": { + "exitCode": 0, + "stdout": "{\"etag\":\"\\\"a2d3350e-58ec-4c7e-a3ee-43af00a0df34\\\"\",\"id\":\"/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/mcpe2-151aca-la-workspace-retention-22/providers/Microsoft.OperationalInsights/workspaces/la-6ec38d10\",\"location\":\"westeurope\",\"name\":\"la-6ec38d10\",\"properties\":{\"createdDate\":\"2026-09-27T20:59:46.1790542786Z\",\"customerId\":\"9447d0af-c38a-4ef4-a9f9-f3bc19d93e18\",\"features\":{\"enableLogAccessUsingOnlyResourcePermissions\":true},\"modifiedDate\":\"2026-09-27T20:59:46.1790542786Z\",\"provisioningState\":\"Succeeded\",\"publicNetworkAccessForIngestion\":\"Enabled\",\"publicNetworkAccessForQuery\":\"Enabled\",\"retentionInDays\":30,\"sku\":{\"lastSkuUpdate\":\"2026-09-27T20:59:46.1790542786Z\",\"name\":\"PerGB2018\"},\"workspaceCapping\":{\"dailyQuotaGb\":-1,\"dataIngestionStatus\":\"RespectQuota\",\"quotaNextResetTime\":\"2026-09-27T20:59:56Z\"}},\"type\":\"Microsoft.OperationalInsights/workspaces\"}", + "stderr": "", + "classId": null + } + } + ] + }, + { + "task": "la-workspace-retention", + "mode": "oracle", + "passed": true, + "wait_s": 90, + "recorded": "2026-09-27T21:27:40Z", + "slots": { + "resource_group": "mcpe2-15123a-la-workspace-retention-22", + "rg": "mcpe2-15123a-la-workspace-retention-22", + "name": "la-cb06402d", + "days": 60, + "_oracle": true + }, + "text": "ORACLE ✅ az monitor log-analytics workspace update --resource-group mcpe2-15123a-la-workspace-retention-22 --workspace-name la-cb06402d --retention-time 60\n```json\n{\n \"createdDate\": \"2026-09-27T21:27:29.1790544449Z\",\n \"customerId\": \"bf9495ca-285c-4c85-9c09-bd9576c59747\",\n \"etag\": \"\\\"eca24178-55f1-4275-8c2f-4c7cc0e86552\\\"\",\n \"features\": {\n \"enableLogAccessUsingOnlyResourcePermissions\": true\n },\n \"id\": \"/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/mcpe2-15123a-la-workspace-retention-22/providers/Microsoft.OperationalInsights/workspaces/la-cb06402d\",\n \"location\": \"westeurope\",\n \"modifiedDate\": \"2026-09-27T21:27:31.1790544451Z\",\n \"name\": \"la-cb06402d\",\n \"provisioningState\": \"Succeeded\",\n \"publicNetworkAccessForIngestion\": \"Enabled\",\n \"publicNetworkAccessForQuery\": \"Enabled\",\n \"resourceGroup\": \"mcpe2-15123a-la-workspace-retention-22\",\n \"retentionInDays\": 60,\n \"sku\": {\n \"lastSkuUpdate\": \"2026-09-27T21:27:31.1790544451Z\",\n \"name\": \"PerGB2018\"\n },\n \"type\": \"Microsoft.OperationalInsights/workspaces\",\n \"workspaceCapping\": {\n \"dailyQuotaGb\": -1.0,\n \"dataIngestionStatus\": \"RespectQuota\",\n \"quotaNextResetTime\": \"2026-09-27T21:27:31Z\"\n }\n}\n```", + "queries": [ + { + "command": "rest --method get --url \"/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/mcpe2-15123a-la-workspace-retention-22/providers/Microsoft.OperationalInsights/workspaces/la-cb06402d?api-version=2026-03-01\"", + "answer": { + "exitCode": 0, + "stdout": "{\"etag\":\"\\\"eca24178-55f1-4275-8c2f-4c7cc0e86552\\\"\",\"id\":\"/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/mcpe2-15123a-la-workspace-retention-22/providers/Microsoft.OperationalInsights/workspaces/la-cb06402d\",\"location\":\"westeurope\",\"name\":\"la-cb06402d\",\"properties\":{\"createdDate\":\"2026-09-27T21:27:29.1790544449Z\",\"customerId\":\"bf9495ca-285c-4c85-9c09-bd9576c59747\",\"features\":{\"enableLogAccessUsingOnlyResourcePermissions\":true},\"modifiedDate\":\"2026-09-27T21:27:31.1790544451Z\",\"provisioningState\":\"Succeeded\",\"publicNetworkAccessForIngestion\":\"Enabled\",\"publicNetworkAccessForQuery\":\"Enabled\",\"retentionInDays\":60,\"sku\":{\"lastSkuUpdate\":\"2026-09-27T21:27:31.1790544451Z\",\"name\":\"PerGB2018\"},\"workspaceCapping\":{\"dailyQuotaGb\":-1,\"dataIngestionStatus\":\"RespectQuota\",\"quotaNextResetTime\":\"2026-09-27T21:27:32Z\"}},\"type\":\"Microsoft.OperationalInsights/workspaces\"}", + "stderr": "", + "classId": null + } + } + ] + }, + { + "task": "lock-delete", + "mode": "negative", + "passed": false, + "wait_s": 10, + "recorded": "2026-09-27T20:52:45Z", + "slots": { + "resource_group": "mcpe2-151aca-lock-delete-4", + "rg": "mcpe2-151aca-lock-delete-4", + "lock_name": "lk-06f028c5" + }, + "text": "", + "queries": [ + { + "command": "rest --method get --url \"/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/mcpe2-151aca-lock-delete-4/providers/Microsoft.Authorization/locks/lk-06f028c5?api-version=2020-05-01\"", + "repeat": 7, + "answer": { + "exitCode": 0, + "stdout": "{\"id\":\"/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/mcpe2-151aca-lock-delete-4/providers/Microsoft.Authorization/locks/lk-06f028c5\",\"name\":\"lk-06f028c5\",\"properties\":{\"level\":\"CanNotDelete\"},\"type\":\"Microsoft.Authorization/locks\"}", + "stderr": "", + "classId": null + } + } + ] + }, + { + "task": "lock-delete", + "mode": "oracle", + "passed": true, + "wait_s": 90, + "recorded": "2026-09-27T21:20:10Z", + "slots": { + "resource_group": "mcpe2-15123a-lock-delete-4", + "rg": "mcpe2-15123a-lock-delete-4", + "lock_name": "lk-621d6fd3", + "_oracle": true + }, + "text": "ORACLE ✅ az resource delete --ids /subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/mcpe2-15123a-lock-delete-4/providers/Microsoft.Authorization/locks/lk-621d6fd3\n```\nThe command succeeded and printed no output.\n```", + "queries": [ + { + "command": "rest --method get --url \"/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/mcpe2-15123a-lock-delete-4/providers/Microsoft.Authorization/locks/lk-621d6fd3?api-version=2020-05-01\"", + "answer": { + "exitCode": 1, + "stdout": "", + "stderr": "ERROR: NOT FOUND({\"error\": {\"code\": \"LockNotFound\", \"message\": \"The lock 'lk-621d6fd3' could not be found.\", \"details\": [], \"additionalInfo\": []}})\r\n", + "classId": "other" + } + } + ] + }, + { + "task": "locks-list", + "mode": "negative", + "passed": false, + "wait_s": 10, + "recorded": "2026-09-27T20:53:01Z", + "slots": { + "resource_group": "mcpe2-151aca-locks-list-5", + "rg": "mcpe2-151aca-locks-list-5", + "lock1": "lk-ops-e50e40", + "lock2": "lk-audit-49e510" + }, + "text": "", + "queries": [] + }, + { + "task": "locks-list", + "mode": "oracle", + "passed": true, + "wait_s": 90, + "recorded": "2026-09-27T21:20:28Z", + "slots": { + "resource_group": "mcpe2-15123a-locks-list-5", + "rg": "mcpe2-15123a-locks-list-5", + "lock1": "lk-ops-2c9f53", + "lock2": "lk-audit-42bd5b", + "_oracle": true + }, + "text": "ORACLE ✅ az lock list --resource-group mcpe2-15123a-locks-list-5\n```json\n[\n {\n \"id\": \"/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/mcpe2-15123a-locks-list-5/providers/Microsoft.Authorization/locks/lk-audit-42bd5b\",\n \"level\": \"CanNotDelete\",\n \"name\": \"lk-audit-42bd5b\",\n \"notes\": null,\n \"owners\": null,\n \"resourceGroup\": \"mcpe2-15123a-locks-list-5\",\n \"type\": \"Microsoft.Authorization/locks\"\n },\n {\n \"id\": \"/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/mcpe2-15123a-locks-list-5/providers/Microsoft.Authorization/locks/lk-ops-2c9f53\",\n \"level\": \"CanNotDelete\",\n \"name\": \"lk-ops-2c9f53\",\n \"notes\": null,\n \"owners\": null,\n \"resourceGroup\": \"mcpe2-15123a-locks-list-5\",\n \"type\": \"Microsoft.Authorization/locks\"\n }\n]\n```", + "queries": [] + }, + { + "task": "private-dns-zones-list", + "mode": "negative", + "passed": false, + "wait_s": 10, + "recorded": "2026-09-27T20:59:26Z", + "slots": { + "resource_group": "mcpe2-151aca-private-dns-zones-list-20", + "rg": "mcpe2-151aca-private-dns-zones-list-20", + "zone1": "d65f75.internal.contoso.com", + "zone2": "d8f838.corp.contoso.com" + }, + "text": "", + "queries": [] + }, + { + "task": "private-dns-zones-list", + "mode": "oracle", + "passed": true, + "wait_s": 90, + "recorded": "2026-09-27T21:27:06Z", + "slots": { + "resource_group": "mcpe2-15123a-private-dns-zones-list-20", + "rg": "mcpe2-15123a-private-dns-zones-list-20", + "zone1": "25fa54.internal.contoso.com", + "zone2": "5e5ac9.corp.contoso.com", + "_oracle": true + }, + "text": "ORACLE ✅ az network private-dns zone list --resource-group mcpe2-15123a-private-dns-zones-list-20\n```json\n[\n {\n \"id\": \"/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/mcpe2-15123a-private-dns-zones-list-20/providers/Microsoft.Network/privateDnsZones/25fa54.internal.contoso.com\",\n \"location\": \"global\",\n \"maxNumberOfRecordSets\": 25000,\n \"maxNumberOfVirtualNetworkLinks\": 1000,\n \"maxNumberOfVirtualNetworkLinksWithRegistration\": 100,\n \"name\": \"25fa54.internal.contoso.com\",\n \"numberOfRecordSets\": 1,\n \"numberOfVirtualNetworkLinks\": 0,\n \"numberOfVirtualNetworkLinksWithRegistration\": 0,\n \"provisioningState\": \"Succeeded\",\n \"resourceGroup\": \"mcpe2-15123a-private-dns-zones-list-20\",\n \"type\": \"Microsoft.Network/privateDnsZones\"\n },\n {\n \"id\": \"/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/mcpe2-15123a-private-dns-zones-list-20/providers/Microsoft.Network/privateDnsZones/5e5ac9.corp.contoso.com\",\n \"location\": \"global\",\n \"maxNumberOfRecordSets\": 25000,\n \"maxNumberOfVirtualNetworkLinks\": 1000,\n \"maxNumberOfVirtualNetworkLinksWithRegistration\": 100,\n \"name\": \"5e5ac9.corp.contoso.com\",\n \"numberOfRecordSets\": 1,\n \"numberOfVirtualNetworkLinks\": 0,\n \"numberOfVirtualNetworkLinksWithRegistration\": 0,\n \"provisioningState\": \"Succeeded\",\n \"resourceGroup\": \"mcpe2-15123a-private-dns-zones-list-20\",\n \"type\": \"Microsoft.Network/privateDnsZones\"\n }\n]\n```", + "queries": [] + }, + { + "task": "role-definition-get", + "mode": "negative", + "passed": false, + "wait_s": 10, + "recorded": "2026-09-27T20:53:13Z", + "slots": { + "resource_group": "mcpe2-151aca-role-definition-get-6", + "rg": "mcpe2-151aca-role-definition-get-6", + "role_definition_name": "97f79227-3696-4a58-a225-547a2d7e1c5b", + "role_id": "97f79227-3696-4a58-a225-547a2d7e1c5b", + "role_name": "Bench Operator dd52", + "action1": "Microsoft.Compute/virtualMachines/start/action", + "action2": "Microsoft.Insights/alertRules/read" + }, + "text": "", + "queries": [] + }, + { + "task": "role-definition-get", + "mode": "oracle", + "passed": true, + "wait_s": 90, + "recorded": "2026-09-27T21:20:44Z", + "slots": { + "resource_group": "mcpe2-15123a-role-definition-get-6", + "rg": "mcpe2-15123a-role-definition-get-6", + "role_definition_name": "5c2d1204-886d-47be-8644-6352ef63b357", + "role_id": "5c2d1204-886d-47be-8644-6352ef63b357", + "role_name": "Bench Operator e196", + "action1": "Microsoft.Insights/alertRules/read", + "action2": "Microsoft.KeyVault/vaults/read", + "_oracle": true + }, + "text": "ORACLE ✅ az role definition list --name 5c2d1204-886d-47be-8644-6352ef63b357\n```json\n[\n {\n \"assignableScopes\": [\n \"/subscriptions/00000000-0000-0000-0000-000000000000\"\n ],\n \"createdBy\": null,\n \"createdOn\": \"2026-09-27T21:20:30.179054+00:00\",\n \"description\": null,\n \"id\": \"/subscriptions/00000000-0000-0000-0000-000000000000/providers/Microsoft.Authorization/roleDefinitions/5c2d1204-886d-47be-8644-6352ef63b357\",\n \"name\": \"5c2d1204-886d-47be-8644-6352ef63b357\",\n \"permissions\": [\n {\n \"actions\": [\n \"Microsoft.Insights/alertRules/read\",\n \"Microsoft.KeyVault/vaults/read\"\n ],\n \"condition\": null,\n \"conditionVersion\": null,\n \"dataActions\": [],\n \"notActions\": [],\n \"notDataActions\": []\n }\n ],\n \"roleName\": \"Bench Operator e196\",\n \"roleType\": \"CustomRole\",\n \"type\": \"Microsoft.Authorization/roleDefinitions\",\n \"updatedBy\": null,\n \"updatedOn\": \"2026-09-27T21:20:30.179054+00:00\"\n }\n]\n```", + "queries": [] + }, + { + "task": "route-get", + "mode": "negative", + "passed": false, + "wait_s": 10, + "recorded": "2026-09-27T20:59:03Z", + "slots": { + "resource_group": "mcpe2-151aca-route-get-19", + "rg": "mcpe2-151aca-route-get-19", + "route_table_name": "rt-c0d8405a", + "route_name": "r-c0d8405a", + "prefix": "10.100.174.0/24", + "next_hop": "10.0.54.4" + }, + "text": "", + "queries": [] + }, + { + "task": "route-get", + "mode": "oracle", + "passed": true, + "wait_s": 90, + "recorded": "2026-09-27T21:26:47Z", + "slots": { + "resource_group": "mcpe2-15123a-route-get-19", + "rg": "mcpe2-15123a-route-get-19", + "route_table_name": "rt-e489c3d1", + "route_name": "r-e489c3d1", + "prefix": "10.35.74.0/24", + "next_hop": "10.0.167.4", + "_oracle": true + }, + "text": "ORACLE ✅ az network route-table route show --name r-e489c3d1 --route-table-name rt-e489c3d1 --resource-group mcpe2-15123a-route-get-19\n```json\n{\n \"addressPrefix\": \"10.35.74.0/24\",\n \"hasBgpOverride\": false,\n \"id\": \"/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/mcpe2-15123a-route-get-19/providers/Microsoft.Network/routeTables/rt-e489c3d1/routes/r-e489c3d1\",\n \"name\": \"r-e489c3d1\",\n \"nextHopIpAddress\": \"10.0.167.4\",\n \"nextHopType\": \"VirtualAppliance\",\n \"provisioningState\": \"Succeeded\",\n \"resourceGroup\": \"mcpe2-15123a-route-get-19\",\n \"type\": \"Microsoft.Network/routeTables/routes\"\n}\n```", + "queries": [] + }, + { + "task": "sb-dp-queue-max-delivery", + "mode": "negative", + "passed": false, + "wait_s": 10, + "recorded": "2026-09-27T21:01:27Z", + "slots": { + "resource_group": "mcpe2-151aca-sb-dp-queue-max-delivery-25", + "rg": "mcpe2-151aca-sb-dp-queue-max-delivery-25", + "name": "sb66c30f50a6", + "namespace_name": "sb66c30f50a6", + "queue_name": "q-66c30f50a6", + "topic_name": "t-66c30f50a6", + "subscription_name": "s-66c30f50a6", + "auth_rule_name": "a-66c30f50a6", + "max_delivery": 3 + }, + "text": "", + "queries": [ + { + "command": "rest --method get --url \"/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/mcpe2-151aca-sb-dp-queue-max-delivery-25/providers/Microsoft.ServiceBus/namespaces/sb66c30f50a6/queues/q-66c30f50a6?api-version=2026-01-01\"", + "answer": { + "exitCode": 0, + "stdout": "{\"id\":\"/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/mcpe2-151aca-sb-dp-queue-max-delivery-25/providers/Microsoft.ServiceBus/namespaces/sb66c30f50a6/queues/q-66c30f50a6\",\"location\":\"westeurope\",\"name\":\"q-66c30f50a6\",\"properties\":{\"autoDeleteOnIdle\":\"P10675199DT2H48M5.4775807S\",\"countDetails\":{\"activeMessageCount\":0,\"deadLetterMessageCount\":0,\"scheduledMessageCount\":0,\"transferDeadLetterMessageCount\":0,\"transferMessageCount\":0},\"deadLetteringOnMessageExpiration\":false,\"defaultMessageTimeToLive\":\"P10675199DT2H48M5.4775807S\",\"duplicateDetectionHistoryTimeWindow\":\"PT10M\",\"enableBatchedOperations\":true,\"enableExpress\":false,\"enablePartitioning\":false,\"lockDuration\":\"PT0H1M0S\",\"maxDeliveryCount\":3,\"maxMessageSizeInKilobytes\":256,\"maxSizeInMegabytes\":1024,\"messageCount\":0,\"requiresDuplicateDetection\":false,\"requiresSession\":false,\"sizeInBytes\":0,\"status\":\"Active\"},\"type\":\"Microsoft.ServiceBus/namespaces/queues\"}", + "stderr": "", + "classId": null + } + } + ] + }, + { + "task": "sb-dp-queue-max-delivery", + "mode": "oracle", + "passed": true, + "wait_s": 90, + "recorded": "2026-09-27T21:28:38Z", + "slots": { + "resource_group": "mcpe2-15123a-sb-dp-queue-max-delivery-25", + "rg": "mcpe2-15123a-sb-dp-queue-max-delivery-25", + "name": "sb8cc84ad642", + "namespace_name": "sb8cc84ad642", + "queue_name": "q-8cc84ad642", + "topic_name": "t-8cc84ad642", + "subscription_name": "s-8cc84ad642", + "auth_rule_name": "a-8cc84ad642", + "max_delivery": 12, + "_oracle": true + }, + "text": "ORACLE ✅ az servicebus queue show --name q-8cc84ad642 --namespace-name sb8cc84ad642 --resource-group mcpe2-15123a-sb-dp-queue-max-delivery-25\n```json\n{\n \"autoDeleteOnIdle\": \"P10675199DT2H48M5.4775807S\",\n \"countDetails\": {\n \"activeMessageCount\": 0,\n \"deadLetterMessageCount\": 0,\n \"scheduledMessageCount\": 0,\n \"transferDeadLetterMessageCount\": 0,\n \"transferMessageCount\": 0\n },\n \"deadLetteringOnMessageExpiration\": false,\n \"defaultMessageTimeToLive\": \"P10675199DT2H48M5.4775807S\",\n \"duplicateDetectionHistoryTimeWindow\": \"PT10M\",\n \"enableBatchedOperations\": true,\n \"enableExpress\": false,\n \"enablePartitioning\": false,\n \"id\": \"/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/mcpe2-15123a-sb-dp-queue-max-delivery-25/providers/Microsoft.ServiceBus/namespaces/sb8cc84ad642/queues/q-8cc84ad642\",\n \"location\": \"westeurope\",\n \"lockDuration\": \"PT0H1M0S\",\n \"maxDeliveryCount\": 12,\n \"maxMessageSizeInKilobytes\": 256,\n \"maxSizeInMegabytes\": 1024,\n \"messageCount\": 0,\n \"name\": \"q-8cc84ad642\",\n \"requiresDuplicateDetection\": false,\n \"requiresSession\": false,\n \"resourceGroup\": \"mcpe2-15123a-sb-dp-queue-max-delivery-25\",\n \"sizeInBytes\": 0,\n \"status\": \"Active\",\n \"type\": \"Microsoft.ServiceBus/namespaces/queues\"\n}\n```", + "queries": [ + { + "command": "rest --method get --url \"/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/mcpe2-15123a-sb-dp-queue-max-delivery-25/providers/Microsoft.ServiceBus/namespaces/sb8cc84ad642/queues/q-8cc84ad642?api-version=2026-01-01\"", + "answer": { + "exitCode": 0, + "stdout": "{\"id\":\"/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/mcpe2-15123a-sb-dp-queue-max-delivery-25/providers/Microsoft.ServiceBus/namespaces/sb8cc84ad642/queues/q-8cc84ad642\",\"location\":\"westeurope\",\"name\":\"q-8cc84ad642\",\"properties\":{\"autoDeleteOnIdle\":\"P10675199DT2H48M5.4775807S\",\"countDetails\":{\"activeMessageCount\":0,\"deadLetterMessageCount\":0,\"scheduledMessageCount\":0,\"transferDeadLetterMessageCount\":0,\"transferMessageCount\":0},\"deadLetteringOnMessageExpiration\":false,\"defaultMessageTimeToLive\":\"P10675199DT2H48M5.4775807S\",\"duplicateDetectionHistoryTimeWindow\":\"PT10M\",\"enableBatchedOperations\":true,\"enableExpress\":false,\"enablePartitioning\":false,\"lockDuration\":\"PT0H1M0S\",\"maxDeliveryCount\":12,\"maxMessageSizeInKilobytes\":256,\"maxSizeInMegabytes\":1024,\"messageCount\":0,\"requiresDuplicateDetection\":false,\"requiresSession\":false,\"sizeInBytes\":0,\"status\":\"Active\"},\"type\":\"Microsoft.ServiceBus/namespaces/queues\"}", + "stderr": "", + "classId": null + } + } + ] + }, + { + "task": "sb-queue-regenerate-key", + "mode": "negative", + "passed": false, + "wait_s": 10, + "recorded": "2026-09-27T21:01:01Z", + "slots": { + "resource_group": "mcpe2-151aca-sb-queue-regenerate-key-24", + "rg": "mcpe2-151aca-sb-queue-regenerate-key-24", + "name": "sbf8a75da63d", + "namespace_name": "sbf8a75da63d", + "queue_name": "q-f8a75da63d", + "topic_name": "t-f8a75da63d", + "subscription_name": "s-f8a75da63d", + "auth_rule_name": "a-f8a75da63d", + "_before_changed": "MTcyZTM4M2UtOGYwZC00ZTMwLThhNDAtYTkxYmM3Yzc5MTJm", + "_before_kept": "OGZiNTdiOWYtNjBiOC00OTUyLWEyOGItZDIxOTJjNjY4NTEw" + }, + "text": "", + "queries": [ + { + "command": "rest --method post --url \"/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/mcpe2-151aca-sb-queue-regenerate-key-24/providers/Microsoft.ServiceBus/namespaces/sbf8a75da63d/queues/q-f8a75da63d/authorizationRules/a-f8a75da63d/listKeys?api-version=2026-01-01\"", + "answer": { + "exitCode": 0, + "stdout": "{\"keyName\":\"a-f8a75da63d\",\"primaryConnectionString\":\"Endpoint=sb://sbf8a75da63d.servicebus.azure.localhost.localstack.cloud:4511/;SharedAccessKeyName=a-f8a75da63d;SharedAccessKey=[redacted];UseDevelopmentEmulator=true\",\"primaryKey\":\"MTcyZTM4M2UtOGYwZC00ZTMwLThhNDAtYTkxYmM3Yzc5MTJm\",\"secondaryConnectionString\":\"Endpoint=sb://sbf8a75da63d.servicebus.azure.localhost.localstack.cloud:4511/;SharedAccessKeyName=a-f8a75da63d;SharedAccessKey=[redacted];UseDevelopmentEmulator=true\",\"secondaryKey\":\"OGZiNTdiOWYtNjBiOC00OTUyLWEyOGItZDIxOTJjNjY4NTEw\"}", + "stderr": "", + "classId": null + } + } + ] + }, + { + "task": "sb-queue-regenerate-key", + "mode": "oracle", + "passed": true, + "wait_s": 90, + "recorded": "2026-09-27T21:28:15Z", + "slots": { + "resource_group": "mcpe2-15123a-sb-queue-regenerate-key-24", + "rg": "mcpe2-15123a-sb-queue-regenerate-key-24", + "name": "sb12d7165099", + "namespace_name": "sb12d7165099", + "queue_name": "q-12d7165099", + "topic_name": "t-12d7165099", + "subscription_name": "s-12d7165099", + "auth_rule_name": "a-12d7165099", + "_before_changed": "MmUwM2FhNzgtM2JkOC00NWQzLWIwYzQtYzdjNDE3NzIwODg2", + "_before_kept": "ZWY3Y2E4YzEtNjQ0ZC00M2Y2LThiY2YtMjU0YWIzYTUxNjYy", + "_oracle": true + }, + "text": "ORACLE ✅ az servicebus queue authorization-rule keys renew --name a-12d7165099 --queue-name q-12d7165099 --namespace-name sb12d7165099 --resource-group mcpe2-15123a-sb-queue-regenerate-key-24 --key PrimaryKey\n```json\n{\n \"keyName\": \"a-12d7165099\",\n \"primaryConnectionString\": \"Endpoint=sb://sb12d7165099.servicebus.azure.localhost.localstack.cloud:4511/;SharedAccessKeyName=a-12d7165099;SharedAccessKey=[redacted];UseDevelopmentEmulator=true\",\n \"primaryKey\": \"ZDVkMDViYmItY2QyMS00ZjY1LWI2NTYtMWRjZDUxMmUxZWI1\",\n \"secondaryConnectionString\": \"Endpoint=sb://sb12d7165099.servicebus.azure.localhost.localstack.cloud:4511/;SharedAccessKeyName=a-12d7165099;SharedAccessKey=[redacted];UseDevelopmentEmulator=true\",\n \"secondaryKey\": \"ZWY3Y2E4YzEtNjQ0ZC00M2Y2LThiY2YtMjU0YWIzYTUxNjYy\"\n}\n```", + "queries": [ + { + "command": "rest --method post --url \"/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/mcpe2-15123a-sb-queue-regenerate-key-24/providers/Microsoft.ServiceBus/namespaces/sb12d7165099/queues/q-12d7165099/authorizationRules/a-12d7165099/listKeys?api-version=2026-01-01\"", + "answer": { + "exitCode": 0, + "stdout": "{\"keyName\":\"a-12d7165099\",\"primaryConnectionString\":\"Endpoint=sb://sb12d7165099.servicebus.azure.localhost.localstack.cloud:4511/;SharedAccessKeyName=a-12d7165099;SharedAccessKey=[redacted];UseDevelopmentEmulator=true\",\"primaryKey\":\"ZDVkMDViYmItY2QyMS00ZjY1LWI2NTYtMWRjZDUxMmUxZWI1\",\"secondaryConnectionString\":\"Endpoint=sb://sb12d7165099.servicebus.azure.localhost.localstack.cloud:4511/;SharedAccessKeyName=a-12d7165099;SharedAccessKey=[redacted];UseDevelopmentEmulator=true\",\"secondaryKey\":\"ZWY3Y2E4YzEtNjQ0ZC00M2Y2LThiY2YtMjU0YWIzYTUxNjYy\"}", + "stderr": "", + "classId": null + } + } + ] + }, + { + "task": "storage-regenerate-key", + "mode": "negative", + "passed": false, + "wait_s": 10, + "recorded": "2026-09-27T21:02:38Z", + "slots": { + "resource_group": "mcpe2-151aca-storage-regenerate-key-26", + "rg": "mcpe2-151aca-storage-regenerate-key-26", + "name": "sa7561599d", + "_before_changed": "ZDkxODExZDMtMTUwZS00MDhjLThkYmMtMmRlMGU4NTRlZTcz", + "_before_kept": "OTA5MzEzN2MtODQyMC00MWMyLWIxMzctMjM0ZjJlNmI3N2Nl" + }, + "text": "", + "queries": [ + { + "command": "rest --method post --url \"/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/mcpe2-151aca-storage-regenerate-key-26/providers/Microsoft.Storage/storageAccounts/sa7561599d/listKeys?api-version=2026-06-01\"", + "answer": { + "exitCode": 0, + "stdout": "{\"keys\":[{\"keyName\":\"key1\",\"permissions\":\"FULL\",\"value\":\"ZDkxODExZDMtMTUwZS00MDhjLThkYmMtMmRlMGU4NTRlZTcz\"},{\"keyName\":\"key2\",\"permissions\":\"FULL\",\"value\":\"OTA5MzEzN2MtODQyMC00MWMyLWIxMzctMjM0ZjJlNmI3N2Nl\"}]}", + "stderr": "", + "classId": null + } + } + ] + }, + { + "task": "storage-regenerate-key", + "mode": "oracle", + "passed": true, + "wait_s": 90, + "recorded": "2026-09-27T21:29:26Z", + "slots": { + "resource_group": "mcpe2-15123a-storage-regenerate-key-26", + "rg": "mcpe2-15123a-storage-regenerate-key-26", + "name": "sae136bf2e", + "_before_changed": "OWY1MGEzNWEtZDRkMy00NzYzLTkyZjYtZmM2NzdmOWNhYjhk", + "_before_kept": "MGJiZDczODMtNmUyZC00YzcyLTg2MmQtYTUzMjZiNWVkMTRl", + "_oracle": true + }, + "text": "ORACLE ✅ az storage account keys renew --account-name sae136bf2e --resource-group mcpe2-15123a-storage-regenerate-key-26 --key primary\n```json\n[]\n```", + "queries": [ + { + "command": "rest --method post --url \"/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/mcpe2-15123a-storage-regenerate-key-26/providers/Microsoft.Storage/storageAccounts/sae136bf2e/listKeys?api-version=2026-06-01\"", + "answer": { + "exitCode": 0, + "stdout": "{\"keys\":[{\"keyName\":\"key1\",\"permissions\":\"FULL\",\"value\":\"NWQ5MWRmNmUtNjdmZi00ZTZmLWE3ZTItZjY0ZmQ4ZWNiZWNk\"},{\"keyName\":\"key2\",\"permissions\":\"FULL\",\"value\":\"MGJiZDczODMtNmUyZC00YzcyLTg2MmQtYTUzMjZiNWVkMTRl\"}]}", + "stderr": "", + "classId": null + } + } + ] + }, + { + "task": "tb-eventgrid-to-eventhub", + "mode": "negative", + "passed": false, + "wait_s": 10, + "recorded": "2026-09-27T21:09:27Z", + "slots": { + "resource_group": "mcpe2-151aca-tb-eventgrid-to-eventhub-35", + "rg": "mcpe2-151aca-tb-eventgrid-to-eventhub-35", + "ehns": "tbeh2d6f71", + "hub": "clicks-f55cb0", + "topic": "tbtopic-d24003", + "esub": "tohub-cbc52b" + }, + "text": "", + "queries": [ + { + "command": "rest --method get --url \"/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/mcpe2-151aca-tb-eventgrid-to-eventhub-35/providers/Microsoft.EventHub/namespaces/tbeh2d6f71?api-version=2026-01-01\"", + "repeat": 3, + "answer": { + "exitCode": 1, + "stdout": "", + "stderr": "ERROR: NOT FOUND({\"error\": {\"code\": \"ResourceNotFound\", \"message\": \"The Resource 'Microsoft.EventHub/namespaces/tbeh2d6f71' under resource group 'mcpe2-151aca-tb-eventgrid-to-eventhub-35' was not found. For more details please go to https://aka.ms/ARMResourceNotFoundFix\", \"details\": [], \"additionalInfo\": []}})\r\n", + "classId": "not-found" + } + }, + { + "command": "rest --method get --url \"/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/mcpe2-151aca-tb-eventgrid-to-eventhub-35/providers/Microsoft.EventHub/namespaces/tbeh2d6f71/eventhubs/clicks-f55cb0?api-version=2026-01-01\"", + "repeat": 3, + "answer": { + "exitCode": 1, + "stdout": "", + "stderr": "ERROR: NOT FOUND({\"error\": {\"code\": \"ParentResourceNotFound\", \"message\": \"Failed to perform 'read' on resource(s) of type 'namespaces/eventhubs', because the parent resource '/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/mcpe2-151aca-tb-eventgrid-to-eventhub-35/providers/Microsoft.EventHub/namespaces/tbeh2d6f71' could not be found.\", \"details\": [], \"additionalInfo\": []}})\r\n", + "classId": "other" + } + }, + { + "command": "rest --method get --url \"/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/mcpe2-151aca-tb-eventgrid-to-eventhub-35/providers/Microsoft.EventGrid/topics/tbtopic-d24003?api-version=2025-02-15\"", + "repeat": 6, + "answer": { + "exitCode": 1, + "stdout": "", + "stderr": "ERROR: NOT FOUND({\"error\": {\"code\": \"ResourceNotFound\", \"message\": \"The Resource 'Microsoft.EventGrid/topics/tbtopic-d24003' under resource group 'mcpe2-151aca-tb-eventgrid-to-eventhub-35' was not found.\", \"details\": [], \"additionalInfo\": []}})\r\n", + "classId": "not-found" + } + }, + { + "command": "rest --method get --url \"/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/mcpe2-151aca-tb-eventgrid-to-eventhub-35/providers/Microsoft.EventGrid/topics/tbtopic-d24003/providers/Microsoft.EventGrid/eventSubscriptions/tohub-cbc52b?api-version=2025-02-15\"", + "repeat": 6, + "answer": { + "exitCode": 1, + "stdout": "", + "stderr": "ERROR: NOT FOUND({\"error\": {\"code\": \"ResourceNotFound\", \"message\": \"Event subscription tohub-cbc52b not found\", \"details\": [], \"additionalInfo\": []}})\r\n", + "classId": "not-found" + } + } + ] + }, + { + "task": "tb-eventgrid-to-eventhub", + "mode": "oracle", + "passed": true, + "wait_s": 90, + "recorded": "2026-09-27T21:34:37Z", + "slots": { + "resource_group": "mcpe2-15123a-tb-eventgrid-to-eventhub-35", + "rg": "mcpe2-15123a-tb-eventgrid-to-eventhub-35", + "ehns": "tbehd3c384", + "hub": "clicks-2433d0", + "topic": "tbtopic-a6ba69", + "esub": "tohub-110456", + "_oracle": true + }, + "text": "ORACLE ✅ az eventhubs namespace create --name tbehd3c384 --resource-group mcpe2-15123a-tb-eventgrid-to-eventhub-35 --location westeurope --sku Standard\n```json\n{\n \"createdAt\": \"2026-09-27T21:34:09.841Z\",\n \"disableLocalAuth\": false,\n \"geoDataReplication\": {\n \"locations\": [\n {\n \"locationName\": \"westeurope\",\n \"replicaState\": \"Ready\",\n \"roleType\": \"Primary\"\n }\n ],\n \"maxReplicationLagDurationInSeconds\": 0\n },\n \"id\": \"/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/mcpe2-15123a-tb-eventgrid-to-eventhub-35/providers/Microsoft.EventHub/namespaces/tbehd3c384\",\n \"isAutoInflateEnabled\": false,\n \"kafkaEnabled\": true,\n \"location\": \"westeurope\",\n \"maximumThroughputUnits\": 0,\n \"metricId\": \"00000000-0000-0000-0000-000000000000:tbehd3c384\",\n \"minimumTlsVersion\": \"1.2\",\n \"name\": \"tbehd3c384\",\n \"provisioningState\": \"Succeeded\",\n \"publicNetworkAccess\": \"Enabled\",\n \"resourceGroup\": \"mcpe2-15123a-tb-eventgrid-to-eventhub-35\",\n \"serviceBusEndpoint\": \"https://tbehd3c384.servicebus.azure.localhost.localstack.cloud:4512\",\n \"sku\": {\n \"capacity\": 1,\n \"name\": \"Standard\",\n \"tier\": \"Standard\"\n },\n \"status\": \"Active\",\n \"tags\": {},\n \"type\": \"Microsoft.EventHub/Namespaces\",\n \"updatedAt\": \"2026-09-27T21:34:09.841Z\",\n \"zoneRedundant\": true\n}\n```\nORACLE ✅ az eventhubs eventhub create --name clicks-2433d0 --namespace-name tbehd3c384 --resource-group mcpe2-15123a-tb-eventgrid-to-eventhub-35\n```json\n{\n \"createdAt\": \"2026-09-27T21:34:20.592Z\",\n \"id\": \"/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/mcpe2-15123a-tb-eventgrid-to-eventhub-35/providers/Microsoft.EventHub/namespaces/tbehd3c384/eventhubs/clicks-2433d0\",\n \"location\": \"westeurope\",\n \"messageRetentionInDays\": 7,\n \"messageTimestampDescription\": {\n \"timestampType\": \"LogAppend\"\n },\n \"name\": \"clicks-2433d0\",\n \"partitionCount\": 4,\n \"partitionIds\": [\n \"0\",\n \"1\",\n \"2\",\n \"3\"\n ],\n \"resourceGroup\": \"mcpe2-15123a-tb-eventgrid-to-eventhub-35\",\n \"retentionDescription\": {\n \"cleanupPolicy\": \"Delete\",\n \"retentionTimeInHours\": 168\n },\n \"status\": \"Active\",\n \"type\": \"Microsoft.EventHub/namespaces/eventhubs\",\n \"updatedAt\": \"2026-09-27T21:34:20.592Z\"\n}\n```\nORACLE ✅ az eventgrid topic create --name tbtopic-a6ba69 --resource-group mcpe2-15123a-tb-eventgrid-to-eventhub-35 --location westeurope\n```json\n{\n \"dataResidencyBoundary\": \"WithinGeopair\",\n \"disableLocalAuth\": false,\n \"endpoint\": \"https://tbtopic-a6ba69.azure.localhost.localstack.cloud:4566/api/events\",\n \"extendedLocation\": null,\n \"id\": \"/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/mcpe2-15123a-tb-eventgrid-to-eventhub-35/providers/Microsoft.EventGrid/topics/tbtopic-a6ba69\",\n \"identity\": null,\n \"inboundIpRules\": null,\n \"inputSchema\": \"EventGridSchema\",\n \"inputSchemaMapping\": null,\n \"kind\": null,\n \"location\": \"westeurope\",\n \"metricResourceId\": \"188f129a-a289-47d8-bcc6-82ea4b16966d\",\n \"name\": \"tbtopic-a6ba69\",\n \"privateEndpointConnections\": null,\n \"provisioningState\": \"Succeeded\",\n \"publicNetworkAccess\": \"Enabled\",\n \"resourceGroup\": \"mcpe2-15123a-tb-eventgrid-to-eventhub-35\",\n \"sku\": null,\n \"systemData\": null,\n \"tags\": null,\n \"type\": \"Microsoft.EventGrid/topics\"\n}\n```\nORACLE ✅ az eventgrid event-subscription create --name tohub-110456 --source-resource-id /subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/mcpe2-15123a-tb-eventgrid-to-eventhub-35/providers/Microsoft.EventGrid/topics/tbtopic-a6ba69 --endpoint-type eventhub --endpoint /subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/mcpe2-15123a-tb-eventgrid-to-eventhub-35/providers/Microsoft.EventHub/namespaces/tbehd3c384/eventhubs/clicks-2433d0\n```json\n{\n \"deadLetterDestination\": null,\n \"deadLetterWithResourceIdentity\": null,\n \"deliveryWithResourceIdentity\": null,\n \"destination\": {\n \"deliveryAttributeMappings\": null,\n \"endpointType\": \"EventHub\",\n \"resourceId\": \"/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/mcpe2-15123a-tb-eventgrid-to-eventhub-35/providers/Microsoft.EventHub/namespaces/tbehd3c384/eventhubs/clicks-2433d0\"\n },\n \"eventDeliverySchema\": \"EventGridSchema\",\n \"expirationTimeUtc\": null,\n \"filter\": {\n \"advancedFilters\": null,\n \"enableAdvancedFilteringOnArrays\": null,\n \"includedEventTypes\": null,\n \"isSubjectCaseSensitive\": null,\n \"subjectBeginsWith\": \"\",\n \"subjectEndsWith\": \"\"\n },\n \"id\": \"/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/mcpe2-15123a-tb-eventgrid-to-eventhub-35/providers/Microsoft.EventGrid/topics/tbtopic-a6ba69/providers/Microsoft.EventGrid/eventSubscriptions/tohub-110456\",\n \"labels\": null,\n \"name\": \"tohub-110456\",\n \"provisioningState\": \"Succeeded\",\n \"resourceGroup\": \"mcpe2-15123a-tb-eventgrid-to-eventhub-35\",\n \"retryPolicy\": {\n \"eventTimeToLiveInMinutes\": 1440,\n \"maxDeliveryAttempts\": 30\n },\n \"systemData\": null,\n \"topic\": \"/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/mcpe2-15123a-tb-eventgrid-to-eventhub-35/providers/microsoft.eventgrid/topics/tbtopic-a6ba69\",\n \"type\": \"Microsoft.EventGrid/eventSubscriptions\"\n}\n```", + "queries": [ + { + "command": "rest --method get --url \"/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/mcpe2-15123a-tb-eventgrid-to-eventhub-35/providers/Microsoft.EventHub/namespaces/tbehd3c384?api-version=2026-01-01\"", + "answer": { + "exitCode": 0, + "stdout": "{\"id\":\"/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/mcpe2-15123a-tb-eventgrid-to-eventhub-35/providers/Microsoft.EventHub/namespaces/tbehd3c384\",\"location\":\"westeurope\",\"name\":\"tbehd3c384\",\"properties\":{\"createdAt\":\"2026-09-27T21:34:09.841Z\",\"disableLocalAuth\":false,\"geoDataReplication\":{\"locations\":[{\"locationName\":\"westeurope\",\"replicaState\":\"Ready\",\"roleType\":\"Primary\"}],\"maxReplicationLagDurationInSeconds\":0},\"ipAddressType\":\"IPv4\",\"isAutoInflateEnabled\":false,\"kafkaEnabled\":true,\"maximumThroughputUnits\":0,\"metricId\":\"00000000-0000-0000-0000-000000000000:tbehd3c384\",\"minimumTlsVersion\":\"1.2\",\"platformCapabilities\":{\"confidentialCompute\":{\"mode\":\"Disabled\"}},\"provisioningState\":\"Succeeded\",\"publicNetworkAccess\":\"Enabled\",\"serviceBusEndpoint\":\"https://tbehd3c384.servicebus.azure.localhost.localstack.cloud:4512\",\"status\":\"Active\",\"updatedAt\":\"2026-09-27T21:34:09.841Z\",\"zoneRedundant\":true},\"sku\":{\"capacity\":1,\"name\":\"Standard\",\"tier\":\"Standard\"},\"tags\":{},\"type\":\"Microsoft.EventHub/Namespaces\"}", + "stderr": "", + "classId": null + } + }, + { + "command": "rest --method get --url \"/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/mcpe2-15123a-tb-eventgrid-to-eventhub-35/providers/Microsoft.EventHub/namespaces/tbehd3c384/eventhubs/clicks-2433d0?api-version=2026-01-01\"", + "answer": { + "exitCode": 0, + "stdout": "{\"id\":\"/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/mcpe2-15123a-tb-eventgrid-to-eventhub-35/providers/Microsoft.EventHub/namespaces/tbehd3c384/eventhubs/clicks-2433d0\",\"location\":\"westeurope\",\"name\":\"clicks-2433d0\",\"properties\":{\"createdAt\":\"2026-09-27T21:34:20.592Z\",\"messageRetentionInDays\":7,\"messageTimestampDescription\":{\"timestampType\":\"LogAppend\"},\"partitionCount\":4,\"partitionIds\":[\"0\",\"1\",\"2\",\"3\"],\"retentionDescription\":{\"cleanupPolicy\":\"Delete\",\"retentionTimeInHours\":168},\"status\":\"Active\",\"updatedAt\":\"2026-09-27T21:34:20.592Z\"},\"type\":\"Microsoft.EventHub/namespaces/eventhubs\"}", + "stderr": "", + "classId": null + } + }, + { + "command": "rest --method get --url \"/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/mcpe2-15123a-tb-eventgrid-to-eventhub-35/providers/Microsoft.EventGrid/topics/tbtopic-a6ba69?api-version=2025-02-15\"", + "answer": { + "exitCode": 0, + "stdout": "{\"id\":\"/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/mcpe2-15123a-tb-eventgrid-to-eventhub-35/providers/Microsoft.EventGrid/topics/tbtopic-a6ba69\",\"location\":\"westeurope\",\"name\":\"tbtopic-a6ba69\",\"properties\":{\"dataResidencyBoundary\":\"WithinGeopair\",\"disableLocalAuth\":false,\"endpoint\":\"https://tbtopic-a6ba69.azure.localhost.localstack.cloud:4566/api/events\",\"inputSchema\":\"EventGridSchema\",\"metricResourceId\":\"188f129a-a289-47d8-bcc6-82ea4b16966d\",\"minimumTlsVersionAllowed\":\"1.0\",\"provisioningState\":\"Succeeded\",\"publicNetworkAccess\":\"Enabled\"},\"type\":\"Microsoft.EventGrid/topics\"}", + "stderr": "", + "classId": null + } + }, + { + "command": "rest --method get --url \"/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/mcpe2-15123a-tb-eventgrid-to-eventhub-35/providers/Microsoft.EventGrid/topics/tbtopic-a6ba69/providers/Microsoft.EventGrid/eventSubscriptions/tohub-110456?api-version=2025-02-15\"", + "answer": { + "exitCode": 0, + "stdout": "{\"id\":\"/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/mcpe2-15123a-tb-eventgrid-to-eventhub-35/providers/Microsoft.EventGrid/topics/tbtopic-a6ba69/providers/Microsoft.EventGrid/eventSubscriptions/tohub-110456\",\"name\":\"tohub-110456\",\"properties\":{\"destination\":{\"endpointType\":\"EventHub\",\"properties\":{\"resourceId\":\"/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/mcpe2-15123a-tb-eventgrid-to-eventhub-35/providers/Microsoft.EventHub/namespaces/tbehd3c384/eventhubs/clicks-2433d0\"}},\"eventDeliverySchema\":\"EventGridSchema\",\"filter\":{\"subjectBeginsWith\":\"\",\"subjectEndsWith\":\"\"},\"provisioningState\":\"Succeeded\",\"retryPolicy\":{\"eventTimeToLiveInMinutes\":1440,\"maxDeliveryAttempts\":30},\"topic\":\"/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/mcpe2-15123a-tb-eventgrid-to-eventhub-35/providers/microsoft.eventgrid/topics/tbtopic-a6ba69\"},\"type\":\"Microsoft.EventGrid/eventSubscriptions\"}", + "stderr": "", + "classId": null + } + } + ] + }, + { + "task": "tb-keyvault-secret-sign", + "mode": "negative", + "passed": false, + "wait_s": 10, + "recorded": "2026-09-27T21:04:56Z", + "slots": { + "resource_group": "mcpe2-151aca-tb-keyvault-secret-sign-30", + "rg": "mcpe2-151aca-tb-keyvault-secret-sign-30", + "kv": "tbkv45c087", + "secret": "db-password-f66e89", + "key": "signing-0c7da7", + "secret_value": "S3cret-53ddaf1bf0", + "digest": "MRkuvoDgkERiTBKGv9kdjGN60Liy88oppZeV45DBu0U", + "digest_b64": "MRkuvoDgkERiTBKGv9kdjGN60Liy88oppZeV45DBu0U=" + }, + "text": "", + "queries": [ + { + "command": "rest --method get --url \"/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/mcpe2-151aca-tb-keyvault-secret-sign-30/providers/Microsoft.KeyVault/vaults/tbkv45c087?api-version=2026-02-01\"", + "repeat": 4, + "answer": { + "exitCode": 1, + "stdout": "", + "stderr": "ERROR: NOT FOUND({\"error\": {\"code\": \"ResourceNotFound\", \"message\": \"The Resource 'Microsoft.KeyVault/vaults/tbkv45c087' under resource group 'mcpe2-151aca-tb-keyvault-secret-sign-30' was not found. For more details please go to https://aka.ms/ARMResourceNotFoundFix\", \"details\": [], \"additionalInfo\": []}})\r\n", + "classId": "not-found" + } + }, + { + "command": "keyvault secret show --vault-name tbkv45c087 --name db-password-f66e89", + "answer": { + "exitCode": 3, + "stdout": "", + "stderr": "ERROR: (404) None\r\nCode: 404\r\nMessage: None\r\n", + "classId": "other" + } + }, + { + "command": "keyvault key list-versions --vault-name tbkv45c087 --name signing-0c7da7", + "answer": { + "exitCode": 1, + "stdout": "", + "stderr": "ERROR: (404) None\r\nCode: 404\r\nMessage: None\r\n", + "classId": "other" + } + }, + { + "command": "keyvault key show --vault-name tbkv45c087 --name signing-0c7da7", + "answer": { + "exitCode": 3, + "stdout": "", + "stderr": "ERROR: (404) None\r\nCode: 404\r\nMessage: None\r\n", + "classId": "other" + } + } + ] + }, + { + "task": "tb-keyvault-secret-sign", + "mode": "oracle", + "passed": true, + "wait_s": 90, + "recorded": "2026-09-27T21:31:04Z", + "slots": { + "resource_group": "mcpe2-15123a-tb-keyvault-secret-sign-30", + "rg": "mcpe2-15123a-tb-keyvault-secret-sign-30", + "kv": "tbkv3f4932", + "secret": "db-password-8deb64", + "key": "signing-c5580a", + "secret_value": "S3cret-f30cbbdb45", + "digest": "oadfkeQZog7ssd5gYEuvHBSegeHKhtn_8GCKLMu5h8w", + "digest_b64": "oadfkeQZog7ssd5gYEuvHBSegeHKhtn/8GCKLMu5h8w=", + "_oracle": true + }, + "text": "ORACLE ✅ az keyvault create --name tbkv3f4932 --resource-group mcpe2-15123a-tb-keyvault-secret-sign-30 --location westeurope\n```json\n{\n \"id\": \"/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/mcpe2-15123a-tb-keyvault-secret-sign-30/providers/Microsoft.KeyVault/vaults/tbkv3f4932\",\n \"location\": \"westeurope\",\n \"name\": \"tbkv3f4932\",\n \"properties\": {\n \"accessPolicies\": [],\n \"createMode\": null,\n \"enablePurgeProtection\": null,\n \"enableRbacAuthorization\": true,\n \"enableSoftDelete\": true,\n \"enabledForDeployment\": false,\n \"enabledForDiskEncryption\": false,\n \"enabledForTemplateDeployment\": false,\n \"hsmPoolResourceId\": null,\n \"networkAcls\": null,\n \"privateEndpointConnections\": null,\n \"provisioningState\": \"Succeeded\",\n \"publicNetworkAccess\": \"Enabled\",\n \"sku\": {\n \"family\": \"A\",\n \"name\": \"standard\"\n },\n \"softDeleteRetentionInDays\": 90,\n \"tenantId\": \"00000000-0000-0000-0000-000000000000\",\n \"vaultUri\": \"https://tbkv3f4932.vault.azure.localhost.localstack.cloud:4566\"\n },\n \"resourceGroup\": \"mcpe2-15123a-tb-keyvault-secret-sign-30\",\n \"systemData\": {\n \"createdAt\": \"2026-09-27T21:30:33.179054+00:00\",\n \"createdBy\": \"any-app\",\n \"createdByType\": \"Application\",\n \"lastModifiedAt\": \"2026-09-27T21:30:33.179054+00:00\",\n \"lastModifiedBy\": \"any-app\",\n \"lastModifiedByType\": \"Application\"\n },\n \"tags\": {},\n \"type\": \"Microsoft.KeyVault/vaults\"\n}\n```\nORACLE ✅ az keyvault secret set --vault-name tbkv3f4932 --name db-password-8deb64 --value S3cret-f30cbbdb45\n```json\n{\n \"attributes\": {\n \"created\": null,\n \"enabled\": true,\n \"expires\": null,\n \"notBefore\": null,\n \"recoverableDays\": 90,\n \"recoveryLevel\": \"Recoverable+Purgeable\",\n \"updated\": null\n },\n \"contentType\": null,\n \"id\": \"https://tbkv3f4932.vault.azure.localhost.localstack.cloud:4566/secrets/db-password-8deb64/6980b67238a548ee9149d1b579015226\",\n \"kid\": null,\n \"managed\": null,\n \"name\": \"db-password-8deb64\",\n \"tags\": {\n \"file-encoding\": \"utf-8\"\n },\n \"value\": \"S3cret-f30cbbdb45\"\n}\n```\nORACLE ✅ az keyvault secret show --vault-name tbkv3f4932 --name db-password-8deb64\n```json\n{\n \"attributes\": {\n \"created\": null,\n \"enabled\": true,\n \"expires\": null,\n \"notBefore\": null,\n \"recoverableDays\": 90,\n \"recoveryLevel\": \"Recoverable+Purgeable\",\n \"updated\": null\n },\n \"contentType\": null,\n \"id\": \"https://tbkv3f4932.vault.azure.localhost.localstack.cloud:4566/secrets/db-password-8deb64/6980b67238a548ee9149d1b579015226\",\n \"kid\": null,\n \"managed\": null,\n \"name\": \"db-password-8deb64\",\n \"tags\": {\n \"file-encoding\": \"utf-8\"\n },\n \"value\": \"S3cret-f30cbbdb45\"\n}\n```\nORACLE ✅ az keyvault key create --vault-name tbkv3f4932 --name signing-c5580a --kty RSA\n```json\n{\n \"attributes\": {\n \"created\": \"2026-09-27T21:30:37+00:00\",\n \"enabled\": true,\n \"expires\": null,\n \"exportable\": false,\n \"hsmPlatform\": \"0\",\n \"notBefore\": null,\n \"recoverableDays\": 90,\n \"recoveryLevel\": \"Recoverable+Purgeable\",\n \"updated\": \"2026-09-27T21:30:37+00:00\"\n },\n \"key\": {\n \"crv\": null,\n \"d\": null,\n \"dp\": null,\n \"dq\": null,\n \"e\": \"AQAB\",\n \"k\": null,\n \"keyOps\": [\n \"encrypt\",\n \"decrypt\",\n \"sign\",\n \"verify\",\n \"wrapKey\",\n \"unwrapKey\"\n ],\n \"kid\": \"https://tbkv3f4932.vault.azure.localhost.localstack.cloud:4566/keys/signing-c5580a/9cc6a9fa6dc94023aebb76a38c089b61\",\n \"kty\": \"RSA\",\n \"n\": \"p1JvidcFfu9LBqvuKskd+zWbNSR57fS3I0S9+GraJQLdCEqMq/DzmUX+hp5zlm4fhn3PeRlOgPhTj7gXwb8c8bpYhVMjlMT1vWSxEuoQCg3T+xQrFEAQI13zsoI5P5o/wQSjn68ck6zaXUcgGGj0QjA+fSLIcug3SP8SK8YBM9ukkgZMug/SCCn8k6tpm+vzp2ueUosH1hcGWusZBiSYXstP+ZClaGs6SPJzfSB+P8219+wROFCoUCfn8ELH5n2ppT4kshkE8OfHQQ1bhZotuWTHgPSRRLPefm0PjxTu0R/sgyymC2yP/X7WxvA6nQWr7imv5Wk0OLdzhcvxg+usoQ==\",\n \"p\": null,\n \"q\": null,\n \"qi\": null,\n \"t\": null,\n \"x\": null,\n \"y\": null\n },\n \"managed\": null,\n \"releasePolicy\": null,\n \"tags\": null\n}\n```\nORACLE ✅ az keyvault key sign --vault-name tbkv3f4932 --name signing-c5580a --algorithm RS256 --digest oadfkeQZog7ssd5gYEuvHBSegeHKhtn/8GCKLMu5h8w=\n```json\n{\n \"algorithm\": \"RS256\",\n \"keyId\": \"https://tbkv3f4932.vault.azure.localhost.localstack.cloud:4566/keys/signing-c5580a/9cc6a9fa6dc94023aebb76a38c089b61\",\n \"signature\": \"funnx6Kzwr9maA64g1qX+Dn6uNQufSg7OCm7bZyS7Lz5mFfTzsfznZdv7w2tQONE4fXSlra73D9J88JF9BeIwdw0RJwN0A3eJSYKQYLbzfr5SZgHWxSP9M2ct+E1ry5dVzW0y1Eh2GSEi9PEumhRgolzsNfzW9NQ1vo/x7hN3SEfk3flrFmt36NYF6xjq3kP7anMVrXBYYSU1+tUOTnE5cv8tRe+ONuXVjsmIOk+SdBWxSIKAVhcKltFHodN1il5U/CGYY+BNu1OTIRAc5jnvZQ9WabB82z1Xc4+QTuvwtcXNu38c90le7YsvZ7ciFCW7Ki0prP+AMiDykXNFc23KA==\"\n}\n```", + "queries": [ + { + "command": "rest --method get --url \"/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/mcpe2-15123a-tb-keyvault-secret-sign-30/providers/Microsoft.KeyVault/vaults/tbkv3f4932?api-version=2026-02-01\"", + "answer": { + "exitCode": 0, + "stdout": "{\"id\":\"/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/mcpe2-15123a-tb-keyvault-secret-sign-30/providers/Microsoft.KeyVault/vaults/tbkv3f4932\",\"location\":\"westeurope\",\"name\":\"tbkv3f4932\",\"properties\":{\"accessPolicies\":[],\"enableRbacAuthorization\":true,\"enableSoftDelete\":true,\"enabledForDeployment\":false,\"enabledForDiskEncryption\":false,\"enabledForTemplateDeployment\":false,\"provisioningState\":\"Succeeded\",\"publicNetworkAccess\":\"Enabled\",\"sku\":{\"family\":\"A\",\"name\":\"standard\"},\"softDeleteRetentionInDays\":90,\"tenantId\":\"00000000-0000-0000-0000-000000000000\",\"vaultUri\":\"https://tbkv3f4932.vault.azure.localhost.localstack.cloud:4566\"},\"systemData\":{\"createdAt\":\"2026-09-27T21:30:33.1790544633+00:00\",\"createdBy\":\"any-app\",\"createdByType\":\"Application\",\"lastModifiedAt\":\"2026-09-27T21:30:33.1790544633+00:00\",\"lastModifiedBy\":\"any-app\",\"lastModifiedByType\":\"Application\"},\"tags\":{},\"type\":\"Microsoft.KeyVault/vaults\"}", + "stderr": "", + "classId": null + } + }, + { + "command": "keyvault secret show --vault-name tbkv3f4932 --name db-password-8deb64", + "answer": { + "exitCode": 0, + "stdout": "{\"attributes\":{\"created\":null,\"enabled\":true,\"expires\":null,\"notBefore\":null,\"recoverableDays\":90,\"recoveryLevel\":\"Recoverable+Purgeable\",\"updated\":null},\"contentType\":null,\"id\":\"https://tbkv3f4932.vault.azure.localhost.localstack.cloud:4566/secrets/db-password-8deb64/6980b67238a548ee9149d1b579015226\",\"kid\":null,\"managed\":null,\"name\":\"db-password-8deb64\",\"tags\":{\"file-encoding\":\"utf-8\"},\"value\":\"S3cret-f30cbbdb45\"}", + "stderr": "", + "classId": null + } + }, + { + "command": "keyvault key list-versions --vault-name tbkv3f4932 --name signing-c5580a", + "answer": { + "exitCode": 0, + "stdout": "[{\"attributes\":{\"attestation\":null,\"created\":\"2026-09-27T21:30:37+00:00\",\"enabled\":true,\"expires\":null,\"exportable\":false,\"hsmPlatform\":\"0\",\"notBefore\":null,\"recoverableDays\":90,\"recoveryLevel\":\"Recoverable+Purgeable\",\"updated\":\"2026-09-27T21:30:37+00:00\"},\"kid\":\"https://tbkv3f4932.vault.azure.localhost.localstack.cloud:4566/keys/signing-c5580a/9cc6a9fa6dc94023aebb76a38c089b61\",\"managed\":null,\"name\":\"signing-c5580a\",\"releasePolicy\":null,\"tags\":null}]", + "stderr": "", + "classId": null + } + }, + { + "command": "keyvault key verify --vault-name tbkv3f4932 --name signing-c5580a --version 9cc6a9fa6dc94023aebb76a38c089b61 --algorithm RS256 --digest oadfkeQZog7ssd5gYEuvHBSegeHKhtn/8GCKLMu5h8w= --signature /subscriptions/00000000+0000+0000+0000+000000000000/resourceGroups/mcpe2+15123a+tb+keyvault+secret+sign+30/providers/Microsoft==", + "answer": { + "exitCode": 0, + "stdout": "{\"algorithm\":\"RS256\",\"isValid\":false,\"keyId\":\"https://tbkv3f4932.vault.azure.localhost.localstack.cloud:4566/keys/signing-c5580a/9cc6a9fa6dc94023aebb76a38c089b61\"}", + "stderr": "", + "classId": null + } + }, + { + "command": "keyvault key verify --vault-name tbkv3f4932 --name signing-c5580a --version 9cc6a9fa6dc94023aebb76a38c089b61 --algorithm RS256 --digest oadfkeQZog7ssd5gYEuvHBSegeHKhtn/8GCKLMu5h8w= --signature 4566/secrets/db+password+8deb64/6980b67238a548ee9149d1b579015226", + "answer": { + "exitCode": 0, + "stdout": "{\"algorithm\":\"RS256\",\"isValid\":false,\"keyId\":\"https://tbkv3f4932.vault.azure.localhost.localstack.cloud:4566/keys/signing-c5580a/9cc6a9fa6dc94023aebb76a38c089b61\"}", + "stderr": "", + "classId": null + } + }, + { + "command": "keyvault key verify --vault-name tbkv3f4932 --name signing-c5580a --version 9cc6a9fa6dc94023aebb76a38c089b61 --algorithm RS256 --digest oadfkeQZog7ssd5gYEuvHBSegeHKhtn/8GCKLMu5h8w= --signature p1JvidcFfu9LBqvuKskd+zWbNSR57fS3I0S9+GraJQLdCEqMq/DzmUX+hp5zlm4fhn3PeRlOgPhTj7gXwb8c8bpYhVMjlMT1vWSxEuoQCg3T+xQrFEAQI13zsoI5P5o/wQSjn68ck6zaXUcgGGj0QjA+fSLIcug3SP8SK8YBM9ukkgZMug/SCCn8k6tpm+vzp2ueUosH1hcGWusZBiSYXstP+ZClaGs6SPJzfSB+P8219+wROFCoUCfn8ELH5n2ppT4kshkE8OfHQQ1bhZotuWTHgPSRRLPefm0PjxTu0R/sgyymC2yP/X7WxvA6nQWr7imv5Wk0OLdzhcvxg+usoQ==", + "answer": { + "exitCode": 0, + "stdout": "{\"algorithm\":\"RS256\",\"isValid\":false,\"keyId\":\"https://tbkv3f4932.vault.azure.localhost.localstack.cloud:4566/keys/signing-c5580a/9cc6a9fa6dc94023aebb76a38c089b61\"}", + "stderr": "", + "classId": null + } + }, + { + "command": "keyvault key verify --vault-name tbkv3f4932 --name signing-c5580a --version 9cc6a9fa6dc94023aebb76a38c089b61 --algorithm RS256 --digest oadfkeQZog7ssd5gYEuvHBSegeHKhtn/8GCKLMu5h8w= --signature funnx6Kzwr9maA64g1qX+Dn6uNQufSg7OCm7bZyS7Lz5mFfTzsfznZdv7w2tQONE4fXSlra73D9J88JF9BeIwdw0RJwN0A3eJSYKQYLbzfr5SZgHWxSP9M2ct+E1ry5dVzW0y1Eh2GSEi9PEumhRgolzsNfzW9NQ1vo/x7hN3SEfk3flrFmt36NYF6xjq3kP7anMVrXBYYSU1+tUOTnE5cv8tRe+ONuXVjsmIOk+SdBWxSIKAVhcKltFHodN1il5U/CGYY+BNu1OTIRAc5jnvZQ9WabB82z1Xc4+QTuvwtcXNu38c90le7YsvZ7ciFCW7Ki0prP+AMiDykXNFc23KA==", + "answer": { + "exitCode": 0, + "stdout": "{\"algorithm\":\"RS256\",\"isValid\":true,\"keyId\":\"https://tbkv3f4932.vault.azure.localhost.localstack.cloud:4566/keys/signing-c5580a/9cc6a9fa6dc94023aebb76a38c089b61\"}", + "stderr": "", + "classId": null + } + } + ] + }, + { + "task": "tb-mysql-firewall", + "mode": "negative", + "passed": false, + "wait_s": 10, + "recorded": "2026-09-27T21:08:23Z", + "slots": { + "resource_group": "mcpe2-151aca-tb-mysql-firewall-34", + "rg": "mcpe2-151aca-tb-mysql-firewall-34", + "my": "tbmy-4aaa31", + "db": "shop617703", + "rule": "office-f26ccc", + "ip_start": "203.0.113.80", + "ip_end": "203.0.113.85" + }, + "text": "", + "queries": [ + { + "command": "rest --method get --url \"/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/mcpe2-151aca-tb-mysql-firewall-34/providers/Microsoft.DBforMySQL/flexibleServers/tbmy-4aaa31?api-version=2023-12-30\"", + "repeat": 6, + "answer": { + "exitCode": 1, + "stdout": "", + "stderr": "ERROR: NOT FOUND({\"error\": {\"code\": \"ResourceNotFound\", \"message\": \"The Resource 'Microsoft.DBforMySQL/flexibleServers/tbmy-4aaa31' under resource group 'mcpe2-151aca-tb-mysql-firewall-34' was not found. For more details please go to https://aka.ms/ARMResourceNotFoundFix\", \"details\": [], \"additionalInfo\": []}})\r\n", + "classId": "not-found" + } + }, + { + "command": "rest --method get --url \"/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/mcpe2-151aca-tb-mysql-firewall-34/providers/Microsoft.DBforMySQL/flexibleServers/tbmy-4aaa31/databases/shop617703?api-version=2023-12-30\"", + "repeat": 6, + "answer": { + "exitCode": 1, + "stdout": "", + "stderr": "ERROR: NOT FOUND({\"error\": {\"code\": \"ResourceNotFound\", \"message\": \"The Resource 'Microsoft.DBforMySQL/flexibleServers/tbmy-4aaa31' under resource group 'mcpe2-151aca-tb-mysql-firewall-34' was not found. For more details please go to https://aka.ms/ARMResourceNotFoundFix\", \"details\": [], \"additionalInfo\": []}})\r\n", + "classId": "not-found" + } + }, + { + "command": "rest --method get --url \"/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/mcpe2-151aca-tb-mysql-firewall-34/providers/Microsoft.DBforMySQL/flexibleServers/tbmy-4aaa31/firewallRules/office-f26ccc?api-version=2023-12-30\"", + "repeat": 7, + "answer": { + "exitCode": 1, + "stdout": "", + "stderr": "ERROR: NOT FOUND({\"error\": {\"code\": \"ResourceNotFound\", \"message\": \"The Resource 'Microsoft.DBforMySQL/flexibleServers/tbmy-4aaa31' under resource group 'mcpe2-151aca-tb-mysql-firewall-34' was not found. For more details please go to https://aka.ms/ARMResourceNotFoundFix\", \"details\": [], \"additionalInfo\": []}})\r\n", + "classId": "not-found" + } + }, + { + "command": "rest --method get --url \"/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/mcpe2-151aca-tb-mysql-firewall-34/providers/Microsoft.DBforMySQL/flexibleServers/tbmy-4aaa31?api-version=2023-12-30\"", + "answer": { + "exitCode": 1, + "stdout": "", + "stderr": "ERROR: NOT FOUND({\"error\": {\"code\": \"ResourceNotFound\", \"message\": \"The Resource 'Microsoft.DBforMySQL/flexibleServers/tbmy-4aaa31' under resource group 'mcpe2-151aca-tb-mysql-firewall-34' was not found. For more details please go to https://aka.ms/ARMResourceNotFoundFix\", \"details\": [], \"additionalInfo\": []}})\r\n", + "classId": "not-found" + } + } + ] + }, + { + "task": "tb-mysql-firewall", + "mode": "oracle", + "passed": true, + "wait_s": 90, + "recorded": "2026-09-27T21:34:00Z", + "slots": { + "resource_group": "mcpe2-15123a-tb-mysql-firewall-34", + "rg": "mcpe2-15123a-tb-mysql-firewall-34", + "my": "tbmy-49108f", + "db": "shop85a9dd", + "rule": "office-f786d3", + "ip_start": "203.0.113.32", + "ip_end": "203.0.113.37", + "_oracle": true + }, + "text": "ORACLE ✅ az mysql flexible-server create --name tbmy-49108f --resource-group mcpe2-15123a-tb-mysql-firewall-34 --location westeurope --tier Burstable --sku-name Standard_B1ms --admin-user myadmin --admin-password LocalStack123! --public-access None --yes\n```json\n{\n \"connectionString\": \"mysql flexibleserverdb --host tbmy-49108f.mysql.database.localhost.localstack.cloud:4513 --user myadmin --password=LocalStack123!\",\n \"databaseName\": \"flexibleserverdb\",\n \"host\": \"tbmy-49108f.mysql.database.localhost.localstack.cloud:4513\",\n \"id\": \"/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/mcpe2-15123a-tb-mysql-firewall-34/providers/Microsoft.DBforMySQL/flexibleServers/tbmy-49108f\",\n \"location\": \"westeurope\",\n \"password\": \"LocalStack123!\",\n \"resourceGroup\": \"mcpe2-15123a-tb-mysql-firewall-34\",\n \"skuname\": \"B_Gen5_1\",\n \"username\": \"myadmin\",\n \"version\": \"8.0.21\"\n}\n```\nORACLE ✅ az mysql flexible-server db create --server-name tbmy-49108f --resource-group mcpe2-15123a-tb-mysql-firewall-34 --database-name shop85a9dd\n```json\n{\n \"charset\": \"utf8\",\n \"collation\": \"utf8_general_ci\",\n \"id\": \"/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/mcpe2-15123a-tb-mysql-firewall-34/providers/Microsoft.DBforMySQL/flexibleServers/tbmy-49108f/databases/shop85a9dd\",\n \"name\": \"shop85a9dd\",\n \"resourceGroup\": \"mcpe2-15123a-tb-mysql-firewall-34\",\n \"systemData\": null,\n \"type\": \"Microsoft.DBforMySQL/flexibleServers/databases\"\n}\n```\nORACLE ✅ az mysql flexible-server firewall-rule create --name tbmy-49108f --resource-group mcpe2-15123a-tb-mysql-firewall-34 --rule-name office-f786d3 --start-ip-address 203.0.113.32 --end-ip-address 203.0.113.37\n```json\n{\n \"endIpAddress\": \"203.0.113.37\",\n \"id\": \"/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/mcpe2-15123a-tb-mysql-firewall-34/providers/Microsoft.DBforMySQL/flexibleServers/tbmy-49108f/firewallRules/office-f786d3\",\n \"name\": \"office-f786d3\",\n \"resourceGroup\": \"mcpe2-15123a-tb-mysql-firewall-34\",\n \"startIpAddress\": \"203.0.113.32\",\n \"systemData\": null,\n \"type\": \"Microsoft.DBforMySQL/flexibleServers/firewallRules\"\n}\n```\nORACLE ✅ az mysql flexible-server show --name tbmy-49108f --resource-group mcpe2-15123a-tb-mysql-firewall-34\n```json\n{\n \"administratorLogin\": \"myadmin\",\n \"administratorLoginPassword\": null,\n \"availabilityZone\": \"\",\n \"backup\": {\n \"backupIntervalHours\": 24,\n \"backupRetentionDays\": 7,\n \"earliestRestoreDate\": null,\n \"geoRedundantBackup\": \"Disabled\"\n },\n \"createMode\": null,\n \"dataEncryption\": null,\n \"databasePort\": 3306,\n \"fullVersion\": null,\n \"fullyQualifiedDomainName\": \"tbmy-49108f.mysql.database.localhost.localstack.cloud:4513\",\n \"highAvailability\": {\n \"mode\": \"Disabled\",\n \"replicationMode\": null,\n \"standbyAvailabilityZone\": \"\",\n \"state\": \"NotEnabled\"\n },\n \"id\": \"/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/mcpe2-15123a-tb-mysql-firewall-34/providers/Microsoft.DBforMySQL/flexibleServers/tbmy-49108f\",\n \"identity\": null,\n \"importSourceProperties\": null,\n \"location\": \"westeurope\",\n \"lowerCaseTableNames\": null,\n \"maintenancePolicy\": {\n \"patchStrategy\": \"VirtualCanary\"\n },\n \"maintenanceWindow\": {\n \"batchOfMaintenance\": \"Default\",\n \"customWindow\": \"Disabled\",\n \"dayOfWeek\": 0,\n \"startHour\": 0,\n \"startMinute\": 0\n },\n \"name\": \"tbmy-49108f\",\n \"network\": {\n \"delegatedSubnetResourceId\": null,\n \"privateDnsZoneResourceId\": null,\n \"publicNetworkAccess\": \"Enabled\"\n },\n \"privateEndpointConnections\": [],\n \"replicaCapacity\": 10,\n \"replicationRole\": \"None\",\n \"resourceGroup\": \"mcpe2-15123a-tb-mysql-firewall-34\",\n \"restorePointInTime\": null,\n \"sku\": {\n \"name\": \"Standard_B1ms\",\n \"tier\": \"Burstable\"\n },\n \"sourceServerResourceId\": null,\n \"state\": \"Ready\",\n \"storage\": {\n \"autoGrow\": \"Enabled\",\n \"autoIoScaling\": \"Enabled\",\n \"iops\": 396,\n \"logOnDisk\": \"Disabled\",\n \"storageRedundancy\": \"LocalRedundancy\",\n \"storageSizeGb\": 32,\n \"storageSku\": \"Premium_LRS\"\n },\n \"systemData\": {\n \"createdAt\": \"2026-09-27T21:32:10.750614+00:00\",\n \"createdBy\": null,\n \"createdByType\": null,\n \"lastModifiedAt\": null,\n \"lastModifiedBy\": null,\n \"lastModifiedByType\": null\n },\n \"tags\": null,\n \"type\": \"Microsoft.DBforMySQL/flexibleServers\",\n \"version\": \"8.0.21\"\n}\n```", + "queries": [ + { + "command": "rest --method get --url \"/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/mcpe2-15123a-tb-mysql-firewall-34/providers/Microsoft.DBforMySQL/flexibleServers/tbmy-49108f?api-version=2023-12-30\"", + "answer": { + "exitCode": 0, + "stdout": "{\"id\":\"/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/mcpe2-15123a-tb-mysql-firewall-34/providers/Microsoft.DBforMySQL/flexibleServers/tbmy-49108f\",\"location\":\"westeurope\",\"name\":\"tbmy-49108f\",\"properties\":{\"administratorLogin\":\"myadmin\",\"availabilityZone\":\"\",\"backup\":{\"backupIntervalHours\":24,\"backupRetentionDays\":7,\"geoRedundantBackup\":\"Disabled\"},\"databasePort\":3306,\"fullyQualifiedDomainName\":\"tbmy-49108f.mysql.database.localhost.localstack.cloud:4513\",\"highAvailability\":{\"mode\":\"Disabled\",\"standbyAvailabilityZone\":\"\",\"state\":\"NotEnabled\"},\"maintenancePolicy\":{\"patchStrategy\":\"VirtualCanary\"},\"maintenanceWindow\":{\"batchOfMaintenance\":\"Default\",\"customWindow\":\"Disabled\",\"dayOfWeek\":0,\"startHour\":0,\"startMinute\":0},\"network\":{\"publicNetworkAccess\":\"Enabled\"},\"privateEndpointConnections\":[],\"replicaCapacity\":10,\"replicationRole\":\"None\",\"state\":\"Ready\",\"storage\":{\"autoGrow\":\"Enabled\",\"autoIoScaling\":\"Enabled\",\"iops\":396,\"logOnDisk\":\"Disabled\",\"storageRedundancy\":\"LocalRedundancy\",\"storageSizeGB\":32,\"storageSku\":\"Premium_LRS\"},\"version\":\"8.0.21\"},\"sku\":{\"name\":\"Standard_B1ms\",\"tier\":\"Burstable\"},\"systemData\":{\"createdAt\":\"2026-09-27T21:32:10.750614+00:00\"},\"type\":\"Microsoft.DBforMySQL/flexibleServers\"}", + "stderr": "", + "classId": null + } + }, + { + "command": "rest --method get --url \"/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/mcpe2-15123a-tb-mysql-firewall-34/providers/Microsoft.DBforMySQL/flexibleServers/tbmy-49108f/databases/shop85a9dd?api-version=2023-12-30\"", + "answer": { + "exitCode": 0, + "stdout": "{\"id\":\"/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/mcpe2-15123a-tb-mysql-firewall-34/providers/Microsoft.DBforMySQL/flexibleServers/tbmy-49108f/databases/shop85a9dd\",\"name\":\"shop85a9dd\",\"properties\":{\"charset\":\"utf8\",\"collation\":\"utf8_general_ci\"},\"type\":\"Microsoft.DBforMySQL/flexibleServers/databases\"}", + "stderr": "", + "classId": null + } + }, + { + "command": "rest --method get --url \"/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/mcpe2-15123a-tb-mysql-firewall-34/providers/Microsoft.DBforMySQL/flexibleServers/tbmy-49108f/firewallRules/office-f786d3?api-version=2023-12-30\"", + "answer": { + "exitCode": 0, + "stdout": "{\"id\":\"/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/mcpe2-15123a-tb-mysql-firewall-34/providers/Microsoft.DBforMySQL/flexibleServers/tbmy-49108f/firewallRules/office-f786d3\",\"name\":\"office-f786d3\",\"properties\":{\"endIpAddress\":\"203.0.113.37\",\"startIpAddress\":\"203.0.113.32\"},\"type\":\"Microsoft.DBforMySQL/flexibleServers/firewallRules\"}", + "stderr": "", + "classId": null + } + }, + { + "command": "rest --method get --url \"/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/mcpe2-15123a-tb-mysql-firewall-34/providers/Microsoft.DBforMySQL/flexibleServers/tbmy-49108f?api-version=2023-12-30\"", + "answer": { + "exitCode": 0, + "stdout": "{\"id\":\"/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/mcpe2-15123a-tb-mysql-firewall-34/providers/Microsoft.DBforMySQL/flexibleServers/tbmy-49108f\",\"location\":\"westeurope\",\"name\":\"tbmy-49108f\",\"properties\":{\"administratorLogin\":\"myadmin\",\"availabilityZone\":\"\",\"backup\":{\"backupIntervalHours\":24,\"backupRetentionDays\":7,\"geoRedundantBackup\":\"Disabled\"},\"databasePort\":3306,\"fullyQualifiedDomainName\":\"tbmy-49108f.mysql.database.localhost.localstack.cloud:4513\",\"highAvailability\":{\"mode\":\"Disabled\",\"standbyAvailabilityZone\":\"\",\"state\":\"NotEnabled\"},\"maintenancePolicy\":{\"patchStrategy\":\"VirtualCanary\"},\"maintenanceWindow\":{\"batchOfMaintenance\":\"Default\",\"customWindow\":\"Disabled\",\"dayOfWeek\":0,\"startHour\":0,\"startMinute\":0},\"network\":{\"publicNetworkAccess\":\"Enabled\"},\"privateEndpointConnections\":[],\"replicaCapacity\":10,\"replicationRole\":\"None\",\"state\":\"Ready\",\"storage\":{\"autoGrow\":\"Enabled\",\"autoIoScaling\":\"Enabled\",\"iops\":396,\"logOnDisk\":\"Disabled\",\"storageRedundancy\":\"LocalRedundancy\",\"storageSizeGB\":32,\"storageSku\":\"Premium_LRS\"},\"version\":\"8.0.21\"},\"sku\":{\"name\":\"Standard_B1ms\",\"tier\":\"Burstable\"},\"systemData\":{\"createdAt\":\"2026-09-27T21:32:10.750614+00:00\"},\"type\":\"Microsoft.DBforMySQL/flexibleServers\"}", + "stderr": "", + "classId": null + } + } + ] + }, + { + "task": "tb-resource-graph-tags", + "mode": "negative", + "passed": false, + "wait_s": 10, + "recorded": "2026-09-27T21:06:21Z", + "slots": { + "resource_group": "mcpe2-151aca-tb-resource-graph-tags-32", + "rg": "mcpe2-151aca-tb-resource-graph-tags-32", + "kv1": "tbkva50c6f4", + "kv2": "tbkvb8ee3be", + "cc": "cc-c02154" + }, + "text": "", + "queries": [ + { + "command": "rest --method get --url \"/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/mcpe2-151aca-tb-resource-graph-tags-32/resources?api-version=2021-04-01\"", + "repeat": 2, + "answer": { + "exitCode": 0, + "stdout": "{\"value\":[]}", + "stderr": "", + "classId": null + } + } + ] + }, + { + "task": "tb-resource-graph-tags", + "mode": "oracle", + "passed": true, + "wait_s": 90, + "recorded": "2026-09-27T21:31:49Z", + "slots": { + "resource_group": "mcpe2-15123a-tb-resource-graph-tags-32", + "rg": "mcpe2-15123a-tb-resource-graph-tags-32", + "kv1": "tbkva40db6f", + "kv2": "tbkvb80c92c", + "cc": "cc-f644b1", + "_oracle": true + }, + "text": "ORACLE ✅ az keyvault create --name tbkva40db6f --resource-group mcpe2-15123a-tb-resource-graph-tags-32 --location westeurope --tags costcenter=cc-f644b1\n```json\n{\n \"id\": \"/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/mcpe2-15123a-tb-resource-graph-tags-32/providers/Microsoft.KeyVault/vaults/tbkva40db6f\",\n \"location\": \"westeurope\",\n \"name\": \"tbkva40db6f\",\n \"properties\": {\n \"accessPolicies\": [],\n \"createMode\": null,\n \"enablePurgeProtection\": null,\n \"enableRbacAuthorization\": true,\n \"enableSoftDelete\": true,\n \"enabledForDeployment\": false,\n \"enabledForDiskEncryption\": false,\n \"enabledForTemplateDeployment\": false,\n \"hsmPoolResourceId\": null,\n \"networkAcls\": null,\n \"privateEndpointConnections\": null,\n \"provisioningState\": \"Succeeded\",\n \"publicNetworkAccess\": \"Enabled\",\n \"sku\": {\n \"family\": \"A\",\n \"name\": \"standard\"\n },\n \"softDeleteRetentionInDays\": 90,\n \"tenantId\": \"00000000-0000-0000-0000-000000000000\",\n \"vaultUri\": \"https://tbkva40db6f.vault.azure.localhost.localstack.cloud:4566\"\n },\n \"resourceGroup\": \"mcpe2-15123a-tb-resource-graph-tags-32\",\n \"systemData\": {\n \"createdAt\": \"2026-09-27T21:31:29.179054+00:00\",\n \"createdBy\": \"any-app\",\n \"createdByType\": \"Application\",\n \"lastModifiedAt\": \"2026-09-27T21:31:29.179054+00:00\",\n \"lastModifiedBy\": \"any-app\",\n \"lastModifiedByType\": \"Application\"\n },\n \"tags\": {},\n \"type\": \"Microsoft.KeyVault/vaults\"\n}\n```\nORACLE ✅ az keyvault create --name tbkvb80c92c --resource-group mcpe2-15123a-tb-resource-graph-tags-32 --location westeurope --tags costcenter=cc-f644b1\n```json\n{\n \"id\": \"/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/mcpe2-15123a-tb-resource-graph-tags-32/providers/Microsoft.KeyVault/vaults/tbkvb80c92c\",\n \"location\": \"westeurope\",\n \"name\": \"tbkvb80c92c\",\n \"properties\": {\n \"accessPolicies\": [],\n \"createMode\": null,\n \"enablePurgeProtection\": null,\n \"enableRbacAuthorization\": true,\n \"enableSoftDelete\": true,\n \"enabledForDeployment\": false,\n \"enabledForDiskEncryption\": false,\n \"enabledForTemplateDeployment\": false,\n \"hsmPoolResourceId\": null,\n \"networkAcls\": null,\n \"privateEndpointConnections\": null,\n \"provisioningState\": \"Succeeded\",\n \"publicNetworkAccess\": \"Enabled\",\n \"sku\": {\n \"family\": \"A\",\n \"name\": \"standard\"\n },\n \"softDeleteRetentionInDays\": 90,\n \"tenantId\": \"00000000-0000-0000-0000-000000000000\",\n \"vaultUri\": \"https://tbkvb80c92c.vault.azure.localhost.localstack.cloud:4566\"\n },\n \"resourceGroup\": \"mcpe2-15123a-tb-resource-graph-tags-32\",\n \"systemData\": {\n \"createdAt\": \"2026-09-27T21:31:31.179054+00:00\",\n \"createdBy\": \"any-app\",\n \"createdByType\": \"Application\",\n \"lastModifiedAt\": \"2026-09-27T21:31:31.179054+00:00\",\n \"lastModifiedBy\": \"any-app\",\n \"lastModifiedByType\": \"Application\"\n },\n \"tags\": {},\n \"type\": \"Microsoft.KeyVault/vaults\"\n}\n```\nORACLE ✅ az resource list --tag costcenter=cc-f644b1\n```json\n[\n {\n \"changedTime\": null,\n \"createdTime\": null,\n \"extendedLocation\": null,\n \"id\": \"/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/mcp-muk8f64jm2of-rg/providers/Microsoft.EventGrid/namespaces/mcp-muk8f64jm2of-egn\",\n \"identity\": null,\n \"kind\": null,\n \"location\": \"westeurope\",\n \"managedBy\": null,\n \"name\": \"mcp-muk8f64jm2of-egn\",\n \"plan\": null,\n \"properties\": null,\n \"provisioningState\": \"Succeeded\",\n \"resourceGroup\": \"mcp-muk8f64jm2of-rg\",\n \"sku\": null,\n \"tags\": {},\n \"type\": \"Microsoft.EventGrid/namespaces\"\n },\n {\n \"changedTime\": null,\n \"createdTime\": null,\n \"extendedLocation\": null,\n \"id\": \"/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/mcp-muk8f64jm2of-rg/providers/Microsoft.EventGrid/namespaces/mcp-muk8f64jm2of-egn/topics/orders/eventSubscriptions/workers\",\n \"identity\": null,\n \"kind\": null,\n \"location\": \"westeurope\",\n \"managedBy\": null,\n \"name\": \"mcp-muk8f64jm2of-egn/orders/workers\",\n \"plan\": null,\n \"properties\": null,\n \"provisioningState\": null,\n \"resourceGroup\": \"mcp-muk8f64jm2of-rg\",\n \"sku\": null,\n \"tags\": {},\n \"type\": \"Microsoft.EventGrid/namespaces/mcp-muk8f64jm2of-egn/topics/orders/eventSubscriptions\"\n },\n {\n \"changedTime\": null,\n \"createdTime\": null,\n \"extendedLocation\": null,\n \"id\": \"/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/mcpe2-15123a-tb-resource-graph-tags-32/providers/Microsoft.KeyVault/vaults/tbkva40db6f\",\n \"identity\": null,\n \"kind\": null,\n \"location\": \"westeurope\",\n \"managedBy\": null,\n \"name\": \"tbkva40db6f\",\n \"plan\": null,\n \"properties\": null,\n \"provisioningState\": \"Succeeded\",\n \"resourceGroup\": \"mcpe2-15123a-tb-resource-graph-tags-32\",\n \"sku\": null,\n \"tags\": {\n \"costcenter\": \"cc-f644b1\"\n },\n \"type\": \"Microsoft.KeyVault/vaults\"\n },\n {\n \"changedTime\": null,\n \"createdTime\": null,\n \"extendedLocation\": null,\n \"id\": \"/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/mcpe2-15123a-tb-resource-graph-tags-32/providers/Microsoft.KeyVault/vaults/tbkvb80c92c\",\n \"identity\": null,\n \"kind\": null,\n \"location\": \"westeurope\",\n \"managedBy\": null,\n \"name\": \"tbkvb80c92c\",\n \"plan\": null,\n \"properties\": null,\n \"provisioningState\": \"Succeeded\",\n \"resourceGroup\": \"mcpe2-15123a-tb-resource-graph-tags-32\",\n \"sku\": null,\n \"tags\": {\n \"costcenter\": \"cc-f644b1\"\n },\n \"type\": \"Microsoft.KeyVault/vaults\"\n }\n]\n```", + "queries": [ + { + "command": "rest --method get --url \"/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/mcpe2-15123a-tb-resource-graph-tags-32/resources?api-version=2021-04-01\"", + "repeat": 2, + "answer": { + "exitCode": 0, + "stdout": "{\"value\":[{\"id\":\"/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/mcpe2-15123a-tb-resource-graph-tags-32/providers/Microsoft.KeyVault/vaults/tbkva40db6f\",\"location\":\"westeurope\",\"name\":\"tbkva40db6f\",\"tags\":{\"costcenter\":\"cc-f644b1\"},\"type\":\"Microsoft.KeyVault/vaults\"},{\"id\":\"/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/mcpe2-15123a-tb-resource-graph-tags-32/providers/Microsoft.KeyVault/vaults/tbkvb80c92c\",\"location\":\"westeurope\",\"name\":\"tbkvb80c92c\",\"tags\":{\"costcenter\":\"cc-f644b1\"},\"type\":\"Microsoft.KeyVault/vaults\"}]}", + "stderr": "", + "classId": null + } + } + ] + }, + { + "task": "tb-servicebus-topic", + "mode": "negative", + "passed": false, + "wait_s": 10, + "recorded": "2026-09-27T21:05:55Z", + "slots": { + "resource_group": "mcpe2-151aca-tb-servicebus-topic-31", + "rg": "mcpe2-151aca-tb-servicebus-topic-31", + "ns": "tbsbf59cc8", + "topic": "orders-b02f93", + "s1": "billing-d13210", + "s2": "shipping-f8eed7" + }, + "text": "", + "queries": [ + { + "command": "rest --method get --url \"/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/mcpe2-151aca-tb-servicebus-topic-31/providers/Microsoft.ServiceBus/namespaces/tbsbf59cc8?api-version=2026-01-01\"", + "repeat": 4, + "answer": { + "exitCode": 1, + "stdout": "", + "stderr": "ERROR: NOT FOUND({\"error\": {\"code\": \"ResourceNotFound\", \"message\": \"The Resource 'Microsoft.ServiceBus/namespaces/tbsbf59cc8' under resource group 'mcpe2-151aca-tb-servicebus-topic-31' was not found. For more details please go to https://aka.ms/ARMResourceNotFoundFix\", \"details\": [], \"additionalInfo\": []}})\r\n", + "classId": "not-found" + } + }, + { + "command": "rest --method get --url \"/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/mcpe2-151aca-tb-servicebus-topic-31/providers/Microsoft.ServiceBus/namespaces/tbsbf59cc8/topics/orders-b02f93?api-version=2026-01-01\"", + "repeat": 5, + "answer": { + "exitCode": 1, + "stdout": "", + "stderr": "ERROR: NOT FOUND({\"error\": {\"code\": \"ParentResourceNotFound\", \"message\": \"Failed to perform 'read' on resource(s) of type 'namespaces/topics', because the parent resource '/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/mcpe2-151aca-tb-servicebus-topic-31/providers/Microsoft.ServiceBus/namespaces/tbsbf59cc8' could not be found.\", \"details\": [], \"additionalInfo\": []}})\r\n", + "classId": "other" + } + }, + { + "command": "rest --method get --url \"/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/mcpe2-151aca-tb-servicebus-topic-31/providers/Microsoft.ServiceBus/namespaces/tbsbf59cc8/topics/orders-b02f93/subscriptions/billing-d13210?api-version=2026-01-01\"", + "repeat": 5, + "answer": { + "exitCode": 1, + "stdout": "", + "stderr": "ERROR: NOT FOUND({\"error\": {\"code\": \"ParentResourceNotFound\", \"message\": \"Failed to perform 'read' on resource(s) of type 'namespaces/topics/subscriptions', because the parent resource '/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/mcpe2-151aca-tb-servicebus-topic-31/providers/Microsoft.ServiceBus/namespaces/tbsbf59cc8/topics/orders-b02f93' could not be found.\", \"details\": [],", + "classId": "other" + } + }, + { + "command": "rest --method get --url \"/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/mcpe2-151aca-tb-servicebus-topic-31/providers/Microsoft.ServiceBus/namespaces/tbsbf59cc8/topics/orders-b02f93/subscriptions/shipping-f8eed7?api-version=2026-01-01\"", + "repeat": 5, + "answer": { + "exitCode": 1, + "stdout": "", + "stderr": "ERROR: NOT FOUND({\"error\": {\"code\": \"ParentResourceNotFound\", \"message\": \"Failed to perform 'read' on resource(s) of type 'namespaces/topics/subscriptions', because the parent resource '/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/mcpe2-151aca-tb-servicebus-topic-31/providers/Microsoft.ServiceBus/namespaces/tbsbf59cc8/topics/orders-b02f93' could not be found.\", \"details\": [],", + "classId": "other" + } + } + ] + }, + { + "task": "tb-servicebus-topic", + "mode": "oracle", + "passed": true, + "wait_s": 90, + "recorded": "2026-09-27T21:31:26Z", + "slots": { + "resource_group": "mcpe2-15123a-tb-servicebus-topic-31", + "rg": "mcpe2-15123a-tb-servicebus-topic-31", + "ns": "tbsbc5f83f", + "topic": "orders-68cabd", + "s1": "billing-2fde7f", + "s2": "shipping-b02149", + "_oracle": true + }, + "text": "ORACLE ✅ az servicebus namespace create --name tbsbc5f83f --resource-group mcpe2-15123a-tb-servicebus-topic-31 --location westeurope --sku Standard\n```json\n{\n \"createdAt\": \"2026-09-27T21:31:08.1790544668+00:00\",\n \"disableLocalAuth\": false,\n \"id\": \"/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/mcpe2-15123a-tb-servicebus-topic-31/providers/Microsoft.ServiceBus/namespaces/tbsbc5f83f\",\n \"location\": \"westeurope\",\n \"metricId\": \"00000000-0000-0000-0000-000000000000:tbsbc5f83f\",\n \"minimumTlsVersion\": \"1.2\",\n \"name\": \"tbsbc5f83f\",\n \"premiumMessagingPartitions\": 0,\n \"provisioningState\": \"Succeeded\",\n \"publicNetworkAccess\": \"Enabled\",\n \"resourceGroup\": \"mcpe2-15123a-tb-servicebus-topic-31\",\n \"serviceBusEndpoint\": \"https://tbsbc5f83f.servicebus.azure.localhost.localstack.cloud:4511\",\n \"sku\": {\n \"name\": \"Standard\",\n \"tier\": \"Standard\"\n },\n \"status\": \"Active\",\n \"tags\": {},\n \"type\": \"Microsoft.ServiceBus/Namespaces\",\n \"updatedAt\": \"2026-09-27T21:31:08.1790544668+00:00\",\n \"zoneRedundant\": true\n}\n```\nORACLE ✅ az servicebus topic create --name orders-68cabd --namespace-name tbsbc5f83f --resource-group mcpe2-15123a-tb-servicebus-topic-31\n```json\n{\n \"accessedAt\": \"2026-09-27T21:31:09.1790544669+00:00\",\n \"autoDeleteOnIdle\": \"P10675199DT2H48M5.4775807S\",\n \"countDetails\": {\n \"activeMessageCount\": 0,\n \"deadLetterMessageCount\": 0,\n \"scheduledMessageCount\": 0,\n \"transferDeadLetterMessageCount\": 0,\n \"transferMessageCount\": 0\n },\n \"createdAt\": \"2026-09-27T21:31:09.1790544669+00:00\",\n \"defaultMessageTimeToLive\": \"P10675199DT2H48M5.4775807S\",\n \"duplicateDetectionHistoryTimeWindow\": \"PT10M\",\n \"enableBatchedOperations\": true,\n \"enableExpress\": false,\n \"enablePartitioning\": false,\n \"id\": \"/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/mcpe2-15123a-tb-servicebus-topic-31/providers/Microsoft.ServiceBus/namespaces/tbsbc5f83f/topics/orders-68cabd\",\n \"location\": \"westeurope\",\n \"maxMessageSizeInKilobytes\": 256,\n \"maxSizeInMegabytes\": 1024,\n \"name\": \"orders-68cabd\",\n \"requiresDuplicateDetection\": false,\n \"resourceGroup\": \"mcpe2-15123a-tb-servicebus-topic-31\",\n \"sizeInBytes\": 0,\n \"status\": \"Active\",\n \"subscriptionCount\": 0,\n \"supportOrdering\": true,\n \"type\": \"Microsoft.ServiceBus/namespaces/topics\",\n \"updatedAt\": \"2026-09-27T21:31:09.1790544669+00:00\"\n}\n```\nORACLE ✅ az servicebus topic subscription create --name billing-2fde7f --topic-name orders-68cabd --namespace-name tbsbc5f83f --resource-group mcpe2-15123a-tb-servicebus-topic-31\n```json\n{\n \"autoDeleteOnIdle\": \"P10675199DT2H48M5.4775807S\",\n \"countDetails\": {\n \"activeMessageCount\": 0,\n \"deadLetterMessageCount\": 0,\n \"scheduledMessageCount\": 0,\n \"transferDeadLetterMessageCount\": 0,\n \"transferMessageCount\": 0\n },\n \"deadLetteringOnFilterEvaluationExceptions\": true,\n \"deadLetteringOnMessageExpiration\": false,\n \"defaultMessageTimeToLive\": \"P10675199DT2H48M5.4775807S\",\n \"enableBatchedOperations\": true,\n \"id\": \"/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/mcpe2-15123a-tb-servicebus-topic-31/providers/Microsoft.ServiceBus/namespaces/tbsbc5f83f/topics/orders-68cabd/subscriptions/billing-2fde7f\",\n \"isClientAffine\": false,\n \"location\": \"westeurope\",\n \"lockDuration\": \"PT1M\",\n \"maxDeliveryCount\": 10,\n \"messageCount\": 0,\n \"name\": \"billing-2fde7f\",\n \"requiresSession\": false,\n \"resourceGroup\": \"mcpe2-15123a-tb-servicebus-topic-31\",\n \"status\": \"Active\",\n \"type\": \"Microsoft.ServiceBus/namespaces/topics/subscriptions\"\n}\n```\nORACLE ✅ az servicebus topic subscription create --name shipping-b02149 --topic-name orders-68cabd --namespace-name tbsbc5f83f --resource-group mcpe2-15123a-tb-servicebus-topic-31\n```json\n{\n \"autoDeleteOnIdle\": \"P10675199DT2H48M5.4775807S\",\n \"countDetails\": {\n \"activeMessageCount\": 0,\n \"deadLetterMessageCount\": 0,\n \"scheduledMessageCount\": 0,\n \"transferDeadLetterMessageCount\": 0,\n \"transferMessageCount\": 0\n },\n \"deadLetteringOnFilterEvaluationExceptions\": true,\n \"deadLetteringOnMessageExpiration\": false,\n \"defaultMessageTimeToLive\": \"P10675199DT2H48M5.4775807S\",\n \"enableBatchedOperations\": true,\n \"id\": \"/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/mcpe2-15123a-tb-servicebus-topic-31/providers/Microsoft.ServiceBus/namespaces/tbsbc5f83f/topics/orders-68cabd/subscriptions/shipping-b02149\",\n \"isClientAffine\": false,\n \"location\": \"westeurope\",\n \"lockDuration\": \"PT1M\",\n \"maxDeliveryCount\": 10,\n \"messageCount\": 0,\n \"name\": \"shipping-b02149\",\n \"requiresSession\": false,\n \"resourceGroup\": \"mcpe2-15123a-tb-servicebus-topic-31\",\n \"status\": \"Active\",\n \"type\": \"Microsoft.ServiceBus/namespaces/topics/subscriptions\"\n}\n```\nORACLE ✅ az servicebus topic subscription list --topic-name orders-68cabd --namespace-name tbsbc5f83f --resource-group mcpe2-15123a-tb-servicebus-topic-31\n```json\n[\n {\n \"autoDeleteOnIdle\": \"P10675199DT2H48M5.4775807S\",\n \"countDetails\": {\n \"activeMessageCount\": 0,\n \"deadLetterMessageCount\": 0,\n \"scheduledMessageCount\": 0,\n \"transferDeadLetterMessageCount\": 0,\n \"transferMessageCount\": 0\n },\n \"deadLetteringOnFilterEvaluationExceptions\": true,\n \"deadLetteringOnMessageExpiration\": false,\n \"defaultMessageTimeToLive\": \"P10675199DT2H48M5.4775807S\",\n \"enableBatchedOperations\": true,\n \"id\": \"/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/mcpe2-15123a-tb-servicebus-topic-31/providers/Microsoft.ServiceBus/namespaces/tbsbc5f83f/topics/orders-68cabd/subscriptions/billing-2fde7f\",\n \"isClientAffine\": false,\n \"location\": \"westeurope\",\n \"lockDuration\": \"PT1M\",\n \"maxDeliveryCount\": 10,\n \"messageCount\": 0,\n \"name\": \"billing-2fde7f\",\n \"requiresSession\": false,\n \"resourceGroup\": \"mcpe2-15123a-tb-servicebus-topic-31\",\n \"status\": \"Active\",\n \"type\": \"Microsoft.ServiceBus/namespaces/topics/subscriptions\"\n },\n {\n \"autoDeleteOnIdle\": \"P10675199DT2H48M5.4775807S\",\n \"countDetails\": {\n \"activeMessageCount\": 0,\n \"deadLetterMessageCount\": 0,\n \"scheduledMessageCount\": 0,\n \"transferDeadLetterMessageCount\": 0,\n \"transferMessageCount\": 0\n },\n \"deadLetteringOnFilterEvaluationExceptions\": true,\n \"deadLetteringOnMessageExpiration\": false,\n \"defaultMessageTimeToLive\": \"P10675199DT2H48M5.4775807S\",\n \"enableBatchedOperations\": true,\n \"id\": \"/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/mcpe2-15123a-tb-servicebus-topic-31/providers/Microsoft.ServiceBus/namespaces/tbsbc5f83f/topics/orders-68cabd/subscriptions/shipping-b02149\",\n \"isClientAffine\": false,\n \"location\": \"westeurope\",\n \"lockDuration\": \"PT1M\",\n \"maxDeliveryCount\": 10,\n \"messageCount\": 0,\n \"name\": \"shipping-b02149\",\n \"requiresSession\": false,\n \"resourceGroup\": \"mcpe2-15123a-tb-servicebus-topic-31\",\n \"status\": \"Active\",\n \"type\": \"Microsoft.ServiceBus/namespaces/topics/subscriptions\"\n }\n]\n```", + "queries": [ + { + "command": "rest --method get --url \"/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/mcpe2-15123a-tb-servicebus-topic-31/providers/Microsoft.ServiceBus/namespaces/tbsbc5f83f?api-version=2026-01-01\"", + "answer": { + "exitCode": 0, + "stdout": "{\"id\":\"/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/mcpe2-15123a-tb-servicebus-topic-31/providers/Microsoft.ServiceBus/namespaces/tbsbc5f83f\",\"location\":\"westeurope\",\"name\":\"tbsbc5f83f\",\"properties\":{\"createdAt\":\"2026-09-27T21:31:08.1790544668+00:00\",\"disableLocalAuth\":false,\"metricId\":\"00000000-0000-0000-0000-000000000000:tbsbc5f83f\",\"minimumTlsVersion\":\"1.2\",\"premiumMessagingPartitions\":0,\"provisioningState\":\"Succeeded\",\"publicNetworkAccess\":\"Enabled\",\"serviceBusEndpoint\":\"https://tbsbc5f83f.servicebus.azure.localhost.localstack.cloud:4511\",\"status\":\"Active\",\"updatedAt\":\"2026-09-27T21:31:08.1790544668+00:00\",\"zoneRedundant\":true},\"sku\":{\"name\":\"Standard\",\"tier\":\"Standard\"},\"tags\":{},\"type\":\"Microsoft.ServiceBus/Namespaces\"}", + "stderr": "", + "classId": null + } + }, + { + "command": "rest --method get --url \"/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/mcpe2-15123a-tb-servicebus-topic-31/providers/Microsoft.ServiceBus/namespaces/tbsbc5f83f/topics/orders-68cabd?api-version=2026-01-01\"", + "answer": { + "exitCode": 0, + "stdout": "{\"id\":\"/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/mcpe2-15123a-tb-servicebus-topic-31/providers/Microsoft.ServiceBus/namespaces/tbsbc5f83f/topics/orders-68cabd\",\"location\":\"westeurope\",\"name\":\"orders-68cabd\",\"properties\":{\"accessedAt\":\"2026-09-27T21:31:09.1790544669+00:00\",\"autoDeleteOnIdle\":\"P10675199DT2H48M5.4775807S\",\"countDetails\":{\"activeMessageCount\":0,\"deadLetterMessageCount\":0,\"scheduledMessageCount\":0,\"transferDeadLetterMessageCount\":0,\"transferMessageCount\":0},\"createdAt\":\"2026-09-27T21:31:09.1790544669+00:00\",\"defaultMessageTimeToLive\":\"P10675199DT2H48M5.4775807S\",\"duplicateDetectionHistoryTimeWindow\":\"PT10M\",\"enableBatchedOperations\":true,\"enableExpress\":false,\"enablePartitioning\":false,\"maxMessageSizeInKilobytes\":256,\"maxSizeInMegabytes\":1024,\"requiresDuplicateDetection\":false,\"sizeInBytes\":0,\"status\":\"Active\",\"subscriptionCount\":0,\"supportOrdering\":true,\"updatedAt\":\"2026-09-27T21:31:09.1790544669+00:00\"},\"type\":\"Microsoft.ServiceBus/namespaces/topics\"}", + "stderr": "", + "classId": null + } + }, + { + "command": "rest --method get --url \"/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/mcpe2-15123a-tb-servicebus-topic-31/providers/Microsoft.ServiceBus/namespaces/tbsbc5f83f/topics/orders-68cabd/subscriptions/billing-2fde7f?api-version=2026-01-01\"", + "answer": { + "exitCode": 0, + "stdout": "{\"id\":\"/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/mcpe2-15123a-tb-servicebus-topic-31/providers/Microsoft.ServiceBus/namespaces/tbsbc5f83f/topics/orders-68cabd/subscriptions/billing-2fde7f\",\"location\":\"westeurope\",\"name\":\"billing-2fde7f\",\"properties\":{\"autoDeleteOnIdle\":\"P10675199DT2H48M5.4775807S\",\"countDetails\":{\"activeMessageCount\":0,\"deadLetterMessageCount\":0,\"scheduledMessageCount\":0,\"transferDeadLetterMessageCount\":0,\"transferMessageCount\":0},\"deadLetteringOnFilterEvaluationExceptions\":true,\"deadLetteringOnMessageExpiration\":false,\"defaultMessageTimeToLive\":\"P10675199DT2H48M5.4775807S\",\"enableBatchedOperations\":true,\"isClientAffine\":false,\"lockDuration\":\"PT1M\",\"maxDeliveryCount\":10,\"messageCount\":0,\"requiresSession\":false,\"status\":\"Active\"},\"type\":\"Microsoft.ServiceBus/namespaces/topics/subscriptions\"}", + "stderr": "", + "classId": null + } + }, + { + "command": "rest --method get --url \"/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/mcpe2-15123a-tb-servicebus-topic-31/providers/Microsoft.ServiceBus/namespaces/tbsbc5f83f/topics/orders-68cabd/subscriptions/shipping-b02149?api-version=2026-01-01\"", + "answer": { + "exitCode": 0, + "stdout": "{\"id\":\"/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/mcpe2-15123a-tb-servicebus-topic-31/providers/Microsoft.ServiceBus/namespaces/tbsbc5f83f/topics/orders-68cabd/subscriptions/shipping-b02149\",\"location\":\"westeurope\",\"name\":\"shipping-b02149\",\"properties\":{\"autoDeleteOnIdle\":\"P10675199DT2H48M5.4775807S\",\"countDetails\":{\"activeMessageCount\":0,\"deadLetterMessageCount\":0,\"scheduledMessageCount\":0,\"transferDeadLetterMessageCount\":0,\"transferMessageCount\":0},\"deadLetteringOnFilterEvaluationExceptions\":true,\"deadLetteringOnMessageExpiration\":false,\"defaultMessageTimeToLive\":\"P10675199DT2H48M5.4775807S\",\"enableBatchedOperations\":true,\"isClientAffine\":false,\"lockDuration\":\"PT1M\",\"maxDeliveryCount\":10,\"messageCount\":0,\"requiresSession\":false,\"status\":\"Active\"},\"type\":\"Microsoft.ServiceBus/namespaces/topics/subscriptions\"}", + "stderr": "", + "classId": null + } + } + ] + }, + { + "task": "tb-storage-blob", + "mode": "negative", + "passed": false, + "wait_s": 10, + "recorded": "2026-09-27T21:04:08Z", + "slots": { + "resource_group": "mcpe2-151aca-tb-storage-blob-29", + "rg": "mcpe2-151aca-tb-storage-blob-29", + "sa": "tbsa3f2110", + "container": "uploads-baa2df" + }, + "text": "", + "queries": [ + { + "command": "rest --method get --url \"/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/mcpe2-151aca-tb-storage-blob-29/providers/Microsoft.Storage/storageAccounts/tbsa3f2110?api-version=2026-06-01\"", + "repeat": 4, + "answer": { + "exitCode": 1, + "stdout": "", + "stderr": "ERROR: NOT FOUND({\"error\": {\"code\": \"ResourceNotFound\", \"message\": \"storageAccount 'tbsa3f2110' could not be found.\", \"details\": [], \"additionalInfo\": []}})\r\n", + "classId": "not-found" + } + }, + { + "command": "rest --method get --url \"/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/mcpe2-151aca-tb-storage-blob-29/providers/Microsoft.Storage/storageAccounts/tbsa3f2110/blobServices/default/containers/uploads-baa2df?api-version=2026-06-01\"", + "repeat": 6, + "answer": { + "exitCode": 1, + "stdout": "", + "stderr": "ERROR: NOT FOUND({})\r\n", + "classId": "other" + } + } + ] + }, + { + "task": "tb-storage-blob", + "mode": "oracle", + "passed": true, + "wait_s": 90, + "recorded": "2026-09-27T21:30:30Z", + "slots": { + "resource_group": "mcpe2-15123a-tb-storage-blob-29", + "rg": "mcpe2-15123a-tb-storage-blob-29", + "sa": "tbsa5452ca", + "container": "uploads-19efb2", + "_oracle": true + }, + "text": "ORACLE ✅ az storage account create --name tbsa5452ca --resource-group mcpe2-15123a-tb-storage-blob-29 --sku Standard_LRS --kind StorageV2\n```json\n{\n \"accessTier\": \"Hot\",\n \"accountMigrationInProgress\": null,\n \"allowBlobPublicAccess\": false,\n \"allowCrossTenantReplication\": false,\n \"allowSharedKeyAccess\": null,\n \"allowedCopyScope\": null,\n \"azureFilesIdentityBasedAuthentication\": null,\n \"blobRestoreStatus\": null,\n \"creationTime\": null,\n \"customDomain\": null,\n \"defaultToOAuthAuthentication\": null,\n \"dnsEndpointType\": null,\n \"dualStackEndpointPreference\": null,\n \"enableExtendedGroups\": null,\n \"enableHttpsTrafficOnly\": true,\n \"enableNfsV3\": null,\n \"encryption\": {\n \"encryptionIdentity\": null,\n \"keySource\": \"Microsoft.Storage\",\n \"keyVaultProperties\": null,\n \"requireInfrastructureEncryption\": null,\n \"services\": {\n \"blob\": {\n \"enabled\": true,\n \"keyType\": \"Account\",\n \"lastEnabledTime\": \"2026-09-27T21:29:57.647759+00:00\"\n },\n \"file\": {\n \"enabled\": true,\n \"keyType\": \"Account\",\n \"lastEnabledTime\": \"2026-09-27T21:29:57.647759+00:00\"\n },\n \"queue\": null,\n \"table\": null\n }\n },\n \"extendedLocation\": null,\n \"failoverInProgress\": null,\n \"geoPriorityReplicationStatus\": null,\n \"geoReplicationStats\": null,\n \"id\": \"/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/mcpe2-15123a-tb-storage-blob-29/providers/Microsoft.Storage/storageAccounts/tbsa5452ca\",\n \"identity\": null,\n \"immutableStorageWithVersioning\": null,\n \"isHnsEnabled\": null,\n \"isLocalUserEnabled\": null,\n \"isSftpEnabled\": null,\n \"isSkuConversionBlocked\": null,\n \"keyCreationTime\": null,\n \"keyPolicy\": null,\n \"kind\": \"StorageV2\",\n \"largeFileSharesState\": null,\n \"lastGeoFailoverTime\": null,\n \"location\": \"westeurope\",\n \"minimumTlsVersion\": \"TLS1_0\",\n \"name\": \"tbsa5452ca\",\n \"networkRuleSet\": {\n \"bypass\": \"None\",\n \"defaultAction\": \"Allow\",\n \"ipRules\": [],\n \"ipv6Rules\": [],\n \"resourceAccessRules\": null,\n \"virtualNetworkRules\": []\n },\n \"placement\": null,\n \"primaryEndpoints\": {\n \"blob\": \"https://tbsa5452ca.blob.core.azure.localhost.localstack.cloud:4566\",\n \"dfs\": \"https://tbsa5452ca.dfs.core.windows.net/\",\n \"file\": \"https://tbsa5452ca.file.core.azure.localhost.localstack.cloud:4566\",\n \"internetEndpoints\": null,\n \"ipv6Endpoints\": null,\n \"microsoftEndpoints\": null,\n \"queue\": \"https://tbsa5452ca.queue.core.azure.localhost.localstack.cloud:4566\",\n \"table\": \"https://tbsa5452ca.table.core.azure.localhost.localstack.cloud:4566\",\n \"web\": \"https://tbsa5452ca.z28.web.core.windows.net/\"\n },\n \"primaryLocation\": \"westeurope\",\n \"privateEndpointConnections\": [],\n \"provisioningState\": \"Succeeded\",\n \"publicNetworkAccess\": null,\n \"resourceGroup\": \"mcpe2-15123a-tb-storage-blob-29\",\n \"routingPreference\": null,\n \"sasPolicy\": null,\n \"secondaryEndpoints\": null,\n \"secondaryLocation\": null,\n \"sku\": {\n \"name\": \"Standard_LRS\",\n \"tier\": \"Standard\"\n },\n \"statusOfPrimary\": \"available\",\n \"statusOfSecondary\": null,\n \"storageAccountSkuConversionStatus\": null,\n \"tags\": {},\n \"type\": \"Microsoft.Storage/storageAccounts\",\n \"zones\": null\n}\n```\nORACLE ✅ az storage container-rm create --storage-account tbsa5452ca --name uploads-19efb2 --resource-group mcpe2-15123a-tb-storage-blob-29\n```json\n{\n \"defaultEncryptionScope\": null,\n \"deleted\": null,\n \"deletedTime\": null,\n \"denyEncryptionScopeOverride\": null,\n \"enableNfsV3AllSquash\": null,\n \"enableNfsV3RootSquash\": null,\n \"etag\": null,\n \"hasImmutabilityPolicy\": null,\n \"hasLegalHold\": null,\n \"id\": \"/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/mcpe2-15123a-tb-storage-blob-29/providers/Microsoft.Storage/storageAccounts/tbsa5452ca/blobServices/default/containers/uploads-19efb2\",\n \"immutabilityPolicy\": null,\n \"immutableStorageWithVersioning\": null,\n \"lastModifiedTime\": null,\n \"leaseDuration\": null,\n \"leaseState\": null,\n \"leaseStatus\": null,\n \"legalHold\": null,\n \"metadata\": null,\n \"name\": \"uploads-19efb2\",\n \"publicAccess\": null,\n \"remainingRetentionDays\": null,\n \"resourceGroup\": \"mcpe2-15123a-tb-storage-blob-29\",\n \"type\": \"Microsoft.Storage/storageAccounts/blobServices/containers\",\n \"version\": null\n}\n```", + "queries": [ + { + "command": "rest --method get --url \"/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/mcpe2-15123a-tb-storage-blob-29/providers/Microsoft.Storage/storageAccounts/tbsa5452ca?api-version=2026-06-01\"", + "repeat": 2, + "answer": { + "exitCode": 0, + "stdout": "{\"id\":\"/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/mcpe2-15123a-tb-storage-blob-29/providers/Microsoft.Storage/storageAccounts/tbsa5452ca\",\"kind\":\"StorageV2\",\"location\":\"westeurope\",\"name\":\"tbsa5452ca\",\"properties\":{\"accessTier\":\"Hot\",\"allowBlobPublicAccess\":false,\"allowCrossTenantReplication\":false,\"encryption\":{\"keySource\":\"Microsoft.Storage\",\"services\":{\"blob\":{\"enabled\":true,\"keyType\":\"Account\",\"lastEnabledTime\":\"2026-09-27T21:29:57.647759+00:00\"},\"file\":{\"enabled\":true,\"keyType\":\"Account\",\"lastEnabledTime\":\"2026-09-27T21:29:57.647759+00:00\"}}},\"minimumTlsVersion\":\"TLS1_0\",\"networkAcls\":{\"bypass\":\"None\",\"defaultAction\":\"Allow\",\"ipRules\":[],\"ipv6Rules\":[],\"virtualNetworkRules\":[]},\"primaryEndpoints\":{\"blob\":\"https://tbsa5452ca.blob.core.azure.localhost.localstack.cloud:4566\",\"dfs\":\"https://tbsa5452ca.dfs.core.windows.net/\",\"file\":\"https://tbsa5452ca.file.core.azure.localhost.localstack.cloud:4566\",\"queue\":\"https://tbsa5452ca.queue.core.azure.localhost.localstack.cloud:4566\",\"table\":\"https://tbsa5452ca.table.core.azure.localhost.localstack.cloud:4566\",\"web\":\"https://tbsa5452ca.z28.web.core.windows.net/\"},\"primaryLocation\":\"westeurope\",\"privateEndpointConnections\":[],\"provisioningState\":\"Succeeded\",\"statusOfPrimary\":\"available\",\"supportsHttpsTrafficOnly\":true},\"sku\":{\"name\":\"Standard_LRS\",\"tier\":\"Standard\"},\"tags\":{},\"type\":\"Microsoft.Storage/storageAccounts\"}", + "stderr": "", + "classId": null + } + }, + { + "command": "rest --method get --url \"/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/mcpe2-15123a-tb-storage-blob-29/providers/Microsoft.Storage/storageAccounts/tbsa5452ca/blobServices/default/containers/uploads-19efb2?api-version=2026-06-01\"", + "answer": { + "exitCode": 0, + "stdout": "{\"id\":\"/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/mcpe2-15123a-tb-storage-blob-29/providers/Microsoft.Storage/storageAccounts/tbsa5452ca/blobServices/default/containers/uploads-19efb2\",\"name\":\"uploads-19efb2\",\"properties\":{\"defaultEncryptionScope\":\"$account-encryption-key\",\"deleted\":false,\"denyEncryptionScopeOverride\":false,\"hasImmutabilityPolicy\":false,\"hasLegalHold\":false,\"lastModifiedTime\":\"2026-09-27T21:30:15.1790544615+00:00\",\"leaseState\":\"Available\",\"leaseStatus\":\"Unlocked\",\"legalHold\":{\"hasLegalHold\":false,\"tags\":[]},\"publicAccess\":\"None\",\"remainingRetentionDays\":0},\"type\":\"Microsoft.Storage/storageAccounts/blobServices/containers\"}", + "stderr": "", + "classId": null + } + } + ] + }, + { + "task": "tb-tag-audit", + "mode": "negative", + "passed": false, + "wait_s": 10, + "recorded": "2026-09-27T21:07:38Z", + "slots": { + "resource_group": "mcpe2-151aca-tb-tag-audit-33", + "rg": "mcpe2-151aca-tb-tag-audit-33", + "t1": "tbvnet-82e594", + "t2": "tbnsg-642d06", + "u1": "tbpip-f612a8", + "u2": "tbrt-8bd985" + }, + "text": "", + "queries": [] + }, + { + "task": "tb-tag-audit", + "mode": "oracle", + "passed": true, + "wait_s": 90, + "recorded": "2026-09-27T21:32:07Z", + "slots": { + "resource_group": "mcpe2-15123a-tb-tag-audit-33", + "rg": "mcpe2-15123a-tb-tag-audit-33", + "t1": "tbvnet-d95564", + "t2": "tbnsg-fc3076", + "u1": "tbpip-8a6a3c", + "u2": "tbrt-224358", + "_oracle": true + }, + "text": "ORACLE ✅ az resource list --resource-group mcpe2-15123a-tb-tag-audit-33\n```json\n[\n {\n \"changedTime\": null,\n \"createdTime\": null,\n \"extendedLocation\": null,\n \"id\": \"/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/mcpe2-15123a-tb-tag-audit-33/providers/Microsoft.Network/virtualNetworks/tbvnet-d95564\",\n \"identity\": null,\n \"kind\": null,\n \"location\": \"westeurope\",\n \"managedBy\": null,\n \"name\": \"tbvnet-d95564\",\n \"plan\": null,\n \"properties\": null,\n \"provisioningState\": \"Succeeded\",\n \"resourceGroup\": \"mcpe2-15123a-tb-tag-audit-33\",\n \"sku\": null,\n \"tags\": {\n \"owner\": \"team-0835\"\n },\n \"type\": \"Microsoft.Network/virtualNetworks\"\n },\n {\n \"changedTime\": null,\n \"createdTime\": null,\n \"extendedLocation\": null,\n \"id\": \"/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/mcpe2-15123a-tb-tag-audit-33/providers/Microsoft.Network/networkSecurityGroups/tbnsg-fc3076\",\n \"identity\": null,\n \"kind\": null,\n \"location\": \"westeurope\",\n \"managedBy\": null,\n \"name\": \"tbnsg-fc3076\",\n \"plan\": null,\n \"properties\": null,\n \"provisioningState\": \"Succeeded\",\n \"resourceGroup\": \"mcpe2-15123a-tb-tag-audit-33\",\n \"sku\": null,\n \"tags\": {\n \"owner\": \"team-0835\"\n },\n \"type\": \"Microsoft.Network/networkSecurityGroups\"\n },\n {\n \"changedTime\": null,\n \"createdTime\": null,\n \"extendedLocation\": null,\n \"id\": \"/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/mcpe2-15123a-tb-tag-audit-33/providers/Microsoft.Network/publicIPAddresses/tbpip-8a6a3c\",\n \"identity\": null,\n \"kind\": null,\n \"location\": \"westeurope\",\n \"managedBy\": null,\n \"name\": \"tbpip-8a6a3c\",\n \"plan\": null,\n \"properties\": null,\n \"provisioningState\": \"Succeeded\",\n \"resourceGroup\": \"mcpe2-15123a-tb-tag-audit-33\",\n \"sku\": {\n \"capacity\": null,\n \"family\": null,\n \"model\": null,\n \"name\": \"Standard\",\n \"size\": null,\n \"tier\": null\n },\n \"tags\": {},\n \"type\": \"Microsoft.Network/publicIPAddresses\"\n },\n {\n \"changedTime\": null,\n \"createdTime\": null,\n \"extendedLocation\": null,\n \"id\": \"/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/mcpe2-15123a-tb-tag-audit-33/providers/Microsoft.Network/routeTables/tbrt-224358\",\n \"identity\": null,\n \"kind\": null,\n \"location\": \"westeurope\",\n \"managedBy\": null,\n \"name\": \"tbrt-224358\",\n \"plan\": null,\n \"properties\": null,\n \"provisioningState\": \"Succeeded\",\n \"resourceGroup\": \"mcpe2-15123a-tb-tag-audit-33\",\n \"sku\": null,\n \"tags\": {},\n \"type\": \"Microsoft.Network/routeTables\"\n }\n]\n```", + "queries": [] + }, + { + "task": "tb-webapp-settings", + "mode": "negative", + "passed": false, + "wait_s": 10, + "recorded": "2026-09-27T21:10:14Z", + "slots": { + "resource_group": "mcpe2-151aca-tb-webapp-settings-36", + "rg": "mcpe2-151aca-tb-webapp-settings-36", + "plan": "tbplan-54cbc2", + "app": "tbweb-97b363", + "flag": "on-1ee9ef" + }, + "text": "", + "queries": [ + { + "command": "rest --method get --url \"/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/mcpe2-151aca-tb-webapp-settings-36/providers/Microsoft.Web/serverfarms/tbplan-54cbc2?api-version=2026-07-15\"", + "repeat": 12, + "answer": { + "exitCode": 1, + "stdout": "", + "stderr": "ERROR: NOT FOUND({\"error\": {\"code\": \"ResourceNotFound\", \"message\": \"The Resource 'Microsoft.Web/serverFarms/tbplan-54cbc2' under resource group 'mcpe2-151aca-tb-webapp-settings-36' was not found.\", \"details\": [], \"additionalInfo\": []}})\r\n", + "classId": "not-found" + } + }, + { + "command": "rest --method get --url \"/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/mcpe2-151aca-tb-webapp-settings-36/providers/Microsoft.Web/sites/tbweb-97b363?api-version=2026-07-15\"", + "repeat": 5, + "answer": { + "exitCode": 1, + "stdout": "", + "stderr": "ERROR: NOT FOUND({\"error\": {\"code\": \"ResourceNotFound\", \"message\": \"The Resource 'Microsoft.Web/sites/tbweb-97b363' under resource group 'mcpe2-151aca-tb-webapp-settings-36' was not found.\", \"details\": [], \"additionalInfo\": []}})\r\n", + "classId": "not-found" + } + }, + { + "command": "rest --method post --url \"/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/mcpe2-151aca-tb-webapp-settings-36/providers/Microsoft.Web/sites/tbweb-97b363/config/appsettings/list?api-version=2026-07-15\"", + "answer": { + "exitCode": 1, + "stdout": "", + "stderr": "ERROR: NOT FOUND({\"error\": {\"code\": \"ResourceNotFound\", \"message\": \"The Resource 'Microsoft.Web/sites/tbweb-97b363' under resource group 'mcpe2-151aca-tb-webapp-settings-36' was not found.\", \"details\": [], \"additionalInfo\": []}})\r\n", + "classId": "not-found" + } + } + ] + }, + { + "task": "tb-webapp-settings", + "mode": "oracle", + "passed": true, + "wait_s": 90, + "recorded": "2026-09-27T21:35:09Z", + "slots": { + "resource_group": "mcpe2-15123a-tb-webapp-settings-36", + "rg": "mcpe2-15123a-tb-webapp-settings-36", + "plan": "tbplan-2eb4a8", + "app": "tbweb-a60646", + "flag": "on-7494a2", + "_oracle": true + }, + "text": "ORACLE ✅ az appservice plan create --name tbplan-2eb4a8 --resource-group mcpe2-15123a-tb-webapp-settings-36 --sku B1 --is-linux\n```json\n{\n \"asyncScalingEnabled\": false,\n \"elasticScaleEnabled\": false,\n \"geoRegion\": \"West Europe\",\n \"hyperV\": false,\n \"id\": \"/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/mcpe2-15123a-tb-webapp-settings-36/providers/Microsoft.Web/serverfarms/tbplan-2eb4a8\",\n \"isCustomMode\": false,\n \"isSpot\": false,\n \"isXenon\": false,\n \"kind\": \"linux\",\n \"location\": \"westeurope\",\n \"maximumElasticWorkerCount\": 1,\n \"maximumNumberOfWorkers\": 0,\n \"name\": \"tbplan-2eb4a8\",\n \"numberOfSites\": 0,\n \"numberOfWorkers\": 1,\n \"perSiteScaling\": false,\n \"provisioningState\": \"Succeeded\",\n \"reserved\": true,\n \"resourceGroup\": \"mcpe2-15123a-tb-webapp-settings-36\",\n \"sku\": {\n \"capacity\": 1,\n \"family\": \"B\",\n \"name\": \"B1\",\n \"size\": \"B1\",\n \"tier\": \"Basic\"\n },\n \"status\": \"Ready\",\n \"subscription\": \"00000000-0000-0000-0000-000000000000\",\n \"targetWorkerCount\": 0,\n \"targetWorkerSizeId\": 0,\n \"type\": \"Microsoft.Web/serverfarms\",\n \"zoneRedundant\": false\n}\n```\nORACLE ✅ az webapp create --name tbweb-a60646 --resource-group mcpe2-15123a-tb-webapp-settings-36 --plan tbplan-2eb4a8 --runtime PYTHON:3.12\n```json\n{\n \"autoGeneratedDomainNameLabelScope\": null,\n \"availabilityState\": \"Normal\",\n \"clientAffinityEnabled\": true,\n \"clientAffinityPartitioningEnabled\": null,\n \"clientAffinityProxyEnabled\": false,\n \"clientCertEnabled\": false,\n \"clientCertExclusionPaths\": null,\n \"clientCertMode\": \"Required\",\n \"cloningInfo\": null,\n \"containerSize\": 0,\n \"customDomainVerificationId\": \"D3LC64FJI32WGRJRSJDFIHK90O3193U5GAQCCMSQ1VQAOHUOTDNRFJSST7BMQYST\",\n \"dailyMemoryTimeQuota\": 0,\n \"daprConfig\": null,\n \"defaultHostName\": \"tbweb-a60646.azurewebsites.azure.localhost.localstack.cloud:4566\",\n \"dnsConfiguration\": {\n \"dnsAltServer\": null,\n \"dnsLegacySortOrder\": null,\n \"dnsMaxCacheTimeout\": null,\n \"dnsRetryAttemptCount\": null,\n \"dnsRetryAttemptTimeout\": null,\n \"dnsServers\": null\n },\n \"enabled\": true,\n \"enabledHostNames\": [\n \"tbweb-a60646.azurewebsites.azure.localhost.localstack.cloud:4566\",\n \"tbweb-a60646.scm.azurewebsites.azure.localhost.localstack.cloud:4566\"\n ],\n \"endToEndEncryptionEnabled\": false,\n \"extendedLocation\": null,\n \"ftpPublishingUrl\": \"ftps://waws-prod-par-025.ftp.azurewebsites.windows.net/site/wwwroot\",\n \"functionAppConfig\": null,\n \"hostNameSslStates\": [\n {\n \"hostType\": \"Standard\",\n \"name\": \"tbweb-a60646.azurewebsites.azure.localhost.localstack.cloud:4566\",\n \"sslState\": \"Disabled\",\n \"thumbprint\": null,\n \"toUpdate\": null,\n \"virtualIp\": null\n },\n {\n \"hostType\": \"Repository\",\n \"name\": \"tbweb-a60646.scm.azurewebsites.azure.localhost.localstack.cloud:4566\",\n \"sslState\": \"Disabled\",\n \"thumbprint\": null,\n \"toUpdate\": null,\n \"virtualIp\": null\n }\n ],\n \"hostNames\": [\n \"tbweb-a60646.azurewebsites.azure.localhost.localstack.cloud:4566\"\n ],\n \"hostNamesDisabled\": false,\n \"hostingEnvironmentProfile\": null,\n \"httpsOnly\": false,\n \"hyperV\": false,\n \"id\": \"/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/mcpe2-15123a-tb-webapp-settings-36/providers/Microsoft.Web/sites/tbweb-a60646\",\n \"identity\": null,\n \"inProgressOperationId\": null,\n \"ipMode\": \"IPv4\",\n \"isDefaultContainer\": null,\n \"isXenon\": false,\n \"keyVaultReferenceIdentity\": \"SystemAssigned\",\n \"kind\": \"app,linux\",\n \"lastModifiedTimeUtc\": \"2026-09-27T21:34:47.179054+00:00\",\n \"location\": \"westeurope\",\n \"managedEnvironmentId\": null,\n \"maxNumberOfWorkers\": null,\n \"name\": \"tbweb-a60646\",\n \"outboundIpAddresses\": \"127.0.0.1\",\n \"outboundVnetRouting\": {\n \"allTraffic\": false,\n \"applicationTraffic\": false,\n \"backupRestoreTraffic\": false,\n \"contentShareTraffic\": false,\n \"imagePullTraffic\": false\n },\n \"possibleOutboundIpAddresses\": \"127.0.0.1\",\n \"publicNetworkAccess\": \"Enabled\",\n \"redundancyMode\": \"None\",\n \"repositorySiteName\": \"tbweb-a60646\",\n \"reserved\": false,\n \"resourceConfig\": null,\n \"resourceGroup\": \"mcpe2-15123a-tb-webapp-settings-36\",\n \"scmSiteAlsoStopped\": false,\n \"serverFarmId\": \"/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/mcpe2-15123a-tb-webapp-settings-36/providers/Microsoft.Web/serverfarms/tbplan-2eb4a8\",\n \"siteConfig\": {\n \"acrUseManagedIdentityCreds\": false,\n \"acrUserManagedIdentityId\": null,\n \"alwaysOn\": false,\n \"apiDefinition\": null,\n \"apiManagementConfig\": null,\n \"appCommandLine\": null,\n \"appSettings\": null,\n \"autoHealEnabled\": null,\n \"autoHealRules\": null,\n \"autoSwapSlotName\": null,\n \"azureStorageAccounts\": null,\n \"connectionStrings\": null,\n \"cors\": null,\n \"defaultDocuments\": null,\n \"detailedErrorLoggingEnabled\": null,\n \"documentRoot\": null,\n \"elasticWebAppScaleLimit\": 0,\n \"experiments\": null,\n \"ftpsState\": null,\n \"functionAppScaleLimit\": null,\n \"functionsRuntimeScaleMonitoringEnabled\": null,\n \"handlerMappings\": null,\n \"healthCheckPath\": null,\n \"http20Enabled\": false,\n \"http20ProxyFlag\": null,\n \"httpLoggingEnabled\": null,\n \"ipSecurityRestrictions\": [\n {\n \"action\": \"Allow\",\n \"description\": \"Allow all access\",\n \"headers\": null,\n \"ipAddress\": \"Any\",\n \"name\": \"Allow all\",\n \"priority\": 2147483647,\n \"subnetMask\": null,\n \"subnetTrafficTag\": null,\n \"tag\": null,\n \"vnetSubnetResourceId\": null,\n \"vnetTrafficTag\": null\n }\n ],\n \"ipSecurityRestrictionsDefaultAction\": null,\n \"javaContainer\": null,\n \"javaContainerVersion\": null,\n \"javaVersion\": null,\n \"keyVaultReferenceIdentity\": null,\n \"limits\": null,\n \"linuxFxVersion\": \"\",\n \"loadBalancing\": null,\n \"localMySqlEnabled\": null,\n \"logsDirectorySizeLimit\": null,\n \"machineKey\": null,\n \"managedPipelineMode\": null,\n \"managedServiceIdentityId\": null,\n \"metadata\": [],\n \"minTlsCipherSuite\": null,\n \"minTlsVersion\": null,\n \"minimumElasticInstanceCount\": 0,\n \"netFrameworkVersion\": null,\n \"nodeVersion\": null,\n \"numberOfWorkers\": 1,\n \"phpVersion\": null,\n \"powerShellVersion\": null,\n \"preWarmedInstanceCount\": null,\n \"publicNetworkAccess\": null,\n \"publishingUsername\": null,\n \"push\": null,\n \"pythonVersion\": null,\n \"remoteDebuggingEnabled\": null,\n \"remoteDebuggingVersion\": null,\n \"requestTracingEnabled\": null,\n \"requestTracingExpirationTime\": null,\n \"scmIpSecurityRestrictions\": [],\n \"scmIpSecurityRestrictionsDefaultAction\": null,\n \"scmIpSecurityRestrictionsUseMain\": null,\n \"scmMinTlsVersion\": null,\n \"scmType\": null,\n \"tracingOptions\": null,\n \"use32BitWorkerProcess\": null,\n \"virtualApplications\": null,\n \"vnetName\": null,\n \"vnetPrivatePortsCount\": null,\n \"vnetRouteAllEnabled\": null,\n \"webSocketsEnabled\": null,\n \"websiteTimeZone\": null,\n \"windowsFxVersion\": null,\n \"xManagedServiceIdentityId\": null\n },\n \"sku\": \"Free\",\n \"slotSwapStatus\": null,\n \"sshEnabled\": null,\n \"state\": \"Running\",\n \"storageAccountRequired\": false,\n \"suspendedTill\": null,\n \"tags\": null,\n \"targetSwapSlot\": null,\n \"trafficManagerHostNames\": null,\n \"type\": \"Microsoft.Web/sites\",\n \"usageState\": \"Normal\",\n \"virtualNetworkSubnetId\": null,\n \"workloadProfileName\": null\n}\n```\nORACLE ✅ az webapp config appsettings set --name tbweb-a60646 --resource-group mcpe2-15123a-tb-webapp-settings-36 --settings FEATURE_FLAG=on-7494a2\n```json\n[\n {\n \"name\": \"FEATURE_FLAG\",\n \"slotSetting\": false,\n \"value\": null\n }\n]\n```\nORACLE ❌ az webapp restart --name tbweb-a60646 --resource-group mcpe2-15123a-tb-webapp-settings-36\n```\n❌ **Command Failed** (exit 1, not-implemented)\n\nERROR: (NotImplemented) The API operation 'POST /subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/mcpe2-15123a-tb-webapp-settings-36/providers/Microsoft.Web/sites/tbweb-a60646/restart' is not yet implemented in LocalStack.\nCode: NotImplemented\nMessage: The API operation 'POST /subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/mcpe2-15123a-tb-webapp-settings-36/providers/Microsoft.Web/sites/tbweb-a60646/restart' is not yet implemented in LocalStack.\n\nThe LocalStack Azure emulator does not implement `POST /subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/mcpe2-15123a-tb-webapp-settings-36/providers/Microsoft.Web/sites/tbweb-a60646/restart` yet. This is an emulator limitation, so retrying will not help. See the implemented operations at http://127.0.0.1:4566/_localstack/coverage and https://docs.localstack.cloud/azure/.\n```", + "queries": [ + { + "command": "rest --method get --url \"/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/mcpe2-15123a-tb-webapp-settings-36/providers/Microsoft.Web/serverfarms/tbplan-2eb4a8?api-version=2026-07-15\"", + "repeat": 3, + "answer": { + "exitCode": 0, + "stdout": "{\"id\":\"/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/mcpe2-15123a-tb-webapp-settings-36/providers/Microsoft.Web/serverfarms/tbplan-2eb4a8\",\"kind\":\"linux\",\"location\":\"West Europe\",\"name\":\"tbplan-2eb4a8\",\"properties\":{\"asyncScalingEnabled\":false,\"elasticScaleEnabled\":false,\"geoRegion\":\"West Europe\",\"hyperV\":false,\"isCustomMode\":false,\"isSpot\":false,\"isXenon\":false,\"maximumElasticWorkerCount\":1,\"maximumNumberOfWorkers\":3,\"numberOfSites\":0,\"numberOfWorkers\":1,\"perSiteScaling\":false,\"provisioningState\":\"Succeeded\",\"reserved\":true,\"resourceGroup\":\"mcpe2-15123a-tb-webapp-settings-36\",\"status\":\"Ready\",\"subscription\":\"00000000-0000-0000-0000-000000000000\",\"targetWorkerCount\":0,\"targetWorkerSizeId\":0,\"zoneRedundant\":false},\"sku\":{\"capacity\":1,\"family\":\"B\",\"name\":\"B1\",\"size\":\"B1\",\"tier\":\"Basic\"},\"type\":\"Microsoft.Web/serverfarms\"}", + "stderr": "", + "classId": null + } + }, + { + "command": "rest --method get --url \"/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/mcpe2-15123a-tb-webapp-settings-36/providers/Microsoft.Web/sites/tbweb-a60646?api-version=2026-07-15\"", + "answer": { + "exitCode": 0, + "stdout": "{\"id\":\"/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/mcpe2-15123a-tb-webapp-settings-36/providers/Microsoft.Web/sites/tbweb-a60646\",\"kind\":\"app,linux\",\"location\":\"West Europe\",\"name\":\"tbweb-a60646\",\"properties\":{\"availabilityState\":\"Normal\",\"clientAffinityEnabled\":true,\"clientAffinityProxyEnabled\":false,\"clientCertEnabled\":false,\"clientCertMode\":\"Required\",\"containerSize\":0,\"customDomainVerificationId\":\"D3LC64FJI32WGRJRSJDFIHK90O3193U5GAQCCMSQ1VQAOHUOTDNRFJSST7BMQYST\",\"dailyMemoryTimeQuota\":0,\"defaultHostName\":\"tbweb-a60646.azurewebsites.azure.localhost.localstack.cloud:4566\",\"dnsConfiguration\":{},\"enabled\":true,\"enabledHostNames\":[\"tbweb-a60646.azurewebsites.azure.localhost.localstack.cloud:4566\",\"tbweb-a60646.scm.azurewebsites.azure.localhost.localstack.cloud:4566\"],\"endToEndEncryptionEnabled\":false,\"hostNameSslStates\":[{\"hostType\":\"Standard\",\"name\":\"tbweb-a60646.azurewebsites.azure.localhost.localstack.cloud:4566\",\"sslState\":\"Disabled\"},{\"hostType\":\"Repository\",\"name\":\"tbweb-a60646.scm.azurewebsites.azure.localhost.localstack.cloud:4566\",\"sslState\":\"Disabled\"}],\"hostNames\":[\"tbweb-a60646.azurewebsites.azure.localhost.localstack.cloud:4566\"],\"hostNamesDisabled\":false,\"httpsOnly\":false,\"hyperV\":false,\"ipMode\":\"IPv4\",\"isXenon\":false,\"keyVaultReferenceIdentity\":\"SystemAssigned\",\"lastModifiedTimeUtc\":\"2026-09-27T21:34:47.1790544887+00:00\",\"outboundIpAddresses\":\"127.0.0.1\",\"outboundVnetRouting\":{\"allTraffic\":false,\"applicationTraffic\":false,\"backupRestoreTraffic\":false,\"contentShareTraffic\":false,\"imagePullTraffic\":false},\"possibleOutboundIpAddresses\":\"127.0.0.1\",\"publicNetworkAccess\":\"Enabled\",\"redundancyMode\":\"None\",\"repositorySiteName\":\"tbweb-a60646\",\"reserved\":false,\"resourceGroup\":\"mcpe2-15123a-tb-webapp-settings-36\",\"scmSiteAlsoStopped\":false,\"serverFarmId\":\"/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/mcpe2-15123a-tb-webapp-settings-36/providers/Microsoft.Web/serverfarms/tbplan-2eb4a8\",\"siteConfig\":{\"acrUseManagedIdentityCreds\":false,\"alwaysOn\":false,\"functionAppScaleLimit\":0,\"http20Enabled\":true,\"ipSecurityRestrictions\":[{\"action\":\"Allow\",\"description\":\"Allow all access\",\"ipAddress\":\"Any\",\"name\":\"Allow all\",\"priority\":2147483647}],\"linuxFxVersion\":\"PYTHON|3.12\",\"metadata\":[],\"minimumElasticInstanceCount\":0,\"numberOfWorkers\":1,\"scmIpSecurityRestrictions\":[]},\"sku\":\"Free\",\"state\":\"Running\",\"storageAccountRequired\":false,\"usageState\":\"Normal\"},\"type\":\"Microsoft.Web/sites\"}", + "stderr": "", + "classId": null + } + }, + { + "command": "rest --method post --url \"/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/mcpe2-15123a-tb-webapp-settings-36/providers/Microsoft.Web/sites/tbweb-a60646/config/appsettings/list?api-version=2026-07-15\"", + "answer": { + "exitCode": 0, + "stdout": "{\"id\":\"/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/mcpe2-15123a-tb-webapp-settings-36/providers/Microsoft.Web/sites/tbweb-a60646/config/appsettings\",\"name\":\"appsettings\",\"properties\":{\"FEATURE_FLAG\":\"on-7494a2\"},\"type\":\"Microsoft.Web/sites/config\"}", + "stderr": "", + "classId": null + } + } + ] + }, + { + "task": "vnet-address-space", + "mode": "negative", + "passed": false, + "wait_s": 10, + "recorded": "2026-09-27T20:59:41Z", + "slots": { + "resource_group": "mcpe2-151aca-vnet-address-space-21", + "rg": "mcpe2-151aca-vnet-address-space-21", + "name": "vnet-core-231897", + "vnet_name": "vnet-core-231897", + "prefix": "10.230.0.0/16" + }, + "text": "", + "queries": [] + }, + { + "task": "vnet-address-space", + "mode": "oracle", + "passed": true, + "wait_s": 90, + "recorded": "2026-09-27T21:27:21Z", + "slots": { + "resource_group": "mcpe2-15123a-vnet-address-space-21", + "rg": "mcpe2-15123a-vnet-address-space-21", + "name": "vnet-core-bddc33", + "vnet_name": "vnet-core-bddc33", + "prefix": "10.242.0.0/16", + "_oracle": true + }, + "text": "ORACLE ✅ az network vnet show --name vnet-core-bddc33 --resource-group mcpe2-15123a-vnet-address-space-21\n```json\n{\n \"addressSpace\": {\n \"addressPrefixes\": [\n \"10.242.0.0/16\"\n ]\n },\n \"enableDdosProtection\": false,\n \"id\": \"/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/mcpe2-15123a-vnet-address-space-21/providers/Microsoft.Network/virtualNetworks/vnet-core-bddc33\",\n \"location\": \"westeurope\",\n \"name\": \"vnet-core-bddc33\",\n \"privateEndpointVNetPolicies\": \"Disabled\",\n \"provisioningState\": \"Succeeded\",\n \"resourceGroup\": \"mcpe2-15123a-vnet-address-space-21\",\n \"resourceGuid\": \"79e27cd7-2bed-4899-b6e9-a71af6b28601\",\n \"subnets\": [],\n \"type\": \"Microsoft.Network/virtualNetworks\",\n \"virtualNetworkPeerings\": []\n}\n```", + "queries": [] + } + ] +} diff --git a/tests/azure/evals/harness.test.ts b/tests/azure/evals/harness.test.ts new file mode 100644 index 0000000..26122c1 --- /dev/null +++ b/tests/azure/evals/harness.test.ts @@ -0,0 +1,591 @@ +/** + * The E2 harness against fake MCP sessions (no emulator, no key): one run in each mode, + * the setup-error path, the cleanup sequence, the agent's tool dispatch (the synthetic + * az_help included), the spend cap, the final sweep and the summary's gate. + */ +import type Anthropic from "@anthropic-ai/sdk"; +import * as A from "./agent"; +import * as H from "./harness"; +import * as V from "./verifiers"; +import * as VT from "./variants"; +import type { McpResult, Task } from "./types"; + +type Answer = { + exitCode: number | null; + stdout?: string; + stderr?: string; + classId?: string | null; +}; +type Handler = (command: string) => Answer; + +const MARKER = "ENVELOPE-MARKER-7f3a"; + +/** A session whose Azure tool answers with text + test envelope, from a handler. */ +function fakeSession(handler: Handler) { + const commands: string[] = []; + const session: H.Session & { commands: string[] } = { + commands, + callTool: async (name, args): Promise => { + if (name !== VT.AZURE_TOOL) throw new Error(`unexpected tool ${name}`); + const command = String(args.command); + commands.push(command); + const a = handler(command); + const text = + a.exitCode === 0 + ? a.stdout || "The command succeeded and printed no output." + : `❌ **Command Failed** (exit ${a.exitCode}, ${a.classId ?? "other"})\n\n${a.stderr ?? ""}`; + const envelope = { + exitCode: a.exitCode, + stdout: a.stdout ?? "", + stderr: a.stderr ?? "", + notes: [], + classId: a.classId ?? null, + truncated: false, + stoppedByTool: false, + emulatorSession: MARKER, + }; + return { + content: [ + { type: "text", text }, + { type: "text", text: JSON.stringify(envelope) }, + ], + }; + }, + }; + return session; +} + +const OK = (stdout = ""): Answer => ({ exitCode: 0, stdout }); +const NOT_FOUND: Answer = { exitCode: 1, stderr: "ERROR: NOT FOUND({})", classId: "not-found" }; + +/** The harness's own commands for group create and cleanup, then the task's. */ +function harnessHandler(rg: string, task: Record Answer)>) { + let exists = 0; + return (command: string): Answer => { + if (command === `group create --name ${rg} --location westeurope`) + return OK(JSON.stringify({ name: rg })); + if (command === `group exists --name ${rg}`) return OK(exists++ === 0 ? "true" : "false"); + if (command.startsWith(`resource list --resource-group ${rg}`)) return OK("[]"); + if (command.includes("/providers/Microsoft.Authorization/locks?")) + return OK(JSON.stringify({ value: [] })); + if (command === `group delete --name ${rg} --yes`) return OK(""); + const hit = task[command]; + if (hit) return typeof hit === "function" ? hit() : hit; + throw new Error(`unscripted harness command: ${command}`); + }; +} + +function fakeClock() { + let t = 0; + return { now: () => t, sleep: async (ms: number) => void (t += ms) }; +} + +function deps( + agentSession: H.Session, + harnessSession: H.Session, + over: Partial = {} +): H.Deps { + const clock = fakeClock(); + return { + lib: V, + agent: A, + variants: VT, + agentSession, + harnessSession, + now: clock.now, + sleep: clock.sleep, + log: () => undefined, + scrub: (s) => s, + ...over, + }; +} + +const TASK: Task = { + id: "fake-read", + tier: "T-A", + caps: "T-A", + opId: "Microsoft.Fake/Things_Get", + verb: "read", + prompts: [ + "What value does the thing {thing} in resource group {rg} hold?", + "Tell me {thing}'s value ({rg}).", + ], + oracle: ["thing show --name {thing} --resource-group {rg}"], + verify: V.claims("{value}"), + setup: async (ctx) => { + ctx.slots.thing = "t1"; + ctx.slots.value = "v-42"; + ctx.slots._secret_snapshot = "before"; + await V.azOk(ctx.q, `thing create --name t1 --resource-group ${ctx.rg}`); + }, +}; + +function spec(mode: H.Mode, rg: string, over: Partial = {}): H.RunSpec { + return { + task: TASK, + mode, + variant: VT.buildVariantTools("compact", [ + { name: VT.AZURE_TOOL, description: "Run az.", inputSchema: { type: "object" } }, + ]), + runIndex: 0, + phrasing: 0, + rg, + model: "claude-opus-5-5", + effort: "high", + price: A.PRICES["claude-opus-5-5"], + prefixTokens: 1000, + budget: { maxUsd: 5, perRunUsd: 1, spent: 0 }, + verifyWaitS: 5, + agentTimeoutMs: 1000, + harnessTimeoutMs: 1000, + ...over, + }; +} + +describe("runOne", () => { + test("oracle: the reference command through the agent session, the verifier through the harness", async () => { + const rg = "mcpe2-t-fake-read-1"; + const harness = fakeSession( + harnessHandler(rg, { [`thing create --name t1 --resource-group ${rg}`]: OK("{}") }) + ); + const agent = fakeSession((c) => + c === `thing show --name t1 --resource-group ${rg}` + ? OK('{\n "value": "v-42"\n}') + : NOT_FOUND + ); + const rec = await H.runOne(spec("oracle", rg), deps(agent, harness)); + expect(rec).toMatchObject({ + task_id: "fake-read", + mode: "oracle", + success: true, + expected: "pass", + failure: null, + tool_calls: 1, + }); + expect(agent.commands).toEqual([`thing show --name t1 --resource-group ${rg}`]); + expect(harness.commands).not.toContain(`thing show --name t1 --resource-group ${rg}`); + expect(rec.transcript).toContain(`ORACLE ✅ az thing show --name t1 --resource-group ${rg}`); + expect(rec.cleanup).toMatchObject({ group_delete: "ok", gone: true }); + const input = rec.verify_input as { slots: Record }; + expect(input.slots).toMatchObject({ _oracle: true, _secret_snapshot: "before", thing: "t1" }); + // Recorded slots leave the internal ones out. + expect(rec.slots).toEqual({ resource_group: rg, rg, thing: "t1", value: "v-42" }); + }); + + test("negative: setup only, and the verifier must fail", async () => { + const rg = "mcpe2-t-fake-read-2"; + const harness = fakeSession( + harnessHandler(rg, { [`thing create --name t1 --resource-group ${rg}`]: OK("{}") }) + ); + const agent = fakeSession(() => { + throw new Error("the agent session must not be used"); + }); + const rec = await H.runOne(spec("negative", rg), deps(agent, harness)); + expect(rec).toMatchObject({ mode: "negative", expected: "fail", success: true, failure: null }); + expect((rec.verifier as { passed: boolean }).passed).toBe(false); + }); + + test("a setup failure is not scored, and the group is still cleaned up", async () => { + const rg = "mcpe2-t-fake-read-3"; + const harness = fakeSession( + harnessHandler(rg, { + [`thing create --name t1 --resource-group ${rg}`]: { exitCode: 1, stderr: "ERROR: boom" }, + }) + ); + const rec = await H.runOne( + spec("oracle", rg), + deps( + fakeSession(() => OK()), + harness + ) + ); + expect(rec).toMatchObject({ success: null, failure: "setup_error" }); + expect(String(rec.reason)).toContain("fixture: thing create --name t1"); + expect(harness.commands).toContain(`group delete --name ${rg} --yes`); + }); + + test("llm: the loop through a fake client; the envelope never reaches it; cost and budget", async () => { + const rg = "mcpe2-t-fake-read-4"; + const harness = fakeSession( + harnessHandler(rg, { [`thing create --name t1 --resource-group ${rg}`]: OK("{}") }) + ); + const agent = fakeSession((c) => + c.startsWith("thing show") ? OK('{"value": "v-42"}') : NOT_FOUND + ); + const calls: unknown[] = []; + const responses = [ + { + content: [ + { + type: "tool_use", + id: "tu1", + name: VT.AZURE_TOOL, + input: { command: `thing show --name t1 --resource-group ${rg}` }, + }, + ], + stop_reason: "tool_use", + usage: { + input_tokens: 1000, + output_tokens: 100, + cache_creation_input_tokens: 0, + cache_read_input_tokens: 0, + }, + }, + { + content: [{ type: "text", text: "The thing t1 holds v-42." }], + stop_reason: "end_turn", + usage: { + input_tokens: 200, + output_tokens: 50, + cache_creation_input_tokens: 0, + cache_read_input_tokens: 1000, + }, + }, + ]; + const client: A.MessagesClient = { + messages: { + create: async (params) => { + calls.push(JSON.parse(JSON.stringify(params))); + return { + id: "m", + type: "message", + role: "assistant", + model: params.model, + stop_sequence: null, + stop_details: null, + ...responses.shift(), + } as unknown as Anthropic.Message; + }, + }, + }; + const budget = { maxUsd: 5, perRunUsd: 1, spent: 0.5 }; + const rec = await H.runOne( + spec("llm", rg, { budget }), + deps(agent, harness, { client, classifyError: () => "api_error" }) + ); + expect(rec).toMatchObject({ + mode: "llm", + success: true, + failure: null, + turns: 2, + tool_calls: 1, + refusal: false, + prompt: `What value does the thing t1 in resource group ${rg} hold?`, + }); + // (1000*4 + 100*20 + 200*4 + 50*20 + 1000*0.2) / 1e6 + expect(rec.cost_usd).toBe(0.008); + expect(budget.spent).toBeCloseTo(0.508, 6); + expect(JSON.stringify(calls)).not.toContain(MARKER); + expect(rec.egress_events).toEqual([]); + expect(rec.final_text).toBe("The thing t1 holds v-42."); + }); + + test("llm without a client is a programming error, not a silent pass", async () => { + const rg = "mcpe2-t-fake-read-5"; + const harness = fakeSession( + harnessHandler(rg, { [`thing create --name t1 --resource-group ${rg}`]: OK("{}") }) + ); + await expect( + H.runOne( + spec("llm", rg), + deps( + fakeSession(() => OK()), + harness + ) + ) + ).rejects.toThrow(/needs a client/); + }); +}); + +describe("cleanupRun", () => { + test("settle, locks, vault delete and purge, group delete, soft-delete purge, gone", async () => { + const rg = "mcpe2-t-kv-1"; + let exists = 0; + const lockId = `/subscriptions/${V.SUBSCRIPTION}/resourceGroups/${rg}/providers/Microsoft.Authorization/locks/lk1`; + const log: string[] = []; + const session = fakeSession((c) => { + log.push(c); + if (c === `group exists --name ${rg}`) return OK(exists++ === 0 ? "true" : "false"); + if (c.startsWith(`resource list --resource-group ${rg} --query`)) return OK("[]"); + if (c.includes("/locks?api-version=2020-05-01") && c.includes("--method get")) + return OK(JSON.stringify({ value: [{ id: lockId }] })); + if (c.includes(`--method delete --url "${lockId}?api-version=2020-05-01"`)) return OK(""); + if ( + c.startsWith( + `resource list --resource-group ${rg} --resource-type Microsoft.KeyVault/vaults` + ) + ) + return OK('["kv1"]'); + if (c === `keyvault delete --name kv1 --resource-group ${rg}`) return OK(""); + if (c === "keyvault purge --name kv1") return OK(""); + if (c === "keyvault purge --name kv2") return NOT_FOUND; + if (c === `group delete --name ${rg} --yes`) return OK(""); + if (c.startsWith("keyvault list-deleted")) return OK("[]"); + throw new Error(`unscripted: ${c}`); + }); + const task: Task = { ...TASK, vaultSlots: ["vault_name", "kv2"] }; + const clock = fakeClock(); + const query = H.sessionQuery(deps(session, session, clock), session, 5, 1000); + const out = await H.cleanupRun(query, V, task, rg, { vault_name: "kv1", kv2: "kv2" }); + expect(out).toMatchObject({ + locks_removed: ["lk1"], + vaults_purged: ["kv1"], + group_delete: "ok", + gone: true, + }); + const order = [ + `group exists --name ${rg}`, + "resource list", + "Microsoft.Authorization/locks?", + "--method delete", + "--resource-type Microsoft.KeyVault/vaults", + "keyvault delete --name kv1", + "keyvault purge --name kv1", + "keyvault purge --name kv2", + `group delete --name ${rg} --yes`, + "keyvault list-deleted", + `group exists --name ${rg}`, + ]; + let i = 0; + for (const want of order) { + while (i < log.length && !log[i].includes(want)) i++; + expect(i).toBeLessThan(log.length); + i++; + } + }); + + test("a group that is already gone needs nothing else", async () => { + const session = fakeSession((c) => (c.startsWith("group exists") ? OK("false") : NOT_FOUND)); + const out = await H.cleanupRun( + H.sessionQuery(deps(session, session), session, 5, 1000), + V, + TASK, + "g", + {} + ); + expect(out.gone).toBe(true); + expect(session.commands).toEqual(["group exists --name g"]); + }); +}); + +describe("the agent's tool dispatch", () => { + const listed: VT.ListedTool[] = [ + { + name: VT.AZURE_TOOL, + description: `x\n${VT.HELP_SENTENCE}\ny`, + inputSchema: { type: "object" }, + }, + { name: "localstack-docs", description: "docs", inputSchema: { type: "object" } }, + ]; + + test("az_help runs ` --help` through the Azure tool", async () => { + const agent = fakeSession(() => OK("Group\n az storage account : Manage storage accounts.")); + const vt = VT.buildVariantTools("help-tool", listed); + const call = H.agentToolCaller( + deps(agent, agent), + vt.tools.map((t) => t.name), + 1000 + ); + const out = await call("az_help", { prefix: "storage account" }); + expect(agent.commands).toEqual(["storage account --help"]); + expect(out.text).toContain("Manage storage accounts"); + expect(out.text).not.toContain(MARKER); + expect(out.envelope?.emulatorSession).toBe(MARKER); + }); + + test("a tool the run did not offer is an error the model sees", async () => { + const agent = fakeSession(() => OK()); + const call = H.agentToolCaller(deps(agent, agent), [VT.AZURE_TOOL], 1000); + expect(await call("localstack-docs", {})).toMatchObject({ + isError: true, + text: "❌ Unknown tool 'localstack-docs' for this run", + }); + expect(agent.commands).toEqual([]); + }); + + test("a transport failure is reported to the model as a failed call", async () => { + const broken: H.Session = { + callTool: async () => { + throw new Error("timeout waiting for tools/call"); + }, + }; + const call = H.agentToolCaller(deps(broken, broken), [VT.AZURE_TOOL], 1000); + expect(await call(VT.AZURE_TOOL, { command: "group list" })).toMatchObject({ + isError: true, + text: "❌ Tool call failed: Error: timeout waiting for tools/call", + command: "group list", + }); + }); +}); + +describe("the spend cap", () => { + test("no new run once the spend reaches the cap", () => { + expect(H.canStart({ maxUsd: 1, perRunUsd: 1, spent: 0.99 })).toBe(true); + expect(H.canStart({ maxUsd: 1, perRunUsd: 1, spent: 1 })).toBe(false); + expect(H.remaining({ maxUsd: 1, perRunUsd: 1, spent: 0.25 })).toBe(0.75); + }); + + test("an exhausted budget stops a run's loop before its first request", async () => { + const rg = "mcpe2-t-fake-read-6"; + const harness = fakeSession( + harnessHandler(rg, { [`thing create --name t1 --resource-group ${rg}`]: OK("{}") }) + ); + let requests = 0; + const client: A.MessagesClient = { + messages: { + create: async () => { + requests++; + throw new Error("must not be called"); + }, + }, + }; + const budget = { maxUsd: 1, perRunUsd: 1, spent: 1 }; + const rec = await H.runOne( + spec("llm", rg, { budget }), + deps( + fakeSession(() => OK()), + harness, + { client, classifyError: () => "api_error" } + ) + ); + expect(requests).toBe(0); + expect(rec).toMatchObject({ success: false, failure: "spend_cap", cost_usd: 0 }); + }); +}); + +describe("finalSweep", () => { + test("deletes what an emptied group index left behind, and checks again", async () => { + const orphan = `/subscriptions/${V.SUBSCRIPTION}/resourceGroups/MCPE2-X-T-1/providers/Microsoft.KeyVault/vaults/kvo`; + let gone = false; + const session = fakeSession((c) => { + if (c.includes("/providers/Microsoft.KeyVault/vaults?api-version=2023-07-01")) + return OK( + JSON.stringify({ + value: [ + { id: orphan }, + { + id: `/subscriptions/${V.SUBSCRIPTION}/resourceGroups/other/providers/Microsoft.KeyVault/vaults/theirs`, + }, + ], + }) + ); + if (c.includes(`--method delete --url "${orphan}?`)) { + gone = true; + return OK(""); + } + if (c.includes(`--method get --url "${orphan}?`)) return gone ? NOT_FOUND : OK("{}"); + if (c.startsWith("group list")) return OK("[]"); + if (c.startsWith("keyvault list-deleted")) return OK('["kvo", "not-mine"]'); + if (c === "keyvault purge --name kvo") return OK(""); + throw new Error(`unscripted: ${c}`); + }); + const out = await H.finalSweep(H.sessionQuery(deps(session, session), session, 5, 1000), V, { + types: ["Microsoft.KeyVault/vaults@2023-07-01"], + groups: new Set(["mcpe2-x-t-1"]), + prefix: "mcpe2-x", + vaults: new Set(["kvo"]), + apim: new Set(), + }); + expect(out.orphans_deleted).toEqual([{ id: orphan, delete: 200, after: 404 }]); + expect(out.vaults_purged).toEqual(["kvo"]); + expect(session.commands.some((c) => c.includes("theirs"))).toBe(false); + }); +}); + +describe("selection, order, names", () => { + const tasks = [ + { ...TASK, id: "a", tier: "T-A" as const }, + { ...TASK, id: "b", tier: "T-B" as const }, + ]; + test("selectTasks by id and tier, refusing unknown ones", () => { + expect(H.selectTasks(tasks, { tiers: ["t-b"] }).map((t) => t.id)).toEqual(["b"]); + expect(H.selectTasks(tasks, { ids: ["a"] }).map((t) => t.id)).toEqual(["a"]); + expect(() => H.selectTasks(tasks, { ids: ["zz"] })).toThrow(/unknown task id/); + expect(() => H.selectTasks(tasks, { tiers: ["T-D"] })).toThrow(/unknown tier/); + }); + + test("the seeded variant order repeats", () => { + const a = H.shuffled(["x", "y", "z", "w"], H.seededRandom(7)); + const b = H.shuffled(["x", "y", "z", "w"], H.seededRandom(7)); + expect(a).toEqual(b); + expect([...a].sort()).toEqual(["w", "x", "y", "z"]); + }); + + test("group names carry the run's prefix and fit Azure's 90 characters", () => { + expect(H.groupName("mcpe2-abc123", "kv-sign", 3)).toBe("mcpe2-abc123-kv-sign-3"); + expect(H.groupName("mcpe2-abc123", "x".repeat(200), 1).length).toBe(90); + }); +}); + +describe("summarize and the gate", () => { + const row = (task: string, success: boolean | null, extra: Record = {}) => ({ + mode: "llm", + variant: "compact", + task_id: task, + success, + cost_usd: 0.02, + cost_warm_usd: 0.015, + time_to_verified_ms: success ? 9000 : null, + egress_events: [], + ...extra, + }); + + test("95% success and zero egress pass the gate; cost per completed task is warm", () => { + const rows = [...Array.from({ length: 19 }, (_, i) => row(`t${i}`, true)), row("t19", false)]; + const s = H.summarize(rows) as { llm: Record }; + expect(s.llm.compact).toMatchObject({ + scored: 20, + successes: 19, + success_rate: 0.95, + gate: { passed: true }, + }); + expect(s.llm.compact.cost_per_completed_task_warm_usd).toBeCloseTo((20 * 0.015) / 19, 6); + expect(s.llm.compact.failed_tasks).toEqual(["t19 0/1"]); + }); + + test("one egress event fails the gate; infrastructure faults are not scored", () => { + const rows = [ + row("a", true), + row("b", true, { egress_events: [{ hosts: ["management.azure.com"] }] }), + row("c", null, { failure: "infra_error" }), + ]; + const s = H.summarize(rows) as { llm: Record }; + expect(s.llm.compact).toMatchObject({ + scored: 2, + success_rate: 1, + egress_events: 1, + infra_errors: 1, + gate: { zero_egress: false, passed: false }, + }); + }); + + test("refusals are counted, and Key Vault crypto refusals apart", () => { + const rows = [ + row("kv-sign", false, { refusal: true, kv_crypto: true }), + row("x", false, { refusal: true }), + row("kv-decrypt", true, { kv_crypto: true }), + ]; + const s = H.summarize(rows) as { llm: Record }; + expect(s.llm.compact).toMatchObject({ refusals: 2, kv_crypto_refusals: 1, kv_crypto_runs: 2 }); + }); + + test("oracle and negative runs report met / not met", () => { + const s = H.summarize([ + { mode: "oracle", task_id: "a", success: true, cleanup: { gone: true } }, + { + mode: "oracle", + task_id: "b", + success: false, + reason: "r", + cleanup: { gone: false }, + rg: "g", + }, + ]) as { oracle: Record }; + expect(s.oracle).toMatchObject({ + runs: 2, + met: 1, + not_met: [{ task: "b", reason: "r" }], + cleanup_not_gone: ["g"], + }); + }); +}); diff --git a/tests/azure/evals/harness.ts b/tests/azure/evals/harness.ts new file mode 100644 index 0000000..7c757cf --- /dev/null +++ b/tests/azure/evals/harness.ts @@ -0,0 +1,809 @@ +/** + * One E2 run, end to end, and what surrounds it: the cleanup, the budget stop and the + * summary. + * + * group create -> the task's setup (harness session) -> the oracle, the agent or nothing + * (agent session) -> the verifier (harness session, every answer recorded) -> teardown + * -> cleanup (settle, locks, vault purge, group delete, soft-delete purges, gone check) + * + * Two server sessions: the agent's (what the model, or the oracle, calls) and the harness's + * own, which does setup, verification and cleanup, so verification never shares the + * agent's session. The modules this file uses at runtime arrive in `deps` (run.mjs loads + * everything with Node's type stripping; see verifiers.ts). + */ +import type * as A from "./agent"; +import type * as V from "./verifiers"; +import type * as VT from "./variants"; +import type { AzAnswer, EgressEvent, McpResult, Price, Query, Slots, Task, Verdict } from "./types"; + +export type Mode = "llm" | "oracle" | "negative"; + +export interface Session { + callTool(name: string, args: Record, timeoutMs?: number): Promise; +} + +export interface Deps { + lib: typeof V; + agent: typeof A; + variants: typeof VT; + agentSession: Session; + harnessSession: Session; + now: () => number; + sleep: (ms: number) => Promise; + log: (line: string) => void; + scrub: (text: string) => string; + client?: A.MessagesClient; + classifyError?: (e: unknown) => A.ErrorClass; +} + +// ── the spend cap ───────────────────────────────────────────────────────────── + +export interface Budget { + /** Stop starting runs once the spend reaches it (--max-usd). */ + maxUsd: number; + /** Stop a run's agent loop once the run's own cost reaches it. */ + perRunUsd: number; + /** Billed so far (pre-warms included). */ + spent: number; +} + +export function canStart(b: Budget): boolean { + return b.spent < b.maxUsd; +} + +export function remaining(b: Budget): number { + return b.maxUsd - b.spent; +} + +// ── sessions as queries ─────────────────────────────────────────────────────── + +function errorMessage(e: unknown): string { + return e instanceof Error ? `${e.name}: ${e.message}` : String(e); +} + +/** An AzAnswer from a split tool result; no envelope means the tool refused the command. */ +export function answerOf(s: A.SplitResult): AzAnswer { + if (s.envelope) { + return { + exitCode: s.envelope.exitCode, + stdout: s.envelope.stdout ?? "", + stderr: s.envelope.stderr ?? "", + classId: s.envelope.classId ?? null, + text: s.text, + notes: s.envelope.notes ?? [], + stoppedByTool: Boolean(s.envelope.stoppedByTool), + }; + } + return { + exitCode: null, + stdout: "", + stderr: s.text, + classId: "tool-refused", + text: s.text, + notes: [], + }; +} + +/** A Query over a server session's Azure tool (egress refusals, which must not happen, are collected). */ +export function sessionQuery( + deps: Deps, + session: Session, + waitS: number, + timeoutMs: number, + egress: EgressEvent[] = [] +): Query { + let calls = 0; + return { + waitS, + now: deps.now, + sleep: deps.sleep, + az: async (command) => { + const index = calls++; + let result: McpResult; + try { + result = await session.callTool(deps.variants.AZURE_TOOL, { command }, timeoutMs); + } catch (e) { + return { + exitCode: null, + stdout: "", + stderr: `tool call failed: ${errorMessage(e)}`, + classId: "tool-error", + text: "", + }; + } + const split = deps.agent.splitEnvelope(result); + egress.push(...deps.agent.egressOf(index, command, split.text, split.envelope).events); + return answerOf(split); + }, + }; +} + +/** The agent's tool calls: the Azure tool, the synthetic az_help, or (all tools) any other. */ +export function agentToolCaller( + deps: Deps, + offered: readonly string[], + timeoutMs: number +): (name: string, input: Record) => Promise { + const names = new Set(offered); + const { AZURE_TOOL, HELP_TOOL, helpCommand } = deps.variants; + return async (name, input) => { + const t0 = deps.now(); + if (!names.has(name)) + return { + text: `❌ Unknown tool '${name}' for this run`, + isError: true, + ms: 0, + envelope: null, + }; + let target = name; + let args = input; + let command: string | undefined; + if (name === HELP_TOOL) { + command = helpCommand(String(input.prefix ?? "")); + target = AZURE_TOOL; + args = { command }; + } else if (name === AZURE_TOOL) { + command = String(input.command ?? ""); + } + try { + const result = await deps.agentSession.callTool(target, args, timeoutMs); + const split = deps.agent.splitEnvelope(result); + return { + text: split.text, + isError: split.isError, + ms: deps.now() - t0, + envelope: split.envelope, + command, + }; + } catch (e) { + return { + text: `❌ Tool call failed: ${errorMessage(e)}`, + isError: true, + ms: deps.now() - t0, + envelope: null, + command, + }; + } + }; +} + +// ── cleanup ─────────────────────────────────────────────────────────────────── + +function list(stdout: string | undefined): string[] { + try { + const v = JSON.parse(stdout || "[]"); + return Array.isArray(v) ? v.filter((x): x is string => typeof x === "string") : []; + } catch { + return []; + } +} + +function slotNames( + task: Task, + key: "vaultSlots" | "apimSlots" | "appConfigSlots", + slots: Slots +): string[] { + return (task[key] ?? []) + .map((s) => slots[s]) + .filter((x): x is string => typeof x === "string" && x !== ""); +} + +/** + * Leaves nothing behind: wait until nothing in the group is + * still provisioning (a group deleted while a storage account is Creating leaks its Azurite + * container), remove its locks, delete and purge its Key Vaults, delete the group and wait, + * purge what soft-deleted, and check the group is gone. + */ +export async function cleanupRun( + q: Query, + lib: typeof V, + task: Task, + rg: string, + slots: Slots +): Promise> { + const t0 = q.now(); + const out: Record = {}; + const exists = await q.az(`group exists --name ${rg}`); + if (exists.exitCode === 0 && exists.stdout.trim() === "false") { + out.gone = true; + out.ms = q.now() - t0; + return out; + } + const pendingQuery = + "[?provisioningState && provisioningState!='Succeeded' && provisioningState!='Failed' && provisioningState!='Canceled'].id"; + const settleDeadline = q.now() + 90_000; + let pending: string[] = []; + for (;;) { + pending = list( + (await q.az(`resource list --resource-group ${rg} --query "${pendingQuery}" -o json`)).stdout + ); + if (pending.length === 0 || q.now() > settleDeadline) break; + await q.sleep(2000); + } + if (pending.length) out.still_provisioning = pending.slice(0, 10); + + const locks = await lib.arm( + q, + "GET", + `/subscriptions/${lib.SUBSCRIPTION}/resourceGroups/${rg}/providers/Microsoft.Authorization/locks`, + "2020-05-01" + ); + const lockIds = + (lib.dotted(locks.body, "value") as unknown[] | null) + ?.map((l) => lib.dotted(l, "id")) + .filter((x): x is string => typeof x === "string") ?? []; + const removedLocks: string[] = []; + for (const id of lockIds) { + const r = await lib.arm(q, "DELETE", id, "2020-05-01"); + if (r.status === 200) removedLocks.push(id.split("/").pop() || id); + } + if (removedLocks.length) out.locks_removed = removedLocks; + + const vaultsInGroup = list( + ( + await q.az( + `resource list --resource-group ${rg} --resource-type Microsoft.KeyVault/vaults --query "[].name" -o json` + ) + ).stdout + ); + const vaults = [...new Set([...vaultsInGroup, ...slotNames(task, "vaultSlots", slots)])]; + const purged: string[] = []; + for (const n of vaultsInGroup) await q.az(`keyvault delete --name ${n} --resource-group ${rg}`); + for (const n of vaults) { + if ((await q.az(`keyvault purge --name ${n}`)).exitCode === 0) purged.push(n); + } + + const del = await q.az(`group delete --name ${rg} --yes`); + out.group_delete = + del.exitCode === 0 ? "ok" : `exit ${del.exitCode}: ${(del.stderr || "").trim().slice(0, 200)}`; + + if (vaults.length) { + const deleted = new Set( + list((await q.az(`keyvault list-deleted --query "[].name" -o json`)).stdout) + ); + for (const n of vaults.filter((x) => deleted.has(x))) { + if ((await q.az(`keyvault purge --name ${n}`)).exitCode === 0) purged.push(n); + } + } + if (purged.length) out.vaults_purged = purged; + const apim = slotNames(task, "apimSlots", slots); + if (apim.length) { + const deleted = new Set( + list((await q.az(`apim deletedservice list --query "[].name" -o json`)).stdout) + ); + const done = []; + for (const n of apim.filter((x) => deleted.has(x))) { + if ( + (await q.az(`apim deletedservice purge --service-name ${n} --location westeurope`)) + .exitCode === 0 + ) + done.push(n); + } + if (done.length) out.apim_purged = done; + } + const stores = slotNames(task, "appConfigSlots", slots); + if (stores.length) { + const deleted = new Set( + list((await q.az(`appconfig list-deleted --query "[].name" -o json`)).stdout) + ); + const done = []; + for (const n of stores.filter((x) => deleted.has(x))) { + if ((await q.az(`appconfig purge --name ${n} --yes`)).exitCode === 0) done.push(n); + } + if (done.length) out.appconfig_purged = done; + } + const after = await q.az(`group exists --name ${rg}`); + out.gone = after.exitCode === 0 && after.stdout.trim() === "false"; + out.ms = q.now() - t0; + return out; +} + +/** + * The end-of-run orphan sweep: a PUT on a group that already + * exists empties the group's resource index, and a later group delete then leaves those + * resources running. The per-type subscription listings still show them, so every listed + * type is read and whatever still sits in one of this run's groups is deleted. Then the + * groups, the soft-deleted vaults and APIM services of this run are checked once more. + */ +export async function finalSweep( + q: Query, + lib: typeof V, + opts: { + types: readonly string[]; + groups: ReadonlySet; + prefix: string; + vaults: ReadonlySet; + apim: ReadonlySet; + } +): Promise> { + const groups = new Set([...opts.groups].map((g) => g.toLowerCase())); + const orphans: Array> = []; + for (const spec of [...new Set(opts.types)]) { + const [type, api] = spec.split("@"); + const r = await lib.arm(q, "GET", `/subscriptions/${lib.SUBSCRIPTION}/providers/${type}`, api); + const items = lib.dotted(r.body, "value"); + for (const item of Array.isArray(items) ? items : []) { + const id = lib.pyStr(lib.dotted(item, "id") ?? ""); + const parts = id.split("/"); + if ( + parts.length < 5 || + parts[3].toLowerCase() !== "resourcegroups" || + !groups.has(parts[4].toLowerCase()) + ) + continue; + const del = await lib.arm(q, "DELETE", id, api); + const gone = await lib.waitStatus(q, id, api, [404, 410], 60); + orphans.push({ id, delete: del.status, after: gone.status }); + } + } + const leftGroups = list( + (await q.az(`group list --query "[?starts_with(name, '${opts.prefix}')].name" -o json`)).stdout + ); + for (const g of leftGroups) await q.az(`group delete --name ${g} --yes`); + const deletedVaults = list( + (await q.az(`keyvault list-deleted --query "[].name" -o json`)).stdout + ).filter((n) => opts.vaults.has(n)); + for (const n of deletedVaults) await q.az(`keyvault purge --name ${n}`); + const deletedApim = opts.apim.size + ? list((await q.az(`apim deletedservice list --query "[].name" -o json`)).stdout).filter((n) => + opts.apim.has(n) + ) + : []; + for (const n of deletedApim) + await q.az(`apim deletedservice purge --service-name ${n} --location westeurope`); + const stillGroups = list( + (await q.az(`group list --query "[?starts_with(name, '${opts.prefix}')].name" -o json`)).stdout + ); + const stillVaults = list( + (await q.az(`keyvault list-deleted --query "[].name" -o json`)).stdout + ).filter((n) => opts.vaults.has(n)); + return { + orphans_deleted: orphans, + groups_left_before: leftGroups, + vaults_purged: deletedVaults, + apim_purged: deletedApim, + groups_left: stillGroups, + deleted_vaults_left: stillVaults, + clean: stillGroups.length === 0 && stillVaults.length === 0, + }; +} + +// ── one run ───────────────────────────────────────────────────────────────── + +export interface RunSpec { + task: Task; + mode: Mode; + variant: VT.VariantTools; + runIndex: number; + phrasing: number; + rg: string; + model: string; + effort: string; + price?: Price; + prefixTokens?: number | null; + budget: Budget; + verifyWaitS: number; + agentTimeoutMs: number; + harnessTimeoutMs: number; +} + +const MAX_RECORDED_QUERIES = 80; +const MAX_RECORDED_STDOUT = 20_000; + +function compactQueries(log: V.RecordedCall[]): Array> { + return log.slice(0, MAX_RECORDED_QUERIES).map((c) => ({ + command: c.command, + ...(c.repeat ? { repeat: c.repeat } : {}), + answer: { + ...c.answer, + stdout: + c.answer.stdout.length > MAX_RECORDED_STDOUT + ? c.answer.stdout.slice(0, MAX_RECORDED_STDOUT) + : c.answer.stdout, + ...(c.answer.stdout.length > MAX_RECORDED_STDOUT ? { stdout_truncated: true } : {}), + }, + })); +} + +function recordSlots(slots: Slots): Record { + const out: Record = {}; + for (const [k, value] of Object.entries(slots)) { + if (!k.startsWith("_") && ["string", "number", "boolean"].includes(typeof value)) { + out[k] = + typeof value === "string" && value.length > 400 + ? `${value.slice(0, 40)}...(${value.length} chars)` + : value; + } + } + return out; +} + +const MAX_VERIFY_TEXT = 64_000; + +/** The slots a verifier read: primitives only, internal ones (snapshots) included. */ +function verifySlots(slots: Slots): Record { + const out: Record = {}; + for (const [k, value] of Object.entries(slots)) { + if (["string", "number", "boolean"].includes(typeof value)) out[k] = value; + } + return out; +} + +function nowIso(): string { + return new Date().toISOString().replace(/\.\d{3}Z$/, "Z"); +} + +/** One run of one task; returns its JSONL record. Never throws for a task's own failure. */ +export async function runOne(spec: RunSpec, deps: Deps): Promise> { + const { task, mode, rg, lib } = { ...spec, lib: deps.lib }; + const rec: Record = { + task_id: task.id, + tier: task.tier, + op_id: task.opId, + provider: task.opId.split("/")[0], + verb: task.verb, + variant: spec.variant.variant, + mode, + model: mode === "llm" ? spec.model : null, + effort: mode === "llm" ? spec.effort : null, + run: spec.runIndex, + phrasing: spec.phrasing, + rg, + guessable: Boolean(task.guessable), + kv_crypto: Boolean(task.kvCrypto), + started: nowIso(), + }; + const harnessEgress: EgressEvent[] = []; + const setupLog: V.RecordedCall[] = []; + const q = sessionQuery( + deps, + deps.harnessSession, + spec.verifyWaitS, + spec.harnessTimeoutMs, + harnessEgress + ); + const setupQ = lib.recordingQuery(q, setupLog); + const slots: Slots = { resource_group: rg, rg }; + + const setupStart = deps.now(); + try { + await lib.azOk(setupQ, `group create --name ${rg} --location westeurope`, "resource group"); + if (task.setup) await task.setup(lib.makeSetupContext(setupQ, slots, rg)); + } catch (e) { + rec.setup = { + ms: deps.now() - setupStart, + calls: setupLog.length, + error: deps.scrub(errorMessage(e)).slice(0, 500), + }; + rec.success = null; + rec.failure = "setup_error"; + rec.reason = `setup failed: ${deps.scrub(errorMessage(e)).slice(0, 300)}`; + rec.slots = recordSlots(slots); + rec.cleanup = await cleanupRun(q, lib, task, rg, slots).catch((err) => ({ + error: errorMessage(err), + })); + rec.harness_egress = harnessEgress; + rec.ended = nowIso(); + return rec; + } + rec.setup = { ms: deps.now() - setupStart, calls: setupLog.length }; + + let text = ""; + let run: A.AgentRun | null = null; + const oracleTrace: Array> = []; + const oracleEgress: EgressEvent[] = []; + let oracleMs = 0; + if (mode === "oracle") { + const blocks: string[] = []; + const t0 = deps.now(); + for (const step of task.oracle) { + const command = typeof step === "string" ? lib.fill(step, slots) : await step(q, slots); + const c0 = deps.now(); + let answer: AzAnswer; + try { + const result = await deps.agentSession.callTool( + deps.variants.AZURE_TOOL, + { command }, + spec.agentTimeoutMs + ); + const split = deps.agent.splitEnvelope(result); + oracleEgress.push( + ...deps.agent.egressOf(oracleTrace.length, command, split.text, split.envelope).events + ); + answer = answerOf(split); + } catch (e) { + answer = { + exitCode: null, + stdout: "", + stderr: `tool call failed: ${errorMessage(e)}`, + classId: "tool-error", + text: "", + }; + } + oracleTrace.push({ + command: command.length > 600 ? `${command.slice(0, 600)}...` : command, + exit_code: answer.exitCode, + class_id: answer.classId, + ms: deps.now() - c0, + }); + blocks.push(lib.oracleBlock(command, answer)); + } + oracleMs = deps.now() - t0; + slots._oracle = true; + text = blocks.join("\n"); + } else if (mode === "llm") { + if (!deps.client || !spec.price || !deps.classifyError) + throw new Error("llm mode needs a client, a price and an error classifier"); + const prompt = lib.fmt(task.prompts[spec.phrasing % task.prompts.length], slots); + rec.prompt = prompt; + try { + run = await deps.agent.runAgent({ + client: deps.client, + model: spec.model, + effort: spec.effort, + tools: spec.variant.tools, + prompt, + caps: deps.agent.CAPS[task.caps], + price: spec.price, + perRunUsd: spec.budget.perRunUsd, + remainingUsd: () => remaining(spec.budget), + // Every turn counts against the campaign's cap at once, so concurrent runs see it. + onCost: (usd) => { + spec.budget.spent += usd; + }, + callTool: agentToolCaller( + deps, + spec.variant.tools.map((t) => t.name), + spec.agentTimeoutMs + ), + classifyError: deps.classifyError, + scrub: deps.scrub, + now: deps.now, + }); + } catch (e) { + // A fault of the harness itself: not scored (as an infrastructure error), and the + // run's group is still verified and cleaned up below. + run = { + final_text: "", + stop: deps.scrub(errorMessage(e)).slice(0, 300), + failure: "infra_error", + refusal: false, + stop_details: null, + turns: [], + tool_calls: [], + egress: [], + housekeeping: [], + wall_ms: 0, + }; + } + text = run.final_text; + } + + const verifyLog: V.RecordedCall[] = []; + const vq = lib.recordingQuery(q, verifyLog); + const v0 = deps.now(); + let verdict: Verdict; + try { + verdict = await task.verify(vq, slots, text); + } catch (e) { + verdict = { passed: false, reason: `verifier error: ${errorMessage(e).slice(0, 200)}` }; + } + const verifyMs = deps.now() - v0; + + let teardown: string | undefined; + if (task.teardown) { + try { + await task.teardown(q, slots); + teardown = "ok"; + } catch (e) { + teardown = errorMessage(e).slice(0, 200); + } + } + const cleanup = await cleanupRun(q, lib, task, rg, slots).catch((err) => ({ + error: errorMessage(err), + })); + if (teardown) (cleanup as Record).teardown = teardown; + + const expectPass = mode !== "negative"; + rec.expected = expectPass ? "pass" : "fail"; + rec.verifier = { + passed: Boolean(verdict.passed), + reason: deps.scrub(String(verdict.reason)).slice(0, 1000), + ...(verdict.steps ? { steps: verdict.steps } : {}), + queries: compactQueries(verifyLog), + }; + if (verdict.steps) { + rec.steps = verdict.steps; + rec.steps_passed = verdict.steps.filter((s) => s.passed).length; + } + rec.verify_ms = verifyMs; + // Everything the verifier read, so a verifier fixed later can re-score the run offline: + // the slots at verification time, snapshots included, and the text it was given. + rec.verify_input = { + slots: verifySlots(slots), + text: text.length > MAX_VERIFY_TEXT ? text.slice(0, MAX_VERIFY_TEXT) : text, + ...(text.length > MAX_VERIFY_TEXT ? { text_truncated: true } : {}), + }; + + if (mode === "llm" && run) { + const infra = run.failure === "infra_error" || run.failure === "account_error"; + const price = spec.price!; + // Already counted into the budget turn by turn (onCost). + const cost = run.turns.length ? deps.agent.costUsd(run.turns, price) : 0; + Object.assign(rec, { + success: infra ? null : Boolean(verdict.passed), + reason: infra + ? `infrastructure: ${run.stop}` + : rec.verifier && (rec.verifier as { reason: string }).reason, + failure: verdict.passed ? null : (run.failure ?? "verify_failed"), + stop: run.stop, + refusal: run.refusal, + stop_details: run.stop_details, + wall_ms: Math.round(run.wall_ms), + time_to_verified_ms: verdict.passed ? Math.round(run.wall_ms + verifyMs) : null, + turns: run.turns.length, + tool_calls: run.tool_calls.length, + tool_ms: run.tool_calls.map((c) => c.ms), + tokens: deps.agent.totals(run.turns), + prefix_tokens: spec.prefixTokens ?? null, + cost_usd: cost, + cost_warm_usd: deps.agent.pricedCostUsd(run.turns, spec.prefixTokens, "warm", price), + cost_cold_usd: deps.agent.pricedCostUsd(run.turns, spec.prefixTokens, "cold", price), + egress_events: run.egress, + housekeeping_blocks: run.housekeeping, + turn_log: run.turns, + tool_trace: run.tool_calls, + final_text: deps.scrub(run.final_text).slice(0, 16000), + }); + } else { + const met = expectPass ? Boolean(verdict.passed) : !verdict.passed; + Object.assign(rec, { + success: met, + reason: rec.verifier && (rec.verifier as { reason: string }).reason, + failure: met ? null : expectPass ? "verify_failed" : "verifier_passed_without_action", + wall_ms: oracleMs, + egress_events: oracleEgress, + tool_calls: oracleTrace.length, + ...(mode === "oracle" ? { oracle: oracleTrace, transcript: text.slice(0, 16000) } : {}), + cost_usd: 0, + }); + } + rec.harness_egress = harnessEgress; + rec.slots = recordSlots(slots); + rec.cleanup = cleanup; + rec.ended = nowIso(); + return rec; +} + +// ── selection, order and the summary ───────────────────────────────────────── + +export function selectTasks(all: Task[], opts: { ids?: string[]; tiers?: string[] }): Task[] { + const byId = new Map(all.map((t) => [t.id, t])); + const unknown = (opts.ids ?? []).filter((id) => !byId.has(id)); + if (unknown.length) throw new Error(`unknown task id(s): ${unknown.join(", ")}`); + const tiers = (opts.tiers ?? []).map((t) => t.toUpperCase()); + const badTier = tiers.filter((t) => !["T-A", "T-B", "T-APIM"].includes(t)); + if (badTier.length) throw new Error(`unknown tier(s): ${badTier.join(", ")} (T-A, T-B, T-APIM)`); + return all.filter( + (t) => + (!opts.ids?.length || opts.ids.includes(t.id)) && (!tiers.length || tiers.includes(t.tier)) + ); +} + +/** mulberry32: a small seeded generator, so the variant order is the same on a rerun. */ +export function seededRandom(seed: number): () => number { + let a = seed >>> 0; + return () => { + a = (a + 0x6d2b79f5) >>> 0; + let t = a; + t = Math.imul(t ^ (t >>> 15), t | 1); + t ^= t + Math.imul(t ^ (t >>> 7), t | 61); + return ((t ^ (t >>> 14)) >>> 0) / 4294967296; + }; +} + +export function shuffled(items: readonly T[], rand: () => number): T[] { + const out = [...items]; + for (let i = out.length - 1; i > 0; i--) { + const j = Math.floor(rand() * (i + 1)); + [out[i], out[j]] = [out[j], out[i]]; + } + return out; +} + +export function groupName(prefix: string, taskId: string, n: number): string { + return `${prefix}-${taskId}-${n}`.slice(0, 90); +} + +function median(xs: number[]): number | null { + if (!xs.length) return null; + const s = [...xs].sort((a, b) => a - b); + const m = Math.floor(s.length / 2); + return s.length % 2 ? s[m] : (s[m - 1] + s[m]) / 2; +} + +type Rec = Record; + +/** Per variant: success, cost per completed task, time, egress, refusals, and the gate. */ +export function summarize(records: Rec[], gate = { success: 0.95 }): Record { + const out: Record = {}; + const modes = [...new Set(records.map((r) => r.mode))]; + for (const mode of modes) { + const rows = records.filter((r) => r.mode === mode); + if (mode !== "llm") { + const met = rows.filter((r) => r.success === true).length; + out[mode] = { + runs: rows.length, + met, + not_met: rows + .filter((r) => r.success === false) + .map((r) => ({ task: r.task_id, reason: r.reason })), + setup_errors: rows + .filter((r) => r.success === null) + .map((r) => ({ task: r.task_id, reason: r.reason })), + egress_events: rows.reduce((n, r) => n + (r.egress_events?.length ?? 0), 0), + harness_egress: rows.reduce((n, r) => n + (r.harness_egress?.length ?? 0), 0), + cleanup_not_gone: rows + .filter((r) => r.cleanup && r.cleanup.gone === false) + .map((r) => r.rg), + }; + continue; + } + const variants: Record = {}; + for (const variant of [...new Set(rows.map((r) => r.variant))]) { + const vr = rows.filter((r) => r.variant === variant); + const scored = vr.filter((r) => r.success !== null && r.success !== undefined); + const ok = scored.filter((r) => r.success === true); + const perTask = new Map(); + for (const r of scored) { + const e = perTask.get(r.task_id) ?? { n: 0, ok: 0 }; + e.n += 1; + e.ok += r.success ? 1 : 0; + perTask.set(r.task_id, e); + } + const taskMeans = [...perTask.values()].map((e) => e.ok / e.n); + const billed = vr.reduce((s, r) => s + (r.cost_usd ?? 0), 0); + const warm = scored.reduce((s, r) => s + (r.cost_warm_usd ?? r.cost_usd ?? 0), 0); + const egress = vr.reduce((n, r) => n + (r.egress_events?.length ?? 0), 0); + const rate = scored.length ? ok.length / scored.length : 0; + variants[variant] = { + runs: vr.length, + scored: scored.length, + successes: ok.length, + success_rate: Math.round(rate * 10000) / 10000, + success_mean_over_tasks: taskMeans.length + ? Math.round((taskMeans.reduce((a, b) => a + b, 0) / taskMeans.length) * 10000) / 10000 + : 0, + billed_usd: Math.round(billed * 1e6) / 1e6, + cost_per_completed_task_warm_usd: ok.length + ? Math.round((warm / ok.length) * 1e6) / 1e6 + : null, + cost_per_completed_task_billed_usd: ok.length + ? Math.round((billed / ok.length) * 1e6) / 1e6 + : null, + median_time_to_verified_ms: median( + ok.map((r) => r.time_to_verified_ms).filter((x) => typeof x === "number") + ), + egress_events: egress, + refusals: vr.filter((r) => r.refusal).length, + kv_crypto_refusals: vr.filter((r) => r.refusal && r.kv_crypto).length, + kv_crypto_runs: vr.filter((r) => r.kv_crypto).length, + setup_errors: vr.filter((r) => r.failure === "setup_error").length, + infra_errors: vr.filter((r) => r.failure === "infra_error" || r.failure === "account_error") + .length, + spend_capped: vr.filter((r) => r.failure === "spend_cap").length, + failed_tasks: [...perTask] + .filter(([, e]) => e.ok < e.n) + .map(([id, e]) => `${id} ${e.ok}/${e.n}`), + gate: { + success_at_least: gate.success, + success_ok: rate >= gate.success, + zero_egress: egress === 0, + passed: rate >= gate.success && egress === 0 && scored.length > 0, + }, + }; + } + out.llm = variants; + } + return out; +} diff --git a/tests/azure/evals/keys.test.ts b/tests/azure/evals/keys.test.ts new file mode 100644 index 0000000..0b3b483 --- /dev/null +++ b/tests/azure/evals/keys.test.ts @@ -0,0 +1,208 @@ +/** + * The API key's handling: where it is read from, how a bad file is refused, that no + * message or record ever carries it, that child processes never inherit it, and that a + * model run without a key refuses at once (exit 2) without a single network connection. + */ +import { spawnSync } from "child_process"; +import { chmodSync, existsSync, mkdtempSync, readFileSync, writeFileSync } from "fs"; +import os from "os"; +import path from "path"; +import { pathToFileURL } from "url"; +import * as K from "./keys"; + +const KEY = "sk-ant-api03-TESTKEY_abcdefghijklmnop-0123456789"; + +function deps( + files: Record, + platform = "linux", + uid: number | null = 1000 +): K.KeyFileDeps { + return { + platform, + uid, + read: (p) => { + if (!(p in files)) throw new Error("ENOENT"); + return files[p].text; + }, + stat: (p) => { + if (!(p in files)) throw new Error("ENOENT"); + return { mode: files[p].mode ?? 0o100600, uid: files[p].uid ?? 1000 }; + }, + }; +} + +/** Runs fn and returns the thrown error's message (never the key itself). */ +function messageOf(fn: () => unknown): string { + try { + fn(); + } catch (e) { + return e instanceof Error ? e.message : String(e); + } + return "(no error)"; +} + +describe("loading the key", () => { + test("ANTHROPIC_API_KEY (an environment variable)", () => { + expect(K.loadKey({ env: { ANTHROPIC_API_KEY: ` ${KEY}\n` } }, deps({}))).toEqual({ + key: KEY, + source: "ANTHROPIC_API_KEY", + }); + }); + + test("--key-file wins over ANTHROPIC_API_KEY_FILE, which wins over ANTHROPIC_API_KEY", () => { + const d = deps({ "/a": { text: KEY }, "/b": { text: KEY.replace("TESTKEY", "OTHERKEY") } }); + expect( + K.loadKey( + { keyFile: "/a", env: { ANTHROPIC_API_KEY_FILE: "/b", ANTHROPIC_API_KEY: "sk-ant-x" } }, + d + )?.source + ).toBe("key file /a"); + expect( + K.loadKey({ env: { ANTHROPIC_API_KEY_FILE: "/b", ANTHROPIC_API_KEY: "sk-ant-x" } }, d)?.source + ).toBe("key file /b"); + }); + + test("no key configured is null (the runner then refuses)", () => { + expect(K.loadKey({ env: {} }, deps({}))).toBeNull(); + expect(K.loadKey({ env: { ANTHROPIC_API_KEY: " " } }, deps({}))).toBeNull(); + }); + + test("a key file may carry a BOM and a newline (Notepad)", () => { + expect(K.readKeyFile("/k", deps({ "/k": { text: `\ufeff${KEY}\r\n` } }))).toBe(KEY); + }); + + test("a file readable by others, or not the user's, is refused on POSIX", () => { + expect( + messageOf(() => K.readKeyFile("/k", deps({ "/k": { text: KEY, mode: 0o100644 } }))) + ).toBe("/k is readable by others; run: chmod 600 /k"); + expect(messageOf(() => K.readKeyFile("/k", deps({ "/k": { text: KEY, uid: 0 } })))).toBe( + "/k is not owned by the current user" + ); + }); + + test("Windows has no such mode bits: the mode is not checked there", () => { + expect( + K.readKeyFile("C:\\k", deps({ "C:\\k": { text: KEY, mode: 0o100666 } }, "win32", null)) + ).toBe(KEY); + }); + + test("a missing file, or one that is not a single key, is refused without echoing its content", () => { + expect(messageOf(() => K.readKeyFile("/none", deps({})))).toBe("no API key file at /none"); + const two = `${KEY}\n${KEY}`; + const m = messageOf(() => K.readKeyFile("/k", deps({ "/k": { text: two } }))); + expect(m).toBe("/k does not contain a single Anthropic API key"); + expect(m).not.toContain(KEY); + const bad = messageOf(() => + K.loadKey({ env: { ANTHROPIC_API_KEY: "not-a-key with spaces" } }, deps({})) + ); + expect(bad).toBe("ANTHROPIC_API_KEY does not contain a single Anthropic API key"); + }); +}); + +describe("keeping the key out of every output", () => { + test("scrubber replaces the key and nothing else", () => { + const scrub = K.scrubber(KEY); + expect(scrub(`error: invalid x-api-key ${KEY} (401)`)).toBe( + "error: invalid x-api-key [redacted] (401)" + ); + expect(scrub("no key here")).toBe("no key here"); + expect(K.scrubber(null)("unchanged")).toBe("unchanged"); + }); + + test("childEnv and stripAnthropicEnv remove every ANTHROPIC_* variable", () => { + const env: Record = { + ANTHROPIC_API_KEY: KEY, + ANTHROPIC_AUTH_TOKEN: "t", + ANTHROPIC_LOG: "debug", + anthropic_api_key_file: "/k", + LOCALSTACK_AUTH_TOKEN: "ls-x", + PATH: "/bin", + }; + expect(Object.keys(K.childEnv(env)).sort()).toEqual(["LOCALSTACK_AUTH_TOKEN", "PATH"]); + expect(K.stripAnthropicEnv(env).sort()).toEqual([ + "ANTHROPIC_API_KEY", + "ANTHROPIC_AUTH_TOKEN", + "ANTHROPIC_LOG", + "anthropic_api_key_file", + ]); + expect(env).toEqual({ LOCALSTACK_AUTH_TOKEN: "ls-x", PATH: "/bin" }); + }); + + test("the refusal message names the ways to provide a key and the no-spend modes", () => { + expect(K.NO_KEY_MESSAGE).toContain("ANTHROPIC_API_KEY"); + expect(K.NO_KEY_MESSAGE).toContain("--key-file"); + expect(K.NO_KEY_MESSAGE).toContain("--oracle, --negative and --dry-run"); + }); +}); + +// run.mjs loads its .ts modules with Node's built-in type stripping (22.18+, 23.6+). +const [major, minor] = process.versions.node.split(".").map(Number); +const TYPE_STRIPPING = major > 23 || (major === 23 && minor >= 6) || (major === 22 && minor >= 18); + +(TYPE_STRIPPING ? describe : describe.skip)("run.mjs without a key", () => { + const runMjs = path.join(__dirname, "run.mjs"); + + function cleanEnv(extra: Record = {}): NodeJS.ProcessEnv { + const env: NodeJS.ProcessEnv = {}; + for (const [k, v] of Object.entries(process.env)) if (!/^ANTHROPIC_/i.test(k)) env[k] = v; + return { ...env, NODE_NO_WARNINGS: "1", ...extra }; + } + + test("exits 2 with the clear message, writes nothing and opens no connection", () => { + const tmp = mkdtempSync(path.join(os.tmpdir(), "e2-nokey-")); + const marker = path.join(tmp, "connect-attempted"); + const hook = path.join(tmp, "no-network.mjs"); + // Preloaded into the runner: any socket connection (http, https, fetch all end in + // net.Socket#connect) writes the marker and throws. + writeFileSync( + hook, + [ + 'import net from "node:net";', + 'import { writeFileSync } from "node:fs";', + `const marker = ${JSON.stringify(marker)};`, + "const connect = net.Socket.prototype.connect;", + "net.Socket.prototype.connect = function (...args) {", + ' writeFileSync(marker, "connect");', + ' throw new Error("network access in a no-key run");', + "};", + "void connect;", + ].join("\n") + ); + const out = path.join(tmp, "results"); + const r = spawnSync( + process.execPath, + ["--import", pathToFileURL(hook).href, runMjs, "--runs", "1", "--out", out], + { + env: cleanEnv(), + encoding: "utf8", + timeout: 60_000, + } + ); + expect(r.status).toBe(2); + expect(r.stderr).toContain( + "E2 needs an Anthropic API key to call the model, and none was found." + ); + expect(existsSync(out)).toBe(false); + expect(existsSync(marker)).toBe(false); + }); + + test("an unreadable key file is refused the same way, and its content is not echoed", () => { + const tmp = mkdtempSync(path.join(os.tmpdir(), "e2-badkey-")); + const keyFile = path.join(tmp, "key.txt"); + writeFileSync(keyFile, "sk-ant-api03-SECRET-1 sk-ant-api03-SECRET-2\n"); + chmodSync(keyFile, 0o600); // private, so POSIX refuses it for its content, not its mode + const r = spawnSync( + process.execPath, + [runMjs, "--key-file", keyFile, "--out", path.join(tmp, "o")], + { + env: cleanEnv(), + encoding: "utf8", + timeout: 60_000, + } + ); + expect(r.status).toBe(2); + expect(r.stderr).toContain("does not contain a single Anthropic API key"); + expect(`${r.stdout}${r.stderr}`).not.toContain("SECRET"); + expect(readFileSync(keyFile, "utf8")).toContain("SECRET-1"); + }); +}); diff --git a/tests/azure/evals/keys.ts b/tests/azure/evals/keys.ts new file mode 100644 index 0000000..6ae41b1 --- /dev/null +++ b/tests/azure/evals/keys.ts @@ -0,0 +1,124 @@ +/** + * The Anthropic API key for E2: it comes + * from ANTHROPIC_API_KEY or a private key file (--key-file or + * ANTHROPIC_API_KEY_FILE), goes straight into the SDK client, and is never printed, logged + * or written. `scrubber` removes it from any error text before that text is recorded, and + * `childEnv` keeps every ANTHROPIC_* variable away from the MCP servers (and their `az`). + * + * No runtime imports of local modules (run.mjs loads this file with type stripping). + */ +import { readFileSync, statSync } from "node:fs"; + +export class KeyError extends Error { + constructor(message: string) { + super(message); + this.name = "KeyError"; + } +} + +export interface LoadedKey { + key: string; + /** Where it came from, without the key: "ANTHROPIC_API_KEY" or "key file ". */ + source: string; +} + +export const NO_KEY_MESSAGE = + "E2 needs an Anthropic API key to call the model, and none was found. Set ANTHROPIC_API_KEY, " + + "or pass --key-file (or ANTHROPIC_API_KEY_FILE=) naming a file that " + + "holds only the key. The key is never printed or logged. Without a key, --oracle, --negative and " + + "--dry-run still run: they call no model."; + +/** One key, as the Console issues it; the file may carry a BOM and a trailing newline. */ +export function parseKey(raw: string, where: string): string { + const key = raw.replace(/^/, "").trim(); + if (!key.startsWith("sk-ant-") || /\s/.test(key)) { + throw new KeyError(`${where} does not contain a single Anthropic API key`); + } + return key; +} + +export interface KeyFileDeps { + read: (path: string) => string; + stat: (path: string) => { mode: number; uid: number }; + platform: string; + uid: number | null; +} + +const realDeps: KeyFileDeps = { + read: (p) => readFileSync(p, "utf8"), + stat: (p) => statSync(p), + platform: process.platform, + uid: typeof process.getuid === "function" ? process.getuid() : null, +}; + +/** + * The key from a file. On POSIX the file must belong to the user and be private (mode + * 600), as ssh demands of private keys; Windows has no such mode bits to check. + */ +export function readKeyFile(path: string, deps: KeyFileDeps = realDeps): string { + let st: { mode: number; uid: number }; + try { + st = deps.stat(path); + } catch { + throw new KeyError(`no API key file at ${path}`); + } + if (deps.platform !== "win32") { + if (deps.uid !== null && st.uid !== deps.uid) + throw new KeyError(`${path} is not owned by the current user`); + if (st.mode & 0o077) + throw new KeyError(`${path} is readable by others; run: chmod 600 ${path}`); + } + return parseKey(deps.read(path), path); +} + +/** + * The key, or null when none is configured. Precedence: --key-file, then + * ANTHROPIC_API_KEY_FILE, then ANTHROPIC_API_KEY. A configured but unusable key is a + * KeyError (its message never contains the key). + */ +export function loadKey( + opts: { keyFile?: string; env: Record }, + deps: KeyFileDeps = realDeps +): LoadedKey | null { + const file = opts.keyFile || opts.env.ANTHROPIC_API_KEY_FILE; + if (file) return { key: readKeyFile(file, deps), source: `key file ${file}` }; + const fromEnv = opts.env.ANTHROPIC_API_KEY; + if (fromEnv !== undefined && fromEnv.trim() !== "") { + return { key: parseKey(fromEnv, "ANTHROPIC_API_KEY"), source: "ANTHROPIC_API_KEY" }; + } + return null; +} + +/** Replaces the key (and nothing else) in a text before it is stored or printed. */ +export function scrubber(key: string | null | undefined): (text: string) => string { + if (!key) return (text) => text; + return (text) => (typeof text === "string" ? text.split(key).join("[redacted]") : text); +} + +/** The environment for child processes: every ANTHROPIC_* variable removed. */ +export function childEnv( + env: Record +): Record { + const out: Record = {}; + for (const [k, value] of Object.entries(env)) { + if (!/^ANTHROPIC_/i.test(k)) out[k] = value; + } + return out; +} + +/** Removes every ANTHROPIC_* variable from this process's environment (in place). */ +export function stripAnthropicEnv(env: Record): string[] { + const removed: string[] = []; + for (const k of Object.keys(env)) { + if (/^ANTHROPIC_/i.test(k)) { + delete env[k]; + removed.push(k); + } + } + return removed; +} + +/** True when a text contains the key anywhere (used to prove a record is clean). */ +export function containsKey(text: string, key: string): boolean { + return key.length > 0 && text.includes(key); +} diff --git a/tests/azure/evals/record-fixtures.mjs b/tests/azure/evals/record-fixtures.mjs new file mode 100644 index 0000000..6617aed --- /dev/null +++ b/tests/azure/evals/record-fixtures.mjs @@ -0,0 +1,102 @@ +#!/usr/bin/env node +// Builds fixtures/recorded-verifications.json, which tasks.test.ts replays, from live E2 runs: +// +// node tests/azure/evals/run.mjs --oracle --out test-results/e2-oracle +// node tests/azure/evals/run.mjs --negative --out test-results/e2-negative +// node tests/azure/evals/record-fixtures.mjs test-results/e2-oracle/runs.jsonl test-results/e2-negative/runs.jsonl +// +// For each (task, mode) it keeps the latest record that met its expectation (--oracle: the +// verifier passed; --negative: it failed) and holds exactly what the verifier read: the slots, +// the text, and every answer the verifier got (JSON stdout minified; connection-string key +// parts in the text, stdout and stderr, which no verifier reads, redacted). Records with a +// truncated input are skipped. +import { mkdirSync, readFileSync, writeFileSync } from "node:fs"; +import { dirname, join } from "node:path"; +import { fileURLToPath } from "node:url"; + +const HERE = dirname(fileURLToPath(import.meta.url)); +const OUT = join(HERE, "fixtures", "recorded-verifications.json"); + +const inputs = process.argv.slice(2); +if (!inputs.length) { + console.error( + "usage: node tests/azure/evals/record-fixtures.mjs [ ...]" + ); + process.exit(2); +} + +const redact = (s) => + String(s ?? "") + .replace(/(SharedAccessKey=)[^;"\\]+/g, "$1[redacted]") + .replace(/(AccountKey=)[^;"\\]+/g, "$1[redacted]"); + +function minify(stdout) { + const t = String(stdout ?? ""); + if (!t.trim()) return t; + try { + return redact(JSON.stringify(JSON.parse(t))); + } catch { + return redact(t); + } +} + +const latest = new Map(); +for (const file of inputs) { + for (const line of readFileSync(file, "utf8").split("\n")) { + if (!line.trim()) continue; + const r = JSON.parse(line); + if (!r.verify_input || r.success !== true || !["oracle", "negative"].includes(r.mode)) continue; + if (r.verify_input.text_truncated) continue; + if ((r.verifier?.queries ?? []).some((q) => q.answer?.stdout_truncated)) continue; + latest.set(`${r.task_id}|${r.mode}`, r); + } +} + +const entries = [...latest.values()] + .sort((a, b) => (a.task_id + a.mode < b.task_id + b.mode ? -1 : 1)) + .map((r) => ({ + task: r.task_id, + mode: r.mode, + passed: r.verifier.passed, + wait_s: r.mode === "negative" ? 10 : 90, + recorded: r.ended, + slots: r.verify_input.slots, + text: redact(r.verify_input.text), + queries: (r.verifier.queries ?? []).map((q) => ({ + command: q.command, + ...(q.repeat ? { repeat: q.repeat } : {}), + answer: { + exitCode: q.answer.exitCode, + stdout: minify(q.answer.stdout), + stderr: redact(q.answer.stderr).slice(0, 400), + classId: q.answer.classId ?? null, + ...(q.answer.stoppedByTool ? { stoppedByTool: true } : {}), + }, + })), + })); + +const doc = { + about: + "Recorded answers of E2's verifiers from live runs against the LocalStack Azure emulator " + + "(run.mjs --oracle: the verifier passed; --negative: setup only, the verifier failed). " + + "tasks.test.ts replays each entry through its task's verifier and expects the recorded " + + "verdict. All values are local emulator test data. Regenerate with record-fixtures.mjs.", + entries, +}; +// Written in the repo's Prettier layout, so `prettier --check` stays clean. +let text = JSON.stringify(doc, null, 2) + "\n"; +try { + const prettier = await import("prettier"); + const options = (await prettier.resolveConfig(OUT)) ?? {}; + text = await prettier.format(text, { ...options, parser: "json" }); +} catch (error) { + console.error(`prettier not available (${error.message}); writing plain JSON`); +} +mkdirSync(dirname(OUT), { recursive: true }); +writeFileSync(OUT, text); +console.log( + `${entries.length} entries, ${Math.round(JSON.stringify(doc).length / 1024)} KB -> ${OUT}` +); +const modes = new Map(); +for (const e of entries) modes.set(e.task, (modes.get(e.task) ?? new Set()).add(e.mode)); +for (const [t, m] of modes) if (m.size < 2) console.log(` ${t}: only ${[...m].join(", ")}`); diff --git a/tests/azure/evals/run.mjs b/tests/azure/evals/run.mjs new file mode 100644 index 0000000..e13126b --- /dev/null +++ b/tests/azure/evals/run.mjs @@ -0,0 +1,569 @@ +#!/usr/bin/env node +// E2: composition evals of the localstack-azure-client tool. +// +// node tests/azure/evals/run.mjs [options] +// +// Runs the E2 tasks (tiers T-A, T-B, T-APIM; see README.md) against `node dist/cli.js`: +// Claude calls the Azure tool over MCP until it ends its turn, then the task's verifier reads the +// outcome from the emulator through a SEPARATE server session, then everything the run created is +// removed. One JSONL record per run; see README.md. +// +// --runs N repetitions per task (default 1) +// --max-usd X stop starting runs once the spend reaches X (default $AZURE_EVALS_MAX_USD or 20) +// --max-usd-per-run Y stop one run's agent loop at Y (default $EVAL_MAX_USD_PER_RUN or 1) +// --out DIR results directory (default test-results/e2) +// --tasks id,id only these tasks --tier T-A,T-B,T-APIM only these tiers +// --model M default $EVAL_MODEL or claude-opus-5-5 +// --effort E default $EVAL_EFFORT or high +// --variant v[,v] compact (default), long, help-tool, no-test-data; several interleave +// --all-tools offer every server tool (localstack-management is withheld), not only Azure +// --key-file PATH the API key file (else ANTHROPIC_API_KEY_FILE, else ANTHROPIC_API_KEY) +// --phrasing alt|0|1 T-A phrasing: alternate by run index (default), or fixed +// --verify-wait S how long polling verifiers wait (default 90; 10 with --negative) +// --seed N seed of the variant order (default 20260924) +// --max-minutes M start no new run after M minutes +// --jobs N concurrent runs, each with its own two servers (default $EVAL_JOBS +// or 1: keep 1 on a shared emulator) +// --no-prewarm skip the max_tokens:0 cache pre-warm per variant +// --oracle no model: run each task's reference commands, the verifier must pass +// --negative no model: setup only, the verifier must fail +// --dry-run no emulator, no model: list the plan and the tool definitions per variant +// --list print the tasks and exit +// +// Exit status: 0 all passed (the gate, or every oracle/negative expectation), 1 not passed or +// stopped early, 2 a usage or configuration error (no API key for a model run, unknown model price, +// no dist/cli.js, Node.js older than 22.18). The API key is never printed, logged or written. +import { appendFileSync, existsSync, mkdirSync, writeFileSync } from "node:fs"; +import { dirname, join, resolve } from "node:path"; +import { performance } from "node:perf_hooks"; +import { fileURLToPath, pathToFileURL } from "node:url"; +import { randomBytes } from "node:crypto"; + +const HERE = dirname(fileURLToPath(import.meta.url)); +const REPO = resolve(HERE, "..", "..", ".."); +const EXIT_USAGE = 2; + +function fail(message, code = EXIT_USAGE) { + console.error(`e2: ${message}`); + process.exit(code); +} + +function parseArgs(argv) { + const env = process.env; + const o = { + runs: 1, + maxUsd: Number(env.AZURE_EVALS_MAX_USD || 20), + maxUsdPerRun: Number(env.EVAL_MAX_USD_PER_RUN || 1), + out: "test-results/e2", + tasks: [], + tiers: [], + model: env.EVAL_MODEL || "claude-opus-5-5", + effort: env.EVAL_EFFORT || "high", + variants: ["compact"], + allTools: false, + keyFile: undefined, + phrasing: "alt", + verifyWait: undefined, + seed: 20260924, + maxMinutes: undefined, + // Concurrent runs, each with its own two server sessions: one, since the emulator may be + // shared; raise it with --jobs on an emulator of your own. + jobs: Number(env.EVAL_JOBS || 1), + prewarm: true, + oracle: false, + negative: false, + dryRun: false, + list: false, + help: false, + }; + const list = (s) => + String(s) + .split(",") + .map((x) => x.trim()) + .filter(Boolean); + for (let i = 0; i < argv.length; i++) { + const arg = argv[i]; + const value = () => { + const next = argv[++i]; + if (next === undefined || next.startsWith("--")) fail(`${arg} needs a value`); + return next; + }; + const num = (v) => { + const n = Number(v); + if (!Number.isFinite(n) || n < 0) fail(`${arg} needs a non-negative number, not ${v}`); + return n; + }; + switch (arg) { + case "--runs": + o.runs = Math.floor(num(value())); + break; + case "--max-usd": + o.maxUsd = num(value()); + break; + case "--max-usd-per-run": + o.maxUsdPerRun = num(value()); + break; + case "--out": + o.out = value(); + break; + case "--tasks": + o.tasks = list(value()); + break; + case "--tier": + o.tiers = list(value()); + break; + case "--model": + o.model = value(); + break; + case "--effort": + o.effort = value(); + break; + case "--variant": + o.variants = list(value()); + break; + case "--all-tools": + o.allTools = true; + break; + case "--key-file": + o.keyFile = value(); + break; + case "--phrasing": + o.phrasing = value(); + break; + case "--verify-wait": + o.verifyWait = num(value()); + break; + case "--seed": + o.seed = Math.floor(num(value())); + break; + case "--max-minutes": + o.maxMinutes = num(value()); + break; + case "--no-prewarm": + o.prewarm = false; + break; + case "--jobs": + o.jobs = Math.floor(num(value())); + break; + case "--oracle": + o.oracle = true; + break; + case "--negative": + o.negative = true; + break; + case "--dry-run": + o.dryRun = true; + break; + case "--list": + o.list = true; + break; + case "-h": + case "--help": + o.help = true; + break; + default: + fail(`unknown argument ${arg} (see --help)`); + } + } + if ([o.oracle, o.negative, o.dryRun].filter(Boolean).length > 1) + fail("--oracle, --negative and --dry-run exclude each other"); + if (!["alt", "0", "1"].includes(o.phrasing)) fail(`--phrasing is alt, 0 or 1, not ${o.phrasing}`); + if (o.runs < 1) fail("--runs must be at least 1"); + if (!Number.isInteger(o.jobs) || o.jobs < 1 || o.jobs > 8) fail("--jobs must be 1 to 8"); + return o; +} + +async function loadModules() { + const load = (file) => import(pathToFileURL(join(HERE, file)).href); + try { + const [lib, agent, keys, variants, harness, ta, tb, tapim] = await Promise.all([ + load("verifiers.ts"), + load("agent.ts"), + load("keys.ts"), + load("variants.ts"), + load("harness.ts"), + load("tasks-ta.ts"), + load("tasks-tb.ts"), + load("tasks-tapim.ts"), + ]); + const tasks = [...ta.taTasks(lib), ...tb.tbTasks(lib), ...tapim.tapimTasks(lib)]; + return { lib, agent, keys, variants, harness, tasks }; + } catch (error) { + fail( + `cannot load the eval modules (${error instanceof Error ? error.message : error}). E2 loads its TypeScript ` + + `modules with Node's built-in type stripping, which needs Node.js 22.18 or newer (this is ${process.version}).` + ); + } +} + +function helpText() { + // The header comment of this file is the help. + return [ + "usage: node tests/azure/evals/run.mjs [--runs N] [--max-usd X] [--max-usd-per-run Y] [--out DIR]", + " [--tasks id,id] [--tier T-A,T-B,T-APIM] [--model M] [--effort E] [--variant v[,v]] [--all-tools]", + " [--key-file PATH] [--phrasing alt|0|1] [--verify-wait S] [--seed N] [--max-minutes M] [--no-prewarm]", + " [--jobs N] [--oracle | --negative | --dry-run] [--list]", + "variants: compact (default), long, help-tool, no-test-data. See tests/azure/evals/README.md.", + ].join("\n"); +} + +async function main() { + const o = parseArgs(process.argv.slice(2)); + if (o.help) { + console.log(helpText()); + return 0; + } + const { lib, agent, keys, variants, harness, tasks: allTasks } = await loadModules(); + let tasks; + try { + tasks = harness.selectTasks(allTasks, { ids: o.tasks, tiers: o.tiers }); + } catch (error) { + fail(error.message); + } + if (!tasks.length) fail("no task matches --tasks/--tier"); + for (const v of o.variants) + if (!variants.isVariant(v)) fail(`unknown variant ${v} (${variants.VARIANTS.join(", ")})`); + if (o.list) { + for (const t of tasks) console.log(`${t.tier.padEnd(7)} ${t.id.padEnd(38)} ${t.opId}`); + console.log(`${tasks.length} tasks`); + return 0; + } + const mode = o.oracle ? "oracle" : o.negative ? "negative" : o.dryRun ? "dry-run" : "llm"; + + // The key first: a model run without one stops here, before any server or SDK is loaded. + let loaded = null; + let price; + if (mode === "llm") { + try { + loaded = keys.loadKey({ keyFile: o.keyFile, env: process.env }); + } catch (error) { + fail(error instanceof keys.KeyError ? error.message : "the API key could not be read"); + } + if (!loaded) fail(keys.NO_KEY_MESSAGE); + price = agent.priceFor(o.model); + if (!price) + fail( + `no price for model ${o.model}; add it to PRICES in tests/azure/evals/agent.ts (the spend cap needs one)` + ); + } + const scrub = keys.scrubber(loaded?.key); + + const cli = join(REPO, "dist", "cli.js"); + if (!existsSync(cli)) fail(`${cli} does not exist; run \`yarn build\` first`); + if (mode !== "dry-run" && !process.env.LOCALSTACK_AUTH_TOKEN) { + fail("LOCALSTACK_AUTH_TOKEN is not set: the Azure tool refuses every call without it"); + } + + const outDir = resolve(REPO, o.out); + mkdirSync(outDir, { recursive: true }); + + // Debug logging could echo request details: never in a process that holds the key. + delete process.env.ANTHROPIC_LOG; + let client; + let classifyError; + if (mode === "llm") { + const { default: Anthropic } = await import("@anthropic-ai/sdk"); + client = new Anthropic({ apiKey: loaded.key, authToken: null, maxRetries: 4, logLevel: "off" }); + classifyError = agent.makeErrorClassifier(Anthropic); + } + // The servers (and every az they run) never see an ANTHROPIC_* variable. + keys.stripAnthropicEnv(process.env); + + const { startServer } = await import( + pathToFileURL(join(REPO, "tests", "azure", "tools", "stdio-client.mjs")).href + ); + const servers = []; + const start = async (label, env) => { + const server = await startServer({ cwd: REPO, env }); + servers.push({ label, server }); + return server; + }; + const closeAll = async () => { + for (const { label, server } of servers.splice(0)) { + await server.close().catch(() => undefined); + try { + writeFileSync(join(outDir, `server-${label}.stderr.log`), scrub(server.stderr())); + } catch { + // the results directory may be gone; the logs are a convenience + } + } + }; + const AGENT_ENV = { LOCALSTACK_AZ_TEST_ENVELOPE: "1", MCP_ANALYTICS_DISABLED: "1" }; + // The harness's own session: slow fixtures (a MySQL server) get more than the default 300 s. + const HARNESS_ENV = { ...AGENT_ENV, LOCALSTACK_AZ_TIMEOUT_SECONDS: "900" }; + const started = new Date(); + const runTag = `mcpe2-${randomBytes(3).toString("hex")}`; + const meta = { + run_id: `${started + .toISOString() + .replace(/[-:]/g, "") + .replace(/\.\d+Z$/, "Z")}-${mode}`, + started: started.toISOString(), + mode, + group_prefix: runTag, + variants: o.variants, + all_tools: o.allTools, + model: mode === "llm" ? o.model : null, + effort: mode === "llm" ? o.effort : null, + thinking: agent.THINKING, + max_tokens: agent.MAX_TOKENS, + system_prompt: agent.SYSTEM_PROMPT, + tasks: tasks.map((t) => t.id), + runs: o.runs, + phrasing: o.phrasing, + seed: o.seed, + max_usd: o.maxUsd, + max_usd_per_run: o.maxUsdPerRun, + price: price ?? null, + key_source: loaded ? loaded.source : null, + node: process.version, + platform: `${process.platform}-${process.arch}`, + }; + const writeMeta = () => + writeFileSync(join(outDir, "meta.json"), scrub(JSON.stringify(meta, null, 2)) + "\n"); + + let interrupted = false; + process.on("SIGINT", () => { + if (interrupted) process.exit(130); + interrupted = true; + console.error("e2: interrupted; stopping after the current run (Ctrl+C again to quit at once)"); + }); + + try { + const agentServer = await start("agent-0", AGENT_ENV); + const listed = await agentServer.listTools(); + const variantTools = {}; + try { + for (const v of o.variants) + variantTools[v] = variants.buildVariantTools(v, listed, o.allTools); + } catch (error) { + fail(error.message); + } + meta.tool_definitions = Object.fromEntries( + Object.entries(variantTools).map(([v, vt]) => [ + v, + { + tools: vt.tools.map((t) => t.name), + bytes: JSON.stringify(vt.tools).length, + azure_description: vt.azureDescription, + }, + ]) + ); + + if (mode === "dry-run") { + const plan = tasks.map((t) => ({ + id: t.id, + tier: t.tier, + op: t.opId, + verb: t.verb, + prompts: t.prompts, + oracle_steps: t.oracle.length, + })); + writeFileSync( + join(outDir, "plan.json"), + JSON.stringify({ meta, tasks: plan }, null, 2) + "\n" + ); + writeMeta(); + for (const [v, d] of Object.entries(meta.tool_definitions)) + console.log(`[${v}] ${d.tools.join(", ")} (${d.bytes} bytes)`); + console.log( + `${tasks.length} tasks x ${o.runs} run(s) x ${o.variants.length} variant(s); plan: ${join(outDir, "plan.json")}` + ); + return 0; + } + + // One agent session and one harness session per worker: verification never shares + // the agent's session, and concurrent runs never share one either. + const workers = []; + for (let w = 0; w < o.jobs; w++) { + const agentSession = w === 0 ? agentServer : await start(`agent-${w}`, AGENT_ENV); + const harnessSession = await start(`harness-${w}`, HARNESS_ENV); + workers.push({ + lib, + agent, + variants, + agentSession, + harnessSession, + now: () => performance.now(), + sleep: (ms) => new Promise((r) => setTimeout(r, ms)), + log: (line) => console.log(line), + scrub, + client, + classifyError, + }); + } + meta.jobs = o.jobs; + const budget = { maxUsd: o.maxUsd, perRunUsd: o.maxUsdPerRun, spent: 0 }; + const prefix = {}; + if (mode === "llm" && o.prewarm) { + meta.prewarm = {}; + for (const [v, vt] of Object.entries(variantTools)) { + try { + const pw = await agent.prewarm(client, o.model, o.effort, vt.tools); + const cost = agent.costUsd([{ usage: pw.usage }], price); + budget.spent += cost; + prefix[v] = pw.prefix_tokens; + meta.prewarm[v] = { usage: pw.usage, prefix_tokens: pw.prefix_tokens, cost_usd: cost }; + } catch (error) { + meta.prewarm[v] = { + error: scrub( + error instanceof Error ? `${error.constructor.name}: ${error.message}` : String(error) + ).slice(0, 300), + }; + } + } + } + writeMeta(); + + const records = []; + const groups = new Set(); + const names = { vaults: new Set(), apim: new Set() }; + const runsPath = join(outDir, "runs.jsonl"); + const rand = harness.seededRandom(o.seed); + const verifyWaitS = o.verifyWait ?? (mode === "negative" ? 10 : 90); + const runVariants = mode === "llm" ? o.variants : [o.variants[0]]; + const t0 = performance.now(); + let stopped = null; + let n = 0; + // The whole schedule first, each task visiting the variants in a fresh seeded order, so a + // rerun, and a run with more jobs, keeps the same order. + const items = []; + for (const task of tasks) { + for (let run = 0; run < o.runs; run++) { + for (const v of harness.shuffled(runVariants, rand)) items.push({ task, run, v }); + } + } + const runWorker = async (deps) => { + for (;;) { + if (stopped) return; + if (interrupted) { + stopped = "interrupted"; + return; + } + if (mode === "llm" && !harness.canStart(budget)) { + stopped = `spend cap: $${budget.spent.toFixed(4)} of $${o.maxUsd}`; + return; + } + if (o.maxMinutes !== undefined && (performance.now() - t0) / 60000 > o.maxMinutes) { + stopped = `time: ${o.maxMinutes} minutes`; + return; + } + const item = items[n]; + if (!item) return; + n += 1; + const { task, run, v } = item; + { + const rg = harness.groupName(runTag, task.id, n); + groups.add(rg); + const phrasing = + o.phrasing === "alt" + ? run % task.prompts.length + : Number(o.phrasing) % task.prompts.length; + let rec; + try { + rec = await harness.runOne( + { + task, + mode, + variant: variantTools[v], + runIndex: run, + phrasing, + rg, + model: o.model, + effort: o.effort, + price, + prefixTokens: prefix[v] ?? null, + budget, + verifyWaitS, + agentTimeoutMs: (agent.TOOL_CALL_TIMEOUT_S + 30) * 1000, + harnessTimeoutMs: 960_000, + }, + deps + ); + } catch (error) { + // A harness fault: recorded unscored; the final sweep removes the run's group. + rec = { + task_id: task.id, + tier: task.tier, + variant: v, + mode, + run, + phrasing, + rg, + success: null, + failure: "harness_error", + reason: scrub( + error instanceof Error ? `${error.name}: ${error.message}` : String(error) + ).slice(0, 300), + }; + } + for (const s of task.vaultSlots ?? []) + if (typeof rec.slots?.[s] === "string") names.vaults.add(rec.slots[s]); + for (const s of task.apimSlots ?? []) + if (typeof rec.slots?.[s] === "string") names.apim.add(rec.slots[s]); + records.push(rec); + appendFileSync(runsPath, scrub(JSON.stringify(rec)) + "\n"); + const mark = rec.success === true ? "PASS " : rec.success === false ? "FAIL " : "SETUP"; + const cost = mode === "llm" ? ` $${Number(rec.cost_usd ?? 0).toFixed(4)}` : ""; + console.log( + `${mark} ${mode === "llm" ? v.padEnd(12) : mode.padEnd(8)} ${task.id.padEnd(38)} r${run} p${phrasing}` + + ` calls=${rec.tool_calls ?? 0} verify=${Math.round(rec.verify_ms ?? 0)}ms${cost} ${String(rec.reason ?? "").slice(0, 160)}` + ); + if (rec.failure === "account_error") { + stopped = "the API key can no longer be used"; + return; + } + } + } + }; + await Promise.all(workers.map((deps) => runWorker(deps))); + + const sweepTypes = [...new Set(tasks.flatMap((t) => t.sweep ?? []))]; + const sweepQuery = harness.sessionQuery(workers[0], workers[0].harnessSession, 60, 960_000); + const sweep = await harness + .finalSweep(sweepQuery, lib, { + types: sweepTypes, + groups, + prefix: runTag, + vaults: names.vaults, + apim: names.apim, + }) + .catch((error) => ({ error: String(error) })); + writeFileSync(join(outDir, "cleanup.json"), scrub(JSON.stringify(sweep, null, 2)) + "\n"); + + const summary = harness.summarize(records); + meta.ended = new Date().toISOString(); + meta.spent_usd = Math.round(budget.spent * 1e6) / 1e6; + meta.stopped = stopped; + meta.runs_recorded = records.length; + writeMeta(); + writeFileSync( + join(outDir, "summary.json"), + scrub( + JSON.stringify( + { stopped, spent_usd: meta.spent_usd, cleanup_clean: sweep.clean ?? false, ...summary }, + null, + 2 + ) + ) + "\n" + ); + console.log(scrub(JSON.stringify(summary, null, 2))); + if (stopped) console.log(`STOPPED: ${stopped}`); + console.log(`records: ${runsPath}`); + + let ok = !stopped && sweep.clean === true; + if (mode === "llm") ok = ok && Object.values(summary.llm ?? {}).every((s) => s.gate.passed); + else ok = ok && records.every((r) => r.success === true); + return ok ? 0 : 1; + } finally { + await closeAll(); + } +} + +main().then( + (code) => process.exit(code), + (error) => { + console.error(`e2: ${error instanceof Error ? error.stack || error.message : error}`); + process.exit(1); + } +); diff --git a/tests/azure/evals/run.test.ts b/tests/azure/evals/run.test.ts new file mode 100644 index 0000000..c97bba7 --- /dev/null +++ b/tests/azure/evals/run.test.ts @@ -0,0 +1,76 @@ +/** + * run.mjs as a program, without dist/, an emulator or a key: Node loads every eval module + * through its built-in type stripping (so no module uses syntax that needs a transform), + * and bad options stop with exit 2 before anything starts. + */ +import { spawnSync } from "child_process"; +import path from "path"; + +const RUN = path.join(__dirname, "run.mjs"); +const [major, minor] = process.versions.node.split(".").map(Number); +/** Built-in type stripping without a flag: Node 22.18+ and 23.6+. */ +const TYPE_STRIPPING = major > 23 || (major === 23 && minor >= 6) || (major === 22 && minor >= 18); +const itNode = TYPE_STRIPPING ? test : test.skip; + +/** This process's environment without any ANTHROPIC_* variable. */ +function cleanEnv(extra: Record = {}): NodeJS.ProcessEnv { + const env: NodeJS.ProcessEnv = {}; + for (const [k, v] of Object.entries(process.env)) if (!/^ANTHROPIC_/i.test(k)) env[k] = v; + return { ...env, NODE_NO_WARNINGS: "1", ...extra }; +} + +function run(args: string[], extra: Record = {}) { + return spawnSync(process.execPath, [RUN, ...args], { + env: cleanEnv(extra), + encoding: "utf8", + timeout: 60_000, + }); +} + +describe("run.mjs", () => { + itNode("--list loads every module and prints the 50 tasks", () => { + const r = run(["--list"]); + expect(r.status).toBe(0); + const lines = r.stdout.trim().split("\n"); + expect(lines[lines.length - 1]).toBe("50 tasks"); + expect(r.stdout).toMatch(/^T-A\s+appconfig-kv-unlock\s+Microsoft\.AppConfiguration\//m); + expect(r.stdout).toMatch(/^T-APIM\s+apim-migrate-stv2\s/m); + }); + + itNode("--list with a tier selects it", () => { + const r = run(["--list", "--tier", "T-B"]); + expect(r.status).toBe(0); + expect(r.stdout.trim().split("\n").pop()).toBe("8 tasks"); + }); + + itNode("--help", () => { + const r = run(["--help"]); + expect(r.status).toBe(0); + expect(r.stdout).toContain("usage: node tests/azure/evals/run.mjs"); + }); + + itNode.each([ + [["--frobnicate"], "unknown argument --frobnicate"], + [["--oracle", "--negative"], "exclude each other"], + [["--list", "--variant", "longer"], "unknown variant longer"], + [["--list", "--tasks", "no-such-task"], "unknown task id(s): no-such-task"], + [["--list", "--tier", "T-D"], "unknown tier(s): T-D"], + [["--jobs", "0"], "--jobs must be 1 to 8"], + [["--runs"], "--runs needs a value"], + [["--phrasing", "2"], "--phrasing is alt, 0 or 1"], + ])("%j exits 2: %s", (args, message) => { + const r = run(args as string[]); + expect(r.status).toBe(2); + expect(r.stderr).toContain(message); + }); + + itNode("a model run without a price for the model refuses before any server starts", () => { + // A fake key: the price check stops the run before any client is made. + const r = run(["--model", "claude-unpriced-1", "--out", "unused"], { + ANTHROPIC_API_KEY: "sk-ant-api03-FAKE-for-the-price-check", + }); + expect(r.status).toBe(2); + expect(r.stderr).toContain("no price for model claude-unpriced-1"); + expect(`${r.stdout}${r.stderr}`).not.toContain("FAKE-for-the-price-check"); + }); +}); diff --git a/tests/azure/evals/tasks-ta.ts b/tests/azure/evals/tasks-ta.ts new file mode 100644 index 0000000..fe0ffe7 --- /dev/null +++ b/tests/azure/evals/tasks-ta.ts @@ -0,0 +1,1077 @@ +/** + * The E2 tasks of tier T-A, single operations: task ids, both phrasings, oracle commands + * and verifiers. The fixtures run as `az` commands through the harness's own server session, + * with an ARM body (`az rest`) where a resource has no `az` command for its setup. + * + * The verifier library arrives as a parameter (no runtime import of a local module: see + * verifiers.ts), so run.mjs can load this file unbundled. + */ +import type * as V from "./verifiers"; +import type { Query, SetupContext, Slots, Task } from "./types"; + +export function taTasks(v: typeof V): Task[] { + const SUB = "/subscriptions/{sub}"; + const RG = SUB + "/resourceGroups/{rg}/providers"; + + // api-versions the verifiers read with + const APPCFG = "2024-06-01"; + const LOCKS = "2020-05-01"; + const ROLES = "2022-04-01"; + const ACR = "2025-11-01"; + const EG = "2025-02-15"; + const EH = "2026-01-01"; + const INS = "2026-01-01"; + const VAULT_API = "2023-07-01"; + const MSI = "2024-11-30"; + const NET = "2025-09-01"; + const PDNS = "2024-06-01"; + const LA = "2026-03-01"; + const DEPLOY_API = "2022-09-01"; + const SB = "2026-01-01"; + const STORAGE = "2026-06-01"; + const DISKS = "2026-03-02"; + const CDN = "2024-02-01"; + + const STORE = RG + "/Microsoft.AppConfiguration/configurationStores/{store_name}"; + const LOCK = RG + "/Microsoft.Authorization/locks/{lock_name}"; + const REG = RG + "/Microsoft.ContainerRegistry/registries/{registry_name}"; + const DOMAIN = RG + "/Microsoft.EventGrid/domains/{domain_name}"; + const EG_TOPIC = RG + "/Microsoft.EventGrid/topics/{topic_name}"; + const EH_NS = RG + "/Microsoft.EventHub/namespaces/{namespace_name}"; + const ALERT = RG + "/Microsoft.Insights/activityLogAlerts/{name}"; + const VAULT = RG + "/Microsoft.KeyVault/vaults/{vault_name}"; + const UAI = RG + "/Microsoft.ManagedIdentity/userAssignedIdentities/{name}"; + const ROUTE_TABLE = RG + "/Microsoft.Network/routeTables/{route_table_name}"; + const VNET = RG + "/Microsoft.Network/virtualNetworks/{vnet_name}"; + const WS = RG + "/Microsoft.OperationalInsights/workspaces/{name}"; + const SUB_DEPLOY = SUB + "/providers/Microsoft.Resources/deployments/{deployment_name}"; + const SB_NS = RG + "/Microsoft.ServiceBus/namespaces/{namespace_name}"; + const ACCT_N = RG + "/Microsoft.Storage/storageAccounts/{name}"; + const DISK = RG + "/Microsoft.Compute/disks/{disk_name}"; + const PROFILE = RG + "/Microsoft.Cdn/profiles/{profile_name}"; + + const SWEEP = { + appcfg: `Microsoft.AppConfiguration/configurationStores@${APPCFG}`, + acr: `Microsoft.ContainerRegistry/registries@${ACR}`, + egTopic: `Microsoft.EventGrid/topics@${EG}`, + egDomain: `Microsoft.EventGrid/domains@${EG}`, + eh: `Microsoft.EventHub/namespaces@${EH}`, + alert: `Microsoft.Insights/activityLogAlerts@${INS}`, + vault: `Microsoft.KeyVault/vaults@${VAULT_API}`, + uai: `Microsoft.ManagedIdentity/userAssignedIdentities@${MSI}`, + routeTable: `Microsoft.Network/routeTables@${NET}`, + vnet: `Microsoft.Network/virtualNetworks@${NET}`, + pdns: `Microsoft.Network/privateDnsZones@${PDNS}`, + la: `Microsoft.OperationalInsights/workspaces@${LA}`, + sb: `Microsoft.ServiceBus/namespaces@${SB}`, + storage: `Microsoft.Storage/storageAccounts@${STORAGE}`, + disk: `Microsoft.Compute/disks@${DISKS}`, + cdn: `Microsoft.Cdn/profiles@${CDN}`, + }; + + // ── fixture builders ── + + /** A store, optionally a data-plane key-value and its lock. */ + async function appcfgStore(ctx: SetupContext, opts: { kv?: boolean; lock?: boolean } = {}) { + const s = ctx.hex(8); + Object.assign(ctx.slots, { + name: `appcfg${s}`, + store_name: `appcfg${s}`, + key: `key${s}`, + replica_name: `rep${s}`, + snapshot_name: `snap${s}`, + }); + const { store_name, key } = ctx.slots as Record; + await v.azOk( + ctx.q, + `appconfig create --name ${store_name} --resource-group ${ctx.rg} --location westeurope --sku Standard` + ); + if (opts.kv || opts.lock) + await v.azOk(ctx.q, `appconfig kv set --name ${store_name} --key ${key} --value v1 --yes`); + if (opts.lock) await v.azOk(ctx.q, `appconfig kv lock --name ${store_name} --key ${key} --yes`); + } + + /** A Basic registry and the names of its would-be children. */ + async function registry(ctx: SetupContext) { + const s = ctx.hex(10); + Object.assign(ctx.slots, { + name: `acr${s}`, + registry_name: `acr${s}`, + webhook_name: `wh${s}`, + replication_name: "northeurope", + }); + await v.azOk(ctx.q, `acr create --name acr${s} --resource-group ${ctx.rg} --sku Basic`); + } + + /** + * A Standard namespace in the run's group, polled until Get-able, with an event hub and an authorization rule on it. + */ + async function ehNamespace(ctx: SetupContext) { + const s = ctx.hex(8); + Object.assign(ctx.slots, { + name: `ehns-${s}`, + namespace_name: `ehns-${s}`, + eventhub_name: `hub-${s}`, + auth_rule_name: `rule-${s}`, + }); + const { + namespace_name: ns, + eventhub_name: hub, + auth_rule_name: rule, + } = ctx.slots as Record; + await v.azOk( + ctx.q, + `eventhubs namespace create --name ${ns} --resource-group ${ctx.rg} --location westeurope --sku Standard` + ); + const ready = await v.waitReady(ctx.q, v.fill(EH_NS, ctx.slots), EH, 240); + if (ready.status !== 200) + throw new v.FixtureError( + `fixture: Event Hubs namespace ${ns} not Get-able (${ready.status})` + ); + await v.azOk( + ctx.q, + `eventhubs eventhub create --name ${hub} --namespace-name ${ns} --resource-group ${ctx.rg}` + ); + await v.azOk( + ctx.q, + `eventhubs eventhub authorization-rule create --name ${rule} --eventhub-name ${hub} --namespace-name ${ns} --resource-group ${ctx.rg} --rights Listen` + ); + } + + /** A vault and (optionally) an RSA key with its version. */ + async function vaultKey(ctx: SetupContext, opts: { key: boolean }) { + const s = ctx.hex(10); + Object.assign(ctx.slots, { vault_name: `kv${s}`, key_name: `k${s}` }); + const { vault_name: vault, key_name: key } = ctx.slots as Record; + await v.azOk( + ctx.q, + `keyvault create --name ${vault} --resource-group ${ctx.rg} --location westeurope` + ); + if (!opts.key) return; + const created = await v.azOk( + ctx.q, + `keyvault key create --vault-name ${vault} --name ${key} --kty RSA` + ); + const version = v.kidVersion(v.dotted(created, "key.kid")); + if (!version) throw new v.FixtureError(`fixture: no key version for ${key}`); + ctx.slots.key_version = version; + } + + /** A namespace (polled), optionally a queue and a queue authorization rule. */ + async function sbNamespace(ctx: SetupContext, opts: { queue?: boolean; queueRule?: boolean }) { + const s = ctx.hex(10); + Object.assign(ctx.slots, { + name: `sb${s}`, + namespace_name: `sb${s}`, + queue_name: `q-${s}`, + topic_name: `t-${s}`, + subscription_name: `s-${s}`, + auth_rule_name: `a-${s}`, + }); + const { + namespace_name: ns, + queue_name: queue, + auth_rule_name: rule, + } = ctx.slots as Record; + await v.azOk( + ctx.q, + `servicebus namespace create --name ${ns} --resource-group ${ctx.rg} --location westeurope` + ); + const ready = await v.waitReady(ctx.q, v.fill(SB_NS, ctx.slots), SB, 60); + if (ready.status !== 200) + throw new v.FixtureError( + `fixture: Service Bus namespace ${ns} not Get-able (${ready.status})` + ); + if (opts.queue || opts.queueRule) + await v.azOk( + ctx.q, + `servicebus queue create --name ${queue} --namespace-name ${ns} --resource-group ${ctx.rg}` + ); + if (opts.queueRule) { + await v.azOk( + ctx.q, + `servicebus queue authorization-rule create --name ${rule} --queue-name ${queue} --namespace-name ${ns} --resource-group ${ctx.rg} --rights Listen Send` + ); + } + } + + const ROLE_ACTIONS = [ + "Microsoft.Storage/storageAccounts/read", + "Microsoft.Network/virtualNetworks/read", + "Microsoft.KeyVault/vaults/read", + "Microsoft.Web/sites/restart/action", + "Microsoft.Compute/virtualMachines/start/action", + "Microsoft.Insights/alertRules/read", + ]; + + const DEPLOY_TEMPLATE = { + $schema: "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#", + contentVersion: "1.0.0.0", + resources: [] as unknown[], + }; + + const tasks: Task[] = []; + const add = (t: Omit) => tasks.push({ tier: "T-A", caps: "T-A", ...t }); + + // ── Microsoft.AppConfiguration ─────────────────────────────────────────── + + add({ + id: "appconfig-kv-unlock", + opId: "Microsoft.AppConfiguration/Microsoft.AppConfiguration_DeleteLock", + verb: "delete", + prompts: [ + "Unlock the key-value {key} in the App Configuration store {store_name} (resource group {rg}).", + "The setting {key} in our App Configuration store {store_name} ({rg}) is locked and we need to edit it. Please unlock it.", + ], + oracle: ["appconfig kv unlock --name {store_name} --key {key} --yes"], + verify: v.fieldIs(STORE + "/keyValues/{key}", APPCFG, "properties.locked", "false"), + setup: (ctx) => appcfgStore(ctx, { lock: true }), + appConfigSlots: ["store_name"], + sweep: [SWEEP.appcfg], + }); + + const kvExists = async (q: Query, slots: Slots, key: string) => + (await v.arm(q, "GET", v.fill(STORE + "/keyValues/" + key, slots), APPCFG)).status === 200; + + add({ + id: "appconfig-check-keys", + opId: "Microsoft.AppConfiguration/Microsoft.AppConfiguration_CheckKeyValues", + verb: "check", + prompts: [ + "Check whether the keys {key} and {missing_key} exist in the App Configuration store {store_name} (resource group {rg}). Say for each whether it exists.", + "Do the settings {key} and {missing_key} exist in our App Configuration store {store_name} ({rg})? Tell me for each one.", + ], + oracle: [ + "appconfig kv list --name {store_name} --key {key}", + "appconfig kv list --name {store_name} --key {missing_key}", + ], + verify: v.existsClaims({ key: true, missing_key: false }, kvExists), + setup: async (ctx) => { + await appcfgStore(ctx, { kv: true }); + ctx.slots.missing_key = ctx.name("legacy-timeout-"); + }, + appConfigSlots: ["store_name"], + sweep: [SWEEP.appcfg], + }); + + add({ + id: "appconfig-regenerate-key", + opId: "Microsoft.AppConfiguration/ConfigurationStores_RegenerateKey", + verb: "action", + prompts: [ + "Regenerate the primary access key of the App Configuration store {store_name} in resource group {rg}. Leave the secondary key as it is.", + "Our App Configuration store {store_name} ({rg}) needs a new primary access key; please rotate it and keep the secondary one unchanged.", + ], + oracle: [ + "appconfig credential regenerate --name {store_name} --resource-group {rg} --id {_primary_id}", + ], + verify: v.changedAndKept( + STORE + "/listKeys", + APPCFG, + "value[name=Primary].value", + "value[name=Secondary].value" + ), + setup: async (ctx) => { + await appcfgStore(ctx); + const keys = await v.arm(ctx.q, "POST", v.fill(STORE + "/listKeys", ctx.slots), APPCFG); + const primary = v.dotted(keys.body, "value[name=Primary]"); + const id = v.dotted(primary, "id"); + if (!id) + throw new v.FixtureError( + `fixture: no Primary key in ${JSON.stringify(keys.body).slice(0, 200)}` + ); + ctx.slots.key_id = id; + ctx.slots._primary_id = id; + await v.snapshot(ctx.q, ctx.slots, STORE + "/listKeys", APPCFG, { + changed: "value[name=Primary].value", + kept: "value[name=Secondary].value", + }); + }, + appConfigSlots: ["store_name"], + sweep: [SWEEP.appcfg], + }); + + // ── Microsoft.Authorization ────────────────────────────────────────────── + + add({ + id: "lock-delete", + opId: "Microsoft.Authorization/ManagementLocks_DeleteAtResourceGroupLevel", + verb: "delete", + prompts: [ + "Remove the management lock {lock_name} from resource group {rg}.", + "We need to delete things in {rg}; please take off the management lock {lock_name}.", + ], + // The natural CLI commands fail here: `az lock delete` and `az group lock delete` first + // list locks at subscription level (501 on this emulator); `resource delete --ids` works. + oracle: [ + "resource delete --ids /subscriptions/{sub}/resourceGroups/{rg}/providers/Microsoft.Authorization/locks/{lock_name}", + ], + verify: v.gone(LOCK, LOCKS), + setup: async (ctx) => { + ctx.slots.lock_name = `lk-${ctx.hex(8)}`; + await v.putOk( + ctx.q, + v.fill(LOCK, ctx.slots), + LOCKS, + { properties: { level: "CanNotDelete" } }, + "lock" + ); + }, + }); + + add({ + id: "locks-list", + opId: "Microsoft.Authorization/ManagementLocks_ListAtResourceGroupLevel", + verb: "list", + prompts: [ + "List the management locks on resource group {rg}.", + "Which management locks are set on the {rg} resource group? Tell me their names.", + ], + oracle: ["lock list --resource-group {rg}"], + verify: v.claims("{lock1}", "{lock2}"), + setup: async (ctx) => { + ctx.slots.lock1 = ctx.name("lk-ops-"); + ctx.slots.lock2 = ctx.name("lk-audit-"); + for (const n of [ctx.slots.lock1, ctx.slots.lock2]) { + await v.putOk( + ctx.q, + v.fill(RG + "/Microsoft.Authorization/locks/" + String(n), ctx.slots), + LOCKS, + { properties: { level: "CanNotDelete", notes: "E2 fixture" } }, + "lock" + ); + } + }, + }); + + const ROLE_DEF = SUB + "/providers/Microsoft.Authorization/roleDefinitions/{role_id}"; + add({ + id: "role-definition-get", + opId: "Microsoft.Authorization/RoleDefinitions_Get", + verb: "read", + prompts: [ + "What is the name of the role definition {role_id}, and which actions does it allow?", + "Can you look up the Azure role definition with ID {role_id} and tell me its role name and the actions it permits?", + ], + oracle: ["role definition list --name {role_id}"], + verify: v.claims("{role_name}", "{action1}", "{action2}"), + setup: async (ctx) => { + const roleId = ctx.uuid(); + const roleName = `Bench Operator ${ctx.hex(4)}`; + const actions = ctx.sample(ROLE_ACTIONS, 2); + Object.assign(ctx.slots, { + role_definition_name: roleId, + role_id: roleId, + role_name: roleName, + action1: actions[0], + action2: actions[1], + }); + await v.putOk( + ctx.q, + v.fill(ROLE_DEF, ctx.slots), + ROLES, + { + properties: { + roleName, + description: "E2 fixture role", + type: "CustomRole", + permissions: [{ actions, notActions: [] }], + assignableScopes: [`/subscriptions/${v.SUBSCRIPTION}`], + }, + }, + "role definition" + ); + }, + teardown: async (q, slots) => { + await v.arm(q, "DELETE", v.fill(ROLE_DEF, slots), ROLES); + }, + }); + + // ── Microsoft.ContainerRegistry ────────────────────────────────────────── + + add({ + id: "acr-login-server", + opId: "Microsoft.ContainerRegistry/Registries_Get", + verb: "read", + prompts: [ + "What is the login server of the container registry {registry_name} in resource group {rg}?", + "Which hostname do I use to docker login to our registry {registry_name} ({rg})?", + ], + oracle: ["acr show --name {registry_name} --resource-group {rg}"], + verify: v.answerHasFields(REG, ACR, ["properties.loginServer"]), + setup: registry, + sweep: [SWEEP.acr], + }); + + add({ + id: "acr-webhook-create", + opId: "Microsoft.ContainerRegistry/Webhooks_Create", + verb: "create", + prompts: [ + "Create a webhook named {webhook_name} on the container registry {registry_name} (resource group {rg}) that calls https://example.com/hooks/{hook_id} on push and delete events.", + "Please add a webhook {webhook_name} to our registry {registry_name} in {rg}: it should notify https://example.com/hooks/{hook_id} whenever an image is pushed or deleted.", + ], + oracle: [ + "acr webhook create --name {webhook_name} --registry {registry_name} --resource-group {rg} --uri https://example.com/hooks/{hook_id} --actions push delete", + ], + verify: v.allOf( + v.readyWith(REG + "/webhooks/{webhook_name}", ACR, {}), + v.listFieldContains(REG + "/webhooks/{webhook_name}", ACR, "properties.actions", [ + "push", + "delete", + ]) + ), + setup: async (ctx) => { + await registry(ctx); + ctx.slots.hook_id = ctx.hex(8); + }, + sweep: [SWEEP.acr], + }); + + // ── Microsoft.EventGrid ────────────────────────────────────────────────── + + add({ + id: "eventgrid-topic-regenerate-key", + opId: "Microsoft.EventGrid/Topics_RegenerateKey", + verb: "action", + prompts: [ + "Regenerate the primary access key (key1) of the Event Grid topic {topic_name} in resource group {rg}. Leave key2 unchanged.", + "key1 of our Event Grid topic {topic_name} ({rg}) was exposed in a log. Please rotate key1 and keep key2 as it is.", + ], + oracle: [ + "eventgrid topic key regenerate --name {topic_name} --resource-group {rg} --key-name key1", + ], + verify: v.changedAndKept(EG_TOPIC + "/listKeys", EG, "key1", "key2"), + setup: async (ctx) => { + ctx.slots.topic_name = `t${ctx.hex(10)}`; + await v.azOk( + ctx.q, + `eventgrid topic create --name ${String(ctx.slots.topic_name)} --resource-group ${ctx.rg} --location westeurope` + ); + await v.snapshot(ctx.q, ctx.slots, EG_TOPIC + "/listKeys", EG, { + changed: "key1", + kept: "key2", + }); + }, + sweep: [SWEEP.egTopic], + }); + + add({ + id: "eventgrid-domain-disable-local-auth", + opId: "Microsoft.EventGrid/Domains_Update", + verb: "update", + prompts: [ + "Disable local (access key) authentication on the Event Grid domain {domain_name} in resource group {rg}.", + "From now on only Entra ID should be able to publish to our Event Grid domain {domain_name} ({rg}): please turn off key-based (local) authentication.", + ], + // az eventgrid domain update has no --disable-local-auth; the generic resource update reaches it. + oracle: [ + "resource update --resource-group {rg} --name {domain_name} --resource-type Microsoft.EventGrid/domains --set properties.disableLocalAuth=true", + ], + verify: v.fieldIs(DOMAIN, EG, "properties.disableLocalAuth", "true"), + setup: async (ctx) => { + ctx.slots.domain_name = `d${ctx.hex(10)}`; + await v.azOk( + ctx.q, + `eventgrid domain create --name ${String(ctx.slots.domain_name)} --resource-group ${ctx.rg} --location westeurope` + ); + const body = await v.getOk(ctx.q, v.fill(DOMAIN, ctx.slots), EG, "domain"); + ctx.slots.location = v.dotted(body, "location") || "westeurope"; + }, + sweep: [SWEEP.egDomain], + }); + + // ── Microsoft.EventHub ─────────────────────────────────────────────────── + + const RIGHTS = [["Listen"], ["Send"], ["Listen", "Send"], ["Listen", "Send", "Manage"]]; + add({ + id: "eventhub-hub-auth-rule-rights", + opId: "Microsoft.EventHub/EventHubs_GetAuthorizationRule", + verb: "read", + prompts: [ + "Which rights does the authorization rule {auth_rule_name} on the event hub {eventhub_name} (namespace {namespace_name}, resource group {rg}) grant?", + "What can clients do with the {auth_rule_name} policy of our event hub {eventhub_name} in namespace {namespace_name} ({rg})? Tell me which rights it grants.", + ], + oracle: [ + "eventhubs eventhub authorization-rule show --name {auth_rule_name} --eventhub-name {eventhub_name} --namespace-name {namespace_name} --resource-group {rg}", + ], + // Verifier defect 1 fixed (verifiers.affirmedRights). + verify: v.rightsClaim( + EH_NS + "/eventhubs/{eventhub_name}/authorizationRules/{auth_rule_name}", + EH + ), + setup: async (ctx) => { + await ehNamespace(ctx); + const rights = ctx.pick(RIGHTS); + ctx.slots._rights = rights.join(","); + await v.putOk( + ctx.q, + v.fill(EH_NS + "/eventhubs/{eventhub_name}/authorizationRules/{auth_rule_name}", ctx.slots), + EH, + { properties: { rights } }, + "authorization rule" + ); + }, + sweep: [SWEEP.eh], + }); + + // ── Microsoft.Insights ─────────────────────────────────────────────────── + + add({ + id: "insights-activity-log-alert-create", + opId: "Microsoft.Insights/ActivityLogAlerts_CreateOrUpdate", + verb: "create", + prompts: [ + "Create an activity log alert named {name} in resource group {rg} that fires on Administrative events, scoped to the resource group {rg} itself.", + "Please set up an activity log alert called {name} in {rg}: it should watch the {rg} resource group and trigger on Administrative-category events.", + ], + oracle: [ + "monitor activity-log alert create --name {name} --resource-group {rg} --scope /subscriptions/{sub}/resourceGroups/{rg} --condition category=Administrative", + ], + verify: v.allOf( + v.readyWith(ALERT, INS, {}), + v.listFieldContains(ALERT, INS, "properties.scopes", [ + "/subscriptions/{sub}/resourceGroups/{rg}", + ]), + v.fieldIs(ALERT, INS, "properties.condition.allOf[field=category].equals", "Administrative") + ), + setup: async (ctx) => { + ctx.slots.name = `activity${ctx.hex(10)}`; + ctx.slots.scope = `/subscriptions/${v.SUBSCRIPTION}/resourceGroups/${ctx.rg}`; + }, + sweep: [SWEEP.alert], + }); + + // ── Microsoft.KeyVault ─────────────────────────────────────────────────── + + add({ + id: "kv-sign", + opId: "Microsoft.KeyVault/Microsoft.KeyVault_Sign", + verb: "action", + prompts: [ + "Sign the SHA-256 digest {digest} (base64url) with version {key_version} of the key {key_name} in the key vault {vault_name}, using RS256, and give me the signature.", + "Please create an RS256 signature with our Key Vault key {key_name} (version {key_version}, vault {vault_name}) over this SHA-256 digest, base64url-encoded: {digest}. I need the signature value.", + ], + oracle: [ + "keyvault key sign --vault-name {vault_name} --name {key_name} --version {key_version} --algorithm RS256 --digest {digest_b64}", + ], + verify: v.signatureVerifies(), + setup: async (ctx) => { + await vaultKey(ctx, { key: true }); + const digest = v.sha256(Buffer.from(ctx.uuid().replace(/-/g, ""), "hex")); + Object.assign(ctx.slots, { + crypto_value: v.b64url(digest), + digest: v.b64url(digest), + digest_b64: v.b64std(digest), + }); + }, + kvCrypto: true, + vaultSlots: ["vault_name"], + sweep: [SWEEP.vault], + }); + + add({ + id: "kv-decrypt", + opId: "Microsoft.KeyVault/Microsoft.KeyVault_Decrypt", + verb: "action", + prompts: [ + "Decrypt this ciphertext with version {key_version} of the key {key_name} in the key vault {vault_name} (algorithm RSA-OAEP) and tell me the plaintext. Ciphertext (base64url): {crypto_value}", + "A partner sent us data encrypted with our Key Vault key {key_name} (version {key_version}) in vault {vault_name}, using RSA-OAEP. Please decrypt it and tell me what it says. Ciphertext, base64url-encoded: {crypto_value}", + ], + oracle: [ + "keyvault key decrypt --vault-name {vault_name} --name {key_name} --version {key_version} --algorithm RSA-OAEP --value {ciphertext_b64} --data-type plaintext", + ], + verify: v.plaintextStated(), + setup: async (ctx) => { + await vaultKey(ctx, { key: true }); + const plaintext = `order-${ctx.hex(10)}`; + const s = ctx.slots as Record; + const enc = await v.azOk( + ctx.q, + `keyvault key encrypt --vault-name ${s.vault_name} --name ${s.key_name} --version ${s.key_version} --algorithm RSA-OAEP --value ${v.b64std(plaintext)} --data-type base64` + ); + const result = v.dotted(enc, "result"); + if (typeof result !== "string" || !result) + throw new v.FixtureError(`fixture: no ciphertext in ${JSON.stringify(enc).slice(0, 200)}`); + Object.assign(ctx.slots, { + crypto_value: v.toB64url(result), + ciphertext_b64: v.toStdB64(result), + plaintext, + }); + }, + kvCrypto: true, + vaultSlots: ["vault_name"], + sweep: [SWEEP.vault], + }); + + add({ + id: "kv-rotation-policy", + opId: "Microsoft.KeyVault/Microsoft.KeyVault_GetKeyRotationPolicy", + verb: "read", + prompts: [ + "What rotation policy does the key {key_name} in the key vault {vault_name} (resource group {rg}) have? After how long is it rotated, and when does it expire?", + "Can you check the automatic rotation settings of the key {key_name} in our vault {vault_name} ({rg}): the rotation interval and the expiry time?", + ], + oracle: ["keyvault key rotation-policy show --vault-name {vault_name} --name {key_name}"], + verify: v.allOf( + v.textMatches("(?:P{rotate_days}D|\\b{rotate_days}\\s*-?\\s*days?\\b)"), + v.textMatches("(?:P{expiry_days}D|\\b{expiry_days}\\s*-?\\s*days?\\b)") + ), + setup: async (ctx) => { + await vaultKey(ctx, { key: true }); + const rotate = ctx.pick([45, 60, 75, 120]); + const expiry = ctx.pick([180, 270, 365]); + const policy = { + lifetimeActions: [ + { trigger: { timeAfterCreate: `P${rotate}D` }, action: { type: "Rotate" } }, + { trigger: { timeBeforeExpiry: "P30D" }, action: { type: "Notify" } }, + ], + attributes: { expiryTime: `P${expiry}D` }, + }; + const s = ctx.slots as Record; + await v.azOk( + ctx.q, + `keyvault key rotation-policy update --vault-name ${s.vault_name} --name ${s.key_name} --value '${v.jsonArg(policy)}'`, + "rotation policy" + ); + Object.assign(ctx.slots, { rotate_days: rotate, expiry_days: expiry }); + }, + vaultSlots: ["vault_name"], + sweep: [SWEEP.vault], + }); + + add({ + id: "kv-access-policy-grant", + opId: "Microsoft.KeyVault/Vaults_UpdateAccessPolicy", + verb: "update", + prompts: [ + "Grant the principal with object ID {object_id} permission to get and list secrets in the key vault {vault_name} (resource group {rg}), using the vault's access policies.", + "Our deployment identity (object ID {object_id}) needs to read secrets from the Key Vault {vault_name} in {rg}: please give it get and list permissions on secrets through an access policy.", + ], + oracle: [ + "keyvault set-policy --name {vault_name} --resource-group {rg} --object-id {object_id} --secret-permissions get list", + ], + verify: v.accessPolicyGrants(VAULT, VAULT_API, "object_id", "secrets", ["get", "list"]), + setup: async (ctx) => { + // A vault with the access-policy model (the default vault uses RBAC), no policies yet. + ctx.slots.vault_name = `kv${ctx.hex(10)}`; + await v.putOk( + ctx.q, + v.fill(VAULT, ctx.slots), + VAULT_API, + { + location: "westeurope", + properties: { + tenantId: v.SUBSCRIPTION, + sku: { family: "A", name: "standard" }, + accessPolicies: [], + enableRbacAuthorization: false, + }, + }, + "vault" + ); + await v.waitReady(ctx.q, v.fill(VAULT, ctx.slots), VAULT_API); + ctx.slots.object_id = ctx.uuid(); + }, + vaultSlots: ["vault_name"], + sweep: [SWEEP.vault], + }); + + // ── Microsoft.ManagedIdentity ──────────────────────────────────────────── + + add({ + id: "identity-create", + opId: "Microsoft.ManagedIdentity/UserAssignedIdentities_CreateOrUpdate", + verb: "create", + prompts: [ + "Create a user-assigned managed identity named {name} in resource group {rg}, and tell me its client ID.", + "Please create a new user-assigned managed identity called {name} in {rg}; I'll need its client ID afterwards.", + ], + oracle: ["identity create --name {name} --resource-group {rg}"], + verify: v.allOf( + v.readyWith(UAI, MSI, {}), + v.answerHasFields(UAI, MSI, ["properties.clientId"]) + ), + setup: async (ctx) => { + ctx.slots.name = `id-${ctx.hex(8)}`; + }, + sweep: [SWEEP.uai], + }); + + add({ + id: "identity-list", + opId: "Microsoft.ManagedIdentity/UserAssignedIdentities_ListBySubscription", + verb: "list", + prompts: [ + "List the user-assigned managed identities in my subscription.", + "Which user-assigned managed identities exist across the whole subscription? Tell me their names.", + ], + oracle: ["identity list"], + verify: v.claims("{id1}", "{id2}"), + setup: async (ctx) => { + ctx.slots.id1 = ctx.name("id-ci-"); + ctx.slots.id2 = ctx.name("id-backup-"); + for (const n of [ctx.slots.id1, ctx.slots.id2]) { + await v.putOk( + ctx.q, + v.fill(RG + "/Microsoft.ManagedIdentity/userAssignedIdentities/" + String(n), ctx.slots), + MSI, + { location: "westeurope" }, + "identity" + ); + } + }, + sweep: [SWEEP.uai], + }); + + // ── Microsoft.Network ──────────────────────────────────────────────────── + + add({ + id: "route-get", + opId: "Microsoft.Network/Routes_Get", + verb: "read", + prompts: [ + "What address prefix and next hop does the route {route_name} in the route table {route_table_name} (resource group {rg}) have?", + "Where does the route {route_name} of our route table {route_table_name} ({rg}) send traffic, and for which address prefix?", + ], + oracle: [ + "network route-table route show --name {route_name} --route-table-name {route_table_name} --resource-group {rg}", + ], + verify: v.claims("{prefix}", "{next_hop}"), + setup: async (ctx) => { + const s = ctx.hex(8); + Object.assign(ctx.slots, { route_table_name: `rt-${s}`, route_name: `r-${s}` }); + await v.azOk( + ctx.q, + `network route-table create --name rt-${s} --resource-group ${ctx.rg} --location westeurope` + ); + const prefix = `10.${ctx.randomInt(20, 200)}.${ctx.randomInt(0, 250)}.0/24`; + const hop = `10.0.${ctx.randomInt(2, 250)}.4`; + const path = v.fill(ROUTE_TABLE + "/routes/{route_name}", ctx.slots); + await v.putOk( + ctx.q, + path, + NET, + { + properties: { + addressPrefix: prefix, + nextHopType: "VirtualAppliance", + nextHopIpAddress: hop, + }, + }, + "route" + ); + await v.waitReady(ctx.q, path, NET); + Object.assign(ctx.slots, { prefix, next_hop: hop }); + }, + sweep: [SWEEP.routeTable], + }); + + add({ + id: "private-dns-zones-list", + opId: "Microsoft.Network/PrivateZones_ListByResourceGroup", + verb: "list", + prompts: [ + "List the private DNS zones in resource group {rg}.", + "Which private DNS zones do we have in {rg}? Tell me their names.", + ], + oracle: ["network private-dns zone list --resource-group {rg}"], + verify: v.claims("{zone1}", "{zone2}"), + setup: async (ctx) => { + ctx.slots.zone1 = `${ctx.hex(6)}.internal.contoso.com`; + ctx.slots.zone2 = `${ctx.hex(6)}.corp.contoso.com`; + for (const z of [ctx.slots.zone1, ctx.slots.zone2]) { + const path = v.fill(RG + "/Microsoft.Network/privateDnsZones/" + String(z), ctx.slots); + await v.putOk(ctx.q, path, PDNS, { location: "global" }, "private DNS zone"); + await v.waitReady(ctx.q, path, PDNS); + } + }, + sweep: [SWEEP.pdns], + }); + + add({ + id: "vnet-address-space", + opId: "Microsoft.Network/VirtualNetworks_Get", + verb: "read", + prompts: [ + "What address space does the virtual network {vnet_name} in resource group {rg} use?", + "Which IP address range is our VNet {vnet_name} ({rg}) configured with?", + ], + oracle: ["network vnet show --name {vnet_name} --resource-group {rg}"], + verify: v.claims("{prefix}"), + setup: async (ctx) => { + const name = ctx.name("vnet-core-"); + const prefix = `10.${ctx.randomInt(10, 250)}.0.0/16`; + Object.assign(ctx.slots, { name, vnet_name: name }); + const path = v.fill(VNET, ctx.slots); + await v.putOk( + ctx.q, + path, + NET, + { location: "westeurope", properties: { addressSpace: { addressPrefixes: [prefix] } } }, + "vnet" + ); + await v.waitReady(ctx.q, path, NET); + ctx.slots.prefix = prefix; + }, + sweep: [SWEEP.vnet], + }); + + // ── Microsoft.OperationalInsights ──────────────────────────────────────── + + add({ + id: "la-workspace-retention", + opId: "Microsoft.OperationalInsights/Workspaces_Update", + verb: "update", + prompts: [ + "Set the data retention of the Log Analytics workspace {name} in resource group {rg} to {days} days.", + "Compliance wants {days} days of log retention: please update our Log Analytics workspace {name} ({rg}) accordingly.", + ], + oracle: [ + "monitor log-analytics workspace update --resource-group {rg} --workspace-name {name} --retention-time {days}", + ], + verify: v.fieldIs(WS, LA, "properties.retentionInDays", "{days}"), + setup: async (ctx) => { + ctx.slots.name = `la-${ctx.hex(8)}`; + await v.azOk( + ctx.q, + `monitor log-analytics workspace create --resource-group ${ctx.rg} --workspace-name ${String(ctx.slots.name)} --location westeurope` + ); + ctx.slots.days = ctx.pick([60, 90, 120, 180]); + }, + sweep: [SWEEP.la], + }); + + // ── Microsoft.Resources (subscription-scope deployments) ──────────────── + + add({ + id: "deployment-sub-validate", + opId: "Microsoft.Resources/Deployments_ValidateAtSubscriptionScope", + verb: "action", + prompts: [ + "Validate, without deploying it, whether this ARM template can be deployed at subscription scope in westeurope as {deployment_name}: {template_json}", + "Before we roll it out, check whether this subscription-level template would deploy cleanly (as {deployment_name}, westeurope) - validation only, please: {template_json}", + ], + // az deployment sub validate needs a template file; the oracle sends it inline with az rest. + oracle: [ + "rest --method post --url /subscriptions/{sub}/providers/Microsoft.Resources/deployments/{deployment_name}/validate?api-version=2022-09-01 --body '{deploy_body}'", + ], + // Verifier defect 2 fixed (verifiers.validationVerdict). + verify: v.validationClaim(SUB_DEPLOY, DEPLOY_API), + setup: async (ctx) => { + ctx.slots.deployment_name = `dep-${ctx.hex(8)}`; + ctx.slots.template_json = v.pyJson(DEPLOY_TEMPLATE); + ctx.slots.deploy_body = v.pyJson({ + location: "westeurope", + properties: { mode: "Incremental", template: DEPLOY_TEMPLATE }, + }); + }, + teardown: async (q, slots) => { + await v.arm(q, "DELETE", v.fill(SUB_DEPLOY, slots), DEPLOY_API); + }, + guessable: true, + }); + + // ── Microsoft.ServiceBus ───────────────────────────────────────────────── + + add({ + id: "sb-queue-regenerate-key", + opId: "Microsoft.ServiceBus/Queues_RegenerateKeys", + verb: "action", + prompts: [ + "Regenerate the primary key of the authorization rule {auth_rule_name} on the queue {queue_name} in the Service Bus namespace {namespace_name} (resource group {rg}). Keep the secondary key.", + "The primary key of the {auth_rule_name} policy on our Service Bus queue {queue_name} (namespace {namespace_name}, {rg}) leaked. Please rotate it and leave the secondary key alone.", + ], + oracle: [ + "servicebus queue authorization-rule keys renew --name {auth_rule_name} --queue-name {queue_name} --namespace-name {namespace_name} --resource-group {rg} --key PrimaryKey", + ], + verify: v.changedAndKept( + SB_NS + "/queues/{queue_name}/authorizationRules/{auth_rule_name}/listKeys", + SB, + "primaryKey", + "secondaryKey" + ), + setup: async (ctx) => { + await sbNamespace(ctx, { queueRule: true }); + await v.snapshot( + ctx.q, + ctx.slots, + SB_NS + "/queues/{queue_name}/authorizationRules/{auth_rule_name}/listKeys", + SB, + { + changed: "primaryKey", + kept: "secondaryKey", + } + ); + }, + sweep: [SWEEP.sb], + }); + + add({ + id: "sb-dp-queue-max-delivery", + opId: "Microsoft.ServiceBus.DataPlane/Entity_Get", + verb: "read", + prompts: [ + "What is the maximum delivery count of the Service Bus queue {queue_name} in the namespace {namespace_name} (resource group {rg})?", + "After how many delivery attempts does our Service Bus queue {queue_name} (namespace {namespace_name}, {rg}) dead-letter a message?", + ], + oracle: [ + "servicebus queue show --name {queue_name} --namespace-name {namespace_name} --resource-group {rg}", + ], + verify: v.allOf( + v.fieldIs( + SB_NS + "/queues/{queue_name}", + SB, + "properties.maxDeliveryCount", + "{max_delivery}" + ), + v.textMatches("\\b{max_delivery}\\b") + ), + setup: async (ctx) => { + await sbNamespace(ctx, { queue: true }); + // Re-PUT the queue with a random maxDeliveryCount. + const count = ctx.pick([3, 5, 7, 12, 15]); + const path = v.fill(SB_NS + "/queues/{queue_name}", ctx.slots); + const body = await v.getOk(ctx.q, path, SB, "queue"); + const props = { ...((v.dotted(body, "properties") as Record) ?? {}) }; + for (const k of [ + "createdAt", + "updatedAt", + "accessedAt", + "countDetails", + "messageCount", + "sizeInBytes", + "status", + "subscriptionCount", + ]) { + delete props[k]; + } + props.maxDeliveryCount = count; + await v.putOk(ctx.q, path, SB, { properties: props }, "queue"); + ctx.slots.max_delivery = count; + }, + sweep: [SWEEP.sb], + }); + + // ── Microsoft.Storage ──────────────────────────────────────────────────── + + add({ + id: "storage-regenerate-key", + opId: "Microsoft.Storage/StorageAccounts_RegenerateKey", + verb: "action", + prompts: [ + "Regenerate the first access key (key1) of the storage account {name} in resource group {rg}. Keep key2.", + "key1 of our storage account {name} ({rg}) was committed to a repository. Please regenerate key1 and leave key2 unchanged.", + ], + oracle: [ + "storage account keys renew --account-name {name} --resource-group {rg} --key primary", + ], + verify: v.changedAndKept( + ACCT_N + "/listKeys", + STORAGE, + "keys[keyName=key1].value", + "keys[keyName=key2].value" + ), + setup: async (ctx) => { + // A private account, waited until Succeeded (its Azurite starts in 10-30 s). + ctx.slots.name = `sa${ctx.hex(8)}`; + await v.azOk( + ctx.q, + `storage account create --name ${String(ctx.slots.name)} --resource-group ${ctx.rg} --location westeurope --sku Standard_LRS` + ); + const state = await v.waitField( + ctx.q, + v.fill(ACCT_N, ctx.slots), + STORAGE, + "properties.provisioningState", + "Succeeded", + 180 + ); + if (String(state).toLowerCase() !== "succeeded") + throw new v.FixtureError(`fixture: storage account still ${String(state)} after 180 s`); + await v.snapshot(ctx.q, ctx.slots, ACCT_N + "/listKeys", STORAGE, { + changed: "keys[keyName=key1].value", + kept: "keys[keyName=key2].value", + }); + }, + sweep: [SWEEP.storage], + }); + + // ── Microsoft.Compute ──────────────────────────────────────────────────── + + add({ + id: "disk-create", + opId: "Microsoft.Compute/Disks_CreateOrUpdate", + verb: "create", + prompts: [ + "Create an empty managed disk named {disk_name} in resource group {rg}: {size_gb} GB, StandardSSD_LRS.", + "I need a new {size_gb} GB managed disk called {disk_name} in the {rg} resource group, on StandardSSD_LRS storage. Please create it.", + ], + oracle: [ + "disk create --name {disk_name} --resource-group {rg} --size-gb {size_gb} --sku StandardSSD_LRS", + ], + verify: v.readyWith(DISK, DISKS, { + "properties.diskSizeGB": "{size_gb}", + "sku.name": "StandardSSD_LRS", + }), + setup: async (ctx) => { + ctx.slots.disk_name = `disk-${ctx.hex(10)}`; + ctx.slots.size_gb = ctx.pick([16, 32, 64, 128]); + }, + sweep: [SWEEP.disk], + }); + + // ── Microsoft.Cdn (classic CDN; needs CDN_CLASSIC_ALLOW_CREATE=1 on the emulator) ── + + add({ + id: "cdn-can-migrate", + opId: "Microsoft.Cdn/Profiles_CdnCanMigrateToAfd", + verb: "action", + prompts: [ + "Can the classic CDN profile {profile_name} in resource group {rg} be migrated to Azure Front Door? If it cannot, tell me what is blocking it.", + "We want to move our classic CDN profile {profile_name} ({rg}) to Azure Front Door: check whether it is eligible for migration, and if it is not, why not.", + ], + oracle: [ + "cdn profile-migration check-compatibility --profile-name {profile_name} --resource-group {rg}", + ], + verify: v.migrationVerdict("can_migrate"), + setup: async (ctx) => { + const s = ctx.hex(10); + Object.assign(ctx.slots, { + profile_name: `cdnp${s}`, + endpoint_name: `cdne${s}`, + origin_name: `or${s}`, + }); + await v.azOk( + ctx.q, + `cdn profile create --name cdnp${s} --resource-group ${ctx.rg} --sku Standard_Microsoft` + ); + // Half the runs give the profile an endpoint (it can migrate). + if (ctx.random() < 0.5) { + const path = v.fill(PROFILE + "/endpoints/{endpoint_name}", ctx.slots); + await v.putOk( + ctx.q, + path, + CDN, + { + location: "global", + properties: { + origins: [ + { name: "origin1", properties: { hostName: `cdne${s}.origin.example.com` } }, + ], + }, + }, + "CDN endpoint" + ); + await v.waitReady(ctx.q, path, CDN); + ctx.slots.can_migrate = true; + } else { + ctx.slots.can_migrate = false; + } + }, + guessable: true, + sweep: [SWEEP.cdn], + }); + + return tasks; +} diff --git a/tests/azure/evals/tasks-tapim.ts b/tests/azure/evals/tasks-tapim.ts new file mode 100644 index 0000000..bfde0a3 --- /dev/null +++ b/tests/azure/evals/tasks-tapim.ts @@ -0,0 +1,543 @@ +/** + * The E2 tasks of tier T-APIM: API Management operations, one phrasing. Each run gets its + * own APIM service (created in about a second, no backing container) in the run's group. + * Most operations have no dedicated CLI command, so the oracles use `az rest`; an XML + * policy body travels JSON-escaped (\u003c, \u003e). + * + * The verifier library arrives as a parameter (see verifiers.ts). + */ +import type * as V from "./verifiers"; +import type { SetupContext, Task } from "./types"; + +/** + * A static self-signed PFX (CN=mcp-t2-cert.contoso.com, no password, + * valid 2026-2036). The emulator parses the material, + * so it must be a genuine PKCS#12 archive. + */ +export const STATIC_TEST_PFX_B64 = + "MIIIzwIBAzCCCIUGCSqGSIb3DQEHAaCCCHYEgghyMIIIbjCCCGoGCSqGSIb3DQEHAaCCCFsEgghX" + + "MIIIUzCCAzsGCyqGSIb3DQEMCgEDoIIC7DCCAugGCiqGSIb3DQEJFgGgggLYBIIC1DCCAtAwggG4" + + "oAMCAQICFASNrd5hHv8MiQlwyQ36cZpl1NFCMA0GCSqGSIb3DQEBCwUAMCIxIDAeBgNVBAMMF21j" + + "cC10Mi1jZXJ0LmNvbnRvc28uY29tMB4XDTI2MDEwMTAwMDAwMFoXDTM2MDEwMTAwMDAwMFowIjEg" + + "MB4GA1UEAwwXbWNwLXQyLWNlcnQuY29udG9zby5jb20wggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAw" + + "ggEKAoIBAQDH5iUNFsw6KLsdeQ/rJLPRRkNWPIkcEKupz8rZqp4gYl+aZVMKfWcazN/DWf2/d/2t" + + "zFsh2iFiCoVAzGzxBDLpBOIStRHGwKGGDG1NsWjLaT6TwAKihyL5g54Qlref2ZewPmcYxqbCMwg6" + + "6TDAY27jP/SWsbh+k/7SNlGc1WbgBcXPWGO6sB5S5lYhRfPfy26Ic3om65eEglev15+vDfDUZy0H" + + "asnFSX+85zda5LlSmR6kWj+r2xdlJDzsd1CJWjvQqhpLGck9nXiYSFslTTzTo0E0AWNMDAlJvZ44" + + "ensHthhbwB7S+S8jpcfSWJfDG2fxGOB3L7dpPjwdGVEUZRspAgMBAAEwDQYJKoZIhvcNAQELBQAD" + + "ggEBADdrXRZR959lr6XGmg6gBodu35V6cqzepZXU5VpbXfLQQlr7HUOpieu7QZQk/jUWKfUR+174" + + "zkCtpfuZOsl6HuYpmbDSIMcPwB2RXOaKUN7bI+ewZopejJ67rT4tWq8DURCkzKSyxLOq7gj0mjYU" + + "zwRIvCMhRP0tyd3gsP32QN3iMkN3ziQZ/MQLOoEiimPfciWW7l1dqM2bFboDLfk3EMyhBP80Iykq" + + "U8HfA1tMO+uRpbjIw5IbOq6A0l0x2j9j4O0TgAfEw28Xrw39WAyWskMPUWnydAqXYQTZz1ZrEwJt" + + "Gx7IoiCacdRrlthIUsJliIWBvCUuYc7BDWhVlBMNRrcxPDAVBgkqhkiG9w0BCRQxCB4GAG0AYwBw" + + "MCMGCSqGSIb3DQEJFTEWBBQ9x6MVcu6BmxRp0nnsvpErHMs8xzCCBRAGCyqGSIb3DQEMCgEBoIIE" + + "wTCCBL0CAQAwDQYJKoZIhvcNAQEBBQAEggSnMIIEowIBAAKCAQEAx+YlDRbMOii7HXkP6ySz0UZD" + + "VjyJHBCrqc/K2aqeIGJfmmVTCn1nGszfw1n9v3f9rcxbIdohYgqFQMxs8QQy6QTiErURxsChhgxt" + + "TbFoy2k+k8ACooci+YOeEJa3n9mXsD5nGMamwjMIOukwwGNu4z/0lrG4fpP+0jZRnNVm4AXFz1hj" + + "urAeUuZWIUXz38tuiHN6JuuXhIJXr9efrw3w1GctB2rJxUl/vOc3WuS5UpkepFo/q9sXZSQ87HdQ" + + "iVo70KoaSxnJPZ14mEhbJU0806NBNAFjTAwJSb2eOHp7B7YYW8Ae0vkvI6XH0liXwxtn8Rjgdy+3" + + "aT48HRlRFGUbKQIDAQABAoIBABGUyEVyb9vdoTdiYmgH/li1mU6IXHa/cJmq0Oh7/RRUEpx9tOQD" + + "S4Ir6rI/w/WHFCSpCIrdqHn6+uVMFXjNKb0c1NVYX4vffVVQ2nwu0kyxoPNVu+WXhRf0334OA3Bn" + + "rtOtaVvgfOIvotWrlMIW/LW39kcasmNPmOXcKSlmC3pAX+xoFFvkW7s+I5sjtHz6Btj5M8o4748e" + + "a9+aeB6eFjd1n3hSyikq2gCW/Wc1J+Xz67jrkYmFtfiv758uJTShmuo0naKVLOXP5B635kuToSPv" + + "dgVfNIoGkxoo5DmfxdhaY/ahATacm9LFyQaCur7RPid9SOeZ4tq74hzVeHDdSRECgYEA5vcHVY25" + + "v1tovfnIfRsjD3lnQESyCeqDl/lolXnNbR+7SVoMOxE59XMBU7O0DKX0n5Qoaaf0csYZMVKeO3tO" + + "+g0a82ZgIzEtp9D5EKUC3KO8KqacVkRt+LqjECrypKk01p33G+TcKFQoZraDNOv5Bvw9mhVXfXsI" + + "bpBknR60LhMCgYEA3ZETuwjd3xLELJ3WqbmIC8iESV18GCtn9w0Gaepci/yg7B5B2EESxlxUer1I" + + "hRjXlh74syWC0uHvI80ckuN1sSlgQhw0yUTlVkI+5tw641C99omcCfR4WhNHZUIdzSAjQ3j9bS87" + + "TB+KW2kfxHtJmOKVQvepN2pyWHGLE0GmiVMCgYBK5WhXcfH9/6JjmJBPIXGpHvFAggZJ+OZbbeFg" + + "N3NKI7QUAUNeFCP7WCPiBLAqOdatuNmyCWjXny2kNOpi14WwyHco3tXmE5h+huF9aEUZT29BVW30" + + "+5O9yKgAIi01ADnGLstnHDvYSBIh3KWHj83dWgaP5MPOSQte7yvxV3eOSQKBgQCSVYMGgHCpKCfQ" + + "n95q8lJcFe7o1YlkHSbpD/0wbsz4170gtUsfnLrdr5VMCz+eSC36xDHVf+zLgtUCfDFnAjt06rLc" + + "duyWEZ62PcJ1jvaFs4oDDhe0q3XhZ+I7ilNMavFfWsVmG0+6kwo6HVAR4KtXAi86r39fTjp/F16X" + + "NUDcaQKBgBjnDKAC2l7PpU4mGMbAQY4ZwKRTvMYMjVa9FmncNE6j5knGIV4TYyd0LVQ23NuwFVOv" + + "vKW5oomOjsfH4hjEwlmuvpKXNLh2s7zsBEC9gqyH3eHP9ubWKldGFaF8ZMCsO1+Mgws10V3f97Ci" + + "xVHwowlJj0SbTPoAOjSA1d12MVPdMTwwFQYJKoZIhvcNAQkUMQgeBgBtAGMAcDAjBgkqhkiG9w0B" + + "CRUxFgQUPcejFXLugZsUadJ57L6RKxzLPMcwQTAxMA0GCWCGSAFlAwQCAQUABCD28J03SDXDV/kH" + + "wdlET/zZHs2JPv2KHRPBcuiWSL+m9gQI/pJ8fI16/iUCAggA"; + +export function tapimTasks(v: typeof V): Task[] { + const APIM = "2024-05-01"; + const SVC = + "/subscriptions/{sub}/resourceGroups/{rg}/providers/Microsoft.ApiManagement/service/{service_name}"; + const SVC_ID = + "/subscriptions/{sub}/resourceGroups/{rg}/providers/Microsoft.ApiManagement/service/{service_name}"; + + /** An `az rest` call on a path of the run's service. */ + const rest = (method: string, path: string, body?: string) => + `rest --method ${method} --url ${SVC_ID}${path}?api-version=${APIM}` + + (body ? ` --body '${body}'` : ""); + + /** A Developer-tier service in the run's group, waited until ready. */ + async function service(ctx: SetupContext) { + const svc = ctx.name("apim"); + Object.assign(ctx.slots, { service_name: svc, name: svc }); + const path = v.fill(SVC, ctx.slots); + await v.putOk( + ctx.q, + path, + APIM, + { + location: "westeurope", + sku: { name: "Developer", capacity: 1 }, + properties: { publisherEmail: "ops@contoso.com", publisherName: "Contoso" }, + }, + "APIM" + ); + await v.waitReady(ctx.q, path, APIM); + } + + /** Random values per slot; `{r}` is 6 random hex characters. */ + function random(ctx: SetupContext, values: Record) { + const r = ctx.hex(6); + for (const [k, tpl] of Object.entries(values)) ctx.slots[k] = tpl.replace(/\{r\}/g, r); + } + + /** + * PUT child resources of the service in order (paths and string values may + * name slots); `{rand:x}` picks a fresh random name for slot x (a prefix of up to six + * characters from x, up to its first underscore, plus 6 hex characters). + */ + async function children(ctx: SetupContext, items: Array<[string, unknown]>) { + const fillRand = (s: string): string => { + const withSlots = s.replace(/\{rand:([a-z0-9_]+)\}/g, (_m, slot: string) => { + if (!(slot in ctx.slots)) ctx.slots[slot] = ctx.name(slot.split("_")[0].slice(0, 6)); + return `{${slot}}`; + }); + return v.fmt(withSlots, ctx.slots); + }; + const deep = (x: unknown): unknown => { + if (typeof x === "string") return fillRand(x); + if (Array.isArray(x)) return x.map(deep); + if (x && typeof x === "object") + return Object.fromEntries(Object.entries(x).map(([k, y]) => [k, deep(y)])); + return x; + }; + for (const [path, body] of items) { + const p = fillRand(path); + await v.putOk(ctx.q, v.fill(SVC, ctx.slots) + p, APIM, deep(body), path); + } + } + + /** An API with one GET operation, and a random rate limit. */ + async function apiOperation(ctx: SetupContext) { + const api = ctx.name("orders"); + const op = ctx.name("getorder"); + const base = v.fill(SVC, ctx.slots); + await v.putOk( + ctx.q, + `${base}/apis/${api}`, + APIM, + { properties: { displayName: api, path: api, protocols: ["https"] } }, + "API" + ); + await v.putOk( + ctx.q, + `${base}/apis/${api}/operations/${op}`, + APIM, + { + properties: { + displayName: "Get order", + method: "GET", + urlTemplate: "/orders/{id}", + templateParameters: [{ name: "id", type: "string", required: true }], + }, + }, + "operation" + ); + Object.assign(ctx.slots, { api, operation: op, calls: ctx.pick([12, 17, 25, 40, 75]) }); + } + + const POLICY = + '' + + ""; + // The same XML with < > " written as JSON escapes. + const POLICY_JSON = POLICY.replace(//g, "\\u003e").replace(/"/g, '\\"'); + + const tasks: Task[] = []; + const add = (t: Omit) => + tasks.push({ + tier: "T-APIM", + caps: "T-A", + apimSlots: ["service_name"], + sweep: [`Microsoft.ApiManagement/service@${APIM}`], + ...t, + }); + + // ── membership check (GroupUser_CheckEntityExists) ────────────────────── + + add({ + id: "apim-group-user-check", + opId: "Microsoft.ApiManagement/GroupUser_CheckEntityExists", + verb: "check", + prompts: [ + "In the API Management service {service_name} (resource group {rg}), are the users {user1} and {user2} members of the group {group}? Check each one.", + ], + oracle: [ + rest("head", "/groups/{group}/users/{user1}"), + rest("head", "/groups/{group}/users/{user2}"), + ], + // Verifier defect 6 fixed (verifiers.membershipVerdicts). + verify: v.membershipStated(), + setup: async (ctx) => { + await service(ctx); + const group = ctx.name("devs"); + const u1 = ctx.name("alice"); + const u2 = ctx.name("bob"); + const base = v.fill(SVC, ctx.slots); + await v.putOk( + ctx.q, + `${base}/groups/${group}`, + APIM, + { properties: { displayName: group } }, + "group" + ); + for (const u of [u1, u2]) { + await v.putOk( + ctx.q, + `${base}/users/${u}`, + APIM, + { properties: { email: `${u}@contoso.com`, firstName: u, lastName: "Test" } }, + "user" + ); + } + const member = ctx.pick([u1, u2]); + const put = await v.arm(ctx.q, "PUT", `${base}/groups/${group}/users/${member}`, APIM); + if (put.status === 0 || put.status >= 300) + throw new v.FixtureError(`fixture: group membership PUT ${put.status}`); + Object.assign(ctx.slots, { group, user1: u1, user2: u2, member }); + }, + }); + + // ── workspaces: product update ────────────────────────────────────────── + + add({ + id: "apim-workspace-product-update", + opId: "Microsoft.ApiManagement/WorkspaceProduct_Update", + verb: "update", + prompts: [ + "Set the description of the product {product} in the workspace {workspace} of the API Management service {service_name} (resource group {rg}) to: {description}", + ], + oracle: [ + rest( + "patch", + "/workspaces/{workspace}/products/{product}", + '{{"properties": {{"description": "{description}"}}}}' + ), + ], + verify: v.fieldIs( + SVC + "/workspaces/{workspace}/products/{product}", + APIM, + "properties.description", + "{description}" + ), + setup: async (ctx) => { + await service(ctx); + random(ctx, { description: "Partner tier, reviewed {r}" }); + await children(ctx, [ + ["/workspaces/{rand:workspace}", { properties: { displayName: "{workspace}" } }], + [ + "/workspaces/{workspace}/products/{rand:product}", + { properties: { displayName: "{product}", state: "notPublished" } }, + ], + ]); + }, + }); + + // ── tenant access and Git access ──────────────────────────────────────── + + add({ + id: "apim-tenant-access", + opId: "Microsoft.ApiManagement/TenantAccess_ListByService", + verb: "list", + prompts: [ + "Is direct management API access enabled on the API Management service {service_name} (resource group {rg})? Check its tenant access settings.", + ], + oracle: [rest("get", "/tenant")], + // Verifier defect 7 fixed (verifiers.tenantAccessVerdict). + verify: v.accessStated(), + setup: async (ctx) => { + await service(ctx); + const enabled = ctx.pick([true, false]); + const r = await v.arm(ctx.q, "PATCH", v.fill(SVC + "/tenant/access", ctx.slots), APIM, { + properties: { enabled }, + }); + if (r.status === 0 || r.status >= 300) + throw new v.FixtureError(`fixture: tenant access PATCH ${r.status}`); + ctx.slots.access_enabled = enabled; + }, + guessable: true, + }); + + add({ + id: "apim-git-regenerate-primary", + opId: "Microsoft.ApiManagement/TenantAccessGit_RegeneratePrimaryKey", + verb: "action", + prompts: [ + "Regenerate the primary key of the Git access configuration of the API Management service {service_name} (resource group {rg}). Leave the secondary key as it is.", + ], + oracle: [rest("post", "/tenant/gitAccess/regeneratePrimaryKey")], + verify: v.changedAndKept( + SVC + "/tenant/gitAccess/listSecrets", + APIM, + "primaryKey", + "secondaryKey" + ), + setup: async (ctx) => { + await service(ctx); + await v.snapshot(ctx.q, ctx.slots, SVC + "/tenant/gitAccess/listSecrets", APIM, { + changed: "primaryKey", + kept: "secondaryKey", + }); + }, + }); + + // ── APIs, operations, tags, tag descriptions, policies, resolvers ─────── + + add({ + id: "apim-api-create", + opId: "Microsoft.ApiManagement/Api_CreateOrUpdate", + verb: "create", + prompts: [ + "In the API Management service {service_name} (resource group {rg}), create an API {api} served under the path {api_path} with the display name {display}.", + ], + oracle: [ + "apim api create --service-name {service_name} --resource-group {rg} --api-id {api} --path {api_path} --display-name {display}", + ], + verify: v.fieldIs(SVC + "/apis/{api}", APIM, "properties.path", "{api_path}"), + setup: async (ctx) => { + await service(ctx); + random(ctx, { api: "inventory-{r}", api_path: "v2/stock-{r}", display: "Inventory{r}" }); + }, + }); + + add({ + id: "apim-operation-delete", + opId: "Microsoft.ApiManagement/ApiOperation_Delete", + verb: "delete", + prompts: [ + "Delete the operation {operation} from the API {api} in the API Management service {service_name} (resource group {rg}).", + ], + oracle: [ + "apim api operation delete --service-name {service_name} --resource-group {rg} --api-id {api} --operation-id {operation}", + ], + verify: v.gone(SVC + "/apis/{api}/operations/{operation}", APIM), + setup: async (ctx) => { + await service(ctx); + await apiOperation(ctx); + }, + }); + + add({ + id: "apim-operation-policy", + opId: "Microsoft.ApiManagement/ApiOperationsPolicy_CreateOrUpdate", + verb: "create", + prompts: [ + "Put a policy on the operation {operation} of the API {api} in the API Management service {service_name} (resource group {rg}) that rate-limits each caller to {calls} calls per 60 seconds.", + ], + oracle: [ + rest( + "put", + "/apis/{api}/operations/{operation}/policies/policy", + '{{"properties": {{"format": "rawxml", "value": "' + POLICY_JSON + '"}}}}' + ), + ], + verify: v.policyRateLimit(SVC + "/apis/{api}/operations/{operation}/policies/policy", APIM), + setup: async (ctx) => { + await service(ctx); + await apiOperation(ctx); + }, + }); + + add({ + id: "apim-tag-assign-api", + opId: "Microsoft.ApiManagement/Tag_AssignToApi", + verb: "action", + prompts: [ + "Assign the tag {tag} to the API {api} in the API Management service {service_name} (resource group {rg}).", + ], + oracle: [rest("put", "/apis/{api}/tags/{tag}")], + verify: v.exists(SVC + "/apis/{api}/tags/{tag}", APIM), + setup: async (ctx) => { + await service(ctx); + await children(ctx, [ + [ + "/apis/{rand:api}", + { properties: { displayName: "{api}", path: "{api}", protocols: ["https"] } }, + ], + ["/tags/{rand:tag}", { properties: { displayName: "{tag}" } }], + ]); + }, + }); + + add({ + id: "apim-tag-detach-product", + opId: "Microsoft.ApiManagement/Tag_DetachFromProduct", + verb: "action", + prompts: [ + "Remove the tag {tag} from the product {product} in the API Management service {service_name} (resource group {rg}). Keep the tag itself.", + ], + oracle: [rest("delete", "/products/{product}/tags/{tag}")], + verify: v.allOf( + v.gone(SVC + "/products/{product}/tags/{tag}", APIM), + v.exists(SVC + "/tags/{tag}", APIM) + ), + setup: async (ctx) => { + await service(ctx); + await children(ctx, [ + [ + "/products/{rand:product}", + { + properties: { + displayName: "{product}", + state: "published", + subscriptionRequired: false, + }, + }, + ], + ["/tags/{rand:tag}", { properties: { displayName: "{tag}" } }], + ["/products/{product}/tags/{tag}", {}], + ]); + }, + }); + + add({ + id: "apim-api-tag-descriptions", + opId: "Microsoft.ApiManagement/ApiTagDescription_ListByService", + verb: "list", + prompts: [ + "List the tag descriptions of the API {api} in the API Management service {service_name} (resource group {rg}), with each description's text.", + ], + oracle: [rest("get", "/apis/{api}/tagDescriptions")], + verify: v.claims("{desc1}", "{desc2}"), + setup: async (ctx) => { + await service(ctx); + random(ctx, { desc1: "Owned by payments {r}", desc2: "Deprecated in 2027 {r}" }); + await children(ctx, [ + [ + "/apis/{rand:api}", + { properties: { displayName: "{api}", path: "{api}", protocols: ["https"] } }, + ], + ["/tags/{rand:tag1}", { properties: { displayName: "{tag1}" } }], + ["/tags/{rand:tag2}", { properties: { displayName: "{tag2}" } }], + ["/apis/{api}/tagDescriptions/{tag1}", { properties: { description: "{desc1}" } }], + ["/apis/{api}/tagDescriptions/{tag2}", { properties: { description: "{desc2}" } }], + ]); + }, + }); + + add({ + id: "apim-graphql-resolver", + opId: "Microsoft.ApiManagement/GraphqlApiResolver_CreateOrUpdate", + verb: "create", + prompts: [ + "Create a resolver {resolver} on the GraphQL API {api} of the API Management service {service_name} (resource group {rg}) for the field Query.{field}.", + ], + oracle: [ + "apim graphql resolver create --service-name {service_name} --resource-group {rg} --api-id {api} --resolver-id {resolver} --display-name {resolver} --path Query/{field}", + ], + verify: v.fieldIs( + SVC + "/apis/{api}/resolvers/{resolver}", + APIM, + "properties.path", + "Query/{field}" + ), + setup: async (ctx) => { + await service(ctx); + random(ctx, { field: "orders{r}", resolver: "orders-resolver-{r}" }); + await children(ctx, [ + [ + "/apis/{rand:api}", + { + properties: { + displayName: "{api}", + path: "{api}", + type: "graphql", + protocols: ["https"], + }, + }, + ], + ]); + }, + }); + + // ── certificates, OpenID Connect providers ────────────────────────────── + + add({ + id: "apim-certificate-create", + opId: "Microsoft.ApiManagement/Certificate_CreateOrUpdate", + verb: "create", + prompts: [ + "Upload a certificate named {cert} to the API Management service {service_name} (resource group {rg}). It is a PFX, base64-encoded, with an empty password: {pfx}", + ], + oracle: [ + rest( + "put", + "/certificates/{cert}", + '{{"properties": {{"data": "{pfx}", "password": "{pfx_password}"}}}}' + ), + ], + verify: v.fieldIs( + SVC + "/certificates/{cert}", + APIM, + "properties.subject", + "CN=mcp-t2-cert.contoso.com" + ), + setup: async (ctx) => { + await service(ctx); + Object.assign(ctx.slots, { + pfx: STATIC_TEST_PFX_B64, + pfx_password: "", + cert: ctx.name("tls"), + }); + }, + }); + + add({ + id: "apim-oidc-secret", + opId: "Microsoft.ApiManagement/OpenIdConnectProvider_ListSecrets", + verb: "list", + prompts: [ + "What is the client secret of the OpenID Connect provider {oidc} in the API Management service {service_name} (resource group {rg})?", + ], + oracle: [rest("post", "/openidConnectProviders/{oidc}/listSecrets")], + verify: v.claims("{secret}"), + setup: async (ctx) => { + await service(ctx); + random(ctx, { secret: "oidc-{r}-s3cr3t" }); + await children(ctx, [ + [ + "/openidConnectProviders/{rand:oidc}", + { + properties: { + displayName: "{oidc}", + clientId: "portal-client", + metadataEndpoint: "https://login.contoso.com/.well-known/openid-configuration", + clientSecret: "{secret}", + }, + }, + ], + ]); + }, + }); + + // ── the service itself ────────────────────────────────────────────────── + + add({ + id: "apim-migrate-stv2", + opId: "Microsoft.ApiManagement/ApiManagementService_MigrateToStv2", + verb: "action", + // Not "Migrate ... to the stv2 compute platform": the emulator creates services on stv2.1, so a + // careful agent read that, rightly said "already on stv2" and did nothing (0 of 3 runs). The + // request now asks for the migration run itself, which is what the task measures. + prompts: [ + "Run the stv2 platform migration on the API Management service {service_name} (resource group {rg}), keeping its IP addresses. Our runbook requires the migration step even when the service already reports an stv2 version.", + ], + oracle: [rest("post", "/migrateToStv2", '{{"mode": "PreserveIp"}}')], + // The emulator creates services on stv2.1, and migrating sets stv2: the only observable change. + verify: v.fieldIs(SVC, APIM, "properties.platformVersion", "stv2"), + setup: service, + }); + + return tasks; +} diff --git a/tests/azure/evals/tasks-tb.ts b/tests/azure/evals/tasks-tb.ts new file mode 100644 index 0000000..a676618 --- /dev/null +++ b/tests/azure/evals/tasks-tb.ts @@ -0,0 +1,323 @@ +/** + * The E2 tasks of tier T-B: multi-step goals with one phrasing, verified step by step + * (partial credit is recorded). The agent gets the goal, the resource group and the names to + * use; the fixture only picks random names, except for tb-tag-audit, whose group it fills. + * + * The verifier library arrives as a parameter (see verifiers.ts). + */ +import type * as V from "./verifiers"; +import type { Query, SetupContext, Slots, Task, Verify } from "./types"; + +export function tbTasks(v: typeof V): Task[] { + const RG = "/subscriptions/{sub}/resourceGroups/{rg}/providers"; + const EG = "2025-02-15"; + const EH = "2026-01-01"; + const KV = "2026-02-01"; + const NET = "2025-09-01"; + const SB = "2026-01-01"; + const STORAGE = "2026-06-01"; + const WEB = "2026-07-15"; + const MYSQL = "2023-12-30"; + + const SA = RG + "/Microsoft.Storage/storageAccounts/{sa}"; + const PLAN = RG + "/Microsoft.Web/serverfarms/{plan}"; + const SITE = RG + "/Microsoft.Web/sites/{app}"; + const SBNS = RG + "/Microsoft.ServiceBus/namespaces/{ns}"; + const EHNS = RG + "/Microsoft.EventHub/namespaces/{ehns}"; + const MYS = RG + "/Microsoft.DBforMySQL/flexibleServers/{my}"; + const HUB_ID = + "/subscriptions/{sub}/resourceGroups/{rg}/providers/Microsoft.EventHub/namespaces/{ehns}/eventhubs/{hub}"; + + /** + * Random names for everything the goal creates; a trailing `#` + * drops the hyphens (storage accounts, vaults). + */ + function names(ctx: SetupContext, prefixes: Record) { + for (const [k, p] of Object.entries(prefixes)) { + ctx.slots[k] = p.endsWith("#") ? ctx.name(p.slice(0, -1)).replace(/-/g, "") : ctx.name(p); + } + } + + const LINUX_PLAN: [string, Verify] = [ + "Linux plan", + v.fieldIs(PLAN, WEB, "properties.reserved", "true"), + ]; + + const tasks: Task[] = []; + const add = (t: Omit) => + tasks.push({ tier: "T-B", caps: "T-B", verb: "workflow", ...t }); + + // ── 1. storage account + container ────────────────────────────────────── + + add({ + id: "tb-storage-blob", + opId: "workflow/storage-blob", + prompts: [ + "In resource group {rg}, create a StorageV2 storage account named {sa} with the Standard_LRS SKU, then create a private blob container named {container} in it.", + ], + oracle: [ + "storage account create --name {sa} --resource-group {rg} --sku Standard_LRS --kind StorageV2", + "storage container-rm create --storage-account {sa} --name {container} --resource-group {rg}", + ], + verify: v.workflow( + ["account", v.readyWith(SA, STORAGE, { "sku.name": "Standard_LRS", kind: "StorageV2" })], + ["container", v.exists(SA + "/blobServices/default/containers/{container}", STORAGE)] + ), + setup: async (ctx) => names(ctx, { sa: "tbsa#", container: "uploads-" }), + sweep: [`Microsoft.Storage/storageAccounts@${STORAGE}`], + }); + + // ── 4. Key Vault: secret round trip, key, signature ───────────────────── + + add({ + id: "tb-keyvault-secret-sign", + opId: "workflow/keyvault-secret-sign", + prompts: [ + "In resource group {rg}: create a key vault {kv}; store a secret {secret} with the value {secret_value} and read it back to confirm the value; create an RSA key {key}; then sign the SHA-256 digest {digest} (base64url) with that key using RS256 and give me the signature.", + ], + oracle: [ + "keyvault create --name {kv} --resource-group {rg} --location westeurope", + "keyvault secret set --vault-name {kv} --name {secret} --value {secret_value}", + "keyvault secret show --vault-name {kv} --name {secret}", + "keyvault key create --vault-name {kv} --name {key} --kty RSA", + "keyvault key sign --vault-name {kv} --name {key} --algorithm RS256 --digest {digest_b64}", + ], + verify: v.workflow( + ["vault", v.exists(RG + "/Microsoft.KeyVault/vaults/{kv}", KV)], + ["secret stored", v.secretIs("kv", "secret", "{secret_value}")], + ["secret read back", v.claims("{secret_value}")], + ["signature", v.signatureVerifiesIn("kv", "key", "digest")] + ), + setup: async (ctx) => { + names(ctx, { kv: "tbkv#", secret: "db-password-", key: "signing-" }); + const raw = v.sha256(Buffer.from(ctx.uuid().replace(/-/g, ""), "hex")); + // The prompt gives Key Vault's own encoding (base64url); az keyvault key sign + // --digest wants standard base64 with padding. + Object.assign(ctx.slots, { + secret_value: `S3cret-${ctx.hex(10)}`, + digest: v.b64url(raw), + digest_b64: v.b64std(raw), + }); + }, + kvCrypto: true, + vaultSlots: ["kv"], + sweep: ["Microsoft.KeyVault/vaults@2023-07-01"], + }); + + // ── 5. Service Bus topic with two subscriptions ───────────────────────── + + add({ + id: "tb-servicebus-topic", + opId: "workflow/servicebus-topic", + prompts: [ + "In resource group {rg}, create a Standard Service Bus namespace {ns} with a topic {topic}, add two subscriptions to the topic named {s1} and {s2}, then list the topic's subscriptions and tell me their names.", + ], + oracle: [ + "servicebus namespace create --name {ns} --resource-group {rg} --location westeurope --sku Standard", + "servicebus topic create --name {topic} --namespace-name {ns} --resource-group {rg}", + "servicebus topic subscription create --name {s1} --topic-name {topic} --namespace-name {ns} --resource-group {rg}", + "servicebus topic subscription create --name {s2} --topic-name {topic} --namespace-name {ns} --resource-group {rg}", + "servicebus topic subscription list --topic-name {topic} --namespace-name {ns} --resource-group {rg}", + ], + verify: v.workflow( + ["namespace", v.exists(SBNS, SB)], + ["topic", v.exists(SBNS + "/topics/{topic}", SB)], + ["subscription 1", v.exists(SBNS + "/topics/{topic}/subscriptions/{s1}", SB)], + ["subscription 2", v.exists(SBNS + "/topics/{topic}/subscriptions/{s2}", SB)], + ["listed", v.claims("{s1}", "{s2}")] + ), + setup: async (ctx) => + names(ctx, { ns: "tbsb", topic: "orders-", s1: "billing-", s2: "shipping-" }), + sweep: [`Microsoft.ServiceBus/namespaces@${SB}`], + }); + + // ── 9. Resource Graph over two tagged creates ─────────────────────────── + + add({ + id: "tb-resource-graph-tags", + opId: "workflow/resource-graph-tags", + prompts: [ + "In resource group {rg}, create two key vaults, {kv1} and {kv2}, both tagged costcenter={cc}. Then use Azure Resource Graph to find every resource in the subscription tagged costcenter={cc}, and tell me their names.", + ], + oracle: [ + "keyvault create --name {kv1} --resource-group {rg} --location westeurope --tags costcenter={cc}", + "keyvault create --name {kv2} --resource-group {rg} --location westeurope --tags costcenter={cc}", + // A KQL filter needs a pipe, which the tool refuses; az resource list --tag answers + // the question (the tagged steps read the ARM resource index). + "resource list --tag costcenter={cc}", + ], + verify: v.workflow( + ["vault 1 tagged", v.indexTagIs("kv1", "costcenter", "{cc}")], + ["vault 2 tagged", v.indexTagIs("kv2", "costcenter", "{cc}")], + ["found by the query", v.claims("{kv1}", "{kv2}")] + ), + setup: async (ctx) => { + names(ctx, { kv1: "tbkva#", kv2: "tbkvb#" }); + ctx.slots.cc = `cc-${ctx.hex(6)}`; + }, + vaultSlots: ["kv1", "kv2"], + sweep: ["Microsoft.KeyVault/vaults@2023-07-01"], + }); + + // ── 10. Tag audit over a group the fixture fills ──────────────────────── + + add({ + id: "tb-tag-audit", + opId: "workflow/tag-audit", + prompts: [ + "Audit resource group {rg}: list every resource in it and tell me which ones have no owner tag.", + ], + oracle: ["resource list --resource-group {rg}"], + // Verifier defect 3 fixed (verifiers.untaggedVerdicts: per line). + verify: v.workflow(["untagged reported", v.untaggedReported(["u1", "u2"], ["t1", "t2"])]), + setup: async (ctx) => { + // Four resources, two with an owner tag and two without. + names(ctx, { t1: "tbvnet-", t2: "tbnsg-", u1: "tbpip-", u2: "tbrt-" }); + const owner = { owner: `team-${ctx.hex(4)}` }; + const items: Array<[string, string, Record]> = [ + [ + "t1", + "virtualNetworks", + { properties: { addressSpace: { addressPrefixes: ["10.8.0.0/16"] } }, tags: owner }, + ], + ["t2", "networkSecurityGroups", { tags: owner }], + [ + "u1", + "publicIPAddresses", + { sku: { name: "Standard" }, properties: { publicIPAllocationMethod: "Static" } }, + ], + ["u2", "routeTables", {}], + ]; + for (const [slot, kind, body] of items) { + const path = v.fill( + `${RG}/Microsoft.Network/${kind}/${String(ctx.slots[slot])}`, + ctx.slots + ); + await v.putOk(ctx.q, path, NET, { location: "westeurope", ...body }, kind); + await v.waitReady(ctx.q, path, NET); + } + }, + sweep: [ + `Microsoft.Network/virtualNetworks@${NET}`, + `Microsoft.Network/networkSecurityGroups@${NET}`, + `Microsoft.Network/publicIPAddresses@${NET}`, + `Microsoft.Network/routeTables@${NET}`, + ], + }); + + // ── 20. MySQL server, database, firewall rule, host name ──────────────── + + const mysqlFqdn = async (q: Query, slots: Slots): Promise => { + const r = await v.arm(q, "GET", v.fill(MYS, slots), MYSQL); + return r.status === 200 + ? v.pyStr(v.dotted(r.body, "properties.fullyQualifiedDomainName") ?? "") + : ""; + }; + + add({ + id: "tb-mysql-firewall", + opId: "workflow/mysql-firewall", + prompts: [ + "In resource group {rg}: create a MySQL flexible server {my} (Burstable Standard_B1ms, admin myadmin, password LocalStack123!), a database {db} on it, and a firewall rule {rule} that allows {ip_start} to {ip_end}. Then tell me the server's fully qualified domain name.", + ], + oracle: [ + "mysql flexible-server create --name {my} --resource-group {rg} --location westeurope --tier Burstable --sku-name Standard_B1ms --admin-user myadmin --admin-password LocalStack123! --public-access None --yes", + "mysql flexible-server db create --server-name {my} --resource-group {rg} --database-name {db}", + "mysql flexible-server firewall-rule create --name {my} --resource-group {rg} --rule-name {rule} --start-ip-address {ip_start} --end-ip-address {ip_end}", + "mysql flexible-server show --name {my} --resource-group {rg}", + ], + verify: v.workflow( + ["server", v.exists(MYS, MYSQL)], + ["database", v.exists(MYS + "/databases/{db}", MYSQL)], + [ + "firewall rule", + v.fieldIs(MYS + "/firewallRules/{rule}", MYSQL, "properties.endIpAddress", "{ip_end}"), + ], + // Verifier defect 4 fixed (verifiers.hostOf: the name with or without the port). + ["host name reported", v.hostStated(mysqlFqdn)] + ), + setup: async (ctx) => { + names(ctx, { my: "tbmy-", db: "shop", rule: "office-" }); + const octet = 20 + ctx.randomInt(0, 199); + Object.assign(ctx.slots, { + ip_start: `203.0.113.${octet}`, + ip_end: `203.0.113.${octet + 5}`, + }); + }, + sweep: [`Microsoft.DBforMySQL/flexibleServers@${MYSQL}`], + notes: "the heaviest task: the emulator runs a MySQL container per server", + }); + + // ── 12 (S15). Event Hub as the destination of an Event Grid topic ─────── + + add({ + id: "tb-eventgrid-to-eventhub", + opId: "workflow/eventgrid-to-eventhub", + prompts: [ + "In resource group {rg}: create a Standard Event Hubs namespace {ehns} with an event hub {hub}, create an Event Grid topic {topic}, and add an event subscription {esub} on the topic that delivers events to that event hub.", + ], + oracle: [ + "eventhubs namespace create --name {ehns} --resource-group {rg} --location westeurope --sku Standard", + "eventhubs eventhub create --name {hub} --namespace-name {ehns} --resource-group {rg}", + "eventgrid topic create --name {topic} --resource-group {rg} --location westeurope", + "eventgrid event-subscription create --name {esub} --source-resource-id /subscriptions/{sub}/resourceGroups/{rg}/providers/Microsoft.EventGrid/topics/{topic} --endpoint-type eventhub --endpoint " + + HUB_ID, + ], + verify: v.workflow( + ["namespace", v.exists(EHNS, EH)], + ["event hub", v.exists(EHNS + "/eventhubs/{hub}", EH)], + ["topic", v.exists(RG + "/Microsoft.EventGrid/topics/{topic}", EG)], + [ + "delivers to the hub", + v.fieldIs( + RG + + "/Microsoft.EventGrid/topics/{topic}/providers/Microsoft.EventGrid/eventSubscriptions/{esub}", + EG, + "properties.destination.properties.resourceId", + HUB_ID + ), + ] + ), + setup: async (ctx) => + names(ctx, { ehns: "tbeh", hub: "clicks-", topic: "tbtopic-", esub: "tohub-" }), + sweep: [`Microsoft.EventHub/namespaces@${EH}`, `Microsoft.EventGrid/topics@${EG}`], + }); + + // ── 7. Web app with a plan, an app setting, a restart ─────────────────── + + add({ + id: "tb-webapp-settings", + opId: "workflow/webapp-settings", + prompts: [ + "In resource group {rg}, create a Linux App Service plan {plan} on the B1 SKU and a Python 3.12 web app {app} on it. Set the application setting FEATURE_FLAG to {flag}, then restart the app.", + ], + oracle: [ + "appservice plan create --name {plan} --resource-group {rg} --sku B1 --is-linux", + "webapp create --name {app} --resource-group {rg} --plan {plan} --runtime PYTHON:3.12", + "webapp config appsettings set --name {app} --resource-group {rg} --settings FEATURE_FLAG={flag}", + "webapp restart --name {app} --resource-group {rg}", + ], + // The restart has no observable result in the emulator, so it is not a checked step. + verify: v.workflow( + ["plan", v.readyWith(PLAN, WEB, { "sku.name": "B1" })], + LINUX_PLAN, + [ + "app on the plan", + v.fieldIs( + SITE, + WEB, + "properties.serverFarmId", + "/subscriptions/{sub}/resourceGroups/{rg}/providers/Microsoft.Web/serverfarms/{plan}" + ), + ], + ["setting", v.appSetting(SITE, WEB, "FEATURE_FLAG", "{flag}")] + ), + setup: async (ctx) => { + names(ctx, { plan: "tbplan-", app: "tbweb-" }); + ctx.slots.flag = `on-${ctx.hex(6)}`; + }, + sweep: [`Microsoft.Web/serverfarms@${WEB}`, `Microsoft.Web/sites@${WEB}`], + }); + + return tasks; +} diff --git a/tests/azure/evals/tasks.test.ts b/tests/azure/evals/tasks.test.ts new file mode 100644 index 0000000..6d5621d --- /dev/null +++ b/tests/azure/evals/tasks.test.ts @@ -0,0 +1,190 @@ +/** + * The E2 task catalogue, and every task's verifier against answers recorded in live runs + * against the LocalStack Azure emulator: the oracle (the verifier must pass) and + * setup alone (the verifier must fail). Replaying the recordings proves the verifiers score + * real emulator answers as they did live, without an emulator. + */ +import { readFileSync } from "fs"; +import path from "path"; +import { tapimTasks } from "./tasks-tapim"; +import { taTasks } from "./tasks-ta"; +import { tbTasks } from "./tasks-tb"; +import * as V from "./verifiers"; +import type { AzAnswer, Task } from "./types"; + +const TASKS: Task[] = [...taTasks(V), ...tbTasks(V), ...tapimTasks(V)]; +const BY_ID = new Map(TASKS.map((t) => [t.id, t])); + +const EXPECTED: Record = { + "T-A": [ + "appconfig-kv-unlock", + "appconfig-check-keys", + "appconfig-regenerate-key", + "lock-delete", + "locks-list", + "role-definition-get", + "acr-login-server", + "acr-webhook-create", + "eventgrid-topic-regenerate-key", + "eventgrid-domain-disable-local-auth", + "eventhub-hub-auth-rule-rights", + "insights-activity-log-alert-create", + "kv-sign", + "kv-decrypt", + "kv-rotation-policy", + "kv-access-policy-grant", + "identity-create", + "identity-list", + "route-get", + "private-dns-zones-list", + "vnet-address-space", + "la-workspace-retention", + "deployment-sub-validate", + "sb-queue-regenerate-key", + "sb-dp-queue-max-delivery", + "storage-regenerate-key", + "disk-create", + "cdn-can-migrate", + ], + "T-B": [ + "tb-storage-blob", + "tb-keyvault-secret-sign", + "tb-servicebus-topic", + "tb-resource-graph-tags", + "tb-tag-audit", + "tb-mysql-firewall", + "tb-eventgrid-to-eventhub", + "tb-webapp-settings", + ], + "T-APIM": [ + "apim-group-user-check", + "apim-workspace-product-update", + "apim-tenant-access", + "apim-git-regenerate-primary", + "apim-api-create", + "apim-operation-delete", + "apim-operation-policy", + "apim-tag-assign-api", + "apim-tag-detach-product", + "apim-api-tag-descriptions", + "apim-graphql-resolver", + "apim-certificate-create", + "apim-oidc-secret", + "apim-migrate-stv2", + ], +}; + +/** Every `{name}` a template names (Python str.format fields; `{{` escapes skipped). */ +function fields(template: string): string[] { + return [...template.replace(/\{\{|\}\}/g, "").matchAll(/\{([A-Za-z_][A-Za-z0-9_]*)\}/g)].map( + (m) => m[1] + ); +} + +describe("the task catalogue", () => { + test("50 tasks in T-A, T-B and T-APIM, with unique ids", () => { + expect(TASKS).toHaveLength(50); + expect(new Set(TASKS.map((t) => t.id)).size).toBe(TASKS.length); + for (const [tier, ids] of Object.entries(EXPECTED)) { + expect(TASKS.filter((t) => t.tier === tier).map((t) => t.id)).toEqual(ids); + } + }); + + test("the tasks of the seven reading pitfalls are all here", () => { + for (const id of [ + "eventhub-hub-auth-rule-rights", + "deployment-sub-validate", + "tb-tag-audit", + "tb-mysql-firewall", + "apim-group-user-check", + "apim-tenant-access", + ]) { + expect(BY_ID.has(id)).toBe(true); + } + }); + + test("the Key Vault crypto tasks whose refusals the summary counts", () => { + expect(TASKS.filter((t) => t.kvCrypto).map((t) => t.id)).toEqual([ + "kv-sign", + "kv-decrypt", + "tb-keyvault-secret-sign", + ]); + }); + + test.each(TASKS.map((t) => [t.id, t] as const))("%s is complete", (_id, t) => { + expect(t.prompts.length).toBeGreaterThanOrEqual(1); + expect(t.prompts.length).toBe(t.tier === "T-A" ? 2 : 1); + expect(t.oracle.length).toBeGreaterThanOrEqual(1); + expect(typeof t.verify).toBe("function"); + expect(t.caps).toBe(t.tier === "T-B" ? "T-B" : "T-A"); + expect(t.opId).toMatch(/^(Microsoft\.[A-Za-z.]+|workflow)\/\S+$/); + for (const step of t.oracle) { + if (typeof step === "string") expect(step).not.toMatch(/^az\s/); + } + }); + + test("a task that has the agent create a vault or an APIM service names it for the purge", () => { + for (const t of TASKS) { + for (const step of t.oracle) { + if (typeof step !== "string") continue; + const vault = /^keyvault create --name \{(\w+)\}/.exec(step); + if (vault) expect([t.id, t.vaultSlots]).toEqual([t.id, expect.arrayContaining([vault[1]])]); + } + if (t.tier === "T-APIM") expect(t.apimSlots).toEqual(["service_name"]); + } + }); +}); + +interface Entry { + task: string; + mode: "oracle" | "negative"; + passed: boolean; + wait_s: number; + slots: Record; + text: string; + queries: Array<{ command: string; repeat?: number; answer: AzAnswer }>; +} + +const RECORDED: { entries: Entry[] } = JSON.parse( + readFileSync(path.join(__dirname, "fixtures", "recorded-verifications.json"), "utf8") +); + +describe("recorded verifications (live oracle and negative runs, replayed)", () => { + test("every task has a passing (oracle) and a failing (negative) recording", () => { + for (const t of TASKS) { + const modes = RECORDED.entries.filter((e) => e.task === t.id).map((e) => [e.mode, e.passed]); + expect([t.id, modes]).toEqual([ + t.id, + expect.arrayContaining([ + ["oracle", true], + ["negative", false], + ]), + ]); + } + }); + + test.each( + RECORDED.entries + .filter((e) => BY_ID.has(e.task)) + .map((e) => [e.task, e.mode, e.passed, e] as const) + )("%s (%s) replays to passed=%s", async (_task, _mode, passed, entry) => { + const task = BY_ID.get(entry.task)!; + const q = V.replayQuery(entry.queries, entry.wait_s); + const verdict = await task.verify(q, { ...entry.slots }, entry.text); + expect([verdict.passed, verdict.reason]).toEqual([passed, expect.any(String)]); + }); + + test("every prompt and oracle template is filled by a recorded setup's slots", () => { + for (const e of RECORDED.entries.filter((x) => x.mode === "oracle")) { + const task = BY_ID.get(e.task); + if (!task) continue; + const have = new Set([...Object.keys(e.slots), "sub"]); + const templates = [ + ...task.prompts, + ...task.oracle.filter((s): s is string => typeof s === "string"), + ]; + const missing = templates.flatMap(fields).filter((f) => !have.has(f)); + expect([e.task, missing]).toEqual([e.task, []]); + } + }); +}); diff --git a/tests/azure/evals/types.ts b/tests/azure/evals/types.ts new file mode 100644 index 0000000..f3e42d1 --- /dev/null +++ b/tests/azure/evals/types.ts @@ -0,0 +1,174 @@ +/** + * Types shared by the E2 eval modules. Type declarations only: every + * module imports this file with `import type`, which Node's type stripping erases, so the + * modules can be loaded unbundled by `run.mjs` (Node 22.18+) and by ts-jest alike. + */ + +/** The E2 task tiers (README.md, The tasks). */ +export type Tier = "T-A" | "T-B" | "T-APIM"; + +/** + * One `az` answer as the tool returns it with LOCALSTACK_AZ_TEST_ENVELOPE=1: az's exact + * stdout, stderr and exit code from the envelope, and the tool's text (what a model sees). + */ +export interface AzAnswer { + exitCode: number | null; + stdout: string; + stderr: string; + classId: string | null; + /** The tool's text answer (first content item), without the envelope. */ + text?: string; + notes?: string[]; + stoppedByTool?: boolean; +} + +/** + * What a verifier or a fixture uses to read and change emulator state. The harness + * implements it with a server session of its own (never the agent's), and the unit tests + * with recorded answers and a fake clock. + */ +export interface Query { + az(command: string): Promise; + sleep(ms: number): Promise; + /** Milliseconds, monotonic. */ + now(): number; + /** How long a polling check waits for a state (90 s by default). */ + waitS: number; +} + +/** Fixture values, filled into prompts, oracle commands and paths. Keys that start with + * an underscore are internal (snapshots, `_oracle`) and never recorded as slots. */ +export type Slots = Record; + +export interface StepResult { + step: string; + passed: boolean; + reason: string; +} + +export interface Verdict { + passed: boolean; + reason: string; + /** A workflow's per-step results (T-B partial credit). */ + steps?: StepResult[]; +} + +/** Reads the outcome from the emulator (and, for read tasks, the answer text). */ +export type Verify = (q: Query, slots: Slots, text: string) => Promise; + +/** An oracle command: a template filled from the slots, or one computed from the + * emulator just before it runs. */ +export type OracleStep = string | ((q: Query, slots: Slots) => Promise); + +/** What a task's setup gets: the harness query, the run's slots and random helpers. */ +export interface SetupContext { + q: Query; + slots: Slots; + rg: string; + /** `prefix` + 6 hex characters. */ + name(prefix: string): string; + hex(chars: number): string; + uuid(): string; + pick(options: readonly T[]): T; + sample(options: readonly T[], count: number): T[]; + randomInt(min: number, max: number): number; + random(): number; +} + +export interface Task { + /** The task id. */ + id: string; + tier: Tier; + /** The sampled operation (provider/OperationId). */ + opId: string; + verb: string; + /** One or two phrasings of the same request (T-A has two). */ + prompts: readonly string[]; + /** The reference (oracle) commands, in order. */ + oracle: readonly OracleStep[]; + verify: Verify; + /** Creates the prerequisites through the harness session and fills the slots. */ + setup?: (ctx: SetupContext) => Promise; + /** Removes what a run leaves outside its resource group (subscription-scope records). */ + teardown?: (q: Query, slots: Slots) => Promise; + /** The caps tier for the agent loop (T-APIM tasks use T-A's). */ + caps: "T-A" | "T-B"; + /** Read tasks whose answer is the same whatever the fixture. */ + guessable?: boolean; + /** Key Vault crypto tasks: the summary counts their refusals apart. */ + kvCrypto?: boolean; + /** Slots naming Key Vaults the run may create, purged in cleanup. */ + vaultSlots?: readonly string[]; + /** Slots naming App Configuration stores, purged in cleanup if soft-deleted. */ + appConfigSlots?: readonly string[]; + /** Slots naming API Management services, purged in cleanup if soft-deleted. */ + apimSlots?: readonly string[]; + /** Resource types (with an api-version) the end-of-run orphan sweep lists. */ + sweep?: readonly string[]; + /** Notes on how the task is set up or verified. */ + notes?: string; +} + +/** Per-tier agent-loop caps. */ +export interface Caps { + turns: number; + toolCalls: number; + seconds: number; +} + +/** Token counts of one Messages API turn. */ +export interface Usage { + input: number; + output: number; + cache_write: number; + cache_read: number; +} + +export interface TurnLog { + ms: number; + stop_reason: string | null; + usage: Usage; + stop_details?: unknown; +} + +/** Prices in USD per million tokens. */ +export interface Price { + input: number; + output: number; + cache_write: number; + cache_read: number; +} + +/** The test envelope the tool appends with LOCALSTACK_AZ_TEST_ENVELOPE=1. */ +export interface Envelope { + exitCode: number | null; + stdout: string; + stderr: string; + notes: string[]; + classId: string | null; + truncated?: boolean; + stoppedByTool?: boolean; + emulatorSession?: string | null; +} + +/** An egress refusal the tool reported during an agent's tool call. */ +export interface EgressEvent { + call: number; + command: string; + classId: string | null; + hosts: string[]; + source: "classId" | "note"; +} + +/** A tool definition as the runner sends it to the Messages API. */ +export interface ToolDef { + name: string; + description: string; + input_schema: Record; +} + +/** An MCP tools/call result (text items only matter here). */ +export interface McpResult { + content?: Array<{ type: string; text?: string }>; + isError?: boolean; +} diff --git a/tests/azure/evals/variants.test.ts b/tests/azure/evals/variants.test.ts new file mode 100644 index 0000000..f68781c --- /dev/null +++ b/tests/azure/evals/variants.test.ts @@ -0,0 +1,126 @@ +/** + * The tool definitions per variant: each variant changes only what it + * says it changes, fails loudly when the server's description no longer has its anchor, + * and the long description's adaptation removes exactly the statements that are false for this tool. + */ +import { buildAzureClientDescription } from "../../../src/lib/azure/description"; +import * as VT from "./variants"; + +/** The server's description as it is built today (src/lib/azure/description.ts). */ +const COMPACT = [ + "Run an Azure CLI (az) command against the local LocalStack for Azure emulator and return its output.", + "", + '- Runs against the local emulator only, never real Azure: the CLI is pre-configured with a "LocalStack" cloud and a dummy login, so all data and secrets are local test data. Subscription: 00000000-0000-0000-0000-000000000000. There is no default location: pass --location (for example westeurope) wherever a command needs one.', + "- Unsure of a command or its parameters? Run it with --help first.", + "Examples: group create --name rg1 --location westeurope", +].join("\n"); + +const LISTED: VT.ListedTool[] = [ + { name: "localstack-docs", description: "Search the docs.", inputSchema: { type: "object" } }, + { + name: VT.AZURE_TOOL, + description: COMPACT, + inputSchema: { + $schema: "http://json-schema.org/draft-07/schema#", + type: "object", + properties: { command: { type: "string" } }, + required: ["command"], + }, + }, + { + name: "localstack-management", + description: "Start and stop emulators.", + inputSchema: { type: "object" }, + }, +]; + +describe("variants", () => { + test("the server's real description still has both anchors the variants edit", () => { + const real = buildAzureClientDescription({ workdir: "/work", maxOutputChars: 30000 }); + expect(real).toContain(VT.HELP_SENTENCE); + expect(real).toContain(VT.TEST_DATA_CLAUSE); + expect(real.split("\n")[2]).toBe(COMPACT.split("\n")[2]); + }); + + test("compact sends the server's own description, without the $schema line", () => { + const vt = VT.buildVariantTools("compact", LISTED); + expect(vt.tools).toHaveLength(1); + expect(vt.tools[0]).toEqual({ + name: VT.AZURE_TOOL, + description: COMPACT, + input_schema: { + type: "object", + properties: { command: { type: "string" } }, + required: ["command"], + }, + }); + expect(vt.helpTool).toBe(false); + }); + + test("no-test-data drops only the local-test-data clause", () => { + const d = VT.buildVariantTools("no-test-data", LISTED).azureDescription; + expect(d).not.toContain("local test data"); + expect(d).toContain( + 'a "LocalStack" cloud and a dummy login. Subscription: 00000000-0000-0000-0000-000000000000.' + ); + expect(COMPACT.replace(VT.TEST_DATA_CLAUSE, "")).toBe(d); + }); + + test("help-tool replaces --help with az_help and adds the az_help tool", () => { + const vt = VT.buildVariantTools("help-tool", LISTED); + expect(vt.tools.map((t) => t.name)).toEqual(["az_help", VT.AZURE_TOOL]); + expect(vt.azureDescription).toContain("Call `az_help` with the command or group"); + expect(vt.azureDescription).not.toContain("Run it with --help first"); + const help = vt.tools[0]; + expect(help.description).toBe(VT.AZ_HELP_DESCRIPTION); + expect(help.input_schema).toMatchObject({ required: ["prefix"] }); + expect(vt.helpTool).toBe(true); + }); + + test("long: the sectioned description says only what this tool does", () => { + const d = VT.buildVariantTools("long", LISTED).azureDescription; + expect(d).toBe(VT.LONG_DESCRIPTION); + // what this tool does not do is gone + expect(d).not.toContain("az_help"); + expect(d).not.toMatch(/list of commands|Batch sequential steps/); + expect(d).not.toContain("is blocked"); + expect(d).not.toContain("JSON output, parsed"); + // and it keeps the rules compact has + expect(d).toContain("Do not run `az login`, `az logout` or `az cloud ...`"); + expect(d).toContain( + "One CLI command per string: no pipes (|), redirects (>, <), chaining (&&, ||, ;)" + ); + expect(d).toContain( + "If you are unsure of a command's name or parameters, run it with --help first." + ); + expect(d.length).toBeGreaterThan(COMPACT.length); + }); + + test("a missing anchor fails loudly instead of testing the default", () => { + const changed = [{ ...LISTED[1], description: "Run az. Help: use --help." }]; + expect(() => VT.buildVariantTools("help-tool", changed)).toThrow(/no longer contains/); + expect(() => VT.buildVariantTools("no-test-data", changed)).toThrow(/no longer contains/); + expect(() => VT.buildVariantTools("compact", [LISTED[0]])).toThrow( + /does not list localstack-azure-client/ + ); + }); + + test("--all-tools offers every tool but the withheld management tool, sorted by name", () => { + const vt = VT.buildVariantTools("compact", LISTED, true); + expect(vt.tools.map((t) => t.name)).toEqual([VT.AZURE_TOOL, "localstack-docs"]); + expect(VT.WITHHELD_TOOLS).toContain("localstack-management"); + }); + + test("helpCommand builds ` --help` once", () => { + expect(VT.helpCommand("storage account create")).toBe("storage account create --help"); + expect(VT.helpCommand("az containerapp env")).toBe("containerapp env --help"); + expect(VT.helpCommand("afd route create --help")).toBe("afd route create --help"); + expect(VT.helpCommand("")).toBe("--help"); + }); + + test("isVariant", () => { + expect(VT.VARIANTS).toEqual(["compact", "long", "help-tool", "no-test-data"]); + expect(VT.isVariant("long")).toBe(true); + expect(VT.isVariant("verbose")).toBe(false); + }); +}); diff --git a/tests/azure/evals/variants.ts b/tests/azure/evals/variants.ts new file mode 100644 index 0000000..e6eddf5 --- /dev/null +++ b/tests/azure/evals/variants.ts @@ -0,0 +1,173 @@ +/** + * The tool definitions E2 sends to Claude, per variant. A variant changes only what the + * runner sends to the Messages API, never the server: the server's own `tools/list` entry is + * the base, and a variant whose anchor text is missing from it fails loudly instead of + * silently testing the default. + * + * compact the server's description as listed (the shipped default) + * long a longer description in sections (Context, Input, Output, Rules, + * Examples) + * help-tool compact, with --help replaced by a separate `az_help` tool + * no-test-data compact without the "local test data" wording + * + * No runtime imports of local modules (run.mjs loads this file with type stripping). + */ +import type { ToolDef } from "./types"; + +export const AZURE_TOOL = "localstack-azure-client"; +export const HELP_TOOL = "az_help"; +export const VARIANTS = ["compact", "long", "help-tool", "no-test-data"] as const; +export type Variant = (typeof VARIANTS)[number]; + +/** Tools never offered with --all-tools: `localstack-management` can stop the emulator + * the evals run on. */ +export const WITHHELD_TOOLS = ["localstack-management"]; + +export const HELP_SENTENCE = "- Unsure of a command or its parameters? Run it with --help first."; +export const HELP_TOOL_SENTENCE = + '- Unsure of a command or its parameters? Call `az_help` with the command or group (for example "containerapp env create") instead of guessing.'; +export const TEST_DATA_CLAUSE = ", so all data and secrets are local test data"; + +/** The `az_help` tool of the help-tool variant. */ +export const AZ_HELP_DESCRIPTION = + "Show the Azure CLI help for a command group or command (`az --help`): " + + "its subcommands, or its parameters with descriptions and examples. Call this " + + "before running a command whose name or parameters you are unsure of. Examples " + + 'of `prefix`: "storage account create", "containerapp env", "afd route create".'; + +export const AZ_HELP_SCHEMA: Record = { + type: "object", + properties: { + prefix: { + type: "string", + description: + "The command group or command, without the leading 'az' (e.g. \"storage account create\").", + }, + }, + required: ["prefix"], + additionalProperties: false, +}; + +/** + * The long description: the same rules as compact, in sections. It states what this tool + * does: one command per call, the CLI's JSON text as output, `rest` with a relative URL (an + * absolute management.azure.com URL is rewritten onto the emulator), and --help for a command + * the model is unsure of. + */ +export const LONG_DESCRIPTION = [ + "Run Azure CLI (`az`) commands against the local LocalStack for Azure emulator and return their output.", + "", + "Context", + '- The CLI is already configured for the emulator: a "LocalStack" cloud is active and a service principal is logged in, so every command reaches only the local emulator, never real Azure. Do not run `az login`, `az logout` or `az cloud ...`: they are rejected because they would break that routing.', + "- Subscription: 00000000-0000-0000-0000-000000000000. There is no default location: pass --location (for example westeurope) wherever a command needs one.", + "- Write the command without the leading `az` (a leading `az` is accepted and ignored).", + "- Extensions that commands need (cdn/afd, graph, k8s-configuration, k8s-extension, fleet, monitor app-insights and others) are pre-installed. Installing or updating extensions is not possible.", + "", + "Input", + "- `command`: one command as a string.", + "", + "Output", + '- On success: the command\'s JSON output. Use `--query` (JMESPath) to return only the fields you need, e.g. --query "{name:name, state:provisioningState}", and `-o tsv` for a single value.', + "- On failure: the CLI's error message, verbatim. Read it: it usually names the missing or invalid argument.", + "", + "Rules", + "- One CLI command per string: no pipes (|), redirects (>, <), chaining (&&, ||, ;), command substitution ($(...) or backticks), variables or newlines. Filter with --query instead of grep or jq.", + '- Quote values that contain spaces or JSON, e.g. --tags "env=dev team=a" or --set properties.x=\'{"a":1}\'.', + "- Long-running creates wait until the resource is ready unless you pass --no-wait. Don't pass --no-wait unless asked, so the next step can see the resource.", + "- For an operation with no dedicated command, `rest` calls the ARM REST API directly. Use a relative URL, which stays on the emulator, e.g. rest --method put --url \"/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/rg1/providers///?api-version=\" --body '{...}'.", + "- If you are unsure of a command's name or parameters, run it with --help first.", + "", + "Examples", + "- group create --name rg1 --location westeurope", + "- keyvault create --name kv1abc --resource-group rg1 --location westeurope", + "- keyvault secret set --vault-name kv1abc --name db-password --value s3cret", + "- network vnet create --resource-group rg1 --name vnet1 --address-prefix 10.0.0.0/16 --subnet-name default --subnet-prefix 10.0.1.0/24", + "- servicebus queue create --resource-group rg1 --namespace-name sb1abc --name orders", + '- storage account show --name st1abc --resource-group rg1 --query "{sku:sku.name, location:location}"', +].join("\n"); + +/** An MCP tools/list entry (the fields used here). */ +export interface ListedTool { + name: string; + description?: string; + inputSchema?: Record; +} + +export interface VariantTools { + variant: Variant; + tools: ToolDef[]; + /** The variant offers the synthetic az_help tool, which the runner implements. */ + helpTool: boolean; + /** The Azure tool's description as sent. */ + azureDescription: string; +} + +/** A Messages API tool from an MCP entry (the `$schema` line is dropped). */ +export function toToolDef(t: ListedTool, description?: string): ToolDef { + const schema = { ...(t.inputSchema ?? { type: "object", properties: {} }) }; + delete schema.$schema; + return { name: t.name, description: description ?? t.description ?? "", input_schema: schema }; +} + +export function isVariant(name: string): name is Variant { + return (VARIANTS as readonly string[]).includes(name); +} + +/** The Azure tool's description for a variant, from the server's own description. */ +export function variantDescription(variant: Variant, listed: string): string { + switch (variant) { + case "compact": + return listed; + case "long": + return LONG_DESCRIPTION; + case "help-tool": + if (!listed.includes(HELP_SENTENCE)) { + throw new Error( + `variant help-tool: the server's description no longer contains "${HELP_SENTENCE}"` + ); + } + return listed.replace(HELP_SENTENCE, HELP_TOOL_SENTENCE); + case "no-test-data": + if (!listed.includes(TEST_DATA_CLAUSE)) { + throw new Error( + `variant no-test-data: the server's description no longer contains "${TEST_DATA_CLAUSE}"` + ); + } + return listed.replace(TEST_DATA_CLAUSE, ""); + } +} + +/** + * The tool list a variant sends: the Azure tool (with the variant's description), the + * synthetic az_help for help-tool, and with `allTools` every other listed tool but the + * withheld ones. Sorted by name, so the prompt-cache prefix stays byte-stable. + */ +export function buildVariantTools( + variant: Variant, + listed: ListedTool[], + allTools = false +): VariantTools { + const azure = listed.find((t) => t.name === AZURE_TOOL); + if (!azure) throw new Error(`the server does not list ${AZURE_TOOL}`); + const azureDescription = variantDescription(variant, azure.description ?? ""); + const tools: ToolDef[] = [toToolDef(azure, azureDescription)]; + const helpTool = variant === "help-tool"; + if (helpTool) + tools.push({ name: HELP_TOOL, description: AZ_HELP_DESCRIPTION, input_schema: AZ_HELP_SCHEMA }); + if (allTools) { + for (const t of listed) { + if (t.name === AZURE_TOOL || WITHHELD_TOOLS.includes(t.name)) continue; + tools.push(toToolDef(t)); + } + } + tools.sort((a, b) => (a.name < b.name ? -1 : a.name > b.name ? 1 : 0)); + return { variant, tools, helpTool, azureDescription }; +} + +/** The Azure command a synthetic az_help call runs: ` --help`. */ +export function helpCommand(prefix: string): string { + const p = String(prefix ?? "") + .trim() + .replace(/^az\s+/, ""); + return /(^|\s)(--help|-h)(\s|$)/.test(p) ? p : `${p} --help`.trim(); +} diff --git a/tests/azure/evals/verifiers.test.ts b/tests/azure/evals/verifiers.test.ts new file mode 100644 index 0000000..d3ba03f --- /dev/null +++ b/tests/azure/evals/verifiers.test.ts @@ -0,0 +1,849 @@ +/** + * The E2 verifier library against recorded answers (no emulator): each verifier with a + * passing and a failing recording, the query helpers, and offline checks of the answer + * readers, case by case. + */ +import * as V from "./verifiers"; +import type { AzAnswer, Slots } from "./types"; + +const SUB = V.SUBSCRIPTION; +const RG = "mcpe2-test-rg"; + +function ok(stdout: unknown): AzAnswer { + return { + exitCode: 0, + stdout: typeof stdout === "string" ? stdout : JSON.stringify(stdout, null, 2), + stderr: "", + classId: null, + }; +} +/** As az rest prints a 404 on this emulator (recorded 2026-09-27: upper-case reason). */ +function notFound(): AzAnswer { + return { + exitCode: 1, + stdout: "", + stderr: + 'ERROR: NOT FOUND({"error": {"code": "ResourceNotFound", "message": "not found", "details": [], "additionalInfo": []}})\r\n', + classId: "not-found", + }; +} +function failed(stderr: string, exitCode = 1, classId: string | null = "other"): AzAnswer { + return { exitCode, stdout: "", stderr, classId }; +} +const get = (path: string, api: string) => V.restCommand("GET", path, api); +const post = (path: string, api: string) => V.restCommand("POST", path, api); +const rgPath = (rest: string) => `/subscriptions/${SUB}/resourceGroups/${RG}/providers${rest}`; +const slots = (extra: Slots = {}): Slots => ({ rg: RG, resource_group: RG, ...extra }); + +function replay(calls: Array<[string, AzAnswer] | [string, AzAnswer, number]>, waitS = 90) { + return V.replayQuery( + calls.map(([command, answer, repeat]) => ({ command, answer, ...(repeat ? { repeat } : {}) })), + waitS + ); +} + +describe("Python-compatible helpers", () => { + test("pyStr renders JSON values as Python's str() did", () => { + expect([ + V.pyStr(true), + V.pyStr(false), + V.pyStr(null), + V.pyStr(undefined), + V.pyStr(14), + V.pyStr("x"), + ]).toEqual(["True", "False", "None", "None", "14", "x"]); + }); + + test("pyJson matches json.dumps layout", () => { + expect(V.pyJson({ a: [1, "b"], c: { d: null, e: true } })).toBe( + '{"a": [1, "b"], "c": {"d": null, "e": true}}' + ); + expect(V.pyJson({ $schema: "s", resources: [] })).toBe('{"$schema": "s", "resources": []}'); + }); + + test("fmt fills {name}, keeps {{ }} as braces and refuses an unknown name", () => { + expect(V.fmt("a {x} {{0,8}} {y}", { x: 1, y: true })).toBe("a 1 {0,8} True"); + expect(() => V.fmt("{nope}", {})).toThrow(/no value for \{nope\}/); + expect(V.fill("/subscriptions/{sub}/resourceGroups/{rg}", { rg: "g" })).toBe( + `/subscriptions/${SUB}/resourceGroups/g` + ); + }); + + test("normText drops whitespace, underscores and hyphens", () => { + expect(V.normText("North Europe")).toBe("northeurope"); + expect(V.normText("General_Purpose-x")).toBe("generalpurposex"); + }); + + // test_verifiers.py: _dotted + test("dotted: selections, list indexes, missing parts", () => { + const body = { + keys: [ + { keyName: "key1", value: "A" }, + { keyName: "key2", value: "B" }, + ], + value: [{ name: "Primary", value: "P" }], + c: { list: [{ image: "nginx" }] }, + }; + expect(V.dotted(body, "keys[keyName=key2].value")).toBe("B"); + expect(V.dotted(body, "value[name=Primary].value")).toBe("P"); + expect(V.dotted(body, "c.list.0.image")).toBe("nginx"); + expect(V.dotted(body, "keys[keyName=nope].value")).toBeNull(); + expect(V.dotted(body, "c.list.5.image")).toBeNull(); + expect(V.dotted({ a: { b: null } }, "a.b.c")).toBeNull(); + }); + + test("base64 forms", () => { + expect(V.toStdB64("ab-_")).toBe("ab+/"); + expect(V.toStdB64("abc")).toBe("abc="); + expect(V.toB64url("ab+/cd==")).toBe("ab-_cd"); + expect(V.b64url("order-1")).toBe("b3JkZXItMQ"); + }); +}); + +describe("az rest status", () => { + test("exit 0 is 200; a reason phrase in stderr gives the status", () => { + expect(V.httpStatus(ok({}))).toBe(200); + expect(V.httpStatus(notFound())).toBe(404); + expect(V.httpStatus(failed("ERROR: GONE({})"))).toBe(410); + expect(V.httpStatus(failed("ERROR: Bad Request({})"))).toBe(400); + expect(V.httpStatus(failed("WARNING: x\nERROR: NOT IMPLEMENTED({})"))).toBe(501); + expect( + V.httpStatus(failed("ERROR: the following arguments are required: --name", 2, "argument")) + ).toBe(0); + }); + + test("a command the tool stopped is not a 200", () => { + expect(V.httpStatus({ ...ok({}), stoppedByTool: true })).toBe(0); + }); + + test("restCommand quotes the URL and refuses a single quote in a body", () => { + expect(V.restCommand("PUT", "/x", "1", { a: 1 })).toBe( + `rest --method put --url "/x?api-version=1" --body '{"a":1}'` + ); + expect(() => V.restCommand("PUT", "/x", "1", { a: "it's" })).toThrow(/single quote/); + }); +}); + +describe("replay and recording", () => { + test("a recorded command replays in order, then keeps its last answer", async () => { + const q = replay([ + ["group exists --name g", ok("true")], + ["group exists --name g", ok("false")], + ]); + expect((await q.az("group exists --name g")).stdout).toBe("true"); + expect((await q.az("group exists --name g")).stdout).toBe("false"); + expect((await q.az("group exists --name g")).stdout).toBe("false"); + }); + + test("an unrecorded command is an error, never a silent default", async () => { + await expect(replay([]).az("group list")).rejects.toThrow(/unrecorded query/); + }); + + test("recordingQuery folds repeated identical answers", async () => { + const log: V.RecordedCall[] = []; + const inner = replay([["a", ok(1)]]); + const q = V.recordingQuery(inner, log); + await q.az("a"); + await q.az("a"); + await q.az("a"); + expect(log).toEqual([{ command: "a", answer: V.recordable(ok(1)), repeat: 3 }]); + }); +}); + +describe("state verifiers", () => { + const path = rgPath("/Microsoft.Compute/disks/{disk_name}"); + const cmd = get(rgPath("/Microsoft.Compute/disks/d1"), "2026-03-02"); + + test("exists: GET 200 with Succeeded passes; Failed or 404 fails", async () => { + const s = slots({ disk_name: "d1" }); + const good = await V.exists(path, "2026-03-02")( + replay([[cmd, ok({ properties: { provisioningState: "Succeeded" } })]]), + s, + "" + ); + expect(good).toEqual({ passed: true, reason: "GET d1 -> 200 Succeeded" }); + const bad = await V.exists(path, "2026-03-02")( + replay([[cmd, ok({ properties: { provisioningState: "Failed" } })]]), + s, + "" + ); + expect(bad.passed).toBe(false); + const missing = await V.exists(path, "2026-03-02")(replay([[cmd, notFound()]], 5), s, ""); + expect(missing).toEqual({ passed: false, reason: "GET d1 -> 404" }); + }); + + test("exists polls until the state is terminal", async () => { + const q = replay([ + [cmd, ok({ properties: { provisioningState: "Creating" } })], + [cmd, ok({ properties: { provisioningState: "Succeeded" } })], + ]); + expect((await V.exists(path, "2026-03-02")(q, slots({ disk_name: "d1" }), "")).passed).toBe( + true + ); + expect(q.asked.length).toBe(2); + }); + + test("gone: 404 or 410 passes, 200 fails at the deadline", async () => { + const s = slots({ disk_name: "d1" }); + expect((await V.gone(path, "2026-03-02")(replay([[cmd, notFound()]]), s, "")).passed).toBe( + true + ); + expect( + (await V.gone(path, "2026-03-02")(replay([[cmd, failed("ERROR: GONE({})")]]), s, "")).passed + ).toBe(true); + expect(await V.gone(path, "2026-03-02")(replay([[cmd, ok({})]], 3), s, "")).toEqual({ + passed: false, + reason: "GET d1 -> 200", + }); + }); + + test("readyWith compares fields as Python's str() did (True, numbers)", async () => { + const body = { + sku: { name: "StandardSSD_LRS" }, + properties: { diskSizeGB: 64, provisioningState: "Succeeded", flag: true }, + }; + const s = slots({ disk_name: "d1", size_gb: 64 }); + const v = V.readyWith(path, "2026-03-02", { + "properties.diskSizeGB": "{size_gb}", + "sku.name": "StandardSSD_LRS", + "properties.flag": "True", + }); + expect((await v(replay([[cmd, ok(body)]]), s, "")).passed).toBe(true); + const wrong = await V.readyWith(path, "2026-03-02", { "properties.diskSizeGB": "{size_gb}" })( + replay([[cmd, ok({ ...body, properties: { ...body.properties, diskSizeGB: 32 } })]]), + s, + "" + ); + expect(wrong.passed).toBe(false); + expect(wrong.reason).toContain('fields differ {"properties.diskSizeGB":32}'); + }); + + test("fieldIs polls until the value arrives, case-insensitively", async () => { + const q = replay([ + [cmd, ok({ properties: { state: "Running" } })], + [cmd, ok({ properties: { state: "STOPPED" } })], + ]); + expect( + await V.fieldIs( + path, + "2026-03-02", + "properties.state", + "Stopped" + )(q, slots({ disk_name: "d1" }), "") + ).toEqual({ + passed: true, + reason: "properties.state -> STOPPED", + }); + }); + + test("fieldIs fails at the deadline with the last value", async () => { + const q = replay([[cmd, ok({ properties: { locked: true } })]], 2); + const r = await V.fieldIs( + path, + "2026-03-02", + "properties.locked", + "false" + )(q, slots({ disk_name: "d1" }), ""); + expect(r).toEqual({ passed: false, reason: "properties.locked -> True" }); + expect(q.asked.length).toBeGreaterThan(3); + }); + + test("listFieldContains: every item, any order, case-insensitive", async () => { + const c1 = get(rgPath("/Microsoft.Compute/disks/d1"), "1"); + const check = V.listFieldContains(path, "1", "properties.actions", ["push", "delete"]); + const has = replay([[c1, ok({ properties: { actions: ["Delete", "push"] } })]]); + expect((await check(has, slots({ disk_name: "d1" }), "")).passed).toBe(true); + const missing = replay([[c1, ok({ properties: { actions: ["push"] } })]], 2); + expect((await check(missing, slots({ disk_name: "d1" }), "")).passed).toBe(false); + }); + + test("changedAndKept: the changed key differs, the kept one does not", async () => { + const keysPath = rgPath("/Microsoft.EventGrid/topics/{topic_name}/listKeys"); + const c = post(rgPath("/Microsoft.EventGrid/topics/t1/listKeys"), "2025-02-15"); + const s = slots({ topic_name: "t1", _before_changed: "old1", _before_kept: "k2" }); + const v = V.changedAndKept(keysPath, "2025-02-15", "key1", "key2"); + expect(await v(replay([[c, ok({ key1: "new1", key2: "k2" })]]), s, "")).toEqual({ + passed: true, + reason: "key1 rotated, key2 kept", + }); + expect((await v(replay([[c, ok({ key1: "old1", key2: "k2" })]]), s, "")).reason).toBe( + "key1 unchanged, key2 kept" + ); + expect((await v(replay([[c, ok({ key1: "new1", key2: "new2" })]]), s, "")).reason).toBe( + "key1 rotated, key2 CHANGED" + ); + }); + + test("answerHasFields: the answer states the emulator's value", async () => { + const regPath = rgPath("/Microsoft.ContainerRegistry/registries/{registry_name}"); + const c = get(rgPath("/Microsoft.ContainerRegistry/registries/acr1"), "2025-11-01"); + const body = { + properties: { loginServer: "acr1.azurecr.azure.localhost.localstack.cloud:4566" }, + }; + const v = V.answerHasFields(regPath, "2025-11-01", ["properties.loginServer"]); + const s = slots({ registry_name: "acr1" }); + expect( + ( + await v( + replay([[c, ok(body)]]), + s, + "Log in to acr1.azurecr.azure.localhost.localstack.cloud:4566." + ) + ).passed + ).toBe(true); + expect((await v(replay([[c, ok(body)]]), s, "Log in to acr1.azurecr.io.")).passed).toBe(false); + expect(await v(replay([[c, notFound()]]), s, "anything")).toEqual({ + passed: false, + reason: "GET -> 404", + }); + }); + + test("allOf runs every check and joins the reasons", async () => { + const r = await V.allOf(V.claims("a"), V.claims("b"))(replay([]), {}, "a"); + expect(r).toEqual({ passed: false, reason: 'all claims present; missing ["b"]' }); + }); +}); + +describe("answer verifiers", () => { + test("claims is a case-insensitive substring check of every template", async () => { + const v = V.claims("{lock1}", "{lock2}"); + expect( + ( + await v( + replay([]), + { lock1: "lk-ops-1", lock2: "lk-audit-2" }, + "Locks: LK-OPS-1 and lk-audit-2" + ) + ).passed + ).toBe(true); + expect( + (await v(replay([]), { lock1: "lk-ops-1", lock2: "lk-audit-2" }, "Locks: lk-ops-1")).reason + ).toBe('missing ["lk-audit-2"]'); + }); + + test("claimsNorm and claimsAnyNorm ignore spaces, case, hyphens and underscores", async () => { + expect( + ( + await V.claimsNorm("{region}")( + replay([]), + { region: "northeurope" }, + "It is in North Europe." + ) + ).passed + ).toBe(true); + expect( + ( + await V.claimsAnyNorm("{tier}", "{sku}")( + replay([]), + { tier: "GeneralPurpose", sku: "GP_Gen5" }, + "General Purpose" + ) + ).passed + ).toBe(true); + expect( + ( + await V.claimsAnyNorm("{tier}", "{sku}")( + replay([]), + { tier: "GeneralPurpose", sku: "GP_Gen5" }, + "Business Critical" + ) + ).passed + ).toBe(false); + }); + + test("textMatches escapes slot values before they reach the pattern", async () => { + const v = V.textMatches("\\b{prefix}\\b"); + expect((await v(replay([]), { prefix: "10.1.2.0/24" }, "prefix 10.1.2.0/24")).passed).toBe( + true + ); + expect((await v(replay([]), { prefix: "10.1.2.0/24" }, "prefix 10x1x2x0/24")).passed).toBe( + false + ); + }); + + test("the rotation-policy patterns read P45D and '45 days'", async () => { + const v = V.allOf( + V.textMatches("(?:P{rotate_days}D|\\b{rotate_days}\\s*-?\\s*days?\\b)"), + V.textMatches("(?:P{expiry_days}D|\\b{expiry_days}\\s*-?\\s*days?\\b)") + ); + const s = { rotate_days: 45, expiry_days: 270 }; + expect( + (await v(replay([]), s, "It rotates 45 days after creation and expires after 270 days.")) + .passed + ).toBe(true); + expect( + (await v(replay([]), s, '"timeAfterCreate": "P45D" ... "expiryTime": "P270D"')).passed + ).toBe(true); + expect( + (await v(replay([]), s, "It rotates after 45 days and expires after 365 days.")).passed + ).toBe(false); + }); + + test("plaintextStated accepts the text or its base64url form", async () => { + const v = V.plaintextStated(); + expect( + (await v(replay([]), { plaintext: "order-abc" }, "The plaintext is order-abc.")).passed + ).toBe(true); + expect((await v(replay([]), { plaintext: "order-abc" }, "b3JkZXItYWJj")).passed).toBe(true); + expect( + (await v(replay([]), { plaintext: "order-abc" }, "I could not decrypt it.")).passed + ).toBe(false); + }); + + test("workflow passes only when every step does, and keeps each step", async () => { + const r = await V.workflow(["one", V.claims("a")], ["two", V.claims("b")])(replay([]), {}, "a"); + expect(r.passed).toBe(false); + expect(r.reason).toBe("1/2 steps; failed: two"); + expect(r.steps).toEqual([ + { step: "one", passed: true, reason: "all claims present" }, + { step: "two", passed: false, reason: 'missing ["b"]' }, + ]); + }); + + test("workflow turns a step's exception into that step's failure", async () => { + const boom = async () => { + throw new Error("kaput"); + }; + const r = await V.workflow(["ok", V.claims("a")], ["boom", boom])(replay([]), {}, "a"); + expect(r.steps?.[1]).toEqual({ + step: "boom", + passed: false, + reason: "verifier error: Error: kaput", + }); + }); +}); + +// ── offline checks of the answer readers ──────────────────────────────────────── + +describe("offline checks: exists_claims", () => { + const HAVE = "feature-flags-7a1"; + const MISSING = "legacy-timeout-3c9"; + const SLOTS = { key: HAVE, missing_key: MISSING }; + const CHECK = V.existsClaims({ key: true, missing_key: false }); + const CASES: Array<[string, boolean]> = [ + [`${HAVE} exists, but ${MISSING} does not exist.`, true], + [`Found ${HAVE}. ${MISSING} was not found in the store.`, true], + [`- \`${HAVE}\`: ✅ present\n- \`${MISSING}\`: ❌ missing`, true], + [ + `| Key | Exists |\n|---|---|\n| ${HAVE} | yes, it exists |\n| ${MISSING} | no such key |`, + true, + ], + [`The key ${HAVE} is there; there is no key named ${MISSING}.`, true], + [`Both ${HAVE} and ${MISSING} exist.`, false], + [`${HAVE} does not exist. ${MISSING} exists.`, false], + [`I looked at ${HAVE} and ${MISSING}.`, false], + [`${HAVE} exists.`, false], + ]; + test.each(CASES)("%s -> %s", async (text, want) => { + expect((await CHECK(replay([]), { ...SLOTS }, text)).passed).toBe(want); + }); + + test("oracle: filtered listings name only the existing key", async () => { + const r = await CHECK( + replay([]), + { ...SLOTS, _oracle: true }, + `[{"key": "${HAVE}", "value": "on"}]\n[]` + ); + expect(r.passed).toBe(true); + }); + + test("oracle: a transcript that echoes both names in its command lines", async () => { + const text = [ + V.oracleBlock(`appconfig kv list --name s --key ${HAVE}`, ok([{ key: HAVE, locked: false }])), + V.oracleBlock(`appconfig kv list --name s --key ${MISSING}`, ok([])), + ].join("\n"); + expect((await CHECK(replay([]), { ...SLOTS, _oracle: true }, text)).passed).toBe(true); + }); + + test("truth: fixture drift fails before the answer is read", async () => { + const truth = async () => false; + const r = await V.existsClaims({ key: true, missing_key: false }, truth)( + replay([]), + { ...SLOTS }, + `${HAVE} exists; ${MISSING} is missing.` + ); + expect(r.passed).toBe(false); + expect(r.reason).toContain("fixture drift"); + }); +}); + +describe("offline checks: migration_verdict", () => { + const MIGRATE = V.migrationVerdict("can_migrate"); + const CASES: Array<[boolean, string, boolean]> = [ + [ + true, + "Yes — the classic CDN profile cdnp1 can be migrated to Azure Front Door. The check returned canMigrate: true with no errors; the default target SKU is Standard_AzureFrontDoor.", + true, + ], + [true, "The profile cdnp1 is eligible for migration; no blocking issues were found.", true], + [ + true, + "Yes, it can be migrated. Note that once you commit the migration it cannot be undone.", + true, + ], + [true, "Yes. Nothing is blocking it.", true], + [ + true, + "No — cdnp1 cannot be migrated yet: it has no endpoints to migrate (NoEndpointsToMigrate).", + false, + ], + [true, "The profile is not eligible for migration.", false], + [true, "I checked the profile cdnp1.", false], + [ + false, + "No — cdnp1 cannot be migrated yet: it has no endpoints to migrate (NoEndpointsToMigrate). Create an endpoint first.", + true, + ], + [false, "It is not eligible because the profile doesn't have any endpoints.", true], + [false, "The compatibility check failed with NoEndpointsToMigrate.", true], + [ + false, + "It can be migrated once it has endpoints; right now the profile has 0 endpoints.", + true, + ], + [false, "Yes, the profile cdnp1 can be migrated to Azure Front Door.", false], + [false, "No, it cannot be migrated.", false], + ]; + test.each(CASES)("can=%s %s -> %s", async (can, text, want) => { + expect((await MIGRATE(replay([]), { can_migrate: can }, text)).passed).toBe(want); + }); + + test("oracle outputs carry the verdict", async () => { + const yes = '{"canMigrate": true, "defaultSku": "Standard_AzureFrontDoor", "errors": []}'; + const no = '{"canMigrate": false, "errors": [{"code": "NoEndpointsToMigrate"}]}'; + expect((await MIGRATE(replay([]), { can_migrate: true, _oracle: true }, yes)).passed).toBe( + true + ); + expect((await MIGRATE(replay([]), { can_migrate: false, _oracle: true }, no)).passed).toBe( + true + ); + expect( + ( + await MIGRATE( + replay([]), + { can_migrate: true, _oracle: true }, + "Profile 'p' checked successfully." + ) + ).passed + ).toBe(false); + }); +}); + +describe("offline checks: sql-elastic-pool and sql-db-audit-policy patterns", () => { + // ta.py's POOL_CAPACITY and AUDIT_RETENTION (the SQL tasks are not in E2; the patterns + // exercise textMatches' Python-format escaping). + const POOL_CAPACITY = + "\\b{capacity}\\s*-?\\s*v?-?cores?\\b|\\b(?:capacity|v-?cores?)\\b(?:[\\s:|=*\"'`]|of|is|set\\s+to){{0,8}}\\b{capacity}\\b"; + const AUDIT_RETENTION = + "\\b{audit_days}\\s*-?\\s*days?\\b|\\bretention(?:\\s*(?:period|days?))?\\b(?:[\\s:|=*\"'`()]|of|is|days?|set\\s+to){{0,10}}\\b{audit_days}\\b"; + const POOL = V.allOf(V.claimsAnyNorm("{pool_tier}", "{pool_sku}"), V.textMatches(POOL_CAPACITY)); + const POOL_SLOTS = { pool_tier: "GeneralPurpose", pool_sku: "GP_Gen5", capacity: 8 }; + const POOL_CASES: Array<[string, boolean]> = [ + ["The pool uses the General Purpose tier (GP_Gen5) with 8 vCores.", true], + ["Tier: GeneralPurpose, capacity: 8", true], + [ + '{"sku": {"name": "GP_Gen5", "tier": "GeneralPurpose", "family": "Gen5", "capacity": 8}}', + true, + ], + ["**Tier:** General Purpose · **Capacity:** 8", true], + ["It is a General Purpose pool with a capacity of 8 vCores.", true], + ["| Tier | Business Critical |\n| vCores | 8 |", false], + ["The pool is General Purpose with 4 vCores (8 GB max per database).", false], + ]; + test.each(POOL_CASES)("pool %s -> %s", async (text, want) => { + expect((await POOL(replay([]), { ...POOL_SLOTS }, text)).passed).toBe(want); + }); + const AUDIT = V.textMatches(AUDIT_RETENTION); + const AUDIT_CASES: Array<[string, boolean]> = [ + ["Audit logs are retained for 45 days.", true], + ["Retention: 45", true], + ["**Retention (days):** 45", true], + ["The retention period is 45 days.", true], + ['{"retentionDays": 45, "state": "Enabled"}', true], + ["Retention is 90 days; 45 GB used.", false], + ]; + test.each(AUDIT_CASES)("audit %s -> %s", async (text, want) => { + expect((await AUDIT(replay([]), { audit_days: 45 }, text)).passed).toBe(want); + }); +}); + +// ── task-specific verifiers ────────────────────────────────────────────────── + +describe("Key Vault verifiers", () => { + const verifyCmd = (vault: string, key: string, version: string, digest: string, sig: string) => + `keyvault key verify --vault-name ${vault} --name ${key} --version ${version} --algorithm RS256 --digest ${V.toStdB64(digest)} --signature ${V.toStdB64(sig)}`; + const SIG = "A".repeat(300) + "+/"; + const s = { vault_name: "kv1", key_name: "k1", key_version: "abc", digest: "ZGlnZXN0" }; + + test("signatureVerifies: a stated signature that the vault verifies", async () => { + const q = replay([[verifyCmd("kv1", "k1", "abc", "ZGlnZXN0", SIG), ok({ isValid: true })]]); + expect(await V.signatureVerifies()(q, s, `Signature: ${SIG}`)).toEqual({ + passed: true, + reason: "a signature in the answer verifies", + }); + }); + + test("signatureVerifies: no candidate, or the vault says invalid", async () => { + expect((await V.signatureVerifies()(replay([]), s, "Signed it, done.")).passed).toBe(false); + const q = replay([[verifyCmd("kv1", "k1", "abc", "ZGlnZXN0", SIG), ok({ isValid: false })]]); + expect((await V.signatureVerifies()(q, s, SIG)).passed).toBe(false); + }); + + test("signatureVerifiesIn tries every key version", async () => { + const sig = "B".repeat(80); + const q = replay([ + [ + "keyvault key list-versions --vault-name v --name k", + ok([{ kid: "https://v.vault/keys/k/v1" }, { kid: "https://v.vault/keys/k/v2" }]), + ], + [verifyCmd("v", "k", "v1", "ZA", sig), ok({ isValid: false })], + [verifyCmd("v", "k", "v2", "ZA", sig), ok({ isValid: true })], + ]); + expect( + ( + await V.signatureVerifiesIn("kv", "key", "digest")( + q, + { kv: "v", key: "k", digest: "ZA" }, + `sig=${sig}` + ) + ).passed + ).toBe(true); + }); + + test("accessPolicyGrants: the principal's policy holds the permissions", async () => { + const vaultPath = rgPath("/Microsoft.KeyVault/vaults/{vault_name}"); + const c = get(rgPath("/Microsoft.KeyVault/vaults/kv1"), "2023-07-01"); + const oid = "11111111-2222-3333-4444-555555555555"; + const body = (perms: string[]) => ({ + properties: { + accessPolicies: [{ objectId: oid.toUpperCase(), permissions: { secrets: perms } }], + }, + }); + const v = V.accessPolicyGrants(vaultPath, "2023-07-01", "object_id", "secrets", [ + "get", + "list", + ]); + expect( + ( + await v( + replay([[c, ok(body(["Get", "List", "Set"]))]]), + slots({ vault_name: "kv1", object_id: oid }), + "" + ) + ).passed + ).toBe(true); + expect( + (await v(replay([[c, ok(body(["get"]))]]), slots({ vault_name: "kv1", object_id: oid }), "")) + .passed + ).toBe(false); + }); + + test("secretIs reads the data plane through az", async () => { + const q = replay([ + ["keyvault secret show --vault-name kv1 --name s1", ok({ value: "S3cret-abc" })], + ]); + expect( + (await V.secretIs("kv", "secret", "{v}")(q, { kv: "kv1", secret: "s1", v: "S3cret-abc" }, "")) + .passed + ).toBe(true); + const missing = replay([ + ["keyvault secret show --vault-name kv1 --name s1", failed("ERROR: (SecretNotFound) no")], + ]); + expect( + ( + await V.secretIs("kv", "secret", "{v}")( + missing, + { kv: "kv1", secret: "s1", v: "S3cret-abc" }, + "" + ) + ).passed + ).toBe(false); + }); + + test("kidVersion", () => { + expect( + V.kidVersion("https://kv.vault.azure.localhost.localstack.cloud:4566/keys/k1/b5f4e5559079") + ).toBe("b5f4e5559079"); + }); +}); + +describe("resource index, web app and API Management verifiers", () => { + test("indexTagIs reads the ARM resource index", async () => { + const c = get(`/subscriptions/${SUB}/resourceGroups/${RG}/resources`, "2021-04-01"); + const q = replay([ + [ + c, + ok({ + value: [ + { name: "kva", tags: { costcenter: "cc-1" } }, + { name: "kvb", tags: null }, + ], + }), + ], + ]); + expect( + (await V.indexTagIs("kv1", "costcenter", "{cc}")(q, slots({ kv1: "kva", cc: "cc-1" }), "")) + .passed + ).toBe(true); + const q2 = replay([[c, ok({ value: [{ name: "kvb", tags: null }] })]]); + expect( + (await V.indexTagIs("kv1", "costcenter", "{cc}")(q2, slots({ kv1: "kvb", cc: "cc-1" }), "")) + .passed + ).toBe(false); + }); + + test("appSetting reads config/appsettings/list", async () => { + const site = "/subscriptions/{sub}/resourceGroups/{rg}/providers/Microsoft.Web/sites/{app}"; + const c = post( + `/subscriptions/${SUB}/resourceGroups/${RG}/providers/Microsoft.Web/sites/web1/config/appsettings/list`, + "2026-07-15" + ); + const q = replay([[c, ok({ properties: { FEATURE_FLAG: "on-1" } })]]); + expect( + ( + await V.appSetting( + site, + "2026-07-15", + "FEATURE_FLAG", + "{flag}" + )(q, slots({ app: "web1", flag: "on-1" }), "") + ).passed + ).toBe(true); + expect( + ( + await V.appSetting( + site, + "2026-07-15", + "FEATURE_FLAG", + "{flag}" + )(q, slots({ app: "web1", flag: "on-2" }), "") + ).passed + ).toBe(false); + }); + + test("policyRateLimit reads calls and renewal-period in either order", async () => { + const p = + "/subscriptions/{sub}/resourceGroups/{rg}/providers/Microsoft.ApiManagement/service/{service_name}/apis/{api}/operations/{operation}/policies/policy"; + const c = get( + `/subscriptions/${SUB}/resourceGroups/${RG}/providers/Microsoft.ApiManagement/service/s1/apis/a1/operations/o1/policies/policy`, + "2024-05-01" + ); + const s = slots({ service_name: "s1", api: "a1", operation: "o1", calls: 25 }); + const xml = (attrs: string) => + ok({ + properties: { value: `` }, + }); + expect( + ( + await V.policyRateLimit(p, "2024-05-01")( + replay([[c, xml('calls="25" renewal-period="60"')]]), + s, + "" + ) + ).passed + ).toBe(true); + expect( + ( + await V.policyRateLimit(p, "2024-05-01")( + replay([[c, xml('renewal-period="60" calls="25"')]]), + s, + "" + ) + ).passed + ).toBe(true); + expect( + ( + await V.policyRateLimit(p, "2024-05-01")( + replay([[c, xml('calls="40" renewal-period="60"')]]), + s, + "" + ) + ).passed + ).toBe(false); + expect( + (await V.policyRateLimit(p, "2024-05-01")(replay([[c, notFound()]], 3), s, "")).passed + ).toBe(false); + }); + + test("membershipStated in oracle mode reads the transcript's marks", async () => { + const s = { user1: "alice1", user2: "bob1", member: "bob1", _oracle: true }; + const text = [ + V.oracleBlock("rest --method head --url x1", notFound()), + V.oracleBlock("rest --method head --url x2", ok("")), + ].join("\n"); + expect((await V.membershipStated()(replay([]), s, text)).passed).toBe(true); + expect((await V.membershipStated()(replay([]), { ...s, member: "alice1" }, text)).passed).toBe( + false + ); + }); + + test("accessStated in oracle mode reads the access entry", async () => { + const tenant = ok({ + value: [ + { name: "access", properties: { enabled: false } }, + { name: "gitAccess", properties: { enabled: true } }, + ], + }); + const text = V.oracleBlock("rest --method get --url /tenant", tenant); + expect( + (await V.accessStated()(replay([]), { access_enabled: false, _oracle: true }, text)).passed + ).toBe(true); + expect( + (await V.accessStated()(replay([]), { access_enabled: true, _oracle: true }, text)).passed + ).toBe(false); + }); +}); + +describe("the oracle transcript", () => { + test("a block marks success or failure and fences az's output", () => { + expect(V.oracleBlock("group list", ok([{ name: "g" }]))).toBe( + 'ORACLE ✅ az group list\n```json\n[\n {\n "name": "g"\n }\n]\n```' + ); + expect( + V.oracleBlock("x", { ...notFound(), text: "❌ **Command Failed** (exit 1, not-found)" }) + ).toBe("ORACLE ❌ az x\n```\n❌ **Command Failed** (exit 1, not-found)\n```"); + }); +}); + +describe("fixture helpers", () => { + test("azOk returns JSON and turns a failure into a FixtureError", async () => { + expect(await V.azOk(replay([["a", ok({ x: 1 })]]), "a")).toEqual({ x: 1 }); + await expect(V.azOk(replay([["a", failed("ERROR: boom")]]), "a")).rejects.toBeInstanceOf( + V.FixtureError + ); + }); + + test("putOk refuses a non-2xx answer", async () => { + const c = V.restCommand("PUT", "/p", "1", { a: 1 }); + await expect( + V.putOk(replay([[c, failed("ERROR: CONFLICT({})")]]), "/p", "1", { a: 1 }, "thing") + ).rejects.toThrow(/thing PUT 409/); + await expect( + V.putOk(replay([[c, ok({ id: "x" })]]), "/p", "1", { a: 1 }, "thing") + ).resolves.toEqual({ id: "x" }); + }); + + test("snapshot stores _before_ slots and refuses an empty value", async () => { + const c = post("/k", "1"); + const s: Slots = {}; + await V.snapshot(replay([[c, ok({ key1: "a", key2: "b" })]]), s, "/k", "1", { + changed: "key1", + kept: "key2", + }); + expect(s).toEqual({ _before_changed: "a", _before_kept: "b" }); + await expect( + V.snapshot(replay([[c, ok({ key1: "" })]]), {}, "/k", "1", { changed: "key1" }) + ).rejects.toThrow(/nothing to snapshot/); + }); + + test("makeSetupContext names and picks", () => { + const ctx = V.makeSetupContext(replay([]), {}, RG); + expect(ctx.name("lk-ops-")).toMatch(/^lk-ops-[0-9a-f]{6}$/); + expect(ctx.hex(10)).toMatch(/^[0-9a-f]{10}$/); + expect([1, 2, 3]).toContain(ctx.pick([1, 2, 3])); + const sample = ctx.sample([1, 2, 3, 4], 2); + expect(new Set(sample).size).toBe(2); + const n = ctx.randomInt(10, 12); + expect(n >= 10 && n <= 12).toBe(true); + }); +}); diff --git a/tests/azure/evals/verifiers.ts b/tests/azure/evals/verifiers.ts new file mode 100644 index 0000000..45b8975 --- /dev/null +++ b/tests/azure/evals/verifiers.ts @@ -0,0 +1,1450 @@ +/** + * The E2 verifier and fixture library. + * + * Every verifier reads the outcome from the emulator at verification time, never from the + * agent's report, through a `Query`: in a run that is a server session of the harness's + * own (never the agent's), in the unit tests recorded answers with a fake clock. Seven + * answer-reading pitfalls are handled here (defects.test.ts shows each): see `affirmedRights` (1), + * `validationVerdict` (2), `untaggedVerdicts` (3), `hostOf` (4), `hostVerdicts` (5), + * `membershipVerdicts` (6) and `tenantAccessVerdict` (7). + * + * Loaded unbundled by run.mjs through Node's type stripping, so this module has no runtime + * import of another local module and no TypeScript syntax that needs a transform (no + * enums, namespaces or parameter properties). The task modules get it as a parameter. + */ +import { createHash, randomBytes, randomUUID } from "node:crypto"; +import type { AzAnswer, Query, SetupContext, Slots, StepResult, Verdict, Verify } from "./types"; + +export const SUBSCRIPTION = "00000000-0000-0000-0000-000000000000"; +export const LOCATION = "westeurope"; + +// ── Python-compatible formatting ──────────────────────────────────────────── + +function isDict(v: unknown): v is Record { + return typeof v === "object" && v !== null && !Array.isArray(v); +} + +/** Python's str() for JSON values: True/False/None, as the task answers print them. */ +export function pyStr(v: unknown): string { + if (v === null || v === undefined) return "None"; + if (v === true) return "True"; + if (v === false) return "False"; + if (typeof v === "string") return v; + if (typeof v === "number") return String(v); + return JSON.stringify(v); +} + +/** Python's json.dumps() layout (", " and ": " separators, ASCII only). */ +export function pyJson(v: unknown): string { + if (v === null || v === undefined) return "null"; + if (typeof v === "boolean" || typeof v === "number") return JSON.stringify(v); + if (typeof v === "string") { + return JSON.stringify(v).replace( + /[\u007f-\uffff]/g, + (c) => "\\u" + c.charCodeAt(0).toString(16).padStart(4, "0") + ); + } + if (Array.isArray(v)) return "[" + v.map(pyJson).join(", ") + "]"; + if (isDict(v)) { + return ( + "{" + + Object.entries(v) + .map(([k, x]) => `${pyJson(k)}: ${pyJson(x)}`) + .join(", ") + + "}" + ); + } + return JSON.stringify(v); +} + +/** + * Python's str.format for `{name}` fields, with `{{` and `}}` as literal braces. An unknown + * name throws, as Python's KeyError did. + */ +export function fmt(template: string, values: Record): string { + return template.replace(/\{\{|\}\}|\{([A-Za-z_][A-Za-z0-9_]*)\}/g, (m, name: string) => { + if (m === "{{") return "{"; + if (m === "}}") return "}"; + if (!(name in values)) throw new Error(`no value for {${name}} in: ${template.slice(0, 120)}`); + return pyStr(values[name]); + }); +} + +/** A path or command template filled with the subscription and the slots. */ +export function fill(template: string, slots: Slots): string { + return fmt(template, { sub: SUBSCRIPTION, ...slots }); +} + +export function escapeRegExp(s: string): string { + return s.replace(/[.*+?^${}()|[\]\\/-]/g, "\\$&"); +} + +/** Normalised text: no whitespace, underscores or hyphens; lower case. */ +export function normText(s: unknown): string { + return pyStr(s) + .replace(/[\s_-]+/g, "") + .toLowerCase(); +} + +const SELECT = /^(\w*)\[(\w+)=([^\]]*)\]$/; + +/** + * A dotted path into a JSON body. A numeric part indexes a list + * (`containers.0.image`); `name[field=value]` picks the list element whose field equals + * value (`keys[keyName=key1].value`). Missing parts give null, as Python's None. + */ +export function dotted(body: unknown, key: string): unknown { + let cur: unknown = body; + for (const part of key.split(".")) { + const m = SELECT.exec(part); + if (m) { + if (m[1]) cur = isDict(cur) ? (cur[m[1]] ?? null) : null; + cur = Array.isArray(cur) + ? (cur.find((e) => isDict(e) && pyStr(e[m[2]]) === m[3]) ?? null) + : null; + } else if (Array.isArray(cur) && /^\d+$/.test(part)) { + const i = Number(part); + cur = i < cur.length ? cur[i] : null; + } else if (isDict(cur)) { + cur = part in cur ? (cur[part] ?? null) : null; + } else { + return null; + } + } + return cur === undefined ? null : cur; +} + +export function parseJson(text: string | undefined): unknown { + if (!text || !text.trim()) return null; + try { + return JSON.parse(text); + } catch { + return null; + } +} + +// ── base64 and random helpers ──────────────────────────────────────────────── + +export function b64url(data: Buffer | string): string { + return Buffer.from(data).toString("base64url"); +} + +export function b64std(data: Buffer | string): string { + return Buffer.from(data).toString("base64"); +} + +/** Standard base64 with padding, from base64 or base64url (padded or not). */ +export function toStdB64(value: string): string { + const s = value.replace(/-/g, "+").replace(/_/g, "/").replace(/=+$/, ""); + return s + "=".repeat((4 - (s.length % 4)) % 4); +} + +/** base64url without padding, from base64 or base64url. */ +export function toB64url(value: string): string { + return value.replace(/\+/g, "-").replace(/\//g, "_").replace(/=+$/, ""); +} + +export function sha256(data: Buffer | string): Buffer { + return createHash("sha256").update(data).digest(); +} + +export function hex(chars: number): string { + return randomBytes(Math.ceil(chars / 2)) + .toString("hex") + .slice(0, chars); +} + +/** The context a task's setup runs with. */ +export function makeSetupContext(q: Query, slots: Slots, rg: string): SetupContext { + const random = () => Math.random(); + const randomInt = (min: number, max: number) => min + Math.floor(random() * (max - min + 1)); + return { + q, + slots, + rg, + name: (prefix: string) => `${prefix}${hex(6)}`, + hex, + uuid: () => randomUUID(), + pick: (options: readonly T[]) => options[Math.floor(random() * options.length)], + sample: (options: readonly T[], count: number) => { + const pool = [...options]; + const out: T[] = []; + while (out.length < count && pool.length > 0) { + out.push(pool.splice(Math.floor(random() * pool.length), 1)[0]); + } + return out; + }, + randomInt, + random, + }; +} + +// ── ARM requests through `az rest` ─────────────────────────────────────────── + +const REASONS: Record = { + "BAD REQUEST": 400, + UNAUTHORIZED: 401, + FORBIDDEN: 403, + "NOT FOUND": 404, + "METHOD NOT ALLOWED": 405, + CONFLICT: 409, + GONE: 410, + "PRECONDITION FAILED": 412, + "UNPROCESSABLE ENTITY": 422, + "TOO MANY REQUESTS": 429, + "INTERNAL SERVER ERROR": 500, + "NOT IMPLEMENTED": 501, + "BAD GATEWAY": 502, + "SERVICE UNAVAILABLE": 503, + "GATEWAY TIMEOUT": 504, +}; + +/** + * The HTTP status behind an `az rest` answer. az exits 0 on 2xx and prints + * `ERROR: ()` otherwise (the emulator's reason phrases are upper case, + * `NOT FOUND`); 0 means the command itself failed before any HTTP answer. + */ +export function httpStatus(a: AzAnswer): number { + if (a.exitCode === 0 && !a.stoppedByTool) return 200; + const m = /^ERROR: ([A-Za-z][A-Za-z ]*?)\s*\(/m.exec(a.stderr || ""); + if (m) { + const code = REASONS[m[1].toUpperCase()]; + if (code) return code; + } + if (a.classId === "not-found") return 404; + if (a.classId === "not-implemented") return 501; + return 0; +} + +export interface ArmResponse { + status: number; + body: unknown; + answer: AzAnswer; +} + +/** A JSON body for `--body '...'`; single quotes cannot travel inside it. */ +export function jsonArg(body: unknown): string { + const text = typeof body === "string" ? body : JSON.stringify(body); + if (text.includes("'")) + throw new Error(`a request body cannot contain a single quote: ${text.slice(0, 80)}`); + return text; +} + +export function restCommand(method: string, path: string, api?: string, body?: unknown): string { + const url = api ? `${path}${path.includes("?") ? "&" : "?"}api-version=${api}` : path; + let cmd = `rest --method ${method.toLowerCase()} --url "${url}"`; + if (body !== undefined) cmd += ` --body '${jsonArg(body)}'`; + return cmd; +} + +/** An ARM request (a relative path, as the tool rewrites it onto the emulator). */ +export async function arm( + q: Query, + method: string, + path: string, + api?: string, + body?: unknown +): Promise { + const answer = await q.az(restCommand(method, path, api, body)); + return { status: httpStatus(answer), body: parseJson(answer.stdout), answer }; +} + +/** properties.provisioningState, when the body has one. */ +export function provisioningState(body: unknown): string | null { + const props = isDict(body) ? body.properties : null; + const state = isDict(props) ? props.provisioningState : null; + return typeof state === "string" ? state : null; +} + +const TERMINAL = new Set(["Succeeded", "Failed", "Canceled"]); + +/** Poll until GET is 200 and any provisioningState is terminal, or the deadline passes. */ +export async function waitReady( + q: Query, + path: string, + api: string, + timeoutS?: number, + intervalMs = 500 +): Promise { + const deadline = q.now() + 1000 * (timeoutS ?? q.waitS); + for (;;) { + const r = await arm(q, "GET", path, api); + const state = r.status === 200 ? provisioningState(r.body) : null; + if ((r.status === 200 && (state === null || TERMINAL.has(state))) || q.now() > deadline) + return r; + await q.sleep(intervalMs); + } +} + +/** Poll a GET until its status is in `want`, or the deadline passes. */ +export async function waitStatus( + q: Query, + path: string, + api: string, + want: readonly number[], + timeoutS?: number, + intervalMs = 500 +): Promise { + const deadline = q.now() + 1000 * (timeoutS ?? q.waitS); + for (;;) { + const r = await arm(q, "GET", path, api); + if (want.includes(r.status) || q.now() > deadline) return r; + await q.sleep(intervalMs); + } +} + +/** Poll until a dotted field equals `want` (case-insensitive); the last value seen. */ +export async function waitField( + q: Query, + path: string, + api: string, + key: string, + want: string, + timeoutS = 60, + intervalMs = 500 +): Promise { + const deadline = q.now() + 1000 * timeoutS; + for (;;) { + const r = await arm(q, "GET", path, api); + const got = r.status === 200 ? dotted(r.body, key) : null; + if (pyStr(got).toLowerCase() === want.toLowerCase() || q.now() > deadline) return got; + await q.sleep(intervalMs); + } +} + +// ── fixture helpers ───────────────────────────────────────────────────────── + +/** A setup step that did not work: the run records a setup error, not an agent failure. */ +export class FixtureError extends Error { + constructor(message: string) { + super(message); + this.name = "FixtureError"; + } +} + +/** Run a fixture command; it must succeed. Returns az's JSON output (or null). */ +export async function azOk(q: Query, command: string, what?: string): Promise { + const a = await q.az(command); + if (a.exitCode !== 0 || a.stoppedByTool) { + const detail = (a.stderr || a.text || "").trim().slice(0, 300); + throw new FixtureError( + `fixture: ${what ?? command.slice(0, 80)} failed (exit ${a.exitCode}): ${detail}` + ); + } + return parseJson(a.stdout); +} + +/** PUT a resource; it must succeed. */ +export async function putOk( + q: Query, + path: string, + api: string, + body: unknown, + what: string +): Promise { + const r = await arm(q, "PUT", path, api, body); + if (r.status >= 300 || r.status === 0) { + const detail = (r.answer.stderr || r.answer.text || "").trim().slice(0, 300); + throw new FixtureError(`fixture: ${what} PUT ${r.status}: ${detail}`); + } + return r.body; +} + +/** A GET that must succeed; the body. */ +export async function getOk(q: Query, path: string, api: string, what: string): Promise { + const r = await arm(q, "GET", path, api); + if (r.status !== 200) { + throw new FixtureError( + `fixture: ${what} GET ${r.status}: ${(r.answer.stderr || "").trim().slice(0, 300)}` + ); + } + return r.body; +} + +/** + * The snapshot setup step: record values before the task, as slots + * `_before_

`) | + +The emulator should run with the CI flags of `scripts/ci/azure-emulator-up.sh`: `FRONT_DOOR_CLASSIC_ALLOW_CREATE=1` +and `CDN_CLASSIC_ALLOW_CREATE=1` (classic Front Door and CDN cases), `MSSQL_ACCEPT_EULA=Y` (SQL), +and the pinned `LS_AZURE_ORYX_BUILD_IMAGE_TO_USE` (web apps). + +### Sharding + +Jest's `--shard` splits test _files_, and the matrix is one file, so the runner shards its cases +itself: `AZURE_MATRIX_SHARD=i/n` assigns each case to a shard by a deterministic +longest-processing-time split (a `backing` case weighs 5, others 1; heaviest first, each to the +lightest shard). The cancel case and the coverage-key check run in shard 1; the egress check runs +in every shard. Each shard needs its own emulator. + +## Local safety + +On a machine where the emulator on 4566 is shared with other agents or people: + +- use it only through the tool (this runner); never start, stop or restart it, and never start a + second emulator beside it (a second instance can remove the first one's storage side-cars); +- run only non-`backing` cases unless whoever shares the emulator agrees, with `--runInBand`; +- every case uses its own names (`mcp--...`, from `runId()`) and cleans up: the runner + waits until nothing in the group is still provisioning, then runs the case's `cleanup` + (Key Vaults are deleted and purged, App Configuration stores and APIM services purged, locks + removed) and deletes the group with `--no-wait`; +- no Docker commands beyond a read-only `docker ps`; never read or change `~/.azure`, `~/.ssh`, + `~/.kube`, `~/.docker` (the tool's private home takes `--generate-ssh-keys` and + `aks get-credentials`); +- the auth token only has to be present: the harness sets a dummy when none is set. + +A failed storage create can leave its `ls-storage-` azurite container running with no +account behind it (seen 2026-09-27); report such containers, do not remove them on a shared engine. + +## Case format + +```yaml +provider: Microsoft.Storage # one of the 29; the file name is derived from it +cases: + - id: storage-account-keys # unique across all files + operations: # coverage keys " " + [Microsoft.Storage StorageAccounts ListKeys, Microsoft.Storage StorageAccounts RegenerateKey] + pr: true # in the PR subset (never backing, never a known gap) + backing: false # true: the case starts emulator side-car containers + note: Why the case looks the way it does (required when operations is empty). + timeout: 120 # optional: cancel each step after this many seconds + setup: # prerequisites; any failure is a real failure + - group create --name {rg} --location {location} + - storage account create --name {storage} --resource-group {rg} --location {location} --sku Standard_LRS + - run: storage account keys list --account-name {storage} --resource-group {rg} --query "[0].value" -o tsv + capture: key_before # stdout (trimmed) into {key_before}; or a map name -> JSON path + until: { stdout: { matches: "." } } # poll every 5 s until this holds + wait: 180 # seconds for `until` (default 180) + - storage account keys renew --account-name {storage} --resource-group {rg} --key key1 + command: storage account keys list --account-name {storage} --resource-group {rg} --query "[0].value != '{key_before}'" + expect: # default { exitCode: 0 } + exitCode: 0 + json: { path: "", equals: true } # one check or a list + cleanup: # best effort, never fails the case + - group delete --name {rg} --yes --no-wait + + - id: storage-account-keys-renew-answer # a known gap (never pr) + operations: [Microsoft.Storage StorageAccounts RegenerateKey] + setup: + - group create --name {rg} --location {location} + - storage account create --name {storage} --resource-group {rg} --location {location} --sku Standard_LRS + command: storage account keys renew --account-name {storage} --resource-group {rg} --key key2 + expect: + # classId: not-found # optional: the answer's class + stdout: { contains: key2 } # on the trimmed stdout; one check or a list + cleanup: + - group delete --name {rg} --yes --no-wait + known_gap: # an expected failure (see Known gaps) + reason: Why it fails. + operations: [Microsoft.Storage StorageAccounts RegenerateKey] # default: all of the case's + date: 2026-09-27 # when it was last seen +``` + +- **Placeholders:** `{id}` (the case run's id), `{rg}`, `{location}`, `{sub}`, `{tenant}`, + `{uuid}`, `{storage}`, `{vault}`, `{container}`, `{port}` (the emulator's gateway port, for + data-plane hosts a case builds itself), `{name:}` (`mcp--`), + `{alnum:}` (lower-case alphanumerics, at most 24: registries, storage), and every name an + earlier setup step captured. An unknown placeholder is a schema error. JSON bodies are safe + (`{"a": 1}` is not a placeholder); a JMESPath multiselect key must not look like one (`{Id:id}`). +- **Checks:** exactly one of `equals` (deep), `contains` (substring, array element, or object + key), `matches` (regular expression), `exists` (`true`/`false`), `length`. `path` is a tiny + resolver: `a.b[0].c`, `[-1]` (last), `""` for the whole document; quote it in YAML when it has + brackets. Narrow with `--query` instead of wildcards. The runner reads az's stdout from the test + envelope, never from the tool's prose. +- **Commands** are one line without the leading `az`; use `>-` for long ones, and a block scalar + for any line with `": "` in it. Unquoted `&`, `|`, `;`, `<`, `>` are refused by the tokenizer: + quote URLs with query strings. Files are relative to the tool's working directory, which the + runner seeds with `hello.txt` and `bicep/` (every `tests/fixtures/azure/bicep/*.bicep`). +- **Schema lints** (`schema.test.ts`): the file parses; ids are unique; operation keys have the + three-part form and the file's provider; every placeholder resolves; every setup, command and + cleanup line passes the real policy (`evaluateAzCommand`); a case that creates `{rg}` deletes it + with `--no-wait`; a case that creates a vault deletes and purges it before the group; the PR + subset has at least 20 cases, none backing or a known gap, and is part of the full run under + every shard split; each of the 127 gap-prone operations listed in `schema.test.ts` is carried by + a case. + +## Results + +- **Known gaps** run as expected failures: the case passes when its command fails (recorded as + `known_gap` with the failure class), a failing _setup_ is still a real failure, and a known gap + that passes fails the test as `gap_fixed`, so the entry gets removed (Jest's `test.failing` + semantics, written out so that setup failures and the class are reported properly). + `AZURE_MATRIX_GAPS=skip` records them as skips instead. +- **az versions:** CI runs `AZ_VERSION` (2.90); cases avoid flags that changed between 2.85 and + 2.90 (`rest` creates and `--ids` reads where az 2.86 renamed PostgreSQL flags). One case is + version-specific by design: `aks-stop-wait` meets its gap only at az 2.90's api-version, so a + local run with az 2.85 reports it `gap_fixed`. +- **Extensions:** a step that fails with class `extension` (a curated extension is not installed) + makes the case a recorded skip on a developer machine and a failure when `CI` is set. +- **Transient classes** (`conn-refused`, `discovery`, `emulator-error`) are retried once per step, + and the retry is logged in the step record. When the tool answers "Emulator Not Ready" (its health + preflight timed out, so az never ran), the step is retried after 5, 10, 20 and 30 s: a busy + emulator stops answering its health check for 20-40 s at a time. +- **JSONL** (`AZURE_MATRIX_RESULTS`): per case `result` (`pass`, `fail`, `setup_failed`, + `known_gap`, `gap_fixed`, `skipped`), class, exit code, duration, each step's command and class, + the problems (with a 400-character stdout excerpt), and the egress hosts seen; plus a + `cancel-in-flight` and an `egress-summary` line. +- **Catalogue** (`AZURE_OP_CATALOGUE_OUT`, for the portal): per coverage key its best + result, the date it was last verified passing, the command template (the case's `command`; the + operation itself may be exercised by one of the case's setup steps), whether the emulator marks + it implemented (read through the tool from `/_localstack/coverage`), and every case that touches + it. Each run rewrites the file with that run's cases only. Shards write one catalogue each; + `scripts/ci/merge-op-catalogue.cjs` merges them by concatenating each key's `cases` and taking + the best result (`pass` > `gap_fixed` > `known_gap` > `fail` > `setup_failed` > `skipped`). +- **Egress:** the last test fails on any host the guard refused, except inside a known-gap case + whose refusal is its documented failure (the App Insights data plane, whose endpoint the tool's + cloud does not map; listed as `refusedInKnownGaps` in the `egress-summary` line), and on any + `app.aladdin.microsoft.com` refusal (a regression of the bootstrap's + `core.error_recommendation=off`); housekeeping hosts such as `aka.ms` (Bicep) are only reported. +- **Coverage keys:** in shard 1 a test checks every key against the emulator's live coverage list; + unknown keys fail the full run and are only reported in the PR run. diff --git a/tests/azure/matrix/apimanagement.yaml b/tests/azure/matrix/apimanagement.yaml new file mode 100644 index 0000000..ee6c0ef --- /dev/null +++ b/tests/azure/matrix/apimanagement.yaml @@ -0,0 +1,717 @@ +# Microsoft.ApiManagement: services (created in about a second, no backing container), APIs, +# operations, policies, releases, version sets, schemas, products, groups, users, subscriptions, +# named values, loggers, diagnostics, backends, caches, certificates, gateways, workspaces, tenant +# access, identity and OpenID providers, GraphQL resolvers, tags. The CLI covers the main groups; +# the rest go through rest (api-version 2024-05-01). A deleted service is soft-deleted, so every case deletes and purges its service. +provider: Microsoft.ApiManagement +cases: + - id: apim-service + operations: + [ + Microsoft.ApiManagement ApiManagementService CreateOrUpdate, + Microsoft.ApiManagement ApiManagementService Get, + Microsoft.ApiManagement ApiManagementService ListByResourceGroup, + Microsoft.ApiManagement ApiManagementService List, + Microsoft.ApiManagement ApiManagementService Update, + Microsoft.ApiManagement ApiManagementService CheckNameAvailability, + ] + pr: true + note: Samples apim create/show. + setup: + - group create --name {rg} --location {location} + - >- + apim create --name {name:apim} --resource-group {rg} --location {location} + --publisher-email l2@example.com --publisher-name L2 --sku-name Developer + - run: apim show --name {name:apim} --resource-group {rg} --query provisioningState -o tsv + until: { stdout: { equals: Succeeded } } + - apim list --resource-group {rg} + - apim list --query "[?name=='{name:apim}'].name" + - apim update --name {name:apim} --resource-group {rg} --tags matrix=l2 + command: apim check-name --name {name:apim} + expect: + json: { path: nameAvailable, equals: false } + cleanup: + - apim delete --name {name:apim} --resource-group {rg} --yes + - apim deletedservice purge --service-name {name:apim} --location {location} + - group delete --name {rg} --yes --no-wait + + - id: apim-deleted-service + operations: + [ + Microsoft.ApiManagement ApiManagementService Delete, + Microsoft.ApiManagement DeletedServices ListBySubscription, + Microsoft.ApiManagement DeletedServices GetByName, + Microsoft.ApiManagement DeletedServices Purge, + ] + note: Samples apim deletedservice show/purge. + setup: + - group create --name {rg} --location {location} + - >- + apim create --name {name:apim} --resource-group {rg} --location {location} + --publisher-email l2@example.com --publisher-name L2 --sku-name Developer + - apim delete --name {name:apim} --resource-group {rg} --yes + - run: apim deletedservice list --query "[?name=='{name:apim}'].name" + until: { json: { path: "", equals: ["{name:apim}"] } } + - apim deletedservice show --service-name {name:apim} --location {location} + command: apim deletedservice purge --service-name {name:apim} --location {location} + expect: + exitCode: 0 + cleanup: + - apim deletedservice purge --service-name {name:apim} --location {location} + - group delete --name {rg} --yes --no-wait + + - id: apim-api + operations: + [ + Microsoft.ApiManagement Api CreateOrUpdate, + Microsoft.ApiManagement Api Get, + Microsoft.ApiManagement Api ListByService, + Microsoft.ApiManagement Api Update, + Microsoft.ApiManagement Api GetEntityTag, + Microsoft.ApiManagement Api ListByTags, + Microsoft.ApiManagement Api Delete, + ] + pr: true + note: Samples apim api show. + setup: + - group create --name {rg} --location {location} + - >- + apim create --name {name:apim} --resource-group {rg} --location {location} + --publisher-email l2@example.com --publisher-name L2 --sku-name Developer + - >- + apim api create --service-name {name:apim} --resource-group {rg} --api-id orders --path orders + --display-name "Orders API" --protocols https + - run: apim api show --service-name {name:apim} --resource-group {rg} --api-id orders --query path -o tsv + until: { stdout: { equals: orders } } + - apim api list --service-name {name:apim} --resource-group {rg} + - apim api update --service-name {name:apim} --resource-group {rg} --api-id orders --description "l2 matrix" + - >- + rest --method head + --url /subscriptions/{sub}/resourceGroups/{rg}/providers/Microsoft.ApiManagement/service/{name:apim}/apis/orders?api-version=2024-05-01 + - >- + rest --method get + --url /subscriptions/{sub}/resourceGroups/{rg}/providers/Microsoft.ApiManagement/service/{name:apim}/apisByTags?api-version=2024-05-01 + command: apim api delete --service-name {name:apim} --resource-group {rg} --api-id orders --yes + expect: + exitCode: 0 + cleanup: + - apim delete --name {name:apim} --resource-group {rg} --yes + - apim deletedservice purge --service-name {name:apim} --location {location} + - group delete --name {rg} --yes --no-wait + + - id: apim-api-operations-policies + operations: + [ + Microsoft.ApiManagement ApiOperation CreateOrUpdate, + Microsoft.ApiManagement ApiOperation Get, + Microsoft.ApiManagement ApiOperation ListByApi, + Microsoft.ApiManagement ApiOperation Update, + Microsoft.ApiManagement ApiOperationPolicy CreateOrUpdate, + Microsoft.ApiManagement ApiOperationPolicy Get, + Microsoft.ApiManagement ApiOperationPolicy ListByOperation, + Microsoft.ApiManagement ApiPolicy CreateOrUpdate, + Microsoft.ApiManagement ApiPolicy Get, + Microsoft.ApiManagement ApiPolicy ListByApi, + Microsoft.ApiManagement ApiOperation Delete, + ] + note: >- + Samples apim api operation. The XML policy travels JSON-escaped (<, >). + setup: + - group create --name {rg} --location {location} + - >- + apim create --name {name:apim} --resource-group {rg} --location {location} + --publisher-email l2@example.com --publisher-name L2 --sku-name Developer + - apim api create --service-name {name:apim} --resource-group {rg} --api-id orders --path orders --display-name Orders + - >- + apim api operation create --service-name {name:apim} --resource-group {rg} --api-id orders + --operation-id list-orders --display-name "List orders" --method GET --url-template /orders + - apim api operation show --service-name {name:apim} --resource-group {rg} --api-id orders --operation-id list-orders + - apim api operation list --service-name {name:apim} --resource-group {rg} --api-id orders + - >- + apim api operation update --service-name {name:apim} --resource-group {rg} --api-id orders + --operation-id list-orders --description "l2 matrix" + - >- + rest --method put + --url /subscriptions/{sub}/resourceGroups/{rg}/providers/Microsoft.ApiManagement/service/{name:apim}/apis/orders/operations/list-orders/policies/policy?api-version=2024-05-01 + --body '{"properties": {"format": "rawxml", "value": ""}}' + - >- + rest --method get + --url /subscriptions/{sub}/resourceGroups/{rg}/providers/Microsoft.ApiManagement/service/{name:apim}/apis/orders/operations/list-orders/policies/policy?api-version=2024-05-01 + - >- + rest --method get + --url /subscriptions/{sub}/resourceGroups/{rg}/providers/Microsoft.ApiManagement/service/{name:apim}/apis/orders/operations/list-orders/policies?api-version=2024-05-01 + - >- + rest --method put + --url /subscriptions/{sub}/resourceGroups/{rg}/providers/Microsoft.ApiManagement/service/{name:apim}/apis/orders/policies/policy?api-version=2024-05-01 + --body '{"properties": {"format": "rawxml", "value": ""}}' + - >- + rest --method get + --url /subscriptions/{sub}/resourceGroups/{rg}/providers/Microsoft.ApiManagement/service/{name:apim}/apis/orders/policies/policy?api-version=2024-05-01 + - >- + rest --method get + --url /subscriptions/{sub}/resourceGroups/{rg}/providers/Microsoft.ApiManagement/service/{name:apim}/apis/orders/policies?api-version=2024-05-01 + command: >- + apim api operation delete --service-name {name:apim} --resource-group {rg} --api-id orders + --operation-id list-orders + expect: + exitCode: 0 + cleanup: + - apim delete --name {name:apim} --resource-group {rg} --yes + - apim deletedservice purge --service-name {name:apim} --location {location} + - group delete --name {rg} --yes --no-wait + + - id: apim-api-releases-versions-schemas + operations: + [ + Microsoft.ApiManagement ApiRelease CreateOrUpdate, + Microsoft.ApiManagement ApiRelease Get, + Microsoft.ApiManagement ApiRelease ListByService, + Microsoft.ApiManagement ApiRelease Update, + Microsoft.ApiManagement ApiRelease Delete, + Microsoft.ApiManagement ApiRevision ListByService, + Microsoft.ApiManagement ApiVersionSet CreateOrUpdate, + Microsoft.ApiManagement ApiVersionSet Get, + Microsoft.ApiManagement ApiVersionSet ListByService, + Microsoft.ApiManagement ApiVersionSet Update, + Microsoft.ApiManagement ApiVersionSet Delete, + Microsoft.ApiManagement ApiSchema CreateOrUpdate, + Microsoft.ApiManagement ApiSchema Get, + Microsoft.ApiManagement ApiSchema ListByApi, + Microsoft.ApiManagement ApiSchema Delete, + ] + setup: + - group create --name {rg} --location {location} + - >- + apim create --name {name:apim} --resource-group {rg} --location {location} + --publisher-email l2@example.com --publisher-name L2 --sku-name Developer + - apim api create --service-name {name:apim} --resource-group {rg} --api-id orders --path orders --display-name Orders + - >- + apim api release create --service-name {name:apim} --resource-group {rg} --api-id orders + --api-revision 1 --release-id r1 --notes l2 + - apim api release show --service-name {name:apim} --resource-group {rg} --api-id orders --release-id r1 + - apim api release list --service-name {name:apim} --resource-group {rg} --api-id orders + - apim api release update --service-name {name:apim} --resource-group {rg} --api-id orders --release-id r1 --notes l2b + - apim api release delete --service-name {name:apim} --resource-group {rg} --api-id orders --release-id r1 + - apim api revision list --service-name {name:apim} --resource-group {rg} --api-id orders + - >- + apim api versionset create --service-name {name:apim} --resource-group {rg} --version-set-id orders-vs + --display-name "Orders versions" --versioning-scheme Segment + - apim api versionset show --service-name {name:apim} --resource-group {rg} --version-set-id orders-vs + - apim api versionset list --service-name {name:apim} --resource-group {rg} + - apim api versionset update --service-name {name:apim} --resource-group {rg} --version-set-id orders-vs --description l2 + - apim api versionset delete --service-name {name:apim} --resource-group {rg} --version-set-id orders-vs + - >- + apim api schema create --service-name {name:apim} --resource-group {rg} --api-id orders --schema-id s1 + --schema-type application/json --schema-content '{"type": "object"}' + - apim api schema show --service-name {name:apim} --resource-group {rg} --api-id orders --schema-id s1 + - apim api schema list --service-name {name:apim} --resource-group {rg} --api-id orders + command: apim api schema delete --service-name {name:apim} --resource-group {rg} --api-id orders --schema-id s1 --yes + expect: + exitCode: 0 + cleanup: + - apim delete --name {name:apim} --resource-group {rg} --yes + - apim deletedservice purge --service-name {name:apim} --location {location} + - group delete --name {rg} --yes --no-wait + + - id: apim-products + operations: + [ + Microsoft.ApiManagement Product CreateOrUpdate, + Microsoft.ApiManagement Product Get, + Microsoft.ApiManagement Product ListByService, + Microsoft.ApiManagement Product Update, + Microsoft.ApiManagement ProductApi CreateOrUpdate, + Microsoft.ApiManagement ProductApi ListByProduct, + Microsoft.ApiManagement ProductApi CheckEntityExists, + Microsoft.ApiManagement ProductApi Delete, + Microsoft.ApiManagement ApiProduct ListByApis, + Microsoft.ApiManagement Product Delete, + ] + note: Samples apim product create/show, apim product api add. + setup: + - group create --name {rg} --location {location} + - >- + apim create --name {name:apim} --resource-group {rg} --location {location} + --publisher-email l2@example.com --publisher-name L2 --sku-name Developer + - apim api create --service-name {name:apim} --resource-group {rg} --api-id orders --path orders --display-name Orders + - >- + apim product create --service-name {name:apim} --resource-group {rg} --product-id starter + --product-name Starter --description l2 --subscription-required true --state published + - apim product show --service-name {name:apim} --resource-group {rg} --product-id starter + - apim product list --service-name {name:apim} --resource-group {rg} + - apim product update --service-name {name:apim} --resource-group {rg} --product-id starter --description l2b + - apim product api add --service-name {name:apim} --resource-group {rg} --product-id starter --api-id orders + - apim product api list --service-name {name:apim} --resource-group {rg} --product-id starter + - apim product api check --service-name {name:apim} --resource-group {rg} --product-id starter --api-id orders + - >- + rest --method get + --url /subscriptions/{sub}/resourceGroups/{rg}/providers/Microsoft.ApiManagement/service/{name:apim}/apis/orders/products?api-version=2024-05-01 + - apim product api delete --service-name {name:apim} --resource-group {rg} --product-id starter --api-id orders + command: apim product delete --service-name {name:apim} --resource-group {rg} --product-id starter --delete-subscriptions true --yes + expect: + exitCode: 0 + cleanup: + - apim delete --name {name:apim} --resource-group {rg} --yes + - apim deletedservice purge --service-name {name:apim} --location {location} + - group delete --name {rg} --yes --no-wait + + - id: apim-named-values + operations: + [ + Microsoft.ApiManagement NamedValue CreateOrUpdate, + Microsoft.ApiManagement NamedValue Get, + Microsoft.ApiManagement NamedValue ListByService, + Microsoft.ApiManagement NamedValue ListValue, + Microsoft.ApiManagement NamedValue Update, + Microsoft.ApiManagement NamedValue Delete, + ] + note: Samples apim nv create/show/show-secret. + setup: + - group create --name {rg} --location {location} + - >- + apim create --name {name:apim} --resource-group {rg} --location {location} + --publisher-email l2@example.com --publisher-name L2 --sku-name Developer + - >- + apim nv create --service-name {name:apim} --resource-group {rg} --named-value-id backend-key + --display-name backend-key --value "l2 secret" --secret true + - apim nv show --service-name {name:apim} --resource-group {rg} --named-value-id backend-key + - apim nv list --service-name {name:apim} --resource-group {rg} + - run: apim nv show-secret --service-name {name:apim} --resource-group {rg} --named-value-id backend-key --query value -o tsv + until: { stdout: { equals: "l2 secret" } } + - apim nv update --service-name {name:apim} --resource-group {rg} --named-value-id backend-key --value "l2 secret 2" + command: apim nv delete --service-name {name:apim} --resource-group {rg} --named-value-id backend-key --yes + expect: + exitCode: 0 + cleanup: + - apim delete --name {name:apim} --resource-group {rg} --yes + - apim deletedservice purge --service-name {name:apim} --location {location} + - group delete --name {rg} --yes --no-wait + + - id: apim-groups-users-subscriptions + operations: + [ + Microsoft.ApiManagement Group CreateOrUpdate, + Microsoft.ApiManagement Group Get, + Microsoft.ApiManagement Group ListByService, + Microsoft.ApiManagement User CreateOrUpdate, + Microsoft.ApiManagement User Get, + Microsoft.ApiManagement User ListByService, + Microsoft.ApiManagement GroupUser Create, + Microsoft.ApiManagement GroupUser List, + Microsoft.ApiManagement GroupUser CheckEntityExists, + Microsoft.ApiManagement Subscription CreateOrUpdate, + Microsoft.ApiManagement Subscription Get, + Microsoft.ApiManagement Subscription List, + Microsoft.ApiManagement Subscription ListSecrets, + Microsoft.ApiManagement Subscription RegeneratePrimaryKey, + Microsoft.ApiManagement Subscription Delete, + ] + setup: + - group create --name {rg} --location {location} + - >- + apim create --name {name:apim} --resource-group {rg} --location {location} + --publisher-email l2@example.com --publisher-name L2 --sku-name Developer + - >- + rest --method put + --url /subscriptions/{sub}/resourceGroups/{rg}/providers/Microsoft.ApiManagement/service/{name:apim}/groups/partners?api-version=2024-05-01 + --body '{"properties": {"displayName": "Partners", "description": "l2"}}' + - >- + rest --method get + --url /subscriptions/{sub}/resourceGroups/{rg}/providers/Microsoft.ApiManagement/service/{name:apim}/groups/partners?api-version=2024-05-01 + - >- + rest --method get + --url /subscriptions/{sub}/resourceGroups/{rg}/providers/Microsoft.ApiManagement/service/{name:apim}/groups?api-version=2024-05-01 + - >- + rest --method put + --url /subscriptions/{sub}/resourceGroups/{rg}/providers/Microsoft.ApiManagement/service/{name:apim}/users/alice?api-version=2024-05-01 + --body '{"properties": {"firstName": "Alice", "lastName": "L2", "email": "alice-{id}@example.com"}}' + - >- + rest --method get + --url /subscriptions/{sub}/resourceGroups/{rg}/providers/Microsoft.ApiManagement/service/{name:apim}/users/alice?api-version=2024-05-01 + - >- + rest --method get + --url /subscriptions/{sub}/resourceGroups/{rg}/providers/Microsoft.ApiManagement/service/{name:apim}/users?api-version=2024-05-01 + - >- + rest --method put + --url /subscriptions/{sub}/resourceGroups/{rg}/providers/Microsoft.ApiManagement/service/{name:apim}/groups/partners/users/alice?api-version=2024-05-01 + - >- + rest --method get + --url /subscriptions/{sub}/resourceGroups/{rg}/providers/Microsoft.ApiManagement/service/{name:apim}/groups/partners/users?api-version=2024-05-01 + - >- + rest --method head + --url /subscriptions/{sub}/resourceGroups/{rg}/providers/Microsoft.ApiManagement/service/{name:apim}/groups/partners/users/alice?api-version=2024-05-01 + - >- + rest --method put + --url /subscriptions/{sub}/resourceGroups/{rg}/providers/Microsoft.ApiManagement/service/{name:apim}/subscriptions/alice-sub?api-version=2024-05-01 + --body '{"properties": {"displayName": "Alice", "scope": "/apis", "ownerId": "/users/alice"}}' + - >- + rest --method get + --url /subscriptions/{sub}/resourceGroups/{rg}/providers/Microsoft.ApiManagement/service/{name:apim}/subscriptions/alice-sub?api-version=2024-05-01 + - >- + rest --method get + --url /subscriptions/{sub}/resourceGroups/{rg}/providers/Microsoft.ApiManagement/service/{name:apim}/subscriptions?api-version=2024-05-01 + - >- + rest --method post + --url /subscriptions/{sub}/resourceGroups/{rg}/providers/Microsoft.ApiManagement/service/{name:apim}/subscriptions/alice-sub/listSecrets?api-version=2024-05-01 + - >- + rest --method post + --url /subscriptions/{sub}/resourceGroups/{rg}/providers/Microsoft.ApiManagement/service/{name:apim}/subscriptions/alice-sub/regeneratePrimaryKey?api-version=2024-05-01 + command: >- + rest --method delete + --url /subscriptions/{sub}/resourceGroups/{rg}/providers/Microsoft.ApiManagement/service/{name:apim}/subscriptions/alice-sub?api-version=2024-05-01 + --headers "If-Match=*" + expect: + exitCode: 0 + cleanup: + - apim delete --name {name:apim} --resource-group {rg} --yes + - apim deletedservice purge --service-name {name:apim} --location {location} + - group delete --name {rg} --yes --no-wait + + - id: apim-backends-loggers-diagnostics + operations: + [ + Microsoft.ApiManagement Backend CreateOrUpdate, + Microsoft.ApiManagement Backend Get, + Microsoft.ApiManagement Backend ListByService, + Microsoft.ApiManagement Logger CreateOrUpdate, + Microsoft.ApiManagement Logger Get, + Microsoft.ApiManagement Logger ListByService, + Microsoft.ApiManagement Diagnostic CreateOrUpdate, + Microsoft.ApiManagement Diagnostic Get, + Microsoft.ApiManagement Diagnostic ListByService, + Microsoft.ApiManagement Cache CreateOrUpdate, + Microsoft.ApiManagement Cache Get, + Microsoft.ApiManagement Cache ListByService, + ] + setup: + - group create --name {rg} --location {location} + - >- + apim create --name {name:apim} --resource-group {rg} --location {location} + --publisher-email l2@example.com --publisher-name L2 --sku-name Developer + - >- + rest --method put + --url /subscriptions/{sub}/resourceGroups/{rg}/providers/Microsoft.ApiManagement/service/{name:apim}/backends/orders-backend?api-version=2024-05-01 + --body '{"properties": {"url": "https://orders.example.com", "protocol": "http"}}' + - >- + rest --method get + --url /subscriptions/{sub}/resourceGroups/{rg}/providers/Microsoft.ApiManagement/service/{name:apim}/backends/orders-backend?api-version=2024-05-01 + - >- + rest --method get + --url /subscriptions/{sub}/resourceGroups/{rg}/providers/Microsoft.ApiManagement/service/{name:apim}/backends?api-version=2024-05-01 + - run: >- + monitor app-insights component create --app {name:ai} --resource-group {rg} --location {location} + --kind web --application-type web --query instrumentationKey -o tsv + capture: ikey + - >- + rest --method put + --url /subscriptions/{sub}/resourceGroups/{rg}/providers/Microsoft.ApiManagement/service/{name:apim}/loggers/ai?api-version=2024-05-01 + --body '{"properties": {"loggerType": "applicationInsights", "credentials": {"instrumentationKey": "{ikey}"}}}' + - >- + rest --method get + --url /subscriptions/{sub}/resourceGroups/{rg}/providers/Microsoft.ApiManagement/service/{name:apim}/loggers/ai?api-version=2024-05-01 + - >- + rest --method get + --url /subscriptions/{sub}/resourceGroups/{rg}/providers/Microsoft.ApiManagement/service/{name:apim}/loggers?api-version=2024-05-01 + - >- + rest --method put + --url /subscriptions/{sub}/resourceGroups/{rg}/providers/Microsoft.ApiManagement/service/{name:apim}/diagnostics/applicationinsights?api-version=2024-05-01 + --body '{"properties": {"loggerId": "/loggers/ai", "alwaysLog": "allErrors"}}' + - >- + rest --method get + --url /subscriptions/{sub}/resourceGroups/{rg}/providers/Microsoft.ApiManagement/service/{name:apim}/diagnostics/applicationinsights?api-version=2024-05-01 + - >- + rest --method get + --url /subscriptions/{sub}/resourceGroups/{rg}/providers/Microsoft.ApiManagement/service/{name:apim}/diagnostics?api-version=2024-05-01 + - >- + rest --method put + --url /subscriptions/{sub}/resourceGroups/{rg}/providers/Microsoft.ApiManagement/service/{name:apim}/caches/default?api-version=2024-05-01 + --body '{"properties": {"connectionString": "redis-{id}.example.com:6380,password=x,ssl=True", "useFromLocation": "default"}}' + - >- + rest --method get + --url /subscriptions/{sub}/resourceGroups/{rg}/providers/Microsoft.ApiManagement/service/{name:apim}/caches/default?api-version=2024-05-01 + command: >- + rest --method get + --url /subscriptions/{sub}/resourceGroups/{rg}/providers/Microsoft.ApiManagement/service/{name:apim}/caches?api-version=2024-05-01 + expect: + exitCode: 0 + cleanup: + - apim delete --name {name:apim} --resource-group {rg} --yes + - apim deletedservice purge --service-name {name:apim} --location {location} + - group delete --name {rg} --yes --no-wait + + - id: apim-tags + operations: + [ + Microsoft.ApiManagement Tag CreateOrUpdate, + Microsoft.ApiManagement Tag Get, + Microsoft.ApiManagement Tag ListByService, + Microsoft.ApiManagement Tag AssignToApi, + Microsoft.ApiManagement Tag ListByApi, + Microsoft.ApiManagement Tag AssignToProduct, + Microsoft.ApiManagement Tag DetachFromProduct, + Microsoft.ApiManagement ApiTagDescription ListByService, + ] + setup: + - group create --name {rg} --location {location} + - >- + apim create --name {name:apim} --resource-group {rg} --location {location} + --publisher-email l2@example.com --publisher-name L2 --sku-name Developer + - apim api create --service-name {name:apim} --resource-group {rg} --api-id orders --path orders --display-name Orders + - apim product create --service-name {name:apim} --resource-group {rg} --product-id starter --product-name Starter + - >- + rest --method put + --url /subscriptions/{sub}/resourceGroups/{rg}/providers/Microsoft.ApiManagement/service/{name:apim}/tags/team-l2?api-version=2024-05-01 + --body '{"properties": {"displayName": "team-l2"}}' + - >- + rest --method get + --url /subscriptions/{sub}/resourceGroups/{rg}/providers/Microsoft.ApiManagement/service/{name:apim}/tags/team-l2?api-version=2024-05-01 + - >- + rest --method get + --url /subscriptions/{sub}/resourceGroups/{rg}/providers/Microsoft.ApiManagement/service/{name:apim}/tags?api-version=2024-05-01 + - >- + rest --method put + --url /subscriptions/{sub}/resourceGroups/{rg}/providers/Microsoft.ApiManagement/service/{name:apim}/apis/orders/tags/team-l2?api-version=2024-05-01 + - >- + rest --method get + --url /subscriptions/{sub}/resourceGroups/{rg}/providers/Microsoft.ApiManagement/service/{name:apim}/apis/orders/tags?api-version=2024-05-01 + - >- + rest --method put + --url /subscriptions/{sub}/resourceGroups/{rg}/providers/Microsoft.ApiManagement/service/{name:apim}/products/starter/tags/team-l2?api-version=2024-05-01 + - >- + rest --method delete + --url /subscriptions/{sub}/resourceGroups/{rg}/providers/Microsoft.ApiManagement/service/{name:apim}/products/starter/tags/team-l2?api-version=2024-05-01 + command: >- + rest --method get + --url /subscriptions/{sub}/resourceGroups/{rg}/providers/Microsoft.ApiManagement/service/{name:apim}/apis/orders/tagDescriptions?api-version=2024-05-01 + expect: + exitCode: 0 + cleanup: + - apim delete --name {name:apim} --resource-group {rg} --yes + - apim deletedservice purge --service-name {name:apim} --location {location} + - group delete --name {rg} --yes --no-wait + + - id: apim-graphql-resolver + operations: + [ + Microsoft.ApiManagement GraphqlApiResolver CreateOrUpdate, + Microsoft.ApiManagement GraphqlApiResolver Get, + Microsoft.ApiManagement GraphqlApiResolver ListByApi, + Microsoft.ApiManagement GraphqlApiResolver Delete, + ] + setup: + - group create --name {rg} --location {location} + - >- + apim create --name {name:apim} --resource-group {rg} --location {location} + --publisher-email l2@example.com --publisher-name L2 --sku-name Developer + - >- + apim api create --service-name {name:apim} --resource-group {rg} --api-id catalog --path catalog + --display-name Catalog --api-type graphql + - >- + apim graphql resolver create --service-name {name:apim} --resource-group {rg} --api-id catalog + --resolver-id products --display-name products --path Query/products + - apim graphql resolver show --service-name {name:apim} --resource-group {rg} --api-id catalog --resolver-id products + - apim graphql resolver list --service-name {name:apim} --resource-group {rg} --api-id catalog + command: >- + apim graphql resolver delete --service-name {name:apim} --resource-group {rg} --api-id catalog + --resolver-id products --yes + expect: + exitCode: 0 + cleanup: + - apim delete --name {name:apim} --resource-group {rg} --yes + - apim deletedservice purge --service-name {name:apim} --location {location} + - group delete --name {rg} --yes --no-wait + + - id: apim-workspaces + operations: + [ + Microsoft.ApiManagement Workspace CreateOrUpdate, + Microsoft.ApiManagement Workspace Get, + Microsoft.ApiManagement Workspace ListByService, + Microsoft.ApiManagement WorkspaceProduct CreateOrUpdate, + Microsoft.ApiManagement WorkspaceProduct Update, + Microsoft.ApiManagement WorkspaceGroup CreateOrUpdate, + Microsoft.ApiManagement WorkspaceGroup Update, + Microsoft.ApiManagement WorkspaceLogger ListByWorkspace, + Microsoft.ApiManagement WorkspaceApi CreateOrUpdate, + Microsoft.ApiManagement WorkspaceApi ListByService, + Microsoft.ApiManagement Workspace Delete, + ] + setup: + - group create --name {rg} --location {location} + - >- + apim create --name {name:apim} --resource-group {rg} --location {location} + --publisher-email l2@example.com --publisher-name L2 --sku-name Premium + - >- + rest --method put + --url /subscriptions/{sub}/resourceGroups/{rg}/providers/Microsoft.ApiManagement/service/{name:apim}/workspaces/team-a?api-version=2024-05-01 + --body '{"properties": {"displayName": "Team A"}}' + - >- + rest --method get + --url /subscriptions/{sub}/resourceGroups/{rg}/providers/Microsoft.ApiManagement/service/{name:apim}/workspaces/team-a?api-version=2024-05-01 + - >- + rest --method get + --url /subscriptions/{sub}/resourceGroups/{rg}/providers/Microsoft.ApiManagement/service/{name:apim}/workspaces?api-version=2024-05-01 + - >- + rest --method put + --url /subscriptions/{sub}/resourceGroups/{rg}/providers/Microsoft.ApiManagement/service/{name:apim}/workspaces/team-a/products/starter?api-version=2024-05-01 + --body '{"properties": {"displayName": "Starter"}}' + - >- + rest --method patch + --url /subscriptions/{sub}/resourceGroups/{rg}/providers/Microsoft.ApiManagement/service/{name:apim}/workspaces/team-a/products/starter?api-version=2024-05-01 + --headers "If-Match=*" --body '{"properties": {"description": "l2 {id}"}}' + - >- + rest --method put + --url /subscriptions/{sub}/resourceGroups/{rg}/providers/Microsoft.ApiManagement/service/{name:apim}/workspaces/team-a/groups/devs?api-version=2024-05-01 + --body '{"properties": {"displayName": "Devs"}}' + - >- + rest --method patch + --url /subscriptions/{sub}/resourceGroups/{rg}/providers/Microsoft.ApiManagement/service/{name:apim}/workspaces/team-a/groups/devs?api-version=2024-05-01 + --headers "If-Match=*" --body '{"properties": {"description": "l2 {id}"}}' + - >- + rest --method get + --url /subscriptions/{sub}/resourceGroups/{rg}/providers/Microsoft.ApiManagement/service/{name:apim}/workspaces/team-a/loggers?api-version=2024-05-01 + - >- + rest --method put + --url /subscriptions/{sub}/resourceGroups/{rg}/providers/Microsoft.ApiManagement/service/{name:apim}/workspaces/team-a/apis/orders?api-version=2024-05-01 + --body '{"properties": {"displayName": "Orders", "path": "team-a/orders", "protocols": ["https"]}}' + - >- + rest --method get + --url /subscriptions/{sub}/resourceGroups/{rg}/providers/Microsoft.ApiManagement/service/{name:apim}/workspaces/team-a/apis?api-version=2024-05-01 + command: >- + rest --method delete + --url /subscriptions/{sub}/resourceGroups/{rg}/providers/Microsoft.ApiManagement/service/{name:apim}/workspaces/team-a?api-version=2024-05-01 + --headers "If-Match=*" + expect: + exitCode: 0 + cleanup: + - apim delete --name {name:apim} --resource-group {rg} --yes + - apim deletedservice purge --service-name {name:apim} --location {location} + - group delete --name {rg} --yes --no-wait + + - id: apim-tenant-access-identity + operations: + [ + Microsoft.ApiManagement TenantAccess ListByService, + Microsoft.ApiManagement TenantAccess Get, + Microsoft.ApiManagement TenantAccessGit RegeneratePrimaryKey, + Microsoft.ApiManagement OpenIdConnectProvider CreateOrUpdate, + Microsoft.ApiManagement OpenIdConnectProvider ListSecrets, + Microsoft.ApiManagement OpenIdConnectProvider ListByService, + Microsoft.ApiManagement IdentityProvider CreateOrUpdate, + Microsoft.ApiManagement IdentityProvider ListByService, + Microsoft.ApiManagement ApiManagementService MigrateToStv2, + ] + setup: + - group create --name {rg} --location {location} + - >- + apim create --name {name:apim} --resource-group {rg} --location {location} + --publisher-email l2@example.com --publisher-name L2 --sku-name Developer + - >- + rest --method get + --url /subscriptions/{sub}/resourceGroups/{rg}/providers/Microsoft.ApiManagement/service/{name:apim}/tenant?api-version=2024-05-01 + - >- + rest --method get + --url /subscriptions/{sub}/resourceGroups/{rg}/providers/Microsoft.ApiManagement/service/{name:apim}/tenant/access?api-version=2024-05-01 + - >- + rest --method post + --url /subscriptions/{sub}/resourceGroups/{rg}/providers/Microsoft.ApiManagement/service/{name:apim}/tenant/gitAccess/regeneratePrimaryKey?api-version=2024-05-01 + - >- + rest --method put + --url /subscriptions/{sub}/resourceGroups/{rg}/providers/Microsoft.ApiManagement/service/{name:apim}/openidConnectProviders/l2oidc?api-version=2024-05-01 + --body '{"properties": {"displayName": "L2", "metadataEndpoint": "https://login.example.com/.well-known/openid-configuration", + "clientId": "l2-client", "clientSecret": "l2-secret"}}' + - >- + rest --method post + --url /subscriptions/{sub}/resourceGroups/{rg}/providers/Microsoft.ApiManagement/service/{name:apim}/openidConnectProviders/l2oidc/listSecrets?api-version=2024-05-01 + - >- + rest --method get + --url /subscriptions/{sub}/resourceGroups/{rg}/providers/Microsoft.ApiManagement/service/{name:apim}/openidConnectProviders?api-version=2024-05-01 + - >- + rest --method put + --url /subscriptions/{sub}/resourceGroups/{rg}/providers/Microsoft.ApiManagement/service/{name:apim}/identityProviders/aad?api-version=2024-05-01 + --body '{"properties": {"clientId": "l2-client", "clientSecret": "l2-secret", "allowedTenants": ["contoso.onmicrosoft.com"]}}' + - >- + rest --method get + --url /subscriptions/{sub}/resourceGroups/{rg}/providers/Microsoft.ApiManagement/service/{name:apim}/identityProviders?api-version=2024-05-01 + command: >- + rest --method post + --url /subscriptions/{sub}/resourceGroups/{rg}/providers/Microsoft.ApiManagement/service/{name:apim}/migrateToStv2?api-version=2024-05-01 + --body '{"mode": "PreserveIp"}' + expect: + exitCode: 0 + cleanup: + - apim delete --name {name:apim} --resource-group {rg} --yes + - apim deletedservice purge --service-name {name:apim} --location {location} + - group delete --name {rg} --yes --no-wait + + - id: apim-gateways-certificates + operations: + [ + Microsoft.ApiManagement Gateway CreateOrUpdate, + Microsoft.ApiManagement Gateway Get, + Microsoft.ApiManagement Gateway ListByService, + Microsoft.ApiManagement Gateway ListKeys, + Microsoft.ApiManagement Gateway RegenerateKey, + Microsoft.ApiManagement Gateway Delete, + Microsoft.ApiManagement Certificate ListByService, + Microsoft.ApiManagement PolicyFragment CreateOrUpdate, + Microsoft.ApiManagement PolicyFragment Get, + Microsoft.ApiManagement PolicyFragment ListByService, + Microsoft.ApiManagement PortalSettings ListByService, + Microsoft.ApiManagement EmailTemplate ListByService, + Microsoft.ApiManagement Notification ListByService, + ] + setup: + - group create --name {rg} --location {location} + - >- + apim create --name {name:apim} --resource-group {rg} --location {location} + --publisher-email l2@example.com --publisher-name L2 --sku-name Premium + - >- + rest --method put + --url /subscriptions/{sub}/resourceGroups/{rg}/providers/Microsoft.ApiManagement/service/{name:apim}/gateways/edge1?api-version=2024-05-01 + --body '{"properties": {"locationData": {"name": "l2-edge"}, "description": "l2"}}' + - >- + rest --method get + --url /subscriptions/{sub}/resourceGroups/{rg}/providers/Microsoft.ApiManagement/service/{name:apim}/gateways/edge1?api-version=2024-05-01 + - >- + rest --method get + --url /subscriptions/{sub}/resourceGroups/{rg}/providers/Microsoft.ApiManagement/service/{name:apim}/gateways?api-version=2024-05-01 + - >- + rest --method post + --url /subscriptions/{sub}/resourceGroups/{rg}/providers/Microsoft.ApiManagement/service/{name:apim}/gateways/edge1/listKeys?api-version=2024-05-01 + - >- + rest --method post + --url /subscriptions/{sub}/resourceGroups/{rg}/providers/Microsoft.ApiManagement/service/{name:apim}/gateways/edge1/regenerateKey?api-version=2024-05-01 + --body '{"keyType": "primary"}' + - >- + rest --method delete + --url /subscriptions/{sub}/resourceGroups/{rg}/providers/Microsoft.ApiManagement/service/{name:apim}/gateways/edge1?api-version=2024-05-01 + --headers "If-Match=*" + - >- + rest --method get + --url /subscriptions/{sub}/resourceGroups/{rg}/providers/Microsoft.ApiManagement/service/{name:apim}/certificates?api-version=2024-05-01 + - >- + rest --method put + --url /subscriptions/{sub}/resourceGroups/{rg}/providers/Microsoft.ApiManagement/service/{name:apim}/policyFragments/cors-l2?api-version=2024-05-01 + --body '{"properties": {"format": "rawxml", "value": "on"}}' + - >- + rest --method get + --url /subscriptions/{sub}/resourceGroups/{rg}/providers/Microsoft.ApiManagement/service/{name:apim}/policyFragments/cors-l2?api-version=2024-05-01 + - >- + rest --method get + --url /subscriptions/{sub}/resourceGroups/{rg}/providers/Microsoft.ApiManagement/service/{name:apim}/policyFragments?api-version=2024-05-01 + - >- + rest --method get + --url /subscriptions/{sub}/resourceGroups/{rg}/providers/Microsoft.ApiManagement/service/{name:apim}/portalsettings?api-version=2024-05-01 + - >- + rest --method get + --url /subscriptions/{sub}/resourceGroups/{rg}/providers/Microsoft.ApiManagement/service/{name:apim}/templates?api-version=2024-05-01 + command: >- + rest --method get + --url /subscriptions/{sub}/resourceGroups/{rg}/providers/Microsoft.ApiManagement/service/{name:apim}/notifications?api-version=2024-05-01 + expect: + exitCode: 0 + cleanup: + - apim delete --name {name:apim} --resource-group {rg} --yes + - apim deletedservice purge --service-name {name:apim} --location {location} + - group delete --name {rg} --yes --no-wait diff --git a/tests/azure/matrix/app.yaml b/tests/azure/matrix/app.yaml new file mode 100644 index 0000000..9256c5c --- /dev/null +++ b/tests/azure/matrix/app.yaml @@ -0,0 +1,362 @@ +# Microsoft.App: Container Apps environments, apps, revisions, replicas, jobs, session pools. +# Environments and apps run on the emulator's Docker engine, so those cases are `backing: true` +# (nginx:latest); the subscription-scope catalogue calls are light. +provider: Microsoft.App +cases: + - id: containerapp-catalogue + operations: + [ + Microsoft.App Operations List, + Microsoft.App Usages List, + Microsoft.App BillingMeters Get, + Microsoft.App AvailableWorkloadProfiles Get, + Microsoft.App Microsoft.App GetCustomDomainVerificationId, + ] + setup: + - rest --method get --url /providers/Microsoft.App/operations?api-version=2025-01-01 + - rest --method get --url /subscriptions/{sub}/providers/Microsoft.App/locations/{location}/usages?api-version=2025-01-01 + - rest --method get --url /subscriptions/{sub}/providers/Microsoft.App/locations/{location}/billingMeters?api-version=2025-01-01 + - >- + rest --method get + --url /subscriptions/{sub}/providers/Microsoft.App/locations/{location}/availableManagedEnvironmentsWorkloadProfileTypes?api-version=2025-01-01 + command: rest --method post --url /subscriptions/{sub}/providers/Microsoft.App/getCustomDomainVerificationId?api-version=2025-01-01 + expect: + exitCode: 0 + + - id: containerapp-env + operations: + [ + Microsoft.App ManagedEnvironments CreateOrUpdate, + Microsoft.App ManagedEnvironments Get, + Microsoft.App ManagedEnvironments ListByResourceGroup, + Microsoft.App ManagedEnvironments ListBySubscription, + Microsoft.App ManagedEnvironments Update, + Microsoft.App ManagedEnvironments GetAuthToken, + Microsoft.App ManagedEnvironments ListWorkloadProfileStates, + Microsoft.App ManagedEnvironmentUsages List, + ] + backing: true + note: Samples containerapp env create/show. + setup: + - group create --name {rg} --location {location} + - containerapp env create --name {name:env} --resource-group {rg} --location {location} + - run: containerapp env show --name {name:env} --resource-group {rg} --query properties.provisioningState -o tsv + until: { stdout: { equals: Succeeded } } + - containerapp env list --resource-group {rg} + - containerapp env list --query "[?name=='{name:env}'].name" + - containerapp env update --name {name:env} --resource-group {rg} --tags matrix=l2 + - >- + rest --method post + --url /subscriptions/{sub}/resourceGroups/{rg}/providers/Microsoft.App/managedEnvironments/{name:env}/getAuthtoken?api-version=2025-01-01 + - >- + rest --method get + --url /subscriptions/{sub}/resourceGroups/{rg}/providers/Microsoft.App/managedEnvironments/{name:env}/workloadProfileStates?api-version=2025-01-01 + command: >- + rest --method get + --url /subscriptions/{sub}/resourceGroups/{rg}/providers/Microsoft.App/managedEnvironments/{name:env}/usages?api-version=2025-01-01 + expect: + exitCode: 0 + cleanup: + - group delete --name {rg} --yes --no-wait + + - id: containerapp-env-children + operations: + [ + Microsoft.App ManagedEnvironmentsStorages CreateOrUpdate, + Microsoft.App ManagedEnvironmentsStorages Get, + Microsoft.App ManagedEnvironmentsStorages List, + Microsoft.App ManagedEnvironmentsStorages Delete, + Microsoft.App DaprComponents CreateOrUpdate, + Microsoft.App DaprComponents Get, + Microsoft.App DaprComponents List, + Microsoft.App DaprComponents ListSecrets, + Microsoft.App DaprComponents Delete, + Microsoft.App HttpRouteConfig List, + Microsoft.App Certificates List, + Microsoft.App ManagedCertificates List, + ] + backing: true + setup: + - group create --name {rg} --location {location} + - containerapp env create --name {name:env} --resource-group {rg} --location {location} + - >- + containerapp env storage set --name {name:env} --resource-group {rg} --storage-name files + --azure-file-account-name {storage} --azure-file-account-key a2V5 --azure-file-share-name share1 + --access-mode ReadOnly + - containerapp env storage show --name {name:env} --resource-group {rg} --storage-name files + - containerapp env storage list --name {name:env} --resource-group {rg} + - containerapp env storage remove --name {name:env} --resource-group {rg} --storage-name files --yes + - >- + rest --method put + --url /subscriptions/{sub}/resourceGroups/{rg}/providers/Microsoft.App/managedEnvironments/{name:env}/daprComponents/statestore?api-version=2025-01-01 + --body '{"properties": {"componentType": "state.in-memory", "version": "v1", "secrets": [{"name": "k", "value": "v"}], + "metadata": [{"name": "mode", "value": "l2"}]}}' + - containerapp env dapr-component show --name {name:env} --resource-group {rg} --dapr-component-name statestore + - containerapp env dapr-component list --name {name:env} --resource-group {rg} + - >- + rest --method post + --url /subscriptions/{sub}/resourceGroups/{rg}/providers/Microsoft.App/managedEnvironments/{name:env}/daprComponents/statestore/listSecrets?api-version=2025-01-01 + - containerapp env dapr-component remove --name {name:env} --resource-group {rg} --dapr-component-name statestore + - containerapp env certificate list --name {name:env} --resource-group {rg} + - >- + rest --method get + --url /subscriptions/{sub}/resourceGroups/{rg}/providers/Microsoft.App/managedEnvironments/{name:env}/managedCertificates?api-version=2025-01-01 + command: containerapp env http-route-config list --name {name:env} --resource-group {rg} + expect: + exitCode: 0 + cleanup: + - group delete --name {rg} --yes --no-wait + + - id: containerapp-env-diagnostics + operations: + [ + Microsoft.App ManagedEnvironmentsDiagnostics GetRoot, + Microsoft.App ManagedEnvironmentDiagnostics ListDetectors, + Microsoft.App ManagedEnvironmentPrivateEndpointConnections List, + Microsoft.App ManagedEnvironmentPrivateLinkResources List, + Microsoft.App ManagedEnvironments Delete, + ] + backing: true + setup: + - group create --name {rg} --location {location} + - containerapp env create --name {name:env} --resource-group {rg} --location {location} + - >- + rest --method get + --url /subscriptions/{sub}/resourceGroups/{rg}/providers/Microsoft.App/managedEnvironments/{name:env}/detectorProperties/rootApi/?api-version=2025-01-01 + - >- + rest --method get + --url /subscriptions/{sub}/resourceGroups/{rg}/providers/Microsoft.App/managedEnvironments/{name:env}/detectors?api-version=2025-01-01 + - >- + rest --method get + --url /subscriptions/{sub}/resourceGroups/{rg}/providers/Microsoft.App/managedEnvironments/{name:env}/privateEndpointConnections?api-version=2025-01-01 + - >- + rest --method get + --url /subscriptions/{sub}/resourceGroups/{rg}/providers/Microsoft.App/managedEnvironments/{name:env}/privateLinkResources?api-version=2025-01-01 + command: containerapp env delete --name {name:env} --resource-group {rg} --yes + expect: + exitCode: 0 + cleanup: + - group delete --name {rg} --yes --no-wait + + - id: containerapp-app + operations: + [ + Microsoft.App ContainerApps CreateOrUpdate, + Microsoft.App ContainerApps Get, + Microsoft.App ContainerApps ListByResourceGroup, + Microsoft.App ContainerApps ListBySubscription, + Microsoft.App ContainerApps Update, + Microsoft.App ContainerApps ListSecrets, + ] + backing: true + note: Samples containerapp create/show/update/secret list. + setup: + - group create --name {rg} --location {location} + - containerapp env create --name {name:env} --resource-group {rg} --location {location} + - >- + containerapp create --name {name:app} --resource-group {rg} --environment {name:env} + --image nginx:latest --target-port 80 --ingress external --secrets api-key=l2-secret --no-wait + - run: containerapp show --name {name:app} --resource-group {rg} --query properties.provisioningState -o tsv + until: { stdout: { equals: Succeeded } } + wait: 600 + - containerapp list --resource-group {rg} + - containerapp list --query "[?name=='{name:app}'].name" + - containerapp update --name {name:app} --resource-group {rg} --set-env-vars MATRIX=l2 --revision-suffix v2 + command: containerapp secret list --name {name:app} --resource-group {rg} --show-values --query "[].name" + expect: + json: { path: "", equals: [api-key] } + cleanup: + - group delete --name {rg} --yes --no-wait + + - id: containerapp-revisions-replicas + operations: + [ + Microsoft.App ContainerAppsRevisions ListRevisions, + Microsoft.App ContainerAppsRevisions GetRevision, + Microsoft.App ContainerAppsRevisions RestartRevision, + Microsoft.App ContainerAppsRevisions DeactivateRevision, + Microsoft.App ContainerAppsRevisions ActivateRevision, + Microsoft.App ContainerAppsRevisionReplicas ListReplicas, + Microsoft.App ContainerAppsRevisionReplicas GetReplica, + ] + backing: true + note: Samples containerapp revision list, containerapp replica list. + setup: + - group create --name {rg} --location {location} + - containerapp env create --name {name:env} --resource-group {rg} --location {location} + - >- + containerapp create --name {name:app} --resource-group {rg} --environment {name:env} + --image nginx:latest --target-port 80 --ingress external --revisions-mode multiple --no-wait + - run: containerapp show --name {name:app} --resource-group {rg} --query properties.provisioningState -o tsv + until: { stdout: { equals: Succeeded } } + wait: 600 + - run: containerapp revision list --name {name:app} --resource-group {rg} --query "[0].name" -o tsv + capture: revision + - containerapp revision show --name {name:app} --resource-group {rg} --revision {revision} + - containerapp revision restart --name {name:app} --resource-group {rg} --revision {revision} + - containerapp revision deactivate --name {name:app} --resource-group {rg} --revision {revision} + - containerapp revision activate --name {name:app} --resource-group {rg} --revision {revision} + - run: containerapp replica list --name {name:app} --resource-group {rg} --revision {revision} --query "[0].name" -o tsv + capture: replica + command: containerapp replica show --name {name:app} --resource-group {rg} --revision {revision} --replica {replica} + expect: + json: { path: name, equals: "{replica}" } + cleanup: + - group delete --name {rg} --yes --no-wait + + - id: containerapp-auth-diagnostics + operations: + [ + Microsoft.App ContainerAppsAuthConfigs CreateOrUpdate, + Microsoft.App ContainerAppsAuthConfigs Get, + Microsoft.App ContainerAppsAuthConfigs ListByContainerApp, + Microsoft.App ContainerAppsAuthConfigs Delete, + Microsoft.App ContainerApps GetAuthToken, + Microsoft.App ContainerApps ListCustomHostNameAnalysis, + Microsoft.App ContainerAppsDiagnostics ListDetectors, + Microsoft.App ContainerAppsDiagnostics ListRevisions, + Microsoft.App ContainerAppsDiagnostics GetRoot, + ] + backing: true + setup: + - group create --name {rg} --location {location} + - containerapp env create --name {name:env} --resource-group {rg} --location {location} + - containerapp create --name {name:app} --resource-group {rg} --environment {name:env} --image nginx:latest --target-port 80 --ingress external --no-wait + - run: containerapp show --name {name:app} --resource-group {rg} --query properties.provisioningState -o tsv + until: { stdout: { equals: Succeeded } } + wait: 600 + - containerapp auth update --name {name:app} --resource-group {rg} --enabled false --unauthenticated-client-action AllowAnonymous --yes + - run: containerapp auth show --name {name:app} --resource-group {rg} --query platform.enabled + until: { json: { path: "", equals: false } } + - >- + rest --method get + --url /subscriptions/{sub}/resourceGroups/{rg}/providers/Microsoft.App/containerApps/{name:app}/authConfigs?api-version=2025-01-01 + - >- + rest --method post + --url /subscriptions/{sub}/resourceGroups/{rg}/providers/Microsoft.App/containerApps/{name:app}/getAuthtoken?api-version=2025-01-01 + - >- + rest --method post + --url "/subscriptions/{sub}/resourceGroups/{rg}/providers/Microsoft.App/containerApps/{name:app}/listCustomHostNameAnalysis?api-version=2025-01-01&customHostname=www.{id}.example.com" + - >- + rest --method get + --url /subscriptions/{sub}/resourceGroups/{rg}/providers/Microsoft.App/containerApps/{name:app}/detectors?api-version=2025-01-01 + - >- + rest --method get + --url /subscriptions/{sub}/resourceGroups/{rg}/providers/Microsoft.App/containerApps/{name:app}/detectorProperties/revisionsApi/revisions/?api-version=2025-01-01 + - >- + rest --method get + --url /subscriptions/{sub}/resourceGroups/{rg}/providers/Microsoft.App/containerApps/{name:app}/detectorProperties/rootApi/?api-version=2025-01-01 + command: >- + rest --method delete + --url /subscriptions/{sub}/resourceGroups/{rg}/providers/Microsoft.App/containerApps/{name:app}/authConfigs/current?api-version=2025-01-01 + expect: + exitCode: 0 + cleanup: + - group delete --name {rg} --yes --no-wait + + - id: containerapp-stop-start-delete + operations: + [ + Microsoft.App ContainerApps Stop, + Microsoft.App ContainerApps Start, + Microsoft.App ContainerAppsSourceControls ListByContainerApp, + Microsoft.App ContainerApps Delete, + ] + backing: true + note: Samples containerapp delete. + setup: + - group create --name {rg} --location {location} + - containerapp env create --name {name:env} --resource-group {rg} --location {location} + - containerapp create --name {name:app} --resource-group {rg} --environment {name:env} --image nginx:latest --target-port 80 --ingress external --no-wait + - run: containerapp show --name {name:app} --resource-group {rg} --query properties.provisioningState -o tsv + until: { stdout: { equals: Succeeded } } + wait: 600 + - >- + rest --method post + --url /subscriptions/{sub}/resourceGroups/{rg}/providers/Microsoft.App/containerApps/{name:app}/stop?api-version=2025-01-01 + - >- + rest --method post + --url /subscriptions/{sub}/resourceGroups/{rg}/providers/Microsoft.App/containerApps/{name:app}/start?api-version=2025-01-01 + - >- + rest --method get + --url /subscriptions/{sub}/resourceGroups/{rg}/providers/Microsoft.App/containerApps/{name:app}/sourcecontrols?api-version=2025-01-01 + command: containerapp delete --name {name:app} --resource-group {rg} --yes + expect: + exitCode: 0 + cleanup: + - group delete --name {rg} --yes --no-wait + + - id: containerapp-job + operations: + [ + Microsoft.App Jobs CreateOrUpdate, + Microsoft.App Jobs Get, + Microsoft.App Jobs ListByResourceGroup, + Microsoft.App Jobs ListBySubscription, + Microsoft.App Jobs Update, + Microsoft.App Jobs ListSecrets, + Microsoft.App Jobs Start, + Microsoft.App JobsExecutions List, + Microsoft.App Jobs StopMultipleExecutions, + Microsoft.App Jobs Delete, + ] + backing: true + setup: + - group create --name {rg} --location {location} + - containerapp env create --name {name:env} --resource-group {rg} --location {location} + - >- + containerapp job create --name {name:job} --resource-group {rg} --environment {name:env} + --trigger-type Manual --replica-timeout 60 --replica-retry-limit 0 --image nginx:latest + --cpu 0.25 --memory 0.5Gi --secrets token=l2 + - run: containerapp job show --name {name:job} --resource-group {rg} --query properties.provisioningState -o tsv + until: { stdout: { equals: Succeeded } } + - containerapp job list --resource-group {rg} + - containerapp job list --query "[?name=='{name:job}'].name" + - containerapp job update --name {name:job} --resource-group {rg} --replica-timeout 90 + - containerapp job secret list --name {name:job} --resource-group {rg} + - containerapp job start --name {name:job} --resource-group {rg} + - containerapp job execution list --name {name:job} --resource-group {rg} + - containerapp job stop --name {name:job} --resource-group {rg} + command: containerapp job delete --name {name:job} --resource-group {rg} --yes + expect: + exitCode: 0 + cleanup: + - group delete --name {rg} --yes --no-wait + + - id: containerapp-session-pool + operations: + [ + Microsoft.App ContainerAppsSessionPools CreateOrUpdate, + Microsoft.App ContainerAppsSessionPools Get, + Microsoft.App ContainerAppsSessionPools ListByResourceGroup, + Microsoft.App ContainerAppsSessionPools ListBySubscription, + Microsoft.App ContainerAppsSessionPools Update, + Microsoft.App ContainerAppsSessionPools Delete, + ] + backing: true + note: The case patches the pool with rest. + setup: + - group create --name {rg} --location {location} + - >- + rest --method put + --url /subscriptions/{sub}/resourceGroups/{rg}/providers/Microsoft.App/sessionPools/{name:pool}?api-version=2025-01-01 + --body '{"location": "{location}", "properties": {"poolManagementType": "Dynamic", "containerType": "PythonLTS", + "scaleConfiguration": {"maxConcurrentSessions": 5}, "dynamicPoolConfiguration": {"lifecycleConfiguration": + {"lifecycleType": "Timed", "cooldownPeriodInSeconds": 300}}}}' + - run: >- + rest --method get + --url /subscriptions/{sub}/resourceGroups/{rg}/providers/Microsoft.App/sessionPools/{name:pool}?api-version=2025-01-01 + until: { json: { path: properties.scaleConfiguration.maxConcurrentSessions, equals: 5 } } + - rest --method get --url /subscriptions/{sub}/resourceGroups/{rg}/providers/Microsoft.App/sessionPools?api-version=2025-01-01 + - rest --method get --url /subscriptions/{sub}/providers/Microsoft.App/sessionPools?api-version=2025-01-01 + - >- + rest --method patch + --url /subscriptions/{sub}/resourceGroups/{rg}/providers/Microsoft.App/sessionPools/{name:pool}?api-version=2025-01-01 + --body '{"properties": {"scaleConfiguration": {"maxConcurrentSessions": 8}}}' + command: >- + rest --method delete + --url /subscriptions/{sub}/resourceGroups/{rg}/providers/Microsoft.App/sessionPools/{name:pool}?api-version=2025-01-01 + expect: + exitCode: 0 + cleanup: + - group delete --name {rg} --yes --no-wait diff --git a/tests/azure/matrix/appconfiguration.yaml b/tests/azure/matrix/appconfiguration.yaml new file mode 100644 index 0000000..bfaace9 --- /dev/null +++ b/tests/azure/matrix/appconfiguration.yaml @@ -0,0 +1,396 @@ +# Microsoft.AppConfiguration: stores (ARM), key-values, locks, labels, revisions and snapshots +# (the data plane, keyed `Microsoft.AppConfiguration Microsoft.AppConfiguration `), replicas. +# A deleted store is soft-deleted, so every case that creates one deletes and purges it. +provider: Microsoft.AppConfiguration +cases: + - id: appconfig-create + operations: [Microsoft.AppConfiguration ConfigurationStores Create] + pr: true + note: Samples appconfig create. + setup: + - group create --name {rg} --location {location} + command: appconfig create --name {name:appcs} --resource-group {rg} --location {location} --sku Standard + expect: + json: + - { path: name, equals: "{name:appcs}" } + - { path: provisioningState, equals: Succeeded } + cleanup: + - appconfig delete --name {name:appcs} --resource-group {rg} --yes + - appconfig purge --name {name:appcs} --yes + - group delete --name {rg} --yes --no-wait + + - id: appconfig-read-update + operations: + [ + Microsoft.AppConfiguration ConfigurationStores Get, + Microsoft.AppConfiguration ConfigurationStores ListByResourceGroup, + Microsoft.AppConfiguration ConfigurationStores List, + Microsoft.AppConfiguration ConfigurationStores Update, + ] + note: Samples appconfig show/update. + setup: + - group create --name {rg} --location {location} + - appconfig create --name {name:appcs} --resource-group {rg} --location {location} --sku Standard + - run: appconfig show --name {name:appcs} --resource-group {rg} --query endpoint -o tsv + until: { stdout: { contains: "{name:appcs}" } } + - run: appconfig list --resource-group {rg} --query "[].name" + until: { json: { path: "", equals: ["{name:appcs}"] } } + - appconfig list --query "[?name=='{name:appcs}'].name" + command: appconfig update --name {name:appcs} --resource-group {rg} --tags matrix=l2 + expect: + json: { path: tags.matrix, equals: l2 } + cleanup: + - appconfig delete --name {name:appcs} --resource-group {rg} --yes + - appconfig purge --name {name:appcs} --yes + - group delete --name {rg} --yes --no-wait + + - id: appconfig-credentials + operations: + [ + Microsoft.AppConfiguration ConfigurationStores ListKeys, + Microsoft.AppConfiguration ConfigurationStores RegenerateKey, + ] + setup: + - group create --name {rg} --location {location} + - appconfig create --name {name:appcs} --resource-group {rg} --location {location} --sku Standard + - run: appconfig credential list --name {name:appcs} --resource-group {rg} --query "[0].id" -o tsv + capture: key_id + command: appconfig credential regenerate --name {name:appcs} --resource-group {rg} --id {key_id} + expect: + json: { path: id, exists: true } + cleanup: + - appconfig delete --name {name:appcs} --resource-group {rg} --yes + - appconfig purge --name {name:appcs} --yes + - group delete --name {rg} --yes --no-wait + + - id: appconfig-soft-delete + operations: + [ + Microsoft.AppConfiguration ConfigurationStores Delete, + Microsoft.AppConfiguration ConfigurationStores ListDeleted, + Microsoft.AppConfiguration ConfigurationStores GetDeleted, + Microsoft.AppConfiguration ConfigurationStores PurgeDeleted, + ] + note: Samples appconfig list-deleted/recover. + setup: + - group create --name {rg} --location {location} + - appconfig create --name {name:appcs} --resource-group {rg} --location {location} --sku Standard + - appconfig delete --name {name:appcs} --resource-group {rg} --yes + - run: appconfig list-deleted --query "[?name=='{name:appcs}'].name" + until: { json: { path: "", equals: ["{name:appcs}"] } } + - appconfig show-deleted --name {name:appcs} + command: appconfig purge --name {name:appcs} --yes + expect: + exitCode: 0 + cleanup: + - appconfig delete --name {name:appcs} --resource-group {rg} --yes + - appconfig purge --name {name:appcs} --yes + - group delete --name {rg} --yes --no-wait + + - id: appconfig-recover + operations: [Microsoft.AppConfiguration ConfigurationStores Create] + note: Samples appconfig recover (a create with createMode Recover). + setup: + - group create --name {rg} --location {location} + - appconfig create --name {name:appcs} --resource-group {rg} --location {location} --sku Standard + - appconfig delete --name {name:appcs} --resource-group {rg} --yes + - appconfig recover --name {name:appcs} --resource-group {rg} --yes + command: appconfig show --name {name:appcs} --resource-group {rg} --query name -o tsv + expect: + stdout: { equals: "{name:appcs}" } + cleanup: + - appconfig delete --name {name:appcs} --resource-group {rg} --yes + - appconfig purge --name {name:appcs} --yes + - group delete --name {rg} --yes --no-wait + + # --- The data plane (samples appconfig kv set/list/show) --- + + - id: appconfig-kv-set-show + operations: + [ + Microsoft.AppConfiguration Microsoft.AppConfiguration PutKeyValue, + Microsoft.AppConfiguration Microsoft.AppConfiguration GetKeyValue, + ] + pr: true + setup: + - group create --name {rg} --location {location} + - appconfig create --name {name:appcs} --resource-group {rg} --location {location} --sku Standard + - appconfig kv set --name {name:appcs} --key ui:color --value teal --label prod --yes + command: appconfig kv show --name {name:appcs} --key ui:color --label prod --query value -o tsv + expect: + stdout: { equals: teal } + cleanup: + - appconfig delete --name {name:appcs} --resource-group {rg} --yes + - appconfig purge --name {name:appcs} --yes + - group delete --name {rg} --yes --no-wait + + - id: appconfig-kv-list + operations: + [ + Microsoft.AppConfiguration Microsoft.AppConfiguration GetKeyValues, + Microsoft.AppConfiguration Microsoft.AppConfiguration GetRevisions, + ] + setup: + - group create --name {rg} --location {location} + - appconfig create --name {name:appcs} --resource-group {rg} --location {location} --sku Standard + - appconfig kv set --name {name:appcs} --key app:one --value 1 --yes + - appconfig kv set --name {name:appcs} --key app:two --value 2 --yes + - run: appconfig kv list --name {name:appcs} --key "app:*" --query "sort([].key)" + until: { json: { path: "", equals: ["app:one", "app:two"] } } + command: appconfig revision list --name {name:appcs} --key app:one --query "[].value" + expect: + json: { path: "", contains: "1" } + cleanup: + - appconfig delete --name {name:appcs} --resource-group {rg} --yes + - appconfig purge --name {name:appcs} --yes + - group delete --name {rg} --yes --no-wait + + - id: appconfig-kv-lock-delete + operations: + [ + Microsoft.AppConfiguration Microsoft.AppConfiguration PutLock, + Microsoft.AppConfiguration Microsoft.AppConfiguration DeleteLock, + Microsoft.AppConfiguration Microsoft.AppConfiguration DeleteKeyValue, + ] + setup: + - group create --name {rg} --location {location} + - appconfig create --name {name:appcs} --resource-group {rg} --location {location} --sku Standard + - appconfig kv set --name {name:appcs} --key feature:beta --value on --yes + - run: appconfig kv lock --name {name:appcs} --key feature:beta --yes + until: { json: { path: locked, equals: true } } + - run: appconfig kv unlock --name {name:appcs} --key feature:beta --yes + until: { json: { path: locked, equals: false } } + command: appconfig kv delete --name {name:appcs} --key feature:beta --yes + expect: + exitCode: 0 + cleanup: + - appconfig delete --name {name:appcs} --resource-group {rg} --yes + - appconfig purge --name {name:appcs} --yes + - group delete --name {rg} --yes --no-wait + + - id: appconfig-dp-keys-labels + operations: + [ + Microsoft.AppConfiguration Microsoft.AppConfiguration GetKeys, + Microsoft.AppConfiguration Microsoft.AppConfiguration GetLabels, + Microsoft.AppConfiguration Microsoft.AppConfiguration CheckKeys, + Microsoft.AppConfiguration Microsoft.AppConfiguration CheckLabels, + Microsoft.AppConfiguration Microsoft.AppConfiguration CheckKeyValues, + Microsoft.AppConfiguration Microsoft.AppConfiguration CheckKeyValue, + Microsoft.AppConfiguration Microsoft.AppConfiguration CheckRevisions, + ] + note: The data-plane REST calls az has no verb for, against the store endpoint (a local host). + setup: + - group create --name {rg} --location {location} + - run: appconfig create --name {name:appcs} --resource-group {rg} --location {location} --sku Standard --query endpoint -o tsv + capture: endpoint + - appconfig kv set --name {name:appcs} --key app:one --value 1 --label prod --yes + - rest --method head --url {endpoint}/keys?api-version=2023-10-01 --resource https://azconfig.io + - rest --method head --url {endpoint}/labels?api-version=2023-10-01 --resource https://azconfig.io + - rest --method head --url {endpoint}/kv?api-version=2023-10-01 --resource https://azconfig.io + - rest --method head --url "{endpoint}/kv/app:one?api-version=2023-10-01&label=prod" --resource https://azconfig.io + - rest --method head --url {endpoint}/revisions?api-version=2023-10-01 --resource https://azconfig.io + - run: rest --method get --url {endpoint}/labels?api-version=2023-10-01 --resource https://azconfig.io + until: { stdout: { contains: prod } } + command: rest --method get --url {endpoint}/keys?api-version=2023-10-01 --resource https://azconfig.io + expect: + json: { path: "items[0].name", equals: "app:one" } + cleanup: + - appconfig delete --name {name:appcs} --resource-group {rg} --yes + - appconfig purge --name {name:appcs} --yes + - group delete --name {rg} --yes --no-wait + + - id: appconfig-snapshots + operations: + [ + Microsoft.AppConfiguration Microsoft.AppConfiguration CreateSnapshot, + Microsoft.AppConfiguration Microsoft.AppConfiguration GetOperationDetails, + Microsoft.AppConfiguration Microsoft.AppConfiguration GetSnapshot, + Microsoft.AppConfiguration Microsoft.AppConfiguration GetSnapshots, + Microsoft.AppConfiguration Microsoft.AppConfiguration CheckSnapshot, + Microsoft.AppConfiguration Microsoft.AppConfiguration CheckSnapshots, + Microsoft.AppConfiguration Microsoft.AppConfiguration UpdateSnapshot, + ] + setup: + - group create --name {rg} --location {location} + - run: appconfig create --name {name:appcs} --resource-group {rg} --location {location} --sku Standard --query endpoint -o tsv + capture: endpoint + - appconfig kv set --name {name:appcs} --key app:one --value 1 --yes + - >- + appconfig snapshot create --name {name:appcs} --snapshot-name release-1 --filters '{"key": "app:*"}' + - run: appconfig snapshot show --name {name:appcs} --snapshot-name release-1 --query status -o tsv + until: { stdout: { equals: ready } } + - run: appconfig snapshot list --name {name:appcs} --query "[].name" + until: { json: { path: "", equals: [release-1] } } + - rest --method head --url {endpoint}/snapshots/release-1?api-version=2023-10-01 --resource https://azconfig.io + - rest --method head --url {endpoint}/snapshots?api-version=2023-10-01 --resource https://azconfig.io + command: appconfig snapshot archive --name {name:appcs} --snapshot-name release-1 + expect: + json: { path: status, equals: archived } + cleanup: + - appconfig delete --name {name:appcs} --resource-group {rg} --yes + - appconfig purge --name {name:appcs} --yes + - group delete --name {rg} --yes --no-wait + + # --- ARM children --- + + - id: appconfig-arm-keyvalues-snapshots + operations: + [ + Microsoft.AppConfiguration KeyValues CreateOrUpdate, + Microsoft.AppConfiguration KeyValues Get, + Microsoft.AppConfiguration KeyValues Delete, + Microsoft.AppConfiguration Snapshots Create, + Microsoft.AppConfiguration Snapshots Get, + ] + setup: + - group create --name {rg} --location {location} + - appconfig create --name {name:appcs} --resource-group {rg} --location {location} --sku Standard + - >- + rest --method put + --url /subscriptions/{sub}/resourceGroups/{rg}/providers/Microsoft.AppConfiguration/configurationStores/{name:appcs}/keyValues/arm-key?api-version=2023-03-01 + --body '{"properties": {"value": "from-arm", "contentType": "text/plain"}}' + - run: >- + rest --method get + --url /subscriptions/{sub}/resourceGroups/{rg}/providers/Microsoft.AppConfiguration/configurationStores/{name:appcs}/keyValues/arm-key?api-version=2023-03-01 + until: { json: { path: properties.value, equals: from-arm } } + - >- + rest --method put + --url /subscriptions/{sub}/resourceGroups/{rg}/providers/Microsoft.AppConfiguration/configurationStores/{name:appcs}/snapshots/arm-snap?api-version=2023-03-01 + --body '{"properties": {"filters": [{"key": "*"}]}}' + - run: >- + rest --method get + --url /subscriptions/{sub}/resourceGroups/{rg}/providers/Microsoft.AppConfiguration/configurationStores/{name:appcs}/snapshots/arm-snap?api-version=2023-03-01 + until: { json: { path: name, equals: arm-snap } } + command: >- + rest --method delete + --url /subscriptions/{sub}/resourceGroups/{rg}/providers/Microsoft.AppConfiguration/configurationStores/{name:appcs}/keyValues/arm-key?api-version=2023-03-01 + expect: + exitCode: 0 + cleanup: + - appconfig delete --name {name:appcs} --resource-group {rg} --yes + - appconfig purge --name {name:appcs} --yes + - group delete --name {rg} --yes --no-wait + + - id: appconfig-replicas + operations: + [ + Microsoft.AppConfiguration Replicas Create, + Microsoft.AppConfiguration Replicas Get, + Microsoft.AppConfiguration Replicas ListByConfigurationStore, + Microsoft.AppConfiguration Replicas Delete, + ] + setup: + - group create --name {rg} --location {location} + - appconfig create --name {name:appcs} --resource-group {rg} --location {location} --sku Standard + - appconfig replica create --store-name {name:appcs} --resource-group {rg} --name neu --location northeurope + - run: appconfig replica show --store-name {name:appcs} --resource-group {rg} --name neu --query location -o tsv + until: { stdout: { equals: northeurope } } + - run: appconfig replica list --store-name {name:appcs} --resource-group {rg} --query "[].name" + until: { json: { path: "", equals: [neu] } } + command: appconfig replica delete --store-name {name:appcs} --resource-group {rg} --name neu --yes + expect: + exitCode: 0 + cleanup: + - appconfig delete --name {name:appcs} --resource-group {rg} --yes + - appconfig purge --name {name:appcs} --yes + - group delete --name {rg} --yes --no-wait + + - id: appconfig-operations-names + operations: + [ + Microsoft.AppConfiguration Operations CheckNameAvailability, + Microsoft.AppConfiguration Operations RegionalCheckNameAvailability, + Microsoft.AppConfiguration Operations List, + ] + setup: + - >- + rest --method post + --url /subscriptions/{sub}/providers/Microsoft.AppConfiguration/locations/{location}/checkNameAvailability?api-version=2023-03-01 + --body '{"name": "{name:appcs}", "type": "Microsoft.AppConfiguration/configurationStores"}' + - rest --method get --url /providers/Microsoft.AppConfiguration/operations?api-version=2023-03-01 + command: >- + rest --method post + --url /subscriptions/{sub}/providers/Microsoft.AppConfiguration/checkNameAvailability?api-version=2023-03-01 + --body '{"name": "{name:appcs}", "type": "Microsoft.AppConfiguration/configurationStores"}' + expect: + json: { path: nameAvailable, equals: true } + + - id: appconfig-private-link-resources + operations: + [ + Microsoft.AppConfiguration PrivateLinkResources ListByConfigurationStore, + Microsoft.AppConfiguration PrivateLinkResources Get, + Microsoft.AppConfiguration PrivateEndpointConnections ListByConfigurationStore, + ] + setup: + - group create --name {rg} --location {location} + - appconfig create --name {name:appcs} --resource-group {rg} --location {location} --sku Standard + - >- + rest --method get + --url /subscriptions/{sub}/resourceGroups/{rg}/providers/Microsoft.AppConfiguration/configurationStores/{name:appcs}/privateLinkResources?api-version=2023-03-01 + - >- + rest --method get + --url /subscriptions/{sub}/resourceGroups/{rg}/providers/Microsoft.AppConfiguration/configurationStores/{name:appcs}/privateLinkResources/configurationStores?api-version=2023-03-01 + command: >- + rest --method get + --url /subscriptions/{sub}/resourceGroups/{rg}/providers/Microsoft.AppConfiguration/configurationStores/{name:appcs}/privateEndpointConnections?api-version=2023-03-01 + expect: + json: { path: value, length: 0 } + cleanup: + - appconfig delete --name {name:appcs} --resource-group {rg} --yes + - appconfig purge --name {name:appcs} --yes + - group delete --name {rg} --yes --no-wait + + - id: appconfig-private-endpoint-connection + operations: + [ + Microsoft.AppConfiguration PrivateEndpointConnections Get, + Microsoft.AppConfiguration PrivateEndpointConnections CreateOrUpdate, + ] + setup: + - group create --name {rg} --location {location} + - >- + network vnet create --name {name:vnet} --resource-group {rg} --address-prefixes 10.31.0.0/16 + --subnet-name endpoints --subnet-prefixes 10.31.1.0/24 + - run: appconfig create --name {name:appcs} --resource-group {rg} --location {location} --sku Standard --query id -o tsv + capture: store_id + - >- + network private-endpoint create --name {name:pe} --resource-group {rg} --vnet-name {name:vnet} + --subnet endpoints --private-connection-resource-id {store_id} --group-id configurationStores + --connection-name appcs + - run: >- + rest --method get + --url /subscriptions/{sub}/resourceGroups/{rg}/providers/Microsoft.AppConfiguration/configurationStores/{name:appcs}/privateEndpointConnections?api-version=2023-03-01 + --query "value[0].name" -o tsv + capture: pec + - >- + rest --method get + --url /subscriptions/{sub}/resourceGroups/{rg}/providers/Microsoft.AppConfiguration/configurationStores/{name:appcs}/privateEndpointConnections/{pec}?api-version=2023-03-01 + command: >- + rest --method put + --url /subscriptions/{sub}/resourceGroups/{rg}/providers/Microsoft.AppConfiguration/configurationStores/{name:appcs}/privateEndpointConnections/{pec}?api-version=2023-03-01 + --body '{"properties": {"privateLinkServiceConnectionState": {"status": "Approved", "description": "l2"}}}' + expect: + json: { path: properties.privateLinkServiceConnectionState.status, equals: Approved } + cleanup: + - appconfig delete --name {name:appcs} --resource-group {rg} --yes + - appconfig purge --name {name:appcs} --yes + - group delete --name {rg} --yes --no-wait + + - id: appconfig-list-key-value + operations: [Microsoft.AppConfiguration ConfigurationStores ListKeyValue] + setup: + - group create --name {rg} --location {location} + - appconfig create --name {name:appcs} --resource-group {rg} --location {location} --sku Standard + - appconfig kv set --name {name:appcs} --key app:one --value 1 --yes + command: >- + rest --method post + --url /subscriptions/{sub}/resourceGroups/{rg}/providers/Microsoft.AppConfiguration/configurationStores/{name:appcs}/listKeyValue?api-version=2019-10-01 + --body '{"key": "app:one"}' + expect: + json: { path: value, equals: "1" } + cleanup: + - appconfig delete --name {name:appcs} --resource-group {rg} --yes + - appconfig purge --name {name:appcs} --yes + - group delete --name {rg} --yes --no-wait diff --git a/tests/azure/matrix/authorization.yaml b/tests/azure/matrix/authorization.yaml new file mode 100644 index 0000000..4efc897 --- /dev/null +++ b/tests/azure/matrix/authorization.yaml @@ -0,0 +1,178 @@ +# Microsoft.Authorization: resource-group locks, role definitions, role assignments. +# A CanNotDelete lock blocks the group delete, so lock cases delete their lock first. Custom role +# definitions live at subscription scope, outside any group, so their cases delete them too. +# Assignees are passed as object ids: az then needs no Microsoft Graph lookup. +provider: Microsoft.Authorization +cases: + - id: lock-create-show + operations: + [ + Microsoft.Authorization ManagementLocks CreateOrUpdateAtResourceGroupLevel, + Microsoft.Authorization ManagementLocks GetAtResourceGroupLevel, + ] + pr: true + note: >- + `lock show` and `lock delete` look the lock up in the subscription-level list, which the + emulator lacks (lock-az-show), so reads and deletes go through the group-level REST routes. + setup: + - group create --name {rg} --location {location} + - lock create --name l2-lock --resource-group {rg} --lock-type CanNotDelete --notes "l2 matrix" + command: rest --method get --url /subscriptions/{sub}/resourceGroups/{rg}/providers/Microsoft.Authorization/locks/l2-lock?api-version=2020-05-01 + expect: + json: + - { path: name, equals: l2-lock } + - { path: properties.level, equals: CanNotDelete } + cleanup: + - rest --method delete --url /subscriptions/{sub}/resourceGroups/{rg}/providers/Microsoft.Authorization/locks/l2-lock?api-version=2020-05-01 + - group delete --name {rg} --yes --no-wait + + - id: lock-notes + operations: [Microsoft.Authorization ManagementLocks CreateOrUpdateAtResourceGroupLevel] + setup: + - group create --name {rg} --location {location} + - lock create --name l2-lock --resource-group {rg} --lock-type CanNotDelete --notes "l2 matrix" + command: rest --method get --url /subscriptions/{sub}/resourceGroups/{rg}/providers/Microsoft.Authorization/locks/l2-lock?api-version=2020-05-01 + expect: + json: { path: properties.notes, equals: l2 matrix } + cleanup: + - rest --method delete --url /subscriptions/{sub}/resourceGroups/{rg}/providers/Microsoft.Authorization/locks/l2-lock?api-version=2020-05-01 + - group delete --name {rg} --yes --no-wait + known_gap: + reason: >- + L2: the emulator drops a lock's `notes`, from `lock create --notes` and from a REST PUT + alike (the stored lock has only `level`); Azure stores and returns them. + date: 2026-09-27 + + - id: lock-list-delete + operations: + [ + Microsoft.Authorization ManagementLocks ListAtResourceGroupLevel, + Microsoft.Authorization ManagementLocks DeleteAtResourceGroupLevel, + ] + setup: + - group create --name {rg} --location {location} + - lock create --name l2-read-only --resource-group {rg} --lock-type ReadOnly + - lock create --name l2-no-delete --resource-group {rg} --lock-type CanNotDelete + - run: lock list --resource-group {rg} --query "sort([].name)" + until: { json: { path: "", equals: [l2-no-delete, l2-read-only] } } + - rest --method delete --url /subscriptions/{sub}/resourceGroups/{rg}/providers/Microsoft.Authorization/locks/l2-read-only?api-version=2020-05-01 + command: rest --method delete --url /subscriptions/{sub}/resourceGroups/{rg}/providers/Microsoft.Authorization/locks/l2-no-delete?api-version=2020-05-01 + expect: + exitCode: 0 + cleanup: + - rest --method delete --url /subscriptions/{sub}/resourceGroups/{rg}/providers/Microsoft.Authorization/locks/l2-read-only?api-version=2020-05-01 + - rest --method delete --url /subscriptions/{sub}/resourceGroups/{rg}/providers/Microsoft.Authorization/locks/l2-no-delete?api-version=2020-05-01 + - group delete --name {rg} --yes --no-wait + + - id: lock-az-show + operations: [Microsoft.Authorization ManagementLocks ListAtSubscriptionLevel] + setup: + - group create --name {rg} --location {location} + - lock create --name l2-lock --resource-group {rg} --lock-type CanNotDelete + command: lock show --name l2-lock --resource-group {rg} + expect: + json: { path: level, equals: CanNotDelete } + cleanup: + - rest --method delete --url /subscriptions/{sub}/resourceGroups/{rg}/providers/Microsoft.Authorization/locks/l2-lock?api-version=2020-05-01 + - group delete --name {rg} --yes --no-wait + known_gap: + reason: >- + Found by L2: `az lock show` and `az lock delete` (az 2.85) find the lock through + GET /subscriptions/{sub}/providers/Microsoft.Authorization/locks, which the emulator answers + with NotImplemented, so both fail even for a group-level lock; the group-level GET, DELETE + and list routes work. A lock left behind blocks the group delete, so cleanups use REST. + date: 2026-09-27 + + - id: lock-delete-missing + operations: [Microsoft.Authorization ManagementLocks DeleteAtResourceGroupLevel] + note: Cleanups delete their locks again after the case did, so this also costs every lock case a retry. + setup: + - group create --name {rg} --location {location} + command: rest --method delete --url /subscriptions/{sub}/resourceGroups/{rg}/providers/Microsoft.Authorization/locks/l2-missing?api-version=2020-05-01 + expect: + exitCode: 0 + cleanup: + - group delete --name {rg} --yes --no-wait + known_gap: + reason: >- + L2: deleting a group-level lock that does not exist answers 500 InternalServerError + ("An unexpected error occurred while processing 'DELETE ...'"); Azure answers 204. + date: 2026-09-27 + + - id: role-definition-builtin + operations: + [Microsoft.Authorization RoleDefinitions List, Microsoft.Authorization RoleDefinitions Get] + pr: true + note: acdd72a7-... is the built-in Reader role. + setup: + - >- + rest --method get + --url /subscriptions/{sub}/providers/Microsoft.Authorization/roleDefinitions/acdd72a7-3385-48ef-bd42-f606fba81ae7?api-version=2022-04-01 + command: role definition list --name Reader --query "[].roleName" + expect: + json: { path: "", equals: [Reader] } + + - id: role-definition-custom + operations: + [ + Microsoft.Authorization RoleDefinitions CreateOrUpdate, + Microsoft.Authorization RoleDefinitions Delete, + ] + setup: + - >- + role definition create --role-definition '{"Name": "L2 Reader {id}", "Description": "L2 matrix role", + "Actions": ["Microsoft.Storage/storageAccounts/read"], "AssignableScopes": ["/subscriptions/{sub}"]}' + - run: role definition list --name "L2 Reader {id}" --custom-role-only true --query "[].roleName" + until: { json: { path: "", equals: ["L2 Reader {id}"] } } + command: role definition delete --name "L2 Reader {id}" + expect: + exitCode: 0 + cleanup: + - role definition delete --name "L2 Reader {id}" + + - id: role-assignment + operations: + [ + Microsoft.Authorization RoleAssignments Create, + Microsoft.Authorization RoleAssignments ListForResourceGroup, + Microsoft.Authorization RoleAssignments ListForScope, + Microsoft.Authorization RoleAssignments ListForSubscription, + Microsoft.Authorization RoleAssignments Get, + ] + pr: true + note: Samples role assignment create/list (22 uses). + setup: + - group create --name {rg} --location {location} + - run: identity create --name {name:id} --resource-group {rg} --location {location} --query principalId -o tsv + capture: principal + - run: >- + role assignment create --assignee-object-id {principal} --assignee-principal-type ServicePrincipal + --role Reader --scope /subscriptions/{sub}/resourceGroups/{rg} --query name -o tsv + capture: assignment + - run: role assignment list --resource-group {rg} --query "[].principalId" + until: { json: { path: "", contains: "{principal}" } } + - role assignment list --scope /subscriptions/{sub}/resourceGroups/{rg} --query "[?principalId=='{principal}'].name" + - role assignment list --all --query "[?principalId=='{principal}'].name" + command: >- + rest --method get + --url /subscriptions/{sub}/resourceGroups/{rg}/providers/Microsoft.Authorization/roleAssignments/{assignment}?api-version=2022-04-01 + expect: + json: { path: properties.principalId, equals: "{principal}" } + cleanup: + - group delete --name {rg} --yes --no-wait + + - id: role-assignment-delete + operations: [Microsoft.Authorization RoleAssignments Delete] + setup: + - group create --name {rg} --location {location} + - run: identity create --name {name:id} --resource-group {rg} --location {location} --query principalId -o tsv + capture: principal + - run: >- + role assignment create --assignee-object-id {principal} --assignee-principal-type ServicePrincipal + --role Reader --scope /subscriptions/{sub}/resourceGroups/{rg} --query id -o tsv + capture: assignment_id + command: role assignment delete --ids {assignment_id} + expect: + exitCode: 0 + cleanup: + - group delete --name {rg} --yes --no-wait diff --git a/tests/azure/matrix/cdn.yaml b/tests/azure/matrix/cdn.yaml new file mode 100644 index 0000000..fd104b1 --- /dev/null +++ b/tests/azure/matrix/cdn.yaml @@ -0,0 +1,832 @@ +# Microsoft.Cdn: Front Door Standard/Premium (afd), classic CDN (cdn; CI starts the emulator with +# CDN_CLASSIC_ALLOW_CREATE=1), WAF policies, log analytics, migration, edge actions. `afd` and +# `cdn` are core (deprecated) in az 2.85-2.87 and extension-only from 2.90; the curated `cdn` +# extension (1.0.0b3) replaces the core commands wherever it is installed, so the cases use its +# syntax (shorthand --conditions/--actions, --parameters, --web-application-firewall, +# --formatted-origins). Edge actions go through rest (the `edge-action` extension is a preview). +provider: Microsoft.Cdn +cases: + - id: afd-profile + operations: + [ + Microsoft.Cdn Profiles Create, + Microsoft.Cdn Profiles Get, + Microsoft.Cdn Profiles ListByResourceGroup, + Microsoft.Cdn Profiles List, + Microsoft.Cdn Profiles Update, + Microsoft.Cdn Profiles ListResourceUsage, + ] + note: Samples afd profile create/show. + setup: + - group create --name {rg} --location {location} + - afd profile create --profile-name {name:afd} --resource-group {rg} --sku Standard_AzureFrontDoor + - run: afd profile show --profile-name {name:afd} --resource-group {rg} --query sku.name -o tsv + until: { stdout: { equals: Standard_AzureFrontDoor } } + - afd profile list --resource-group {rg} + - afd profile list --query "[?name=='{name:afd}'].name" + - afd profile usage --profile-name {name:afd} --resource-group {rg} + command: afd profile update --profile-name {name:afd} --resource-group {rg} --tags matrix=l2 + expect: + json: { path: tags.matrix, equals: l2 } + cleanup: + - group delete --name {rg} --yes --no-wait + + - id: afd-endpoint + operations: + [ + Microsoft.Cdn AfdEndpoints Create, + Microsoft.Cdn AfdEndpoints Get, + Microsoft.Cdn AfdEndpoints ListByProfile, + Microsoft.Cdn AfdEndpoints Update, + Microsoft.Cdn AfdEndpoints PurgeContent, + Microsoft.Cdn AfdEndpoints ValidateCustomDomain, + Microsoft.Cdn AfdEndpoints ListResourceUsage, + Microsoft.Cdn AfdEndpoints Delete, + ] + note: Samples afd endpoint create/show/purge/update. + setup: + - group create --name {rg} --location {location} + - afd profile create --profile-name {name:afd} --resource-group {rg} --sku Standard_AzureFrontDoor + - afd endpoint create --endpoint-name {name:ep} --profile-name {name:afd} --resource-group {rg} --enabled-state Enabled + - run: afd endpoint show --endpoint-name {name:ep} --profile-name {name:afd} --resource-group {rg} --query enabledState -o tsv + until: { stdout: { equals: Enabled } } + - afd endpoint list --profile-name {name:afd} --resource-group {rg} + - afd endpoint update --endpoint-name {name:ep} --profile-name {name:afd} --resource-group {rg} --enabled-state Disabled + - afd endpoint purge --endpoint-name {name:ep} --profile-name {name:afd} --resource-group {rg} --content-paths "/*" + - >- + rest --method post + --url /subscriptions/{sub}/resourceGroups/{rg}/providers/Microsoft.Cdn/profiles/{name:afd}/afdEndpoints/{name:ep}/validateCustomDomain?api-version=2024-02-01 + --body '{"hostName": "www.{id}.example.com"}' + - >- + rest --method post + --url /subscriptions/{sub}/resourceGroups/{rg}/providers/Microsoft.Cdn/profiles/{name:afd}/afdEndpoints/{name:ep}/usages?api-version=2024-02-01 + command: afd endpoint delete --endpoint-name {name:ep} --profile-name {name:afd} --resource-group {rg} --yes + expect: + exitCode: 0 + cleanup: + - group delete --name {rg} --yes --no-wait + + - id: afd-origins + operations: + [ + Microsoft.Cdn AfdOriginGroups Create, + Microsoft.Cdn AfdOriginGroups Get, + Microsoft.Cdn AfdOriginGroups ListByProfile, + Microsoft.Cdn AfdOriginGroups Update, + Microsoft.Cdn AfdOriginGroups ListResourceUsage, + Microsoft.Cdn AfdOrigins Create, + Microsoft.Cdn AfdOrigins Get, + Microsoft.Cdn AfdOrigins ListByOriginGroup, + Microsoft.Cdn AfdOrigins Update, + Microsoft.Cdn AfdOrigins Delete, + Microsoft.Cdn AfdOriginGroups Delete, + ] + note: Samples afd origin-group/origin create. + setup: + - group create --name {rg} --location {location} + - afd profile create --profile-name {name:afd} --resource-group {rg} --sku Standard_AzureFrontDoor + - >- + afd origin-group create --origin-group-name og --profile-name {name:afd} --resource-group {rg} + --probe-request-type GET --probe-protocol Http --probe-interval-in-seconds 60 --probe-path / + --sample-size 4 --successful-samples-required 3 --additional-latency-in-milliseconds 50 + - afd origin-group show --origin-group-name og --profile-name {name:afd} --resource-group {rg} + - afd origin-group list --profile-name {name:afd} --resource-group {rg} + - afd origin-group update --origin-group-name og --profile-name {name:afd} --resource-group {rg} --probe-path /health + - >- + rest --method post + --url /subscriptions/{sub}/resourceGroups/{rg}/providers/Microsoft.Cdn/profiles/{name:afd}/originGroups/og/usages?api-version=2024-02-01 + - >- + afd origin create --origin-name web1 --origin-group-name og --profile-name {name:afd} --resource-group {rg} + --host-name www.example.com --origin-host-header www.example.com --http-port 80 --https-port 443 + --priority 1 --weight 1000 --enabled-state Enabled + - run: afd origin show --origin-name web1 --origin-group-name og --profile-name {name:afd} --resource-group {rg} --query hostName -o tsv + until: { stdout: { equals: www.example.com } } + - afd origin list --origin-group-name og --profile-name {name:afd} --resource-group {rg} + - afd origin update --origin-name web1 --origin-group-name og --profile-name {name:afd} --resource-group {rg} --weight 500 + - afd origin delete --origin-name web1 --origin-group-name og --profile-name {name:afd} --resource-group {rg} --yes + command: afd origin-group delete --origin-group-name og --profile-name {name:afd} --resource-group {rg} --yes + expect: + exitCode: 0 + cleanup: + - group delete --name {rg} --yes --no-wait + + - id: afd-route + operations: [Microsoft.Cdn Routes Create, Microsoft.Cdn Routes ListByEndpoint] + note: Samples afd route create. + setup: + - group create --name {rg} --location {location} + - afd profile create --profile-name {name:afd} --resource-group {rg} --sku Standard_AzureFrontDoor + - afd endpoint create --endpoint-name {name:ep} --profile-name {name:afd} --resource-group {rg} --enabled-state Enabled + - >- + afd origin-group create --origin-group-name og --profile-name {name:afd} --resource-group {rg} + --probe-request-type GET --probe-protocol Http --probe-interval-in-seconds 60 --probe-path / + --sample-size 4 --successful-samples-required 3 --additional-latency-in-milliseconds 50 + - >- + afd origin create --origin-name web1 --origin-group-name og --profile-name {name:afd} --resource-group {rg} + --host-name www.example.com --origin-host-header www.example.com --priority 1 --weight 1000 --enabled-state Enabled + - >- + afd route create --route-name r1 --endpoint-name {name:ep} --profile-name {name:afd} --resource-group {rg} + --origin-group og --supported-protocols Http Https --link-to-default-domain Enabled + --forwarding-protocol MatchRequest --https-redirect Enabled + command: afd route list --endpoint-name {name:ep} --profile-name {name:afd} --resource-group {rg} --query "[].name" + expect: + json: { path: "", equals: [r1] } + cleanup: + - group delete --name {rg} --yes --no-wait + + - id: afd-route-read + operations: [Microsoft.Cdn Routes Get, Microsoft.Cdn Routes Update, Microsoft.Cdn Routes Delete] + setup: + - group create --name {rg} --location {location} + - afd profile create --profile-name {name:afd} --resource-group {rg} --sku Standard_AzureFrontDoor + - afd endpoint create --endpoint-name {name:ep} --profile-name {name:afd} --resource-group {rg} --enabled-state Enabled + - >- + afd origin-group create --origin-group-name og --profile-name {name:afd} --resource-group {rg} + --probe-request-type GET --probe-protocol Http --probe-interval-in-seconds 60 --probe-path / + --sample-size 4 --successful-samples-required 3 --additional-latency-in-milliseconds 50 + - >- + afd origin create --origin-name web1 --origin-group-name og --profile-name {name:afd} --resource-group {rg} + --host-name www.example.com --origin-host-header www.example.com --priority 1 --weight 1000 --enabled-state Enabled + - >- + afd route create --route-name r1 --endpoint-name {name:ep} --profile-name {name:afd} --resource-group {rg} + --origin-group og --supported-protocols Http Https --link-to-default-domain Enabled + --forwarding-protocol MatchRequest --https-redirect Enabled + - run: afd route show --route-name r1 --endpoint-name {name:ep} --profile-name {name:afd} --resource-group {rg} --query name -o tsv + until: { stdout: { equals: r1 } } + - run: >- + afd route update --route-name r1 --endpoint-name {name:ep} --profile-name {name:afd} --resource-group {rg} + --https-redirect Disabled --query httpsRedirect -o tsv + until: { stdout: { equals: Disabled } } + command: afd route delete --route-name r1 --endpoint-name {name:ep} --profile-name {name:afd} --resource-group {rg} --yes + expect: + exitCode: 0 + cleanup: + - group delete --name {rg} --yes --no-wait + + - id: afd-rule-sets + operations: + [ + Microsoft.Cdn RuleSets Create, + Microsoft.Cdn RuleSets Get, + Microsoft.Cdn RuleSets ListByProfile, + Microsoft.Cdn RuleSets ListResourceUsage, + Microsoft.Cdn Rules Create, + Microsoft.Cdn Rules Get, + Microsoft.Cdn Rules ListByRuleSet, + Microsoft.Cdn Rules Update, + Microsoft.Cdn Rules Delete, + Microsoft.Cdn RuleSets Delete, + ] + note: Samples afd rule-set create/show, afd rule create (7 uses). + setup: + - group create --name {rg} --location {location} + - afd profile create --profile-name {name:afd} --resource-group {rg} --sku Standard_AzureFrontDoor + - afd rule-set create --rule-set-name headers --profile-name {name:afd} --resource-group {rg} + - afd rule-set show --rule-set-name headers --profile-name {name:afd} --resource-group {rg} + - afd rule-set list --profile-name {name:afd} --resource-group {rg} + - >- + rest --method post + --url /subscriptions/{sub}/resourceGroups/{rg}/providers/Microsoft.Cdn/profiles/{name:afd}/ruleSets/headers/usages?api-version=2024-02-01 + # The curated cdn extension (1.0.0b3) takes conditions and actions in shorthand syntax; the + # older --match-variable/--action-name flags are gone (L2 flag lint, 2026-09-27). + - >- + afd rule create --rule-name addheader --rule-set-name headers --profile-name {name:afd} --resource-group {rg} + --order 1 --conditions "[{request-method:{parameters:{operator:Equal,match-values:[GET]}}}]" + --actions "[{modify-response-header:{parameters:{header-action:Append,header-name:X-L2,value:matrix}}}]" + - run: afd rule show --rule-name addheader --rule-set-name headers --profile-name {name:afd} --resource-group {rg} --query order + until: { json: { path: "", equals: 1 } } + - afd rule list --rule-set-name headers --profile-name {name:afd} --resource-group {rg} + - afd rule update --rule-name addheader --rule-set-name headers --profile-name {name:afd} --resource-group {rg} --order 2 + - afd rule delete --rule-name addheader --rule-set-name headers --profile-name {name:afd} --resource-group {rg} --yes + command: afd rule-set delete --rule-set-name headers --profile-name {name:afd} --resource-group {rg} --yes + expect: + exitCode: 0 + cleanup: + - group delete --name {rg} --yes --no-wait + + - id: afd-custom-domain + operations: + [ + Microsoft.Cdn AfdCustomDomains Create, + Microsoft.Cdn AfdCustomDomains Get, + Microsoft.Cdn AfdCustomDomains ListByProfile, + Microsoft.Cdn AfdCustomDomains Update, + Microsoft.Cdn AfdCustomDomains RefreshValidationToken, + Microsoft.Cdn AfdCustomDomains Delete, + ] + setup: + - group create --name {rg} --location {location} + - afd profile create --profile-name {name:afd} --resource-group {rg} --sku Standard_AzureFrontDoor + - >- + afd custom-domain create --custom-domain-name www --profile-name {name:afd} --resource-group {rg} + --host-name www.{id}.example.com --minimum-tls-version TLS12 --certificate-type ManagedCertificate + - run: afd custom-domain show --custom-domain-name www --profile-name {name:afd} --resource-group {rg} --query hostName -o tsv + until: { stdout: { equals: "www.{id}.example.com" } } + - afd custom-domain list --profile-name {name:afd} --resource-group {rg} + - afd custom-domain update --custom-domain-name www --profile-name {name:afd} --resource-group {rg} --minimum-tls-version TLS12 + - afd custom-domain regenerate-validation-token --custom-domain-name www --profile-name {name:afd} --resource-group {rg} + command: afd custom-domain delete --custom-domain-name www --profile-name {name:afd} --resource-group {rg} --yes + expect: + exitCode: 0 + cleanup: + - group delete --name {rg} --yes --no-wait + + - id: afd-secret-security-policy + operations: + [ + Microsoft.Cdn Secrets Create, + Microsoft.Cdn Secrets Get, + Microsoft.Cdn Secrets ListByProfile, + Microsoft.Cdn Secrets Update, + Microsoft.Cdn Secrets Delete, + Microsoft.Cdn SecurityPolicies Create, + Microsoft.Cdn SecurityPolicies Get, + Microsoft.Cdn SecurityPolicies ListByProfile, + Microsoft.Cdn SecurityPolicies Patch, + Microsoft.Cdn SecurityPolicies Delete, + Microsoft.Cdn AfdProfiles ValidateSecret, + Microsoft.Cdn Validate Secret, + ] + setup: + - group create --name {rg} --location {location} + - afd profile create --profile-name {name:afd} --resource-group {rg} --sku Premium_AzureFrontDoor + - run: afd endpoint create --endpoint-name {name:ep} --profile-name {name:afd} --resource-group {rg} --enabled-state Enabled --query id -o tsv + capture: endpoint_id + - keyvault create --name {vault} --resource-group {rg} --location {location} + - run: keyvault certificate get-default-policy -o json + capture: policy + - run: keyvault certificate create --vault-name {vault} --name tls --policy '{policy}' --query id -o tsv + capture: cert_id + - run: keyvault show --name {vault} --query id -o tsv + capture: vault_id + # Shorthand --parameters (cdn extension 1.0.0b3); the updates go through rest so that the + # PATCH operations (Secrets Update, SecurityPolicies Patch) are the ones exercised. + - >- + afd secret create --profile-name {name:afd} --resource-group {rg} --secret-name tls + --parameters "{customer-certificate:{secret-source:{id:{vault_id}/secrets/tls},use-latest-version:true}}" + - afd secret show --profile-name {name:afd} --resource-group {rg} --secret-name tls + - afd secret list --profile-name {name:afd} --resource-group {rg} + # The PATCH answers 200 but keeps the old parameters (afd-secret-patch). + - >- + rest --method patch + --url /subscriptions/{sub}/resourceGroups/{rg}/providers/Microsoft.Cdn/profiles/{name:afd}/secrets/tls?api-version=2024-02-01 + --body '{"properties": {"parameters": {"type": "CustomerCertificate", "secretSource": {"id": "{vault_id}/secrets/tls"}, "useLatestVersion": false}}}' + - >- + rest --method post + --url /subscriptions/{sub}/resourceGroups/{rg}/providers/Microsoft.Cdn/profiles/{name:afd}/validateSecret?api-version=2024-02-01 + --body '{"secretType": "CustomerCertificate", "secretSource": {"id": "{vault_id}/secrets/tls"}}' + - >- + rest --method post --url /subscriptions/{sub}/providers/Microsoft.Cdn/validateSecret?api-version=2024-02-01 + --body '{"secretType": "CustomerCertificate", "secretSource": {"id": "{vault_id}/secrets/tls"}}' + - run: >- + network front-door waf-policy create --name {alnum:waf} --resource-group {rg} + --sku Premium_AzureFrontDoor --mode Detection --query id -o tsv + capture: waf_id + - >- + afd security-policy create --security-policy-name waf --profile-name {name:afd} --resource-group {rg} + --web-application-firewall "{waf-policy:{id:{waf_id}},associations:[{domains:[{id:{endpoint_id}}],patterns-to-match:['/*']}]}" + - afd security-policy show --security-policy-name waf --profile-name {name:afd} --resource-group {rg} + - afd security-policy list --profile-name {name:afd} --resource-group {rg} + - >- + rest --method patch + --url /subscriptions/{sub}/resourceGroups/{rg}/providers/Microsoft.Cdn/profiles/{name:afd}/securityPolicies/waf?api-version=2024-02-01 + --body '{"properties": {"parameters": {"type": "WebApplicationFirewall", "wafPolicy": {"id": "{waf_id}"}, "associations": [{"domains": [{"id": "{endpoint_id}"}], "patternsToMatch": ["/*", "/l2/*"]}]}}}' + - afd security-policy delete --security-policy-name waf --profile-name {name:afd} --resource-group {rg} --yes + command: afd secret delete --profile-name {name:afd} --resource-group {rg} --secret-name tls --yes + expect: + exitCode: 0 + cleanup: + - keyvault delete --name {vault} + - keyvault purge --name {vault} + - group delete --name {rg} --yes --no-wait + + - id: afd-secret-patch + operations: [Microsoft.Cdn Secrets Update] + note: "`afd secret update` does a GET and a PUT, which applies; the PATCH is what does not." + setup: + - group create --name {rg} --location {location} + - afd profile create --profile-name {name:afd} --resource-group {rg} --sku Premium_AzureFrontDoor + - run: keyvault create --name {vault} --resource-group {rg} --location {location} --query id -o tsv + capture: vault_id + - >- + afd secret create --profile-name {name:afd} --resource-group {rg} --secret-name tls + --parameters "{customer-certificate:{secret-source:{id:{vault_id}/secrets/tls},use-latest-version:true}}" + command: >- + rest --method patch + --url /subscriptions/{sub}/resourceGroups/{rg}/providers/Microsoft.Cdn/profiles/{name:afd}/secrets/tls?api-version=2024-02-01 + --body '{"properties": {"parameters": {"type": "CustomerCertificate", "secretSource": {"id": "{vault_id}/secrets/tls"}, "useLatestVersion": false}}}' + --query properties.parameters.useLatestVersion + expect: + json: { path: "", equals: false } + cleanup: + - keyvault delete --name {vault} + - keyvault purge --name {vault} + - group delete --name {rg} --yes --no-wait + known_gap: + reason: >- + L2: Secrets_Update (PATCH) answers 200 with the old parameters and never applies the new + ones (useLatestVersion stayed true for 3 minutes of polling). + date: 2026-09-27 + + - id: afd-log-analytics + operations: + [ + Microsoft.Cdn LogAnalytics GetLogAnalyticsMetrics, + Microsoft.Cdn LogAnalytics GetLogAnalyticsRankings, + Microsoft.Cdn LogAnalytics GetLogAnalyticsResources, + Microsoft.Cdn LogAnalytics GetLogAnalyticsLocations, + Microsoft.Cdn LogAnalytics GetWafLogAnalyticsMetrics, + Microsoft.Cdn LogAnalytics GetWafLogAnalyticsRankings, + ] + setup: + - group create --name {rg} --location {location} + - afd profile create --profile-name {name:afd} --resource-group {rg} --sku Premium_AzureFrontDoor + - >- + afd log-analytic metric list --profile-name {name:afd} --resource-group {rg} --metrics clientRequestCount + --custom-domains www.{id}.example.com --protocols https + --date-time-begin 2026-01-01T00:00:00Z --date-time-end 2026-01-08T00:00:00Z --granularity P1D + - >- + afd log-analytic ranking list --profile-name {name:afd} --resource-group {rg} --rankings url + --metrics clientRequestCount --max-ranking 5 --date-time-begin 2026-01-01T00:00:00Z --date-time-end 2026-01-08T00:00:00Z + - afd log-analytic resource list --profile-name {name:afd} --resource-group {rg} + - afd log-analytic location list --profile-name {name:afd} --resource-group {rg} + - >- + afd waf-log-analytic metric list --profile-name {name:afd} --resource-group {rg} --metrics clientRequestCount + --date-time-begin 2026-01-01T00:00:00Z --date-time-end 2026-01-08T00:00:00Z --granularity P1D + command: >- + afd waf-log-analytic ranking list --profile-name {name:afd} --resource-group {rg} --rankings action + --metrics clientRequestCount --max-ranking 5 --date-time-begin 2026-01-01T00:00:00Z --date-time-end 2026-01-08T00:00:00Z + expect: + exitCode: 0 + cleanup: + - group delete --name {rg} --yes --no-wait + + - id: cdn-catalogue + operations: + [ + Microsoft.Cdn Microsoft.Cdn CheckNameAvailability, + Microsoft.Cdn Microsoft.Cdn CheckNameAvailabilityWithSubscription, + Microsoft.Cdn Microsoft.Cdn CheckEndpointNameAvailability, + Microsoft.Cdn Microsoft.Cdn ValidateProbe, + Microsoft.Cdn Operations List, + Microsoft.Cdn ResourceUsage List, + Microsoft.Cdn EdgeNodes List, + Microsoft.Cdn ManagedRuleSets List, + ] + setup: + - group create --name {rg} --location {location} + - cdn name-exists --name {name:cdn} + - >- + rest --method post --url /subscriptions/{sub}/providers/Microsoft.Cdn/checkNameAvailability?api-version=2024-02-01 + --body '{"name": "{name:cdn}", "type": "Microsoft.Cdn/Profiles/Endpoints"}' + - >- + rest --method post + --url /subscriptions/{sub}/resourceGroups/{rg}/providers/Microsoft.Cdn/checkEndpointNameAvailability?api-version=2024-02-01 + --body '{"name": "{name:cdn}", "type": "Microsoft.Cdn/Profiles/AfdEndpoints", "autoGeneratedDomainNameLabelScope": "TenantReuse"}' + - >- + rest --method post --url /subscriptions/{sub}/providers/Microsoft.Cdn/validateProbe?api-version=2024-02-01 + --body '{"probeURL": "https://www.example.com/health"}' + - rest --method get --url /providers/Microsoft.Cdn/operations?api-version=2024-02-01 + - rest --method post --url /subscriptions/{sub}/providers/Microsoft.Cdn/checkResourceUsage?api-version=2024-02-01 + - rest --method get --url /providers/Microsoft.Cdn/edgenodes?api-version=2024-02-01 + command: rest --method get --url /subscriptions/{sub}/providers/Microsoft.Cdn/CdnWebApplicationFirewallManagedRuleSets?api-version=2024-02-01 + expect: + exitCode: 0 + cleanup: + - group delete --name {rg} --yes --no-wait + + - id: afd-host-name-check + operations: + [ + Microsoft.Cdn AfdProfiles CheckHostNameAvailability, + Microsoft.Cdn AfdProfiles ListResourceUsage, + ] + setup: + - group create --name {rg} --location {location} + - afd profile create --profile-name {name:afd} --resource-group {rg} --sku Standard_AzureFrontDoor + - >- + rest --method post + --url /subscriptions/{sub}/resourceGroups/{rg}/providers/Microsoft.Cdn/profiles/{name:afd}/usages?api-version=2024-02-01 + command: >- + rest --method post + --url /subscriptions/{sub}/resourceGroups/{rg}/providers/Microsoft.Cdn/profiles/{name:afd}/checkHostNameAvailability?api-version=2024-02-01 + --body '{"hostName": "www.{id}.example.com"}' + expect: + json: { path: nameAvailable, equals: true } + cleanup: + - group delete --name {rg} --yes --no-wait + + - id: afd-endpoint-name-check + operations: [Microsoft.Cdn AfdProfiles CheckEndpointNameAvailability] + setup: + - group create --name {rg} --location {location} + - afd profile create --profile-name {name:afd} --resource-group {rg} --sku Standard_AzureFrontDoor + command: >- + rest --method post + --url /subscriptions/{sub}/resourceGroups/{rg}/providers/Microsoft.Cdn/profiles/{name:afd}/checkEndpointNameAvailability?api-version=2024-02-01 + --body '{"name": "ep{id}", "type": "Microsoft.Cdn/Profiles/AfdEndpoints", "autoGeneratedDomainNameLabelScope": "TenantReuse"}' + expect: + json: { path: nameAvailable, equals: true } + cleanup: + - group delete --name {rg} --yes --no-wait + + - id: afd-profile-upgrade-delete + operations: [Microsoft.Cdn AfdProfiles Upgrade, Microsoft.Cdn Profiles Delete] + setup: + - group create --name {rg} --location {location} + - afd profile create --profile-name {name:afd} --resource-group {rg} --sku Standard_AzureFrontDoor + - >- + rest --method post + --url /subscriptions/{sub}/resourceGroups/{rg}/providers/Microsoft.Cdn/profiles/{name:afd}/upgrade?api-version=2024-02-01 + --body '{"wafMappingList": []}' + command: afd profile delete --profile-name {name:afd} --resource-group {rg} + expect: + exitCode: 0 + cleanup: + - group delete --name {rg} --yes --no-wait + + - id: afd-generate-sso-uri + operations: [Microsoft.Cdn Profiles GenerateSsoUri] + setup: + - group create --name {rg} --location {location} + - afd profile create --profile-name {name:afd} --resource-group {rg} --sku Standard_AzureFrontDoor + command: >- + rest --method post + --url /subscriptions/{sub}/resourceGroups/{rg}/providers/Microsoft.Cdn/profiles/{name:afd}/generateSsoUri?api-version=2024-02-01 + expect: + json: { path: ssoUriValue, exists: true } + cleanup: + - group delete --name {rg} --yes --no-wait + known_gap: + reason: >- + GenerateSsoUri exists only for classic Verizon SKUs, which are + retired; on a Front Door profile the emulator answers 400 "action isn't allowed on this + resource for this sku". + date: 2026-09-27 + + # --- Classic CDN (CDN_CLASSIC_ALLOW_CREATE=1) --- + + - id: cdn-classic-endpoint + operations: + [ + Microsoft.Cdn Endpoints Create, + Microsoft.Cdn Endpoints Get, + Microsoft.Cdn Endpoints ListByProfile, + Microsoft.Cdn Endpoints Update, + Microsoft.Cdn Endpoints Stop, + Microsoft.Cdn Endpoints Start, + Microsoft.Cdn Endpoints PurgeContent, + Microsoft.Cdn Endpoints LoadContent, + Microsoft.Cdn Endpoints ValidateCustomDomain, + Microsoft.Cdn Endpoints ListResourceUsage, + Microsoft.Cdn Profiles ListSupportedOptimizationTypes, + Microsoft.Cdn Endpoints Delete, + ] + setup: + - group create --name {rg} --location {location} + - cdn profile create --name {name:cdn} --resource-group {rg} --sku Standard_Microsoft + - cdn endpoint create --name {name:ep} --profile-name {name:cdn} --resource-group {rg} --origin www.example.com + - run: cdn endpoint show --name {name:ep} --profile-name {name:cdn} --resource-group {rg} --query hostName -o tsv + until: { stdout: { contains: "{name:ep}" } } + - cdn endpoint list --profile-name {name:cdn} --resource-group {rg} + - cdn endpoint update --name {name:ep} --profile-name {name:cdn} --resource-group {rg} --no-http false + - cdn endpoint stop --name {name:ep} --profile-name {name:cdn} --resource-group {rg} + - cdn endpoint start --name {name:ep} --profile-name {name:cdn} --resource-group {rg} + - cdn endpoint purge --name {name:ep} --profile-name {name:cdn} --resource-group {rg} --content-paths "/*" + - cdn endpoint load --name {name:ep} --profile-name {name:cdn} --resource-group {rg} --content-paths /index.html + - cdn endpoint validate-custom-domain --name {name:ep} --profile-name {name:cdn} --resource-group {rg} --host-name www.{id}.example.com + - >- + rest --method post + --url /subscriptions/{sub}/resourceGroups/{rg}/providers/Microsoft.Cdn/profiles/{name:cdn}/endpoints/{name:ep}/checkResourceUsage?api-version=2024-02-01 + # No `cdn profile list-supported-optimization-types` in the curated cdn extension (1.0.0b3). + - run: >- + rest --method post + --url /subscriptions/{sub}/resourceGroups/{rg}/providers/Microsoft.Cdn/profiles/{name:cdn}/getSupportedOptimizationTypes?api-version=2024-02-01 + until: { json: { path: supportedOptimizationTypes, contains: GeneralWebDelivery } } + command: cdn endpoint delete --name {name:ep} --profile-name {name:cdn} --resource-group {rg} + expect: + exitCode: 0 + cleanup: + - group delete --name {rg} --yes --no-wait + + - id: cdn-classic-origins-domains + operations: + [ + Microsoft.Cdn Origins Get, + Microsoft.Cdn Origins ListByEndpoint, + Microsoft.Cdn Origins Update, + Microsoft.Cdn Origins Create, + Microsoft.Cdn Origins Delete, + Microsoft.Cdn OriginGroups Create, + Microsoft.Cdn OriginGroups Get, + Microsoft.Cdn OriginGroups ListByEndpoint, + Microsoft.Cdn OriginGroups Update, + Microsoft.Cdn OriginGroups Delete, + Microsoft.Cdn CustomDomains Create, + Microsoft.Cdn CustomDomains Get, + Microsoft.Cdn CustomDomains ListByEndpoint, + Microsoft.Cdn CustomDomains EnableCustomHttps, + Microsoft.Cdn CustomDomains DisableCustomHttps, + Microsoft.Cdn CustomDomains Delete, + ] + setup: + - group create --name {rg} --location {location} + - cdn profile create --name {name:cdn} --resource-group {rg} --sku Standard_Microsoft + - cdn endpoint create --name {name:ep} --profile-name {name:cdn} --resource-group {rg} --origin www.example.com + - run: cdn origin list --endpoint-name {name:ep} --profile-name {name:cdn} --resource-group {rg} --query "[0].name" -o tsv + capture: origin + - cdn origin show --endpoint-name {name:ep} --profile-name {name:cdn} --resource-group {rg} --name {origin} + - cdn origin update --endpoint-name {name:ep} --profile-name {name:cdn} --resource-group {rg} --name {origin} --http-port 8080 + - run: >- + cdn origin create --endpoint-name {name:ep} --profile-name {name:cdn} --resource-group {rg} --name backup + --host-name backup.example.com --query id -o tsv + capture: backup_id + # Shorthand --formatted-origins and --health-probe-settings (cdn extension 1.0.0b3; the + # older --origins and --probe-interval are gone). + - >- + cdn origin-group create --endpoint-name {name:ep} --profile-name {name:cdn} --resource-group {rg} --name og + --formatted-origins "[{id:{backup_id}}]" + --health-probe-settings "{probe-interval-in-seconds:120,probe-path:/,probe-protocol:Http,probe-request-type:GET}" + - cdn origin-group show --endpoint-name {name:ep} --profile-name {name:cdn} --resource-group {rg} --name og + - cdn origin-group list --endpoint-name {name:ep} --profile-name {name:cdn} --resource-group {rg} + - run: >- + cdn origin-group update --endpoint-name {name:ep} --profile-name {name:cdn} --resource-group {rg} --name og + --health-probe-settings "{probe-interval-in-seconds:60,probe-path:/health,probe-protocol:Http,probe-request-type:HEAD}" + --query healthProbeSettings.probeIntervalInSeconds + until: { json: { path: "", equals: 60 } } + - cdn origin-group delete --endpoint-name {name:ep} --profile-name {name:cdn} --resource-group {rg} --name og --yes + - cdn origin delete --endpoint-name {name:ep} --profile-name {name:cdn} --resource-group {rg} --name backup --yes + - >- + cdn custom-domain create --endpoint-name {name:ep} --profile-name {name:cdn} --resource-group {rg} + --name www --hostname www.{id}.example.com + - cdn custom-domain show --endpoint-name {name:ep} --profile-name {name:cdn} --resource-group {rg} --name www + - cdn custom-domain list --endpoint-name {name:ep} --profile-name {name:cdn} --resource-group {rg} + - cdn custom-domain enable-https --endpoint-name {name:ep} --profile-name {name:cdn} --resource-group {rg} --name www + - cdn custom-domain disable-https --endpoint-name {name:ep} --profile-name {name:cdn} --resource-group {rg} --name www + command: cdn custom-domain delete --endpoint-name {name:ep} --profile-name {name:cdn} --resource-group {rg} --name www + expect: + exitCode: 0 + cleanup: + - group delete --name {rg} --yes --no-wait + + - id: cdn-waf-policy + operations: + [ + Microsoft.Cdn Policies CreateOrUpdate, + Microsoft.Cdn Policies Get, + Microsoft.Cdn Policies List, + Microsoft.Cdn Policies Update, + Microsoft.Cdn Policies Delete, + ] + note: >- + Through rest: az 2.85 with the curated cdn extension (1.0.0b3) has no `cdn waf policy` group + (L2 flag lint, 2026-09-27). + setup: + - group create --name {rg} --location {location} + - >- + rest --method put + --url /subscriptions/{sub}/resourceGroups/{rg}/providers/Microsoft.Cdn/CdnWebApplicationFirewallPolicies/{alnum:waf}?api-version=2024-02-01 + --body '{"location": "Global", "sku": {"name": "Standard_Microsoft"}, "properties": {"policySettings": {"enabledState": "Enabled", "mode": "Detection"}}}' + - run: >- + rest --method get + --url /subscriptions/{sub}/resourceGroups/{rg}/providers/Microsoft.Cdn/CdnWebApplicationFirewallPolicies/{alnum:waf}?api-version=2024-02-01 + until: { json: { path: properties.policySettings.mode, equals: Detection } } + - run: >- + rest --method get + --url /subscriptions/{sub}/resourceGroups/{rg}/providers/Microsoft.Cdn/CdnWebApplicationFirewallPolicies?api-version=2024-02-01 + --query "value[].name" + until: { json: { path: "", equals: ["{alnum:waf}"] } } + - >- + rest --method patch + --url /subscriptions/{sub}/resourceGroups/{rg}/providers/Microsoft.Cdn/CdnWebApplicationFirewallPolicies/{alnum:waf}?api-version=2024-02-01 + --body '{"tags": {"matrix": "l2"}}' + command: >- + rest --method delete + --url /subscriptions/{sub}/resourceGroups/{rg}/providers/Microsoft.Cdn/CdnWebApplicationFirewallPolicies/{alnum:waf}?api-version=2024-02-01 + expect: + exitCode: 0 + cleanup: + - group delete --name {rg} --yes --no-wait + + - id: cdn-migration + operations: [Microsoft.Cdn Profiles CdnCanMigrateToAfd, Microsoft.Cdn Profiles CdnMigrateToAfd] + setup: + - group create --name {rg} --location {location} + - cdn profile create --name {name:cdn} --resource-group {rg} --sku Standard_Microsoft + # The emulator refuses to migrate a profile without endpoints ("has no endpoints to migrate"). + - cdn endpoint create --name {name:ep} --profile-name {name:cdn} --resource-group {rg} --origin www.example.com + - cdn profile-migration check-compatibility --profile-name {name:cdn} --resource-group {rg} + command: cdn profile-migration migrate --profile-name {name:cdn} --resource-group {rg} --sku Standard_AzureFrontDoor + expect: + json: { path: migratedProfileResourceId.id, contains: "{name:cdn}" } + cleanup: + - group delete --name {rg} --yes --no-wait + + - id: cdn-migration-abort + operations: [Microsoft.Cdn Profiles MigrationAbort] + setup: + - group create --name {rg} --location {location} + - cdn profile create --name {name:cdn} --resource-group {rg} --sku Standard_Microsoft + - cdn endpoint create --name {name:ep} --profile-name {name:cdn} --resource-group {rg} --origin www.example.com + - cdn profile-migration migrate --profile-name {name:cdn} --resource-group {rg} --sku Standard_AzureFrontDoor + command: cdn profile-migration abort --profile-name {name:cdn} --resource-group {rg} + expect: + exitCode: 0 + cleanup: + - group delete --name {rg} --yes --no-wait + known_gap: + reason: >- + L2: POST .../migrationAbort answers 202 with a Location header, and polling that Location + answers 200 with `{"status": "Succeeded"}`. A 200 from a Location URL is the operation's + final result, and abort has none (ARM: 204), so the CLI's poller (cdn extension 1.0.0b3) + crashes with TypeError "'NoneType' object is not callable" (seen in --debug). + date: 2026-09-27 + + - id: cdn-migration-commit + operations: [Microsoft.Cdn Profiles MigrationCommit] + setup: + - group create --name {rg} --location {location} + - cdn profile create --name {name:cdn} --resource-group {rg} --sku Standard_Microsoft + # The emulator refuses to migrate a profile without endpoints ("has no endpoints to migrate"). + - cdn endpoint create --name {name:ep} --profile-name {name:cdn} --resource-group {rg} --origin www.example.com + - cdn profile-migration migrate --profile-name {name:cdn} --resource-group {rg} --sku Standard_AzureFrontDoor + command: cdn profile-migration commit --profile-name {name:cdn} --resource-group {rg} + expect: + exitCode: 0 + cleanup: + - group delete --name {rg} --yes --no-wait + + - id: afd-classic-migration + operations: [Microsoft.Cdn Profiles CanMigrate, Microsoft.Cdn Profiles Migrate] + note: Classic Front Door (Microsoft.Network) to Front Door Standard; CI sets FRONT_DOOR_CLASSIC_ALLOW_CREATE=1. + setup: + - group create --name {rg} --location {location} + - run: network front-door create --name {name:fd} --resource-group {rg} --backend-address www.example.com --query id -o tsv + capture: fd_id + - >- + rest --method post --url /subscriptions/{sub}/resourceGroups/{rg}/providers/Microsoft.Cdn/canMigrate?api-version=2024-02-01 + --body '{"classicResourceReference": {"id": "{fd_id}"}}' + command: >- + rest --method post --url /subscriptions/{sub}/resourceGroups/{rg}/providers/Microsoft.Cdn/migrate?api-version=2024-02-01 + --body '{"sku": {"name": "Standard_AzureFrontDoor"}, "classicResourceReference": {"id": "{fd_id}"}, "profileName": "{name:afd}"}' + expect: + exitCode: 0 + cleanup: + - group delete --name {rg} --yes --no-wait + + # --- Edge actions (rest; no stable az command group) --- + + - id: cdn-edge-action + operations: + [ + Microsoft.Cdn EdgeActions Create, + Microsoft.Cdn EdgeActions Get, + Microsoft.Cdn EdgeActions ListByResourceGroup, + Microsoft.Cdn EdgeActions ListBySubscription, + Microsoft.Cdn EdgeActions Update, + Microsoft.Cdn EdgeActionVersions Create, + Microsoft.Cdn EdgeActionVersions Get, + Microsoft.Cdn EdgeActionVersions ListByEdgeAction, + Microsoft.Cdn EdgeActionVersions Update, + Microsoft.Cdn EdgeActionVersions DeployVersionCode, + Microsoft.Cdn EdgeActionVersions GetVersionCode, + Microsoft.Cdn EdgeActionVersions SwapDefault, + Microsoft.Cdn EdgeActionExecutionFilters Create, + Microsoft.Cdn EdgeActionExecutionFilters Get, + Microsoft.Cdn EdgeActionExecutionFilters ListByEdgeAction, + Microsoft.Cdn EdgeActionExecutionFilters Update, + Microsoft.Cdn EdgeActionExecutionFilters Delete, + Microsoft.Cdn EdgeActionVersions Delete, + Microsoft.Cdn EdgeActions Delete, + ] + setup: + - group create --name {rg} --location {location} + - >- + rest --method put + --url /subscriptions/{sub}/resourceGroups/{rg}/providers/Microsoft.Cdn/edgeActions/{alnum:ea}?api-version=2025-09-01-preview + --body '{"location": "global", "sku": {"name": "Standard", "tier": "Standard"}, "properties": {}}' + - run: >- + rest --method get + --url /subscriptions/{sub}/resourceGroups/{rg}/providers/Microsoft.Cdn/edgeActions/{alnum:ea}?api-version=2025-09-01-preview + until: { json: { path: name, equals: "{alnum:ea}" } } + - >- + rest --method get + --url /subscriptions/{sub}/resourceGroups/{rg}/providers/Microsoft.Cdn/edgeActions?api-version=2025-09-01-preview + - rest --method get --url /subscriptions/{sub}/providers/Microsoft.Cdn/edgeActions?api-version=2025-09-01-preview + - >- + rest --method patch + --url /subscriptions/{sub}/resourceGroups/{rg}/providers/Microsoft.Cdn/edgeActions/{alnum:ea}?api-version=2025-09-01-preview + --body '{"tags": {"matrix": "l2"}}' + - >- + rest --method put + --url /subscriptions/{sub}/resourceGroups/{rg}/providers/Microsoft.Cdn/edgeActions/{alnum:ea}/versions/v1?api-version=2025-09-01-preview + --body '{"location": "global", "properties": {"deploymentType": "file", "isDefaultVersion": "True"}}' + - >- + rest --method get + --url /subscriptions/{sub}/resourceGroups/{rg}/providers/Microsoft.Cdn/edgeActions/{alnum:ea}/versions/v1?api-version=2025-09-01-preview + - >- + rest --method get + --url /subscriptions/{sub}/resourceGroups/{rg}/providers/Microsoft.Cdn/edgeActions/{alnum:ea}/versions?api-version=2025-09-01-preview + - >- + rest --method patch + --url /subscriptions/{sub}/resourceGroups/{rg}/providers/Microsoft.Cdn/edgeActions/{alnum:ea}/versions/v1?api-version=2025-09-01-preview + --body '{"tags": {"matrix": "l2"}}' + - >- + rest --method post + --url /subscriptions/{sub}/resourceGroups/{rg}/providers/Microsoft.Cdn/edgeActions/{alnum:ea}/versions/v1/deployVersionCode?api-version=2025-09-01-preview + --body '{"content": "ZXhwb3J0IGRlZmF1bHQge30=", "name": "index.js"}' + - >- + rest --method post + --url /subscriptions/{sub}/resourceGroups/{rg}/providers/Microsoft.Cdn/edgeActions/{alnum:ea}/versions/v1/getVersionCode?api-version=2025-09-01-preview + - >- + rest --method post + --url /subscriptions/{sub}/resourceGroups/{rg}/providers/Microsoft.Cdn/edgeActions/{alnum:ea}/versions/v1/swapDefault?api-version=2025-09-01-preview + - >- + rest --method put + --url /subscriptions/{sub}/resourceGroups/{rg}/providers/Microsoft.Cdn/edgeActions/{alnum:ea}/executionFilters/f1?api-version=2025-09-01-preview + --body '{"location": "global", "properties": {"versionId": "v1", "executionFilterIdentifierHeaderName": "x-l2", + "executionFilterIdentifierHeaderValue": "on"}}' + - >- + rest --method get + --url /subscriptions/{sub}/resourceGroups/{rg}/providers/Microsoft.Cdn/edgeActions/{alnum:ea}/executionFilters/f1?api-version=2025-09-01-preview + - >- + rest --method get + --url /subscriptions/{sub}/resourceGroups/{rg}/providers/Microsoft.Cdn/edgeActions/{alnum:ea}/executionFilters?api-version=2025-09-01-preview + # The PATCH is validated as a full PUT (cdn-edge-action-filter-patch-tags), so this one + # repeats every required property. + - >- + rest --method patch + --url /subscriptions/{sub}/resourceGroups/{rg}/providers/Microsoft.Cdn/edgeActions/{alnum:ea}/executionFilters/f1?api-version=2025-09-01-preview + --body '{"tags": {"matrix": "l2"}, "properties": {"versionId": "v1", "executionFilterIdentifierHeaderName": "x-l2", + "executionFilterIdentifierHeaderValue": "on"}}' + - >- + rest --method delete + --url /subscriptions/{sub}/resourceGroups/{rg}/providers/Microsoft.Cdn/edgeActions/{alnum:ea}/executionFilters/f1?api-version=2025-09-01-preview + - >- + rest --method delete + --url /subscriptions/{sub}/resourceGroups/{rg}/providers/Microsoft.Cdn/edgeActions/{alnum:ea}/versions/v1?api-version=2025-09-01-preview + command: >- + rest --method delete + --url /subscriptions/{sub}/resourceGroups/{rg}/providers/Microsoft.Cdn/edgeActions/{alnum:ea}?api-version=2025-09-01-preview + expect: + exitCode: 0 + cleanup: + - group delete --name {rg} --yes --no-wait + + - id: cdn-edge-action-filter-patch-tags + operations: [Microsoft.Cdn EdgeActionExecutionFilters Update] + setup: + - group create --name {rg} --location {location} + - >- + rest --method put + --url /subscriptions/{sub}/resourceGroups/{rg}/providers/Microsoft.Cdn/edgeActions/{alnum:ea}?api-version=2025-09-01-preview + --body '{"location": "global", "sku": {"name": "Standard", "tier": "Standard"}, "properties": {}}' + - >- + rest --method put + --url /subscriptions/{sub}/resourceGroups/{rg}/providers/Microsoft.Cdn/edgeActions/{alnum:ea}/versions/v1?api-version=2025-09-01-preview + --body '{"location": "global", "properties": {"deploymentType": "file", "isDefaultVersion": "True"}}' + - >- + rest --method put + --url /subscriptions/{sub}/resourceGroups/{rg}/providers/Microsoft.Cdn/edgeActions/{alnum:ea}/executionFilters/f1?api-version=2025-09-01-preview + --body '{"location": "global", "properties": {"versionId": "v1", "executionFilterIdentifierHeaderName": "x-l2", + "executionFilterIdentifierHeaderValue": "on"}}' + command: >- + rest --method patch + --url /subscriptions/{sub}/resourceGroups/{rg}/providers/Microsoft.Cdn/edgeActions/{alnum:ea}/executionFilters/f1?api-version=2025-09-01-preview + --body '{"tags": {"matrix": "l2"}}' + expect: + json: { path: tags.matrix, equals: l2 } + cleanup: + - group delete --name {rg} --yes --no-wait + known_gap: + reason: >- + L2: the execution filter PATCH is validated as a full PUT: tags alone answer 400 "Property + 'EdgeActionExecutionFilter.Properties.VersionId' is required but it was not set", and with + the version id the next required property is asked for. A PATCH is partial (the edge action + and version PATCHes accept tags alone). + date: 2026-09-27 + + - id: cdn-edge-action-attachment + operations: + [Microsoft.Cdn EdgeActions AddAttachment, Microsoft.Cdn EdgeActions DeleteAttachment] + setup: + - group create --name {rg} --location {location} + - >- + rest --method put + --url /subscriptions/{sub}/resourceGroups/{rg}/providers/Microsoft.Cdn/edgeActions/{alnum:ea}?api-version=2025-09-01-preview + --body '{"location": "global", "sku": {"name": "Standard", "tier": "Standard"}, "properties": {}}' + command: >- + rest --method post + --url /subscriptions/{sub}/resourceGroups/{rg}/providers/Microsoft.Cdn/edgeActions/{alnum:ea}/addAttachment?api-version=2025-09-01-preview + --body '{"attachedResourceId": "/subscriptions/{sub}/resourceGroups/{rg}/providers/Microsoft.Cdn/profiles/p/ruleSets/rs/rules/r"}' + expect: + exitCode: 0 + cleanup: + - group delete --name {rg} --yes --no-wait + known_gap: + reason: >- + The handler dereferences its request model unguarded + (`A=parameters; D.require_edge_action(A.subscription_id, ...)`) and the parser passes None, so + AddAttachment and DeleteAttachment answer HTTP 500 AttributeError for every body and + api-version (verified 2026-09-22 with four bodies and four api-versions). + date: 2026-09-27 diff --git a/tests/azure/matrix/compute.yaml b/tests/azure/matrix/compute.yaml new file mode 100644 index 0000000..f975db2 --- /dev/null +++ b/tests/azure/matrix/compute.yaml @@ -0,0 +1,212 @@ +# Microsoft.Compute: managed disks, the image catalogue, sizes and SKUs, virtual machines and +# scale sets. VMs and scale sets are `backing: true` (the emulator may run them as containers). +# `--generate-ssh-keys` writes into the tool's private home, never the real ~/.ssh. +provider: Microsoft.Compute +cases: + - id: compute-disk + operations: [Microsoft.Compute Disks CreateOrUpdate, Microsoft.Compute Disks Get] + pr: true + setup: + - group create --name {rg} --location {location} + - disk create --name {name:disk} --resource-group {rg} --location {location} --size-gb 8 --sku StandardSSD_LRS + command: disk show --name {name:disk} --resource-group {rg} + expect: + json: + - { path: diskSizeGB, equals: 8 } + - { path: sku.name, equals: StandardSSD_LRS } + cleanup: + - group delete --name {rg} --yes --no-wait + + - id: compute-disk-manage + operations: + [ + Microsoft.Compute Disks ListByResourceGroup, + Microsoft.Compute Disks List, + Microsoft.Compute Disks Update, + Microsoft.Compute Disks Delete, + ] + setup: + - group create --name {rg} --location {location} + - disk create --name {name:disk} --resource-group {rg} --location {location} --size-gb 8 + - run: disk list --resource-group {rg} --query "[].name" + until: { json: { path: "", equals: ["{name:disk}"] } } + - rest --method get --url /subscriptions/{sub}/providers/Microsoft.Compute/disks?api-version=2023-10-02 + - run: disk update --name {name:disk} --resource-group {rg} --size-gb 16 + until: { json: { path: diskSizeGB, equals: 16 } } + command: disk delete --name {name:disk} --resource-group {rg} --yes + expect: + exitCode: 0 + cleanup: + - group delete --name {rg} --yes --no-wait + + - id: compute-image-catalogue + operations: + [ + Microsoft.Compute VirtualMachineImages ListPublishers, + Microsoft.Compute VirtualMachineImages ListOffers, + Microsoft.Compute VirtualMachineImages ListSkus, + Microsoft.Compute VirtualMachineImages List, + ] + setup: + - run: vm image list-publishers --location {location} --query "[?name=='Canonical'].name" + until: { json: { path: "", equals: [Canonical] } } + - vm image list-offers --location {location} --publisher Canonical --query "[].name" + - vm image list-skus --location {location} --publisher Canonical --offer 0001-com-ubuntu-server-jammy --query "[].name" + command: >- + vm image list --location {location} --publisher Canonical --offer 0001-com-ubuntu-server-jammy + --sku 22_04-lts-gen2 --all --query "length(@) > `0`" + expect: + json: { path: "", equals: true } + + - id: compute-image-get + operations: [Microsoft.Compute VirtualMachineImages Get] + setup: + - run: >- + vm image list --location {location} --publisher Canonical --offer 0001-com-ubuntu-server-jammy + --sku 22_04-lts-gen2 --all --query "[0].version" -o tsv + capture: version + command: vm image show --location {location} --urn Canonical:0001-com-ubuntu-server-jammy:22_04-lts-gen2:{version} + expect: + json: { path: name, equals: "{version}" } + + - id: compute-sizes-skus + operations: [Microsoft.Compute VirtualMachineSizes List, Microsoft.Compute ResourceSkus List] + setup: + - run: vm list-sizes --location {location} --query "[?name=='Standard_B2s'].name" + until: { json: { path: "", equals: [Standard_B2s] } } + command: vm list-skus --location {location} --resource-type virtualMachines --query "length(@) > `0`" + expect: + json: { path: "", equals: true } + + - id: compute-vm + operations: + [ + Microsoft.Compute VirtualMachines CreateOrUpdate, + Microsoft.Compute VirtualMachines Get, + Microsoft.Compute VirtualMachines List, + Microsoft.Compute VirtualMachines InstanceView, + Microsoft.Compute VirtualMachines ListAvailableSizes, + ] + backing: true + setup: + - group create --name {rg} --location {location} + - >- + vm create --name {name:vm} --resource-group {rg} --location {location} + --image Canonical:0001-com-ubuntu-server-jammy:22_04-lts-gen2:latest --size Standard_B1s + --admin-username azureuser --generate-ssh-keys --public-ip-address "" --nsg "" + - run: vm show --name {name:vm} --resource-group {rg} --query provisioningState -o tsv + until: { stdout: { equals: Succeeded } } + - vm list --resource-group {rg} + - vm list-vm-resize-options --name {name:vm} --resource-group {rg} --query "length(@)" + command: vm get-instance-view --name {name:vm} --resource-group {rg} --query "instanceView.statuses[].code" + expect: + json: { path: "", contains: ProvisioningState/succeeded } + cleanup: + - group delete --name {rg} --yes --no-wait + + - id: compute-vm-delete + operations: [Microsoft.Compute VirtualMachines Delete] + backing: true + setup: + - group create --name {rg} --location {location} + - >- + vm create --name {name:vm} --resource-group {rg} --location {location} + --image Canonical:0001-com-ubuntu-server-jammy:22_04-lts-gen2:latest --size Standard_B1s + --admin-username azureuser --generate-ssh-keys --public-ip-address "" --nsg "" + command: vm delete --name {name:vm} --resource-group {rg} --yes + expect: + exitCode: 0 + cleanup: + - group delete --name {rg} --yes --no-wait + + - id: compute-vm-attach-detach-disks + operations: [Microsoft.Compute VirtualMachines AttachDetachDataDisks] + backing: true + setup: + - group create --name {rg} --location {location} + - >- + vm create --name {name:vm} --resource-group {rg} --location {location} + --image Canonical:0001-com-ubuntu-server-jammy:22_04-lts-gen2:latest --size Standard_B1s + --admin-username azureuser --generate-ssh-keys --public-ip-address "" --nsg "" + - run: disk create --name {name:disk} --resource-group {rg} --location {location} --size-gb 8 --query id -o tsv + capture: disk_id + command: >- + rest --method post + --url /subscriptions/{sub}/resourceGroups/{rg}/providers/Microsoft.Compute/virtualMachines/{name:vm}/attachDetachDataDisks?api-version=2024-03-01 + --body '{"dataDisksToAttach": [{"lun": 1, "diskId": "{disk_id}"}]}' + expect: + exitCode: 0 + cleanup: + - group delete --name {rg} --yes --no-wait + + - id: compute-vmss + operations: + [ + Microsoft.Compute VirtualMachineScaleSets CreateOrUpdate, + Microsoft.Compute VirtualMachineScaleSets Get, + Microsoft.Compute VirtualMachineScaleSets List, + Microsoft.Compute VirtualMachineScaleSets ListSkus, + Microsoft.Compute VirtualMachineScaleSets UpdateInstances, + Microsoft.Compute VirtualMachineScaleSetVMs List, + ] + backing: true + setup: + - group create --name {rg} --location {location} + - >- + vmss create --name {name:vmss} --resource-group {rg} --location {location} + --image Canonical:0001-com-ubuntu-server-jammy:22_04-lts-gen2:latest --vm-sku Standard_B1s + --instance-count 1 --admin-username azureuser --generate-ssh-keys --lb "" --public-ip-address "" + - run: vmss show --name {name:vmss} --resource-group {rg} --query provisioningState -o tsv + until: { stdout: { equals: Succeeded } } + - vmss list --resource-group {rg} + - vmss list-instances --name {name:vmss} --resource-group {rg} + - >- + rest --method get + --url /subscriptions/{sub}/resourceGroups/{rg}/providers/Microsoft.Compute/virtualMachineScaleSets/{name:vmss}/skus?api-version=2024-03-01 + command: vmss update-instances --name {name:vmss} --resource-group {rg} --instance-ids "*" + expect: + exitCode: 0 + cleanup: + - group delete --name {rg} --yes --no-wait + + - id: compute-vmss-delete + operations: [Microsoft.Compute VirtualMachineScaleSets Delete] + backing: true + setup: + - group create --name {rg} --location {location} + - >- + vmss create --name {name:vmss} --resource-group {rg} --location {location} + --image Canonical:0001-com-ubuntu-server-jammy:22_04-lts-gen2:latest --vm-sku Standard_B1s + --instance-count 1 --admin-username azureuser --generate-ssh-keys --lb "" --public-ip-address "" + command: vmss delete --name {name:vmss} --resource-group {rg} + expect: + exitCode: 0 + cleanup: + - group delete --name {rg} --yes --no-wait + + - id: compute-vmss-instances + operations: + [ + Microsoft.Compute VirtualMachineScaleSetVMs Get, + Microsoft.Compute VirtualMachineScaleSetVMs Update, + Microsoft.Compute NetworkInterfaces ListVirtualMachineScaleSetVMNetworkInterfaces, + Microsoft.Compute NetworkInterfaces GetVirtualMachineScaleSetNetworkInterface, + ] + backing: true + setup: + - group create --name {rg} --location {location} + - >- + vmss create --name {name:vmss} --resource-group {rg} --location {location} + --image Canonical:0001-com-ubuntu-server-jammy:22_04-lts-gen2:latest --vm-sku Standard_B1s + --instance-count 1 --admin-username azureuser --generate-ssh-keys --lb "" --public-ip-address "" + command: vmss show --name {name:vmss} --resource-group {rg} --instance-id 0 + expect: + json: { path: instanceId, equals: "0" } + cleanup: + - group delete --name {rg} --yes --no-wait + known_gap: + reason: >- + The emulator's scale sets never materialize VM instances, so + GET/PATCH .../virtualMachines/{instanceId} and the per-instance network-interface calls answer + 404 (the instance list is empty). + date: 2026-09-27 diff --git a/tests/azure/matrix/containerinstance.yaml b/tests/azure/matrix/containerinstance.yaml new file mode 100644 index 0000000..a1c8fc0 --- /dev/null +++ b/tests/azure/matrix/containerinstance.yaml @@ -0,0 +1,119 @@ +# Microsoft.ContainerInstance: container groups. Every case runs a real container on the +# emulator's Docker engine, so all are `backing: true` (nginx:latest). +# `container exec` and `container attach` are refused by the policy (they open a shell or a +# stream), so Attach and ExecuteCommand go through rest, which only returns the connection info. +# ExecuteCommand needs the inner container's name, not the group's; az names the container +# after the group, so the case below passes that name. +provider: Microsoft.ContainerInstance +cases: + - id: aci-create + operations: + [ + Microsoft.ContainerInstance ContainerGroups CreateOrUpdate, + Microsoft.ContainerInstance ContainerGroups Get, + ] + backing: true + note: Samples container create/show (aci-blob-storage). + setup: + - group create --name {rg} --location {location} + - >- + container create --name {name:aci} --resource-group {rg} --location {location} --image nginx:latest + --os-type Linux --cpu 1 --memory 1 --ports 80 --restart-policy Never + command: container show --name {name:aci} --resource-group {rg} + expect: + json: + - { path: provisioningState, equals: Succeeded } + - { path: "containers[0].image", equals: "nginx:latest" } + cleanup: + - group delete --name {rg} --yes --no-wait + + - id: aci-list-logs + operations: + [ + Microsoft.ContainerInstance ContainerGroups ListByResourceGroup, + Microsoft.ContainerInstance ContainerGroups List, + Microsoft.ContainerInstance Containers ListLogs, + ] + backing: true + note: Samples container list/logs. + setup: + - group create --name {rg} --location {location} + - container create --name {name:aci} --resource-group {rg} --image nginx:latest --os-type Linux --ports 80 + - run: container list --resource-group {rg} --query "[].name" + until: { json: { path: "", equals: ["{name:aci}"] } } + - container list --query "[?name=='{name:aci}'].name" + command: container logs --name {name:aci} --resource-group {rg} + expect: + exitCode: 0 + cleanup: + - group delete --name {rg} --yes --no-wait + + - id: aci-stop-start-restart + operations: + [ + Microsoft.ContainerInstance ContainerGroups Stop, + Microsoft.ContainerInstance ContainerGroups Start, + Microsoft.ContainerInstance ContainerGroups Restart, + ] + backing: true + note: Samples container stop/start/restart. + setup: + - group create --name {rg} --location {location} + - container create --name {name:aci} --resource-group {rg} --image nginx:latest --os-type Linux --ports 80 + - container stop --name {name:aci} --resource-group {rg} + - container start --name {name:aci} --resource-group {rg} + command: container restart --name {name:aci} --resource-group {rg} + expect: + exitCode: 0 + cleanup: + - group delete --name {rg} --yes --no-wait + + - id: aci-update-attach + operations: + [ + Microsoft.ContainerInstance ContainerGroups Update, + Microsoft.ContainerInstance Containers Attach, + ] + backing: true + setup: + - group create --name {rg} --location {location} + - container create --name {name:aci} --resource-group {rg} --image nginx:latest --os-type Linux --ports 80 + - >- + rest --method patch + --url /subscriptions/{sub}/resourceGroups/{rg}/providers/Microsoft.ContainerInstance/containerGroups/{name:aci}?api-version=2023-05-01 + --body '{"tags": {"matrix": "l2"}}' + command: >- + rest --method post + --url /subscriptions/{sub}/resourceGroups/{rg}/providers/Microsoft.ContainerInstance/containerGroups/{name:aci}/containers/{name:aci}/attach?api-version=2023-05-01 + expect: + json: { path: webSocketUri, exists: true } + cleanup: + - group delete --name {rg} --yes --no-wait + + - id: aci-execute-command + operations: [Microsoft.ContainerInstance Containers ExecuteCommand] + backing: true + setup: + - group create --name {rg} --location {location} + - container create --name {name:aci} --resource-group {rg} --image nginx:latest --os-type Linux --ports 80 + command: >- + rest --method post + --url /subscriptions/{sub}/resourceGroups/{rg}/providers/Microsoft.ContainerInstance/containerGroups/{name:aci}/containers/{name:aci}/exec?api-version=2023-05-01 + --body '{"command": "/bin/sh", "terminalSize": {"rows": 24, "cols": 80}}' + expect: + json: { path: webSocketUri, exists: true } + cleanup: + - group delete --name {rg} --yes --no-wait + + - id: aci-delete + operations: [Microsoft.ContainerInstance ContainerGroups Delete] + backing: true + note: Samples container delete. + setup: + - group create --name {rg} --location {location} + - container create --name {name:aci} --resource-group {rg} --image nginx:latest --os-type Linux --ports 80 + command: container delete --name {name:aci} --resource-group {rg} --yes + expect: + exitCode: 0 + cleanup: + - group delete --name {rg} --yes --no-wait diff --git a/tests/azure/matrix/containerregistry.yaml b/tests/azure/matrix/containerregistry.yaml new file mode 100644 index 0000000..e8b416c --- /dev/null +++ b/tests/azure/matrix/containerregistry.yaml @@ -0,0 +1,454 @@ +# Microsoft.ContainerRegistry: registries (control plane), credentials, webhooks, replications, +# scope maps and tokens, cache rules, credential sets, connected registries, private link. +# `acr login` is allowed only with --expose-token (the policy), which returns a token instead of +# running `docker login` on this machine. Task runs and image imports can make the emulator start +# containers or pull images, so those cases are `backing: true`. +provider: Microsoft.ContainerRegistry +cases: + - id: acr-create + operations: [Microsoft.ContainerRegistry Registries Create] + pr: true + note: Samples acr create (web-app-custom-image). + setup: + - group create --name {rg} --location {location} + command: acr create --name {alnum:acr} --resource-group {rg} --location {location} --sku Basic --admin-enabled true + expect: + json: + - { path: name, equals: "{alnum:acr}" } + - { path: loginServer, contains: "{alnum:acr}" } + - { path: adminUserEnabled, equals: true } + cleanup: + - group delete --name {rg} --yes --no-wait + + - id: acr-read + operations: + [ + Microsoft.ContainerRegistry Registries Get, + Microsoft.ContainerRegistry Registries ListByResourceGroup, + Microsoft.ContainerRegistry Registries List, + Microsoft.ContainerRegistry Registries CheckNameAvailability, + ] + note: Samples acr show. + setup: + - group create --name {rg} --location {location} + - acr create --name {alnum:acr} --resource-group {rg} --location {location} --sku Basic + - run: acr show --name {alnum:acr} --resource-group {rg} --query loginServer -o tsv + until: { stdout: { contains: "{alnum:acr}" } } + - run: acr list --resource-group {rg} --query "[].name" + until: { json: { path: "", equals: ["{alnum:acr}"] } } + - acr list --query "[?name=='{alnum:acr}'].name" + command: acr check-name --name {alnum:acr} + expect: + json: { path: nameAvailable, equals: false } + cleanup: + - group delete --name {rg} --yes --no-wait + + - id: acr-credentials + operations: + [ + Microsoft.ContainerRegistry Registries ListCredentials, + Microsoft.ContainerRegistry Registries RegenerateCredential, + ] + pr: true + note: Samples acr credential show. + setup: + - group create --name {rg} --location {location} + - acr create --name {alnum:acr} --resource-group {rg} --location {location} --sku Basic --admin-enabled true + - run: acr credential show --name {alnum:acr} --resource-group {rg} --query username -o tsv + until: { stdout: { equals: "{alnum:acr}" } } + command: acr credential renew --name {alnum:acr} --resource-group {rg} --password-name password2 + expect: + json: { path: "passwords[1].name", equals: password2 } + cleanup: + - group delete --name {rg} --yes --no-wait + + - id: acr-update + operations: [Microsoft.ContainerRegistry Registries Update] + setup: + - group create --name {rg} --location {location} + - acr create --name {alnum:acr} --resource-group {rg} --location {location} --sku Premium + command: acr update --name {alnum:acr} --resource-group {rg} --tags matrix=l2 + expect: + json: { path: tags.matrix, equals: l2 } + cleanup: + - group delete --name {rg} --yes --no-wait + known_gap: + reason: >- + Found by L2: `acr update` (the PATCH az 2.85 sends for a Premium registry) answers HTTP 500 + InternalServerError "An unexpected error occurred while processing 'PATCH .../registries/...'". + date: 2026-09-27 + + - id: acr-usage-policies + operations: + [ + Microsoft.ContainerRegistry Registries ListUsages, + Microsoft.ContainerRegistry Registries UpdatePolicies, + ] + note: >- + The case posts updatePolicies with rest; this + enables the trust policy (disabled is the default) and reads it back from the registry. + setup: + - group create --name {rg} --location {location} + - acr create --name {alnum:acr} --resource-group {rg} --location {location} --sku Premium + - acr show-usage --name {alnum:acr} --resource-group {rg} + - >- + rest --method post + --url /subscriptions/{sub}/resourceGroups/{rg}/providers/Microsoft.ContainerRegistry/registries/{alnum:acr}/updatePolicies?api-version=2017-10-01 + --body '{"trustPolicy": {"type": "Notary", "status": "enabled"}}' + command: acr show --name {alnum:acr} --resource-group {rg} --query policies.trustPolicy.status -o tsv + expect: + stdout: { equals: enabled } + cleanup: + - group delete --name {rg} --yes --no-wait + + - id: acr-list-policies + operations: [Microsoft.ContainerRegistry Registries ListPolicies] + setup: + - group create --name {rg} --location {location} + - acr create --name {alnum:acr} --resource-group {rg} --location {location} --sku Premium + command: >- + rest --method post + --url /subscriptions/{sub}/resourceGroups/{rg}/providers/Microsoft.ContainerRegistry/registries/{alnum:acr}/listPolicies?api-version=2017-10-01 + expect: + json: { path: trustPolicy.type, equals: Notary } + cleanup: + - group delete --name {rg} --yes --no-wait + known_gap: + reason: >- + L2: the emulator routes Registries_ListPolicies as GET; the 2017-10-01 spec and its SDK + send POST .../listPolicies, which answers 404 (no route). The GET answers the policies. + date: 2026-09-27 + + - id: acr-login-expose-token + operations: [] + note: >- + The registry's token exchange is data plane (not in the coverage list); the policy requires + --expose-token so no `docker login` runs on this machine. + setup: + - group create --name {rg} --location {location} + - acr create --name {alnum:acr} --resource-group {rg} --location {location} --sku Basic --admin-enabled true + command: acr login --name {alnum:acr} --expose-token + expect: + json: + - { path: accessToken, exists: true } + - { path: loginServer, contains: "{alnum:acr}" } + cleanup: + - group delete --name {rg} --yes --no-wait + + - id: acr-webhooks + operations: + [ + Microsoft.ContainerRegistry Webhooks Create, + Microsoft.ContainerRegistry Webhooks Get, + Microsoft.ContainerRegistry Webhooks List, + Microsoft.ContainerRegistry Webhooks Update, + Microsoft.ContainerRegistry Webhooks Ping, + Microsoft.ContainerRegistry Webhooks GetCallbackConfig, + Microsoft.ContainerRegistry Webhooks ListEvents, + Microsoft.ContainerRegistry Webhooks Delete, + ] + setup: + - group create --name {rg} --location {location} + - acr create --name {alnum:acr} --resource-group {rg} --location {location} --sku Standard + - >- + acr webhook create --name {alnum:hook} --registry {alnum:acr} --resource-group {rg} + --uri https://example.com/hooks/{id} --actions push delete + - run: acr webhook show --name {alnum:hook} --registry {alnum:acr} --resource-group {rg} --query "sort(actions)" + until: { json: { path: "", equals: [delete, push] } } + - acr webhook list --registry {alnum:acr} --resource-group {rg} + - acr webhook update --name {alnum:hook} --registry {alnum:acr} --resource-group {rg} --status disabled + - acr webhook ping --name {alnum:hook} --registry {alnum:acr} --resource-group {rg} + - acr webhook get-config --name {alnum:hook} --registry {alnum:acr} --resource-group {rg} + - acr webhook list-events --name {alnum:hook} --registry {alnum:acr} --resource-group {rg} + command: acr webhook delete --name {alnum:hook} --registry {alnum:acr} --resource-group {rg} + expect: + exitCode: 0 + cleanup: + - group delete --name {rg} --yes --no-wait + + - id: acr-replications + operations: + [ + Microsoft.ContainerRegistry Replications Create, + Microsoft.ContainerRegistry Replications Get, + Microsoft.ContainerRegistry Replications List, + Microsoft.ContainerRegistry Replications Update, + Microsoft.ContainerRegistry Replications Delete, + ] + setup: + - group create --name {rg} --location {location} + - acr create --name {alnum:acr} --resource-group {rg} --location {location} --sku Premium + - acr replication create --registry {alnum:acr} --resource-group {rg} --location northeurope --name northeurope + - run: acr replication show --registry {alnum:acr} --resource-group {rg} --name northeurope --query location -o tsv + until: { stdout: { equals: northeurope } } + - acr replication list --registry {alnum:acr} --resource-group {rg} + - acr replication update --registry {alnum:acr} --resource-group {rg} --name northeurope --tags matrix=l2 + command: acr replication delete --registry {alnum:acr} --resource-group {rg} --name northeurope + expect: + exitCode: 0 + cleanup: + - group delete --name {rg} --yes --no-wait + + - id: acr-scope-maps-tokens + operations: + [ + Microsoft.ContainerRegistry ScopeMaps Create, + Microsoft.ContainerRegistry ScopeMaps Get, + Microsoft.ContainerRegistry ScopeMaps List, + Microsoft.ContainerRegistry ScopeMaps Update, + Microsoft.ContainerRegistry Tokens Create, + Microsoft.ContainerRegistry Tokens Get, + Microsoft.ContainerRegistry Tokens List, + Microsoft.ContainerRegistry Tokens Update, + ] + setup: + - group create --name {rg} --location {location} + - acr create --name {alnum:acr} --resource-group {rg} --location {location} --sku Premium + - >- + acr scope-map create --name readers --registry {alnum:acr} --resource-group {rg} + --repository app content/read --description l2 + - acr scope-map show --name readers --registry {alnum:acr} --resource-group {rg} + - acr scope-map list --registry {alnum:acr} --resource-group {rg} + - acr scope-map update --name readers --registry {alnum:acr} --resource-group {rg} --add-repository app metadata/read + - acr token create --name ci --registry {alnum:acr} --resource-group {rg} --scope-map readers --no-passwords + - acr token show --name ci --registry {alnum:acr} --resource-group {rg} + - acr token list --registry {alnum:acr} --resource-group {rg} + command: acr token update --name ci --registry {alnum:acr} --resource-group {rg} --status disabled + expect: + json: { path: status, equals: disabled } + cleanup: + - group delete --name {rg} --yes --no-wait + + - id: acr-token-credential-generate + operations: [Microsoft.ContainerRegistry Registries GenerateCredentials] + setup: + - group create --name {rg} --location {location} + - acr create --name {alnum:acr} --resource-group {rg} --location {location} --sku Premium + - acr scope-map create --name readers --registry {alnum:acr} --resource-group {rg} --repository app content/read + - acr token create --name ci --registry {alnum:acr} --resource-group {rg} --scope-map readers --no-passwords + command: acr token credential generate --name ci --registry {alnum:acr} --resource-group {rg} --password1 + expect: + json: { path: "passwords[0].name", equals: password1 } + cleanup: + - group delete --name {rg} --yes --no-wait + known_gap: + reason: >- + Found by L2: POST .../generateCredentials answers 200 with an empty body, so az prints + {"passwords": null, "username": null}: no password is generated for the token. + date: 2026-09-27 + + - id: acr-scope-map-token-delete + operations: + [Microsoft.ContainerRegistry Tokens Delete, Microsoft.ContainerRegistry ScopeMaps Delete] + setup: + - group create --name {rg} --location {location} + - acr create --name {alnum:acr} --resource-group {rg} --location {location} --sku Premium + - acr scope-map create --name readers --registry {alnum:acr} --resource-group {rg} --repository app content/read + - acr token create --name ci --registry {alnum:acr} --resource-group {rg} --scope-map readers --no-passwords + - acr token delete --name ci --registry {alnum:acr} --resource-group {rg} --yes + command: acr scope-map delete --name readers --registry {alnum:acr} --resource-group {rg} --yes + expect: + exitCode: 0 + cleanup: + - group delete --name {rg} --yes --no-wait + + - id: acr-cache-credential-sets + operations: + [ + Microsoft.ContainerRegistry CacheRules Create, + Microsoft.ContainerRegistry CacheRules Get, + Microsoft.ContainerRegistry CacheRules List, + Microsoft.ContainerRegistry CacheRules Update, + Microsoft.ContainerRegistry CacheRules Delete, + Microsoft.ContainerRegistry CredentialSets Create, + Microsoft.ContainerRegistry CredentialSets Get, + Microsoft.ContainerRegistry CredentialSets List, + Microsoft.ContainerRegistry CredentialSets Update, + Microsoft.ContainerRegistry CredentialSets Delete, + ] + note: The credential set only names Key Vault secret URIs; nothing is pulled. + setup: + - group create --name {rg} --location {location} + - acr create --name {alnum:acr} --resource-group {rg} --location {location} --sku Standard + - >- + acr credential-set create --registry {alnum:acr} --resource-group {rg} --name dockerhub + --login-server docker.io --username-id https://{vault}.vault.azure.net/secrets/user + --password-id https://{vault}.vault.azure.net/secrets/pass + - acr credential-set show --registry {alnum:acr} --resource-group {rg} --name dockerhub + - acr credential-set list --registry {alnum:acr} --resource-group {rg} + - >- + acr cache create --registry {alnum:acr} --resource-group {rg} --name nginx + --source-repo docker.io/library/nginx --target-repo cached/nginx --cred-set dockerhub + - acr cache show --registry {alnum:acr} --resource-group {rg} --name nginx + - acr cache list --registry {alnum:acr} --resource-group {rg} + - acr cache update --registry {alnum:acr} --resource-group {rg} --name nginx --remove-cred-set + - acr cache delete --registry {alnum:acr} --resource-group {rg} --name nginx --yes + - >- + acr credential-set update --registry {alnum:acr} --resource-group {rg} --name dockerhub + --password-id https://{vault}.vault.azure.net/secrets/pass2 + command: acr credential-set delete --registry {alnum:acr} --resource-group {rg} --name dockerhub --yes + expect: + exitCode: 0 + cleanup: + - group delete --name {rg} --yes --no-wait + + - id: acr-connected-registry + operations: + [ + Microsoft.ContainerRegistry ConnectedRegistries Create, + Microsoft.ContainerRegistry ConnectedRegistries Get, + Microsoft.ContainerRegistry ConnectedRegistries List, + Microsoft.ContainerRegistry ConnectedRegistries Update, + Microsoft.ContainerRegistry ConnectedRegistries Deactivate, + Microsoft.ContainerRegistry ConnectedRegistries Delete, + ] + setup: + - group create --name {rg} --location {location} + - acr create --name {alnum:acr} --resource-group {rg} --location {location} --sku Premium + - acr update --name {alnum:acr} --resource-group {rg} --data-endpoint-enabled true + - acr connected-registry create --registry {alnum:acr} --resource-group {rg} --name edge1 --repository hello-world --yes + - acr connected-registry show --registry {alnum:acr} --resource-group {rg} --name edge1 + - acr connected-registry list --registry {alnum:acr} --resource-group {rg} + - acr connected-registry update --registry {alnum:acr} --resource-group {rg} --name edge1 --log-level Warning + - acr connected-registry deactivate --registry {alnum:acr} --resource-group {rg} --name edge1 --yes + command: acr connected-registry delete --registry {alnum:acr} --resource-group {rg} --name edge1 --yes + expect: + exitCode: 0 + cleanup: + - group delete --name {rg} --yes --no-wait + + - id: acr-private-link + operations: + [ + Microsoft.ContainerRegistry Registries ListPrivateLinkResources, + Microsoft.ContainerRegistry Registries GetPrivateLinkResource, + Microsoft.ContainerRegistry PrivateEndpointConnections List, + ] + setup: + - group create --name {rg} --location {location} + - acr create --name {alnum:acr} --resource-group {rg} --location {location} --sku Premium + - acr private-link-resource list --name {alnum:acr} --resource-group {rg} + - >- + rest --method get + --url /subscriptions/{sub}/resourceGroups/{rg}/providers/Microsoft.ContainerRegistry/registries/{alnum:acr}/privateLinkResources/registry?api-version=2023-07-01 + command: acr private-endpoint-connection list --registry-name {alnum:acr} --resource-group {rg} + expect: + json: { path: "", length: 0 } + cleanup: + - group delete --name {rg} --yes --no-wait + + - id: acr-private-endpoint-connection + operations: + [ + Microsoft.ContainerRegistry PrivateEndpointConnections Get, + Microsoft.ContainerRegistry PrivateEndpointConnections CreateOrUpdate, + Microsoft.ContainerRegistry PrivateEndpointConnections Delete, + ] + setup: + - group create --name {rg} --location {location} + - >- + network vnet create --name {name:vnet} --resource-group {rg} --address-prefixes 10.32.0.0/16 + --subnet-name endpoints --subnet-prefixes 10.32.1.0/24 + - run: acr create --name {alnum:acr} --resource-group {rg} --location {location} --sku Premium --query id -o tsv + capture: acr_id + - >- + network private-endpoint create --name {name:pe} --resource-group {rg} --vnet-name {name:vnet} + --subnet endpoints --private-connection-resource-id {acr_id} --group-id registry --connection-name acr + command: acr private-endpoint-connection list --registry-name {alnum:acr} --resource-group {rg} --query "length(@)" + expect: + json: { path: "", equals: 1 } + cleanup: + - group delete --name {rg} --yes --no-wait + known_gap: + reason: >- + Found by L2: a private endpoint to a registry (group id registry) succeeds, but no connection + is registered on the registry (its list stays empty), so Get, approve (CreateOrUpdate) and + Delete of the connection have nothing to act on (the case lists the connections). + date: 2026-09-27 + + - id: acr-operations-delete + operations: + [Microsoft.ContainerRegistry Operations List, Microsoft.ContainerRegistry Registries Delete] + note: Samples acr delete. + setup: + - group create --name {rg} --location {location} + - acr create --name {alnum:acr} --resource-group {rg} --location {location} --sku Basic + - rest --method get --url /providers/Microsoft.ContainerRegistry/operations?api-version=2023-07-01 + command: acr delete --name {alnum:acr} --resource-group {rg} --yes + expect: + exitCode: 0 + cleanup: + - group delete --name {rg} --yes --no-wait + + - id: acr-run + operations: + [ + Microsoft.ContainerRegistry Registries ScheduleRun, + Microsoft.ContainerRegistry Registries GetBuildSourceUploadUrl, + Microsoft.ContainerRegistry Runs GetLogSasUrl, + ] + backing: true + note: >- + /dev/null is ACR's empty context. The run id comes from a rest scheduleRun: `acr run + --no-wait` prints it only in a stderr warning, so a capture of its output is empty. The encoded + task is "version: v1.1.0, steps: - cmd: echo hello from l2". + setup: + - group create --name {rg} --location {location} + - acr create --name {alnum:acr} --resource-group {rg} --location {location} --sku Standard + - >- + rest --method post + --url /subscriptions/{sub}/resourceGroups/{rg}/providers/Microsoft.ContainerRegistry/registries/{alnum:acr}/listBuildSourceUploadUrl?api-version=2019-06-01-preview + # --no-wait: waiting polls Runs Get, which is NotImplemented (acr-run-wait). + - acr run --registry {alnum:acr} --cmd "echo hello from l2" /dev/null --no-logs --no-wait + - run: >- + rest --method post + --url /subscriptions/{sub}/resourceGroups/{rg}/providers/Microsoft.ContainerRegistry/registries/{alnum:acr}/scheduleRun?api-version=2019-06-01-preview + --body '{"type": "EncodedTaskRunRequest", "encodedTaskContent": "dmVyc2lvbjogdjEuMS4wCnN0ZXBzOgogIC0gY21kOiBlY2hvIGhlbGxvIGZyb20gbDIK", "platform": {"os": "linux"}}' + --query properties.runId -o tsv + capture: run_id + command: >- + rest --method post + --url /subscriptions/{sub}/resourceGroups/{rg}/providers/Microsoft.ContainerRegistry/registries/{alnum:acr}/runs/{run_id}/listLogSasUrl?api-version=2019-06-01-preview + expect: + json: { path: logLink, contains: "/logs/{run_id}/" } + cleanup: + - group delete --name {rg} --yes --no-wait + + - id: acr-run-wait + operations: [Microsoft.ContainerRegistry Runs Get] + backing: true + note: A plain `acr run` queues the run and then polls it until it finishes. + setup: + - group create --name {rg} --location {location} + - acr create --name {alnum:acr} --resource-group {rg} --location {location} --sku Standard + command: acr run --registry {alnum:acr} --cmd "echo hello from l2" /dev/null --no-logs --query status -o tsv + expect: + stdout: { equals: Succeeded } + cleanup: + - group delete --name {rg} --yes --no-wait + known_gap: + reason: >- + L2: ScheduleRun queues the run ("Queued a run with ID: qs1"), but polling it, + GET .../registries/{registry}/runs/{runId}, answers NotImplemented (the coverage list marks + Runs Get not implemented), so a waiting `acr run` fails after "Waiting for an agent...". + date: 2026-09-27 + + - id: acr-import + operations: [Microsoft.ContainerRegistry Registries ImportImage] + backing: true + note: The emulator would pull the source image itself; the tool never contacts docker.io. + setup: + - group create --name {rg} --location {location} + - acr create --name {alnum:acr} --resource-group {rg} --location {location} --sku Standard + command: >- + acr import --name {alnum:acr} --resource-group {rg} --source docker.io/library/hello-world:latest + --image hello:l2 + expect: + exitCode: 0 + cleanup: + - group delete --name {rg} --yes --no-wait + known_gap: + reason: >- + L2: ImportImage (marked implemented in the coverage list) answers an error with code + NotImplementedError and an empty message. + date: 2026-09-27 diff --git a/tests/azure/matrix/containerservice.yaml b/tests/azure/matrix/containerservice.yaml new file mode 100644 index 0000000..e968626 --- /dev/null +++ b/tests/azure/matrix/containerservice.yaml @@ -0,0 +1,402 @@ +# Microsoft.ContainerService: AKS managed clusters, agent pools, maintenance configurations, +# snapshots, and Fleet (extension fleet). A cluster runs a k3d cluster on the emulator's Docker +# engine (about 3 minutes to start), so every cluster case is `backing: true` and among the +# heaviest; fleet cases are too, to be safe on a shared engine. +# `aks get-credentials` writes the kubeconfig into the tool's private home, never the real ~/.kube. +provider: Microsoft.ContainerService +cases: + - id: aks-catalogue + operations: + [ + Microsoft.ContainerService Operations List, + Microsoft.ContainerService ContainerService ListNodeImageVersions, + ] + setup: + - rest --method get --url /providers/Microsoft.ContainerService/operations?api-version=2025-02-01 + command: >- + rest --method get + --url /subscriptions/{sub}/providers/Microsoft.ContainerService/locations/{location}/nodeImageVersions?api-version=2025-02-01 + expect: + exitCode: 0 + + - id: aks-cluster + operations: + [ + Microsoft.ContainerService ManagedClusters CreateOrUpdate, + Microsoft.ContainerService ManagedClusters Get, + Microsoft.ContainerService ManagedClusters ListByResourceGroup, + Microsoft.ContainerService ManagedClusters List, + Microsoft.ContainerService ManagedClusters ListKubernetesVersions, + Microsoft.ContainerService ManagedClusters GetUpgradeProfile, + Microsoft.ContainerService ManagedClusters ListClusterUserCredentials, + Microsoft.ContainerService ManagedClusters ListClusterAdminCredentials, + ] + backing: true + note: The kubeconfig lands in the private home. + setup: + - group create --name {rg} --location {location} + - aks create --name {name:aks} --resource-group {rg} --node-count 1 --generate-ssh-keys --no-wait --yes + - run: aks show --name {name:aks} --resource-group {rg} --query provisioningState -o tsv + until: { stdout: { equals: Succeeded } } + wait: 900 + - aks list --resource-group {rg} + - aks list --query "[?name=='{name:aks}'].name" + - aks get-versions --location {location} + - aks get-upgrades --name {name:aks} --resource-group {rg} + - aks get-credentials --name {name:aks} --resource-group {rg} --overwrite-existing + command: aks get-credentials --name {name:aks} --resource-group {rg} --admin --file kubeconfig-{id}.yaml + expect: + exitCode: 0 + cleanup: + - aks delete --name {name:aks} --resource-group {rg} --yes --no-wait + - group delete --name {rg} --yes --no-wait + + - id: aks-nodepool + operations: + [ + Microsoft.ContainerService AgentPools CreateOrUpdate, + Microsoft.ContainerService AgentPools List, + Microsoft.ContainerService AgentPools GetAvailableAgentPoolVersions, + ] + backing: true + note: GetUpgradeProfile is its own known gap below. + setup: + - group create --name {rg} --location {location} + - aks create --name {name:aks} --resource-group {rg} --node-count 1 --generate-ssh-keys --no-wait --yes + - run: aks show --name {name:aks} --resource-group {rg} --query provisioningState -o tsv + until: { stdout: { equals: Succeeded } } + wait: 900 + - aks nodepool add --cluster-name {name:aks} --resource-group {rg} --name np2 --node-count 1 + - >- + rest --method get + --url /subscriptions/{sub}/resourceGroups/{rg}/providers/Microsoft.ContainerService/managedClusters/{name:aks}/availableAgentPoolVersions?api-version=2025-02-01 + command: aks nodepool list --cluster-name {name:aks} --resource-group {rg} --query "sort([].name)" + expect: + json: { path: "", contains: np2 } + cleanup: + - aks delete --name {name:aks} --resource-group {rg} --yes --no-wait + - group delete --name {rg} --yes --no-wait + + - id: aks-nodepool-upgrade-profile + operations: [Microsoft.ContainerService AgentPools GetUpgradeProfile] + backing: true + setup: + - group create --name {rg} --location {location} + - aks create --name {name:aks} --resource-group {rg} --node-count 1 --generate-ssh-keys --no-wait --yes + - run: aks show --name {name:aks} --resource-group {rg} --query provisioningState -o tsv + until: { stdout: { equals: Succeeded } } + wait: 900 + command: aks nodepool get-upgrades --cluster-name {name:aks} --resource-group {rg} --nodepool-name nodepool1 + expect: + json: { path: name, equals: default } + cleanup: + - aks delete --name {name:aks} --resource-group {rg} --yes --no-wait + - group delete --name {rg} --yes --no-wait + known_gap: + reason: >- + HTTP 500 for every Linux pool. agent_pools__get_upgrade_profile raises + "ValueError: is not a valid + AgentPoolUpgradeProfilePropertiesOsType" (it passes the pool's enum to another enum type). + date: 2026-09-28 + + - id: aks-nodepool-read-delete + operations: + [Microsoft.ContainerService AgentPools Get, Microsoft.ContainerService AgentPools Delete] + backing: true + note: >- + The matrix waits for the cluster before it adds the pool. + setup: + - group create --name {rg} --location {location} + - aks create --name {name:aks} --resource-group {rg} --node-count 1 --generate-ssh-keys --no-wait --yes + - run: aks show --name {name:aks} --resource-group {rg} --query provisioningState -o tsv + until: { stdout: { equals: Succeeded } } + wait: 900 + - aks nodepool add --cluster-name {name:aks} --resource-group {rg} --name np2 --node-count 1 + - aks nodepool show --cluster-name {name:aks} --resource-group {rg} --name np2 + command: aks nodepool delete --cluster-name {name:aks} --resource-group {rg} --name np2 + expect: + exitCode: 0 + cleanup: + - aks delete --name {name:aks} --resource-group {rg} --yes --no-wait + - group delete --name {rg} --yes --no-wait + + - id: aks-maintenance-configuration + operations: + [ + Microsoft.ContainerService MaintenanceConfigurations CreateOrUpdate, + Microsoft.ContainerService MaintenanceConfigurations ListByManagedCluster, + ] + backing: true + note: >- + The auto-upgrade schedule (a + maintenanceWindow), not `default` (timeInWeek), which is its own known gap below. + setup: + - group create --name {rg} --location {location} + - aks create --name {name:aks} --resource-group {rg} --node-count 1 --generate-ssh-keys --no-wait --yes + - run: aks show --name {name:aks} --resource-group {rg} --query provisioningState -o tsv + until: { stdout: { equals: Succeeded } } + wait: 900 + - >- + aks maintenanceconfiguration add --cluster-name {name:aks} --resource-group {rg} + --name aksManagedAutoUpgradeSchedule --schedule-type Weekly --day-of-week Monday + --interval-weeks 1 --start-time 01:00 --duration 4 + command: aks maintenanceconfiguration list --cluster-name {name:aks} --resource-group {rg} --query "[].name" + expect: + json: { path: "", equals: [aksManagedAutoUpgradeSchedule] } + cleanup: + - aks delete --name {name:aks} --resource-group {rg} --yes --no-wait + - group delete --name {rg} --yes --no-wait + + - id: aks-maintenance-configuration-default-window + operations: [Microsoft.ContainerService MaintenanceConfigurations CreateOrUpdate] + backing: true + setup: + - group create --name {rg} --location {location} + - aks create --name {name:aks} --resource-group {rg} --node-count 1 --generate-ssh-keys --no-wait --yes + - run: aks show --name {name:aks} --resource-group {rg} --query provisioningState -o tsv + until: { stdout: { equals: Succeeded } } + wait: 900 + command: aks maintenanceconfiguration add --cluster-name {name:aks} --resource-group {rg} --name default --weekday Monday --start-hour 1 + expect: + json: { path: name, equals: default } + cleanup: + - aks delete --name {name:aks} --resource-group {rg} --yes --no-wait + - group delete --name {rg} --yes --no-wait + known_gap: + reason: >- + HTTP 500 for the `default` configuration (timeInWeek, from --weekday and + --start-hour). MaintenanceConfiguration.__init__ assumes a maintenanceWindow: + "AttributeError: 'NoneType' object has no attribute 'not_allowed_dates'". + date: 2026-09-28 + + - id: aks-maintenance-configuration-read-delete + operations: + [ + Microsoft.ContainerService MaintenanceConfigurations Get, + Microsoft.ContainerService MaintenanceConfigurations Delete, + ] + backing: true + setup: + - group create --name {rg} --location {location} + - aks create --name {name:aks} --resource-group {rg} --node-count 1 --generate-ssh-keys --no-wait --yes + - run: aks show --name {name:aks} --resource-group {rg} --query provisioningState -o tsv + until: { stdout: { equals: Succeeded } } + wait: 900 + - >- + aks maintenanceconfiguration add --cluster-name {name:aks} --resource-group {rg} + --name aksManagedAutoUpgradeSchedule --schedule-type Weekly --day-of-week Monday + --interval-weeks 1 --start-time 01:00 --duration 4 + - aks maintenanceconfiguration show --cluster-name {name:aks} --resource-group {rg} --name aksManagedAutoUpgradeSchedule + command: aks maintenanceconfiguration delete --cluster-name {name:aks} --resource-group {rg} --name aksManagedAutoUpgradeSchedule + expect: + exitCode: 0 + cleanup: + - aks delete --name {name:aks} --resource-group {rg} --yes --no-wait + - group delete --name {rg} --yes --no-wait + + - id: aks-stop-and-snapshots + operations: + [ + Microsoft.ContainerService ManagedClusters Stop, + Microsoft.ContainerService Snapshots CreateOrUpdate, + Microsoft.ContainerService Snapshots Get, + Microsoft.ContainerService Snapshots ListByResourceGroup, + Microsoft.ContainerService Snapshots List, + Microsoft.ContainerService Snapshots UpdateTags, + Microsoft.ContainerService Snapshots Delete, + ] + backing: true + setup: + - group create --name {rg} --location {location} + - aks create --name {name:aks} --resource-group {rg} --node-count 1 --generate-ssh-keys --no-wait --yes + - run: aks show --name {name:aks} --resource-group {rg} --query provisioningState -o tsv + until: { stdout: { equals: Succeeded } } + wait: 900 + - run: aks nodepool show --cluster-name {name:aks} --resource-group {rg} --name nodepool1 --query id -o tsv + capture: pool_id + - aks nodepool snapshot create --name {name:snap} --resource-group {rg} --nodepool-id {pool_id} + - aks nodepool snapshot show --name {name:snap} --resource-group {rg} + - aks nodepool snapshot list --resource-group {rg} + - aks nodepool snapshot list --query "[?name=='{name:snap}'].name" + - aks nodepool snapshot update --name {name:snap} --resource-group {rg} --tags owner=l2 + - aks nodepool snapshot delete --name {name:snap} --resource-group {rg} --yes + command: aks stop --name {name:aks} --resource-group {rg} --no-wait + expect: + exitCode: 0 + cleanup: + - aks delete --name {name:aks} --resource-group {rg} --yes --no-wait + - group delete --name {rg} --yes --no-wait + + - id: aks-start-run-command + operations: + [ + Microsoft.ContainerService ManagedClusters Start, + Microsoft.ContainerService ManagedClusters RunCommand, + ] + backing: true + setup: + - group create --name {rg} --location {location} + - aks create --name {name:aks} --resource-group {rg} --node-count 1 --generate-ssh-keys --no-wait --yes + - run: aks show --name {name:aks} --resource-group {rg} --query provisioningState -o tsv + until: { stdout: { equals: Succeeded } } + wait: 900 + # --no-wait and a power-state poll: a waited stop fails on az 2.90's api-version (aks-stop-wait). + - aks stop --name {name:aks} --resource-group {rg} --no-wait + - run: aks show --name {name:aks} --resource-group {rg} --query powerState.code -o tsv + until: { stdout: { equals: Stopped } } + command: aks start --name {name:aks} --resource-group {rg} + expect: + exitCode: 0 + cleanup: + - aks delete --name {name:aks} --resource-group {rg} --yes --no-wait + - group delete --name {rg} --yes --no-wait + known_gap: + reason: >- + The ARM start route is synchronous and the cluster's power state + never reaches Running, so Start does not complete and RunCommand is always refused with 400 + "PowerState=Stopped" (the case starts the cluster). + date: 2026-09-27 + + - id: aks-stop-wait + operations: [Microsoft.ContainerService ManagedClusters Stop] + backing: true + note: >- + A waited stop (aks-stop-and-snapshots stops with --no-wait, which never reads the outcome). + API-version specific: az 2.85 (api-version 2026-01-01) gets 204 from the operation result, so a + local 2.85 run reports this case gap_fixed; CI's az 2.90 (2026-05-01) meets the gap. + setup: + - group create --name {rg} --location {location} + - aks create --name {name:aks} --resource-group {rg} --node-count 1 --generate-ssh-keys --no-wait --yes + - run: aks show --name {name:aks} --resource-group {rg} --query provisioningState -o tsv + until: { stdout: { equals: Succeeded } } + wait: 900 + command: aks stop --name {name:aks} --resource-group {rg} + expect: + exitCode: 0 + cleanup: + - aks delete --name {name:aks} --resource-group {rg} --yes --no-wait + - group delete --name {rg} --yes --no-wait + known_gap: + reason: >- + With api-version 2026-05-01 (az 2.90) the stop's operation result answers + 500 OperationFailed "Cluster ... not tracked" on every poll, so a waited `aks stop` fails; with + 2026-01-01 (az 2.85) it answers 204. A new cluster also reports powerState Stopped at once. + date: 2026-09-28 + + - id: aks-machines + operations: [Microsoft.ContainerService Machines List, Microsoft.ContainerService Machines Get] + backing: true + setup: + - group create --name {rg} --location {location} + - aks create --name {name:aks} --resource-group {rg} --node-count 1 --generate-ssh-keys --no-wait --yes + - run: aks show --name {name:aks} --resource-group {rg} --query provisioningState -o tsv + until: { stdout: { equals: Succeeded } } + wait: 900 + # One node, so Azure lists one machine; a machine name would then let the case read it. + command: >- + rest --method get + --url /subscriptions/{sub}/resourceGroups/{rg}/providers/Microsoft.ContainerService/managedClusters/{name:aks}/agentPools/nodepool1/machines?api-version=2025-02-01 + --query "length(value)" + expect: + json: { path: "", equals: 1 } + cleanup: + - aks delete --name {name:aks} --resource-group {rg} --yes --no-wait + - group delete --name {rg} --yes --no-wait + known_gap: + reason: >- + The list of a one-node pool's machines is empty (Azure lists one machine per + node), so Get has nothing to read. + date: 2026-09-28 + + - id: aks-machines-create + operations: [Microsoft.ContainerService Machines CreateOrUpdate] + backing: true + setup: + - group create --name {rg} --location {location} + - aks create --name {name:aks} --resource-group {rg} --node-count 1 --generate-ssh-keys --no-wait --yes + - run: aks show --name {name:aks} --resource-group {rg} --query provisioningState -o tsv + until: { stdout: { equals: Succeeded } } + wait: 900 + command: >- + rest --method put + --url /subscriptions/{sub}/resourceGroups/{rg}/providers/Microsoft.ContainerService/managedClusters/{name:aks}/agentPools/nodepool1/machines/uatmachine1?api-version=2025-02-01 + --body '{"properties": {}}' + expect: + json: { path: name, equals: uatmachine1 } + cleanup: + - aks delete --name {name:aks} --resource-group {rg} --yes --no-wait + - group delete --name {rg} --yes --no-wait + known_gap: + reason: >- + The AKS emulator has no scale-set-backed machines, and + CreateOrUpdate fails with HTTP 500 ConfigException "Invalid kube-config file". + date: 2026-09-28 + + - id: aks-delete + operations: [Microsoft.ContainerService ManagedClusters Delete] + backing: true + note: >- + The delete can take minutes, so the case allows 600 s. + timeout: 600 + setup: + - group create --name {rg} --location {location} + - aks create --name {name:aks} --resource-group {rg} --node-count 1 --generate-ssh-keys --no-wait --yes + - run: aks show --name {name:aks} --resource-group {rg} --query provisioningState -o tsv + until: { stdout: { equals: Succeeded } } + wait: 900 + command: aks delete --name {name:aks} --resource-group {rg} --yes + expect: + exitCode: 0 + cleanup: + - aks delete --name {name:aks} --resource-group {rg} --yes --no-wait + - group delete --name {rg} --yes --no-wait + + - id: fleet + operations: + [ + Microsoft.ContainerService Fleets CreateOrUpdate, + Microsoft.ContainerService Fleets Get, + Microsoft.ContainerService Fleets ListByResourceGroup, + Microsoft.ContainerService Fleets ListBySubscription, + Microsoft.ContainerService Fleets Update, + Microsoft.ContainerService Fleets Delete, + ] + backing: true + note: Extension fleet. A hubless fleet. + setup: + - group create --name {rg} --location {location} + - fleet create --name {name:fleet} --resource-group {rg} --location {location} + - run: fleet show --name {name:fleet} --resource-group {rg} --query provisioningState -o tsv + until: { stdout: { equals: Succeeded } } + - fleet list --resource-group {rg} + - fleet list --query "[?name=='{name:fleet}'].name" + - fleet update --name {name:fleet} --resource-group {rg} --tags matrix=l2 + command: fleet delete --name {name:fleet} --resource-group {rg} --yes + expect: + exitCode: 0 + cleanup: + - group delete --name {rg} --yes --no-wait + + - id: fleet-update-strategy + operations: + [ + Microsoft.ContainerService FleetUpdateStrategies CreateOrUpdate, + Microsoft.ContainerService FleetUpdateStrategies Get, + Microsoft.ContainerService FleetUpdateStrategies ListByFleet, + Microsoft.ContainerService FleetUpdateStrategies Delete, + ] + backing: true + setup: + - group create --name {rg} --location {location} + - fleet create --name {name:fleet} --resource-group {rg} --location {location} + - >- + rest --method put + --url /subscriptions/{sub}/resourceGroups/{rg}/providers/Microsoft.ContainerService/fleets/{name:fleet}/updateStrategies/staged?api-version=2025-03-01 + --body '{"properties": {"strategy": {"stages": [{"name": "stage1", "groups": [{"name": "group1"}]}]}}}' + - fleet updatestrategy show --fleet-name {name:fleet} --resource-group {rg} --name staged + - fleet updatestrategy list --fleet-name {name:fleet} --resource-group {rg} + command: fleet updatestrategy delete --fleet-name {name:fleet} --resource-group {rg} --name staged --yes + expect: + exitCode: 0 + cleanup: + - group delete --name {rg} --yes --no-wait diff --git a/tests/azure/matrix/dbformysql.yaml b/tests/azure/matrix/dbformysql.yaml new file mode 100644 index 0000000..861a2a6 --- /dev/null +++ b/tests/azure/matrix/dbformysql.yaml @@ -0,0 +1,260 @@ +# Microsoft.DBforMySQL: flexible servers, databases, firewall rules, parameters, backups. A server +# runs a real MySQL container (ls-mysql-flexible-*), so every server case is `backing: true`; the +# runner waits for the group's resources to settle before deleting (an early delete was seen to +# leave the container running). +provider: Microsoft.DBforMySQL +cases: + - id: mysql-catalogue + operations: + [ + Microsoft.DBforMySQL LocationBasedCapabilities List, + Microsoft.DBforMySQL CheckNameAvailability Execute, + Microsoft.DBforMySQL CheckNameAvailabilityWithoutLocation Execute, + ] + setup: + - mysql flexible-server list-skus --location {location} --query "length(@) > `0`" + - >- + rest --method post + --url /subscriptions/{sub}/providers/Microsoft.DBforMySQL/locations/{location}/checkNameAvailability?api-version=2023-12-30 + --body '{"name": "{name:my}", "type": "Microsoft.DBforMySQL/flexibleServers"}' + command: >- + rest --method post + --url /subscriptions/{sub}/providers/Microsoft.DBforMySQL/checkNameAvailability?api-version=2023-12-30 + --body '{"name": "{name:my}", "type": "Microsoft.DBforMySQL/flexibleServers"}' + expect: + json: { path: nameAvailable, equals: true } + + - id: mysql-server + operations: + [ + Microsoft.DBforMySQL FlexibleServers Servers_Create, + Microsoft.DBforMySQL FlexibleServers Servers_Get, + Microsoft.DBforMySQL FlexibleServers Servers_ListByResourceGroup, + Microsoft.DBforMySQL FlexibleServers Servers_List, + Microsoft.DBforMySQL FlexibleServers Servers_Update, + ] + backing: true + note: Samples mysql flexible-server create/show. + setup: + - group create --name {rg} --location {location} + - >- + mysql flexible-server create --name {name:my} --resource-group {rg} --location {location} + --tier Burstable --sku-name Standard_B1ms --admin-user myadmin --admin-password LocalStack123! + --public-access None --yes + - run: mysql flexible-server show --name {name:my} --resource-group {rg} --query state -o tsv + until: { stdout: { equals: Ready } } + - mysql flexible-server list --resource-group {rg} + - mysql flexible-server list --query "[?name=='{name:my}'].name" + command: mysql flexible-server update --name {name:my} --resource-group {rg} --tags matrix=l2 + expect: + json: { path: tags.matrix, equals: l2 } + cleanup: + - group delete --name {rg} --yes --no-wait + + - id: mysql-databases-firewall + operations: + [ + Microsoft.DBforMySQL FlexibleServers Databases_CreateOrUpdate, + Microsoft.DBforMySQL FlexibleServers Databases_Get, + Microsoft.DBforMySQL FlexibleServers Databases_ListByServer, + Microsoft.DBforMySQL FlexibleServers Databases_Delete, + Microsoft.DBforMySQL FlexibleServers FirewallRules_CreateOrUpdate, + Microsoft.DBforMySQL FlexibleServers FirewallRules_Get, + Microsoft.DBforMySQL FlexibleServers FirewallRules_ListByServer, + Microsoft.DBforMySQL FlexibleServers FirewallRules_Delete, + ] + backing: true + note: Samples mysql flexible-server db/firewall-rule create. + setup: + - group create --name {rg} --location {location} + - >- + mysql flexible-server create --name {name:my} --resource-group {rg} --location {location} + --tier Burstable --sku-name Standard_B1ms --admin-user myadmin --admin-password LocalStack123! + --public-access None --yes + - mysql flexible-server db create --server-name {name:my} --resource-group {rg} --database-name shop + - mysql flexible-server db show --server-name {name:my} --resource-group {rg} --database-name shop + - mysql flexible-server db list --server-name {name:my} --resource-group {rg} + - >- + mysql flexible-server firewall-rule create --name {name:my} --resource-group {rg} --rule-name office + --start-ip-address 10.0.0.1 --end-ip-address 10.0.0.9 + - mysql flexible-server firewall-rule show --name {name:my} --resource-group {rg} --rule-name office + - mysql flexible-server firewall-rule list --name {name:my} --resource-group {rg} + - mysql flexible-server firewall-rule delete --name {name:my} --resource-group {rg} --rule-name office --yes + command: mysql flexible-server db delete --server-name {name:my} --resource-group {rg} --database-name shop --yes + expect: + exitCode: 0 + cleanup: + - group delete --name {rg} --yes --no-wait + + - id: mysql-parameters + operations: + [ + Microsoft.DBforMySQL FlexibleServers Configurations_Get, + Microsoft.DBforMySQL FlexibleServers Configurations_ListByServer, + Microsoft.DBforMySQL FlexibleServers Configurations_Update, + Microsoft.DBforMySQL FlexibleServers Configurations_CreateOrUpdate, + Microsoft.DBforMySQL FlexibleServers Configurations_BatchUpdate, + ] + backing: true + setup: + - group create --name {rg} --location {location} + - >- + mysql flexible-server create --name {name:my} --resource-group {rg} --location {location} + --tier Burstable --sku-name Standard_B1ms --admin-user myadmin --admin-password LocalStack123! + --public-access None --yes + - mysql flexible-server parameter show --server-name {name:my} --resource-group {rg} --name max_connections + - mysql flexible-server parameter list --server-name {name:my} --resource-group {rg} --query "length(@) > `0`" + - mysql flexible-server parameter set --server-name {name:my} --resource-group {rg} --name wait_timeout --value 300 + - >- + rest --method put + --url /subscriptions/{sub}/resourceGroups/{rg}/providers/Microsoft.DBforMySQL/flexibleServers/{name:my}/configurations/interactive_timeout?api-version=2023-12-30 + --body '{"properties": {"value": "600", "source": "user-override"}}' + command: >- + mysql flexible-server parameter set-batch --server-name {name:my} --resource-group {rg} + --source user-override --args max_connections=120 wait_timeout=400 + expect: + exitCode: 0 + cleanup: + - group delete --name {rg} --yes --no-wait + + - id: mysql-lifecycle + operations: + [ + Microsoft.DBforMySQL FlexibleServers Servers_Stop, + Microsoft.DBforMySQL FlexibleServers Servers_Start, + Microsoft.DBforMySQL FlexibleServers Servers_Restart, + Microsoft.DBforMySQL FlexibleServers Servers_Delete, + ] + backing: true + setup: + - group create --name {rg} --location {location} + - >- + mysql flexible-server create --name {name:my} --resource-group {rg} --location {location} + --tier Burstable --sku-name Standard_B1ms --admin-user myadmin --admin-password LocalStack123! + --public-access None --yes + - mysql flexible-server stop --name {name:my} --resource-group {rg} + - mysql flexible-server start --name {name:my} --resource-group {rg} + - mysql flexible-server restart --name {name:my} --resource-group {rg} + command: mysql flexible-server delete --name {name:my} --resource-group {rg} --yes + expect: + exitCode: 0 + cleanup: + - group delete --name {rg} --yes --no-wait + + - id: mysql-backups-replicas-admins + operations: + [ + Microsoft.DBforMySQL FlexibleServers Backups_Put, + Microsoft.DBforMySQL FlexibleServers Backups_Get, + Microsoft.DBforMySQL FlexibleServers Backups_ListByServer, + Microsoft.DBforMySQL FlexibleServers LongRunningBackup_Create, + Microsoft.DBforMySQL FlexibleServers LongRunningBackups_List, + Microsoft.DBforMySQL FlexibleServers Replicas_ListByServer, + Microsoft.DBforMySQL FlexibleServers LogFiles_ListByServer, + Microsoft.DBforMySQL FlexibleServers AzureAdAdministrators_ListByServer, + Microsoft.DBforMySQL FlexibleServers AdvancedThreatProtectionSettings_Get, + Microsoft.DBforMySQL FlexibleServers AdvancedThreatProtectionSettings_List, + Microsoft.DBforMySQL FlexibleServers AdvancedThreatProtectionSettings_Update, + Microsoft.DBforMySQL FlexibleServers PrivateLinkResources_ListByServer, + Microsoft.DBforMySQL FlexibleServers Maintenances_List, + ] + backing: true + note: >- + The backup is read back by the name the PUT returned, not by manual1: + the service names an on-demand backup customer-