diff --git a/.github/azure-live-tests/azure-oidc-login.sh b/.github/azure-live-tests/azure-oidc-login.sh new file mode 100644 index 00000000..f991e808 --- /dev/null +++ b/.github/azure-live-tests/azure-oidc-login.sh @@ -0,0 +1,45 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# Renew the existing Azure CLI login through the job's id-token permission. +# The initial azure/login assertion expires long before a delayed cleanup. +# Requesting a fresh assertion with the same audience and logging in as the +# same service principal adds no credential source or permission. +set -euo pipefail + +: "${AZURE_CLIENT_ID:?Set the existing Azure client ID}" +: "${AZURE_TENANT_ID:?Set the existing Azure tenant ID}" +: "${ACTIONS_ID_TOKEN_REQUEST_URL:?The job needs id-token: write}" +: "${ACTIONS_ID_TOKEN_REQUEST_TOKEN:?The job needs id-token: write}" + +token_response=$(curl --fail --silent --show-error \ + --header "Authorization: bearer ${ACTIONS_ID_TOKEN_REQUEST_TOKEN}" \ + "${ACTIONS_ID_TOKEN_REQUEST_URL}&audience=api%3A%2F%2FAzureADTokenExchange") +federated_token=$(jq --raw-output '.value // empty' <<< "$token_response") +unset token_response +test -n "$federated_token" +if [[ "${GITHUB_ACTIONS:-}" == true ]]; then + echo "::add-mask::$federated_token" +fi +az login \ + --service-principal \ + --username "$AZURE_CLIENT_ID" \ + --tenant "$AZURE_TENANT_ID" \ + --federated-token "$federated_token" \ + --output none +unset federated_token diff --git a/.github/azure-live-tests/azure-pipelines.sh b/.github/azure-live-tests/azure-pipelines.sh index 78911bda..c0bab449 100644 --- a/.github/azure-live-tests/azure-pipelines.sh +++ b/.github/azure-live-tests/azure-pipelines.sh @@ -18,6 +18,7 @@ # Keep the fixture alive while Azure Pipelines uses its read-only identity. set -euo pipefail +script_dir=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd) run_id="" completed=false @@ -59,14 +60,30 @@ read_run_status() { done } +# A delayed cleanup can outlive the GitHub OIDC assertion used by the initial +# login. Retry once through a fresh login as the same service principal. +# Set the global access_token directly instead of returning it: the login +# helper must write its ::add-mask:: command to the runner's stdout, not into +# the captured token. +load_devops_access_token() { + if access_token=$(az account get-access-token \ + --resource https://app.vssps.visualstudio.com/ \ + --query accessToken \ + --output tsv); then + return 0 + fi + bash "$script_dir/azure-oidc-login.sh" || return + access_token=$(az account get-access-token \ + --resource https://app.vssps.visualstudio.com/ \ + --query accessToken \ + --output tsv) +} + # Called by the EXIT handler. # shellcheck disable=SC2329 cancel_unfinished_run() { local response - access_token=$(az account get-access-token \ - --resource https://app.vssps.visualstudio.com/ \ - --query accessToken \ - --output tsv) || return + load_devops_access_token || return response=$(read_run_status "$access_token") || return $? state=$(jq --raw-output '.state' <<< "$response") || return if [[ "$state" != "completed" ]]; then @@ -140,10 +157,7 @@ echo "Queued Azure Pipelines run: $run_url" deadline=$((SECONDS + 3300)) while (( SECONDS < deadline )); do - access_token=$(az account get-access-token \ - --resource https://app.vssps.visualstudio.com/ \ - --query accessToken \ - --output tsv) + load_devops_access_token || exit $? response=$(read_run_status "$access_token") || exit $? state=$(jq --raw-output '.state' <<< "$response") result=$(jq --raw-output '.result // empty' <<< "$response") diff --git a/.github/azure-live-tests/test_scripts.py b/.github/azure-live-tests/test_scripts.py index cba9fee3..a9de6420 100644 --- a/.github/azure-live-tests/test_scripts.py +++ b/.github/azure-live-tests/test_scripts.py @@ -31,9 +31,25 @@ root = pathlib.Path(os.environ['FAKE_AZ_STATE']) def option(name, default=''): return args[args.index(name)+1] if name in args else default +def calls_with(prefix): + return [json.loads(line) for line in (root / 'calls.jsonl').read_text().splitlines() + if json.loads(line)[:len(prefix)] == prefix] +def logged_in(): + return bool(calls_with(['login'])) with (root / 'calls.jsonl').open('a') as f: f.write(json.dumps(args) + '\n') -if args[:2] == ['account', 'get-access-token']: +if args[:1] == ['login']: + assert '--service-principal' in args + assert option('--username') == 'fake-client' + assert option('--tenant') == 'fake-tenant' + assert option('--federated-token') == 'fake-federated-token' +elif args[:2] == ['account', 'get-access-token']: + expire_after = int(os.getenv('EXPIRE_DEVOPS_AUTH_AFTER', '0')) + expired = (expire_after and + len(calls_with(['account', 'get-access-token'])) > expire_after) + if expired and not logged_in(): + print('AADSTS700... assertion expired', file=sys.stderr) + sys.exit(1) print('fake-devops-token') elif args[:3] == ['storage', 'blob', 'upload']: if os.getenv('FAIL_UPLOAD'): sys.exit(1) @@ -46,6 +62,9 @@ def option(name, default=''): print('fake-read-only-token') elif args[:3] == ['storage', 'blob', 'delete']: if os.getenv('FAIL_DELETE'): sys.exit(1) + if os.getenv('EXPIRE_BLOB_AUTH') and not logged_in(): + print('AADSTS700... assertion expired', file=sys.stderr) + sys.exit(1) (root / option('--name')).unlink() elif args[:2] == ['vm', 'create']: name = option('--name') @@ -89,6 +108,16 @@ def respond(body, code='200'): else: print(body) method = option('--request', 'GET') +url = args[-1] +if url.startswith('https://oidc.example.test/token'): + if os.getenv('FAIL_OIDC'): sys.exit(35) + assert 'audience=api%3A%2F%2FAzureADTokenExchange' in url + assert option('--header') == 'Authorization: bearer fake-oidc-request-token' + print(json.dumps({'value': 'fake-federated-token'})) + sys.exit(0) +if url.startswith('https://dev.azure.com/'): + assert option('--header') == 'Authorization: Bearer fake-devops-token', \ + 'unexpected DevOps authorization header' # Both waiting and cancellation must finish before fixture cleanup. assert list(root.glob('ci-probe-*')) with (root / 'curl-calls.jsonl').open('a') as f: @@ -162,6 +191,10 @@ def setUp(self): 'AZURE_DEVOPS_ORGANIZATION': 'fixture', 'AZURE_DEVOPS_PROJECT': 'tests', 'AZURE_DEVOPS_PIPELINE_ID': '1', + 'AZURE_CLIENT_ID': 'fake-client', + 'AZURE_TENANT_ID': 'fake-tenant', + 'ACTIONS_ID_TOKEN_REQUEST_URL': 'https://oidc.example.test/token?api-version=2.0', + 'ACTIONS_ID_TOKEN_REQUEST_TOKEN': 'fake-oidc-request-token', 'AZURE_RUNTIME_RESOURCE_GROUP': 'fixture-rg', 'AZURE_IMDS_IDENTITY_ID': '/fake/identity', 'RUNNER_TEMP': str(self.root), @@ -260,6 +293,7 @@ def test_pipeline_reads_temporary_probe_until_completed(self): self.assertEqual(result.returncode, 0, result.stderr) self.assertTrue((self.state / 'queued.json').exists()) self.assertFalse((self.state / 'cancelled').exists()) + self.assertEqual([c for c in self.calls() if c[:1] == ['login']], []) self.assertEqual(list(self.state.glob('ci-probe-*')), []) def test_failed_pipeline_fails_job_and_cleans_probe(self): @@ -315,6 +349,59 @@ def test_cleanup_status_reset_is_retried_and_cancels_once(self): self.assertTrue(all('/runs/23?api-version=7.1' in c['url'] for c in gets)) self.assertEqual(list(self.state.glob('ci-probe-*')), []) + def test_pipeline_cleanup_reauthenticates_after_assertion_expiry(self): + self.env.update(EXPIRE_DEVOPS_AUTH_AFTER='2', + FAIL_PIPELINE_POLL='1') + result = self.run_script('with-probe.sh', 'bash', str(SCRIPTS / 'azure-pipelines.sh')) + self.assertEqual(result.returncode, 22, result.stderr) + logins = [c for c in self.calls() if c[:1] == ['login']] + self.assertEqual(len(logins), 1) + self.assertEqual(logins[0][logins[0].index('--username') + 1], 'fake-client') + self.assertTrue((self.state / 'cancelled').exists()) + self.assertNotIn('fake-federated-token', result.stdout + result.stderr) + self.assertNotIn('fake-devops-token', result.stdout + result.stderr) + self.assertEqual(list(self.state.glob('ci-probe-*')), []) + + def test_pipeline_cleanup_reauthenticates_in_actions_mode(self): + self.env.update(EXPIRE_DEVOPS_AUTH_AFTER='2', + FAIL_PIPELINE_POLL='1', GITHUB_ACTIONS='true') + result = self.run_script('with-probe.sh', 'bash', str(SCRIPTS / 'azure-pipelines.sh')) + self.assertEqual(result.returncode, 22, result.stderr) + calls = self.curl_calls() + gets = [c for c in calls if c['method'] == 'GET'] + self.assertEqual(len(gets), 2) + self.assertEqual(len([c for c in calls if c['method'] == 'POST']), 1) + self.assertEqual(len([c for c in calls if c['method'] == 'PATCH']), 1) + self.assertTrue((self.state / 'cancelled').exists()) + masks = [line for line in result.stdout.splitlines() + if line.startswith('::add-mask::')] + self.assertEqual(masks, ['::add-mask::fake-devops-token', + '::add-mask::fake-federated-token']) + self.assertNotIn('::add-mask::', result.stderr) + unmasked_stdout = '\n'.join(line for line in result.stdout.splitlines() + if not line.startswith('::add-mask::')) + self.assertNotIn('fake-federated-token', unmasked_stdout + result.stderr) + self.assertNotIn('fake-devops-token', unmasked_stdout + result.stderr) + self.assertEqual(list(self.state.glob('ci-probe-*')), []) + + def test_blob_cleanup_reauthenticates_after_assertion_expiry(self): + self.env['EXPIRE_BLOB_AUTH'] = '1' + result = self.run_script('with-probe.sh', 'true') + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(len([c for c in self.calls() if c[:1] == ['login']]), 1) + self.assertEqual(self.resources(), []) + self.assertNotIn('fake-federated-token', result.stdout + result.stderr) + + def test_failed_reauth_keeps_original_exit_and_cleans_probe(self): + self.env.update(EXPIRE_DEVOPS_AUTH_AFTER='2', + FAIL_PIPELINE_POLL='1', FAIL_OIDC='1') + result = self.run_script('with-probe.sh', 'bash', str(SCRIPTS / 'azure-pipelines.sh')) + self.assertEqual(result.returncode, 22, result.stderr) + self.assertFalse((self.state / 'cancelled').exists()) + self.assertEqual([c for c in self.calls() if c[:1] == ['login']], []) + self.assertIn('Failed to cancel', result.stderr) + self.assertEqual(list(self.state.glob('ci-probe-*')), []) + def test_pipeline_status_503_is_retried(self): self.env['HTTP_503_PIPELINE_POLL_ONCE'] = '1' result = self.run_script('with-probe.sh', 'bash', str(SCRIPTS / 'azure-pipelines.sh')) diff --git a/.github/azure-live-tests/with-probe.sh b/.github/azure-live-tests/with-probe.sh index a3e572bb..8b408eaf 100644 --- a/.github/azure-live-tests/with-probe.sh +++ b/.github/azure-live-tests/with-probe.sh @@ -24,6 +24,8 @@ set -euo pipefail : "${AZURE_STORAGE_CONTAINER:?Set the fixture container}" (( $# > 0 )) || { echo "Usage: with-probe.sh command [args...]" >&2; exit 2; } +script_dir=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd) + auth_mode=${AZURE_STORAGE_AUTH_MODE:-login} if [[ "$auth_mode" == key ]]; then export AZURE_STORAGE_KEY="${REQSIGN_AZURE_STORAGE_ACCOUNT_KEY:?Set the fixture account key}" @@ -33,11 +35,22 @@ storage_args=(--auth-mode "$auth_mode" --account-name "$AZURE_STORAGE_ACCOUNT" blob_name="ci-probe-$(python3 -c 'import uuid; print(uuid.uuid4())').txt" probe_dir=$(mktemp -d) created=false +delete_probe() { + az storage blob delete "${storage_args[@]}" --name "$blob_name" --output none && return 0 + # The initial federated login can outlive its GitHub OIDC assertion. Renew + # through the same job permission, then retry this probe's delete once. + if [[ "$auth_mode" == login ]]; then + bash "$script_dir/azure-oidc-login.sh" && + az storage blob delete "${storage_args[@]}" --name "$blob_name" --output none + return $? + fi + return 1 +} cleanup() { local result=$? trap - EXIT if [[ "$created" == true ]]; then - if ! az storage blob delete "${storage_args[@]}" --name "$blob_name" --output none; then + if ! delete_probe; then echo "Failed to delete the live-test probe" >&2 if (( result == 0 )); then result=1; fi fi