From dbeb9affe0b2a473744b31c3c1ab1e89227569b0 Mon Sep 17 00:00:00 2001 From: Xuanwo Date: Thu, 19 Nov 2026 14:35:56 +0100 Subject: [PATCH 1/2] fix(ci): renew Azure login for delayed probe cleanup A 55-minute Azure Pipelines run can outlive the GitHub OIDC assertion used by azure/login. Token refresh then fails during cleanup, leaving the unfinished run and probe behind. Request a fresh assertion with the job existing id-token permission and audience, and log in again as the same service principal. Use it only after a failed token or blob delete, and retry that operation once. The queue POST, run ownership, polling timeout, and original exit status remain unchanged. Cover normal cleanup, expired assertion cleanup for both the pipeline run and blob-only path, and reauthentication failure. --- .github/azure-live-tests/azure-oidc-login.sh | 45 ++++++++++++++ .github/azure-live-tests/azure-pipelines.sh | 26 +++++--- .github/azure-live-tests/test_scripts.py | 63 +++++++++++++++++++- .github/azure-live-tests/with-probe.sh | 15 ++++- 4 files changed, 139 insertions(+), 10 deletions(-) create mode 100644 .github/azure-live-tests/azure-oidc-login.sh diff --git a/.github/azure-live-tests/azure-oidc-login.sh b/.github/azure-live-tests/azure-oidc-login.sh new file mode 100644 index 00000000..f991e808 --- /dev/null +++ b/.github/azure-live-tests/azure-oidc-login.sh @@ -0,0 +1,45 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# Renew the existing Azure CLI login through the job's id-token permission. +# The initial azure/login assertion expires long before a delayed cleanup. +# Requesting a fresh assertion with the same audience and logging in as the +# same service principal adds no credential source or permission. +set -euo pipefail + +: "${AZURE_CLIENT_ID:?Set the existing Azure client ID}" +: "${AZURE_TENANT_ID:?Set the existing Azure tenant ID}" +: "${ACTIONS_ID_TOKEN_REQUEST_URL:?The job needs id-token: write}" +: "${ACTIONS_ID_TOKEN_REQUEST_TOKEN:?The job needs id-token: write}" + +token_response=$(curl --fail --silent --show-error \ + --header "Authorization: bearer ${ACTIONS_ID_TOKEN_REQUEST_TOKEN}" \ + "${ACTIONS_ID_TOKEN_REQUEST_URL}&audience=api%3A%2F%2FAzureADTokenExchange") +federated_token=$(jq --raw-output '.value // empty' <<< "$token_response") +unset token_response +test -n "$federated_token" +if [[ "${GITHUB_ACTIONS:-}" == true ]]; then + echo "::add-mask::$federated_token" +fi +az login \ + --service-principal \ + --username "$AZURE_CLIENT_ID" \ + --tenant "$AZURE_TENANT_ID" \ + --federated-token "$federated_token" \ + --output none +unset federated_token diff --git a/.github/azure-live-tests/azure-pipelines.sh b/.github/azure-live-tests/azure-pipelines.sh index ce1d02d8..c8a7d7b3 100644 --- a/.github/azure-live-tests/azure-pipelines.sh +++ b/.github/azure-live-tests/azure-pipelines.sh @@ -18,6 +18,7 @@ # Keep the fixture alive while Azure Pipelines uses its read-only identity. set -euo pipefail +script_dir=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd) run_id="" completed=false @@ -46,14 +47,26 @@ read_run_status() { done } +# A delayed cleanup can outlive the GitHub OIDC assertion used by the initial +# login. Retry once through a fresh login as the same service principal. +get_devops_access_token() { + local token + token=$(az account get-access-token \ + --resource https://app.vssps.visualstudio.com/ \ + --query accessToken \ + --output tsv) && { printf '%s\n' "$token"; return 0; } + bash "$script_dir/azure-oidc-login.sh" || return + az account get-access-token \ + --resource https://app.vssps.visualstudio.com/ \ + --query accessToken \ + --output tsv +} + # Called by the EXIT handler. # shellcheck disable=SC2329 cancel_unfinished_run() { local response - access_token=$(az account get-access-token \ - --resource https://app.vssps.visualstudio.com/ \ - --query accessToken \ - --output tsv) || return + access_token=$(get_devops_access_token) || return response=$(read_run_status "$access_token") || return $? state=$(jq --raw-output '.state' <<< "$response") || return if [[ "$state" != "completed" ]]; then @@ -127,10 +140,7 @@ echo "Queued Azure Pipelines run: $run_url" deadline=$((SECONDS + 3300)) while (( SECONDS < deadline )); do - access_token=$(az account get-access-token \ - --resource https://app.vssps.visualstudio.com/ \ - --query accessToken \ - --output tsv) + access_token=$(get_devops_access_token) || exit $? response=$(read_run_status "$access_token") || exit $? state=$(jq --raw-output '.state' <<< "$response") result=$(jq --raw-output '.result // empty' <<< "$response") diff --git a/.github/azure-live-tests/test_scripts.py b/.github/azure-live-tests/test_scripts.py index 64352957..1b52a3ca 100644 --- a/.github/azure-live-tests/test_scripts.py +++ b/.github/azure-live-tests/test_scripts.py @@ -31,9 +31,25 @@ root = pathlib.Path(os.environ['FAKE_AZ_STATE']) def option(name, default=''): return args[args.index(name)+1] if name in args else default +def calls_with(prefix): + return [json.loads(line) for line in (root / 'calls.jsonl').read_text().splitlines() + if json.loads(line)[:len(prefix)] == prefix] +def logged_in(): + return bool(calls_with(['login'])) with (root / 'calls.jsonl').open('a') as f: f.write(json.dumps(args) + '\n') -if args[:2] == ['account', 'get-access-token']: +if args[:1] == ['login']: + assert '--service-principal' in args + assert option('--username') == 'fake-client' + assert option('--tenant') == 'fake-tenant' + assert option('--federated-token') == 'fake-federated-token' +elif args[:2] == ['account', 'get-access-token']: + expire_after = int(os.getenv('EXPIRE_DEVOPS_AUTH_AFTER', '0')) + expired = (expire_after and + len(calls_with(['account', 'get-access-token'])) > expire_after) + if expired and not logged_in(): + print('AADSTS700... assertion expired', file=sys.stderr) + sys.exit(1) print('fake-devops-token') elif args[:3] == ['storage', 'blob', 'upload']: if os.getenv('FAIL_UPLOAD'): sys.exit(1) @@ -46,6 +62,9 @@ def option(name, default=''): print('fake-read-only-token') elif args[:3] == ['storage', 'blob', 'delete']: if os.getenv('FAIL_DELETE'): sys.exit(1) + if os.getenv('EXPIRE_BLOB_AUTH') and not logged_in(): + print('AADSTS700... assertion expired', file=sys.stderr) + sys.exit(1) (root / option('--name')).unlink() elif args[:2] == ['vm', 'create']: name = option('--name') @@ -79,6 +98,13 @@ def option(name, default=''): def option(name, default=''): return args[args.index(name)+1] if name in args else default method = option('--request', 'GET') +url = args[-1] +if url.startswith('https://oidc.example.test/token'): + if os.getenv('FAIL_OIDC'): sys.exit(35) + assert 'audience=api%3A%2F%2FAzureADTokenExchange' in url + assert option('--header') == 'Authorization: bearer fake-oidc-request-token' + print(json.dumps({'value': 'fake-federated-token'})) + sys.exit(0) # Both waiting and cancellation must finish before fixture cleanup. assert list(root.glob('ci-probe-*')) with (root / 'curl-calls.jsonl').open('a') as f: @@ -143,6 +169,10 @@ def setUp(self): 'AZURE_DEVOPS_ORGANIZATION': 'fixture', 'AZURE_DEVOPS_PROJECT': 'tests', 'AZURE_DEVOPS_PIPELINE_ID': '1', + 'AZURE_CLIENT_ID': 'fake-client', + 'AZURE_TENANT_ID': 'fake-tenant', + 'ACTIONS_ID_TOKEN_REQUEST_URL': 'https://oidc.example.test/token?api-version=2.0', + 'ACTIONS_ID_TOKEN_REQUEST_TOKEN': 'fake-oidc-request-token', 'AZURE_RUNTIME_RESOURCE_GROUP': 'fixture-rg', 'AZURE_IMDS_IDENTITY_ID': '/fake/identity', 'RUNNER_TEMP': str(self.root), @@ -241,6 +271,7 @@ def test_pipeline_reads_temporary_probe_until_completed(self): self.assertEqual(result.returncode, 0, result.stderr) self.assertTrue((self.state / 'queued.json').exists()) self.assertFalse((self.state / 'cancelled').exists()) + self.assertEqual([c for c in self.calls() if c[:1] == ['login']], []) self.assertEqual(list(self.state.glob('ci-probe-*')), []) def test_failed_pipeline_fails_job_and_cleans_probe(self): @@ -294,6 +325,36 @@ def test_cleanup_status_reset_is_retried_and_cancels_once(self): patches = [c for c in calls if c['method'] == 'PATCH'] self.assertEqual(len(patches), 1) self.assertTrue(all('/runs/23?api-version=7.1' in c['url'] for c in gets)) + + def test_pipeline_cleanup_reauthenticates_after_assertion_expiry(self): + self.env.update(EXPIRE_DEVOPS_AUTH_AFTER='2', + FAIL_PIPELINE_POLL='1') + result = self.run_script('with-probe.sh', 'bash', str(SCRIPTS / 'azure-pipelines.sh')) + self.assertEqual(result.returncode, 22, result.stderr) + logins = [c for c in self.calls() if c[:1] == ['login']] + self.assertEqual(len(logins), 1) + self.assertEqual(logins[0][logins[0].index('--username') + 1], 'fake-client') + self.assertTrue((self.state / 'cancelled').exists()) + self.assertNotIn('fake-federated-token', result.stdout + result.stderr) + self.assertNotIn('fake-devops-token', result.stdout + result.stderr) + self.assertEqual(list(self.state.glob('ci-probe-*')), []) + + def test_blob_cleanup_reauthenticates_after_assertion_expiry(self): + self.env['EXPIRE_BLOB_AUTH'] = '1' + result = self.run_script('with-probe.sh', 'true') + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(len([c for c in self.calls() if c[:1] == ['login']]), 1) + self.assertEqual(self.resources(), []) + self.assertNotIn('fake-federated-token', result.stdout + result.stderr) + + def test_failed_reauth_keeps_original_exit_and_cleans_probe(self): + self.env.update(EXPIRE_DEVOPS_AUTH_AFTER='2', + FAIL_PIPELINE_POLL='1', FAIL_OIDC='1') + result = self.run_script('with-probe.sh', 'bash', str(SCRIPTS / 'azure-pipelines.sh')) + self.assertEqual(result.returncode, 22, result.stderr) + self.assertFalse((self.state / 'cancelled').exists()) + self.assertEqual([c for c in self.calls() if c[:1] == ['login']], []) + self.assertIn('Failed to cancel', result.stderr) self.assertEqual(list(self.state.glob('ci-probe-*')), []) diff --git a/.github/azure-live-tests/with-probe.sh b/.github/azure-live-tests/with-probe.sh index a3e572bb..8b408eaf 100644 --- a/.github/azure-live-tests/with-probe.sh +++ b/.github/azure-live-tests/with-probe.sh @@ -24,6 +24,8 @@ set -euo pipefail : "${AZURE_STORAGE_CONTAINER:?Set the fixture container}" (( $# > 0 )) || { echo "Usage: with-probe.sh command [args...]" >&2; exit 2; } +script_dir=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd) + auth_mode=${AZURE_STORAGE_AUTH_MODE:-login} if [[ "$auth_mode" == key ]]; then export AZURE_STORAGE_KEY="${REQSIGN_AZURE_STORAGE_ACCOUNT_KEY:?Set the fixture account key}" @@ -33,11 +35,22 @@ storage_args=(--auth-mode "$auth_mode" --account-name "$AZURE_STORAGE_ACCOUNT" blob_name="ci-probe-$(python3 -c 'import uuid; print(uuid.uuid4())').txt" probe_dir=$(mktemp -d) created=false +delete_probe() { + az storage blob delete "${storage_args[@]}" --name "$blob_name" --output none && return 0 + # The initial federated login can outlive its GitHub OIDC assertion. Renew + # through the same job permission, then retry this probe's delete once. + if [[ "$auth_mode" == login ]]; then + bash "$script_dir/azure-oidc-login.sh" && + az storage blob delete "${storage_args[@]}" --name "$blob_name" --output none + return $? + fi + return 1 +} cleanup() { local result=$? trap - EXIT if [[ "$created" == true ]]; then - if ! az storage blob delete "${storage_args[@]}" --name "$blob_name" --output none; then + if ! delete_probe; then echo "Failed to delete the live-test probe" >&2 if (( result == 0 )); then result=1; fi fi From 75bc750b75516fcec6684786c93a700f728f1079 Mon Sep 17 00:00:00 2001 From: Xuanwo Date: Thu, 19 Nov 2026 17:31:04 +0100 Subject: [PATCH 2/2] fix(ci): keep Azure OIDC mask out of the captured token The login helper ::add-mask:: command was captured by command substitution together with the refreshed DevOps token. In Actions mode the Authorization header then contained the mask command and a newline, so cleanup status reads failed and the run was not cancelled. Set the global access_token directly and call the helper outside command substitution so the workflow command reaches the runner on stdout. Add Actions-mode expiry coverage with exact DevOps bearer checks, and restore the #959 cleanup probe-deletion assertion. --- .github/azure-live-tests/azure-pipelines.sh | 20 +++++++++------- .github/azure-live-tests/test_scripts.py | 26 +++++++++++++++++++++ 2 files changed, 38 insertions(+), 8 deletions(-) diff --git a/.github/azure-live-tests/azure-pipelines.sh b/.github/azure-live-tests/azure-pipelines.sh index c8a7d7b3..ca47a396 100644 --- a/.github/azure-live-tests/azure-pipelines.sh +++ b/.github/azure-live-tests/azure-pipelines.sh @@ -49,24 +49,28 @@ read_run_status() { # A delayed cleanup can outlive the GitHub OIDC assertion used by the initial # login. Retry once through a fresh login as the same service principal. -get_devops_access_token() { - local token - token=$(az account get-access-token \ +# Set the global access_token directly instead of returning it: the login +# helper must write its ::add-mask:: command to the runner's stdout, not into +# the captured token. +load_devops_access_token() { + if access_token=$(az account get-access-token \ --resource https://app.vssps.visualstudio.com/ \ --query accessToken \ - --output tsv) && { printf '%s\n' "$token"; return 0; } + --output tsv); then + return 0 + fi bash "$script_dir/azure-oidc-login.sh" || return - az account get-access-token \ + access_token=$(az account get-access-token \ --resource https://app.vssps.visualstudio.com/ \ --query accessToken \ - --output tsv + --output tsv) } # Called by the EXIT handler. # shellcheck disable=SC2329 cancel_unfinished_run() { local response - access_token=$(get_devops_access_token) || return + load_devops_access_token || return response=$(read_run_status "$access_token") || return $? state=$(jq --raw-output '.state' <<< "$response") || return if [[ "$state" != "completed" ]]; then @@ -140,7 +144,7 @@ echo "Queued Azure Pipelines run: $run_url" deadline=$((SECONDS + 3300)) while (( SECONDS < deadline )); do - access_token=$(get_devops_access_token) || exit $? + load_devops_access_token || exit $? response=$(read_run_status "$access_token") || exit $? state=$(jq --raw-output '.state' <<< "$response") result=$(jq --raw-output '.result // empty' <<< "$response") diff --git a/.github/azure-live-tests/test_scripts.py b/.github/azure-live-tests/test_scripts.py index 1b52a3ca..ba268911 100644 --- a/.github/azure-live-tests/test_scripts.py +++ b/.github/azure-live-tests/test_scripts.py @@ -105,6 +105,9 @@ def option(name, default=''): assert option('--header') == 'Authorization: bearer fake-oidc-request-token' print(json.dumps({'value': 'fake-federated-token'})) sys.exit(0) +if url.startswith('https://dev.azure.com/'): + assert option('--header') == 'Authorization: Bearer fake-devops-token', \ + 'unexpected DevOps authorization header' # Both waiting and cancellation must finish before fixture cleanup. assert list(root.glob('ci-probe-*')) with (root / 'curl-calls.jsonl').open('a') as f: @@ -325,6 +328,7 @@ def test_cleanup_status_reset_is_retried_and_cancels_once(self): patches = [c for c in calls if c['method'] == 'PATCH'] self.assertEqual(len(patches), 1) self.assertTrue(all('/runs/23?api-version=7.1' in c['url'] for c in gets)) + self.assertEqual(list(self.state.glob('ci-probe-*')), []) def test_pipeline_cleanup_reauthenticates_after_assertion_expiry(self): self.env.update(EXPIRE_DEVOPS_AUTH_AFTER='2', @@ -339,6 +343,28 @@ def test_pipeline_cleanup_reauthenticates_after_assertion_expiry(self): self.assertNotIn('fake-devops-token', result.stdout + result.stderr) self.assertEqual(list(self.state.glob('ci-probe-*')), []) + def test_pipeline_cleanup_reauthenticates_in_actions_mode(self): + self.env.update(EXPIRE_DEVOPS_AUTH_AFTER='2', + FAIL_PIPELINE_POLL='1', GITHUB_ACTIONS='true') + result = self.run_script('with-probe.sh', 'bash', str(SCRIPTS / 'azure-pipelines.sh')) + self.assertEqual(result.returncode, 22, result.stderr) + calls = self.curl_calls() + gets = [c for c in calls if c['method'] == 'GET'] + self.assertEqual(len(gets), 2) + self.assertEqual(len([c for c in calls if c['method'] == 'POST']), 1) + self.assertEqual(len([c for c in calls if c['method'] == 'PATCH']), 1) + self.assertTrue((self.state / 'cancelled').exists()) + masks = [line for line in result.stdout.splitlines() + if line.startswith('::add-mask::')] + self.assertEqual(masks, ['::add-mask::fake-devops-token', + '::add-mask::fake-federated-token']) + self.assertNotIn('::add-mask::', result.stderr) + unmasked_stdout = '\n'.join(line for line in result.stdout.splitlines() + if not line.startswith('::add-mask::')) + self.assertNotIn('fake-federated-token', unmasked_stdout + result.stderr) + self.assertNotIn('fake-devops-token', unmasked_stdout + result.stderr) + self.assertEqual(list(self.state.glob('ci-probe-*')), []) + def test_blob_cleanup_reauthenticates_after_assertion_expiry(self): self.env['EXPIRE_BLOB_AUTH'] = '1' result = self.run_script('with-probe.sh', 'true')