From dbeb9affe0b2a473744b31c3c1ab1e89227569b0 Mon Sep 17 00:00:00 2001 From: Xuanwo Date: Thu, 19 Nov 2026 14:35:56 +0100 Subject: [PATCH 1/3] fix(ci): renew Azure login for delayed probe cleanup A 55-minute Azure Pipelines run can outlive the GitHub OIDC assertion used by azure/login. Token refresh then fails during cleanup, leaving the unfinished run and probe behind. Request a fresh assertion with the job existing id-token permission and audience, and log in again as the same service principal. Use it only after a failed token or blob delete, and retry that operation once. The queue POST, run ownership, polling timeout, and original exit status remain unchanged. Cover normal cleanup, expired assertion cleanup for both the pipeline run and blob-only path, and reauthentication failure. --- .github/azure-live-tests/azure-oidc-login.sh | 45 ++++++++++++++ .github/azure-live-tests/azure-pipelines.sh | 26 +++++--- .github/azure-live-tests/test_scripts.py | 63 +++++++++++++++++++- .github/azure-live-tests/with-probe.sh | 15 ++++- 4 files changed, 139 insertions(+), 10 deletions(-) create mode 100644 .github/azure-live-tests/azure-oidc-login.sh diff --git a/.github/azure-live-tests/azure-oidc-login.sh b/.github/azure-live-tests/azure-oidc-login.sh new file mode 100644 index 00000000..f991e808 --- /dev/null +++ b/.github/azure-live-tests/azure-oidc-login.sh @@ -0,0 +1,45 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# Renew the existing Azure CLI login through the job's id-token permission. +# The initial azure/login assertion expires long before a delayed cleanup. +# Requesting a fresh assertion with the same audience and logging in as the +# same service principal adds no credential source or permission. +set -euo pipefail + +: "${AZURE_CLIENT_ID:?Set the existing Azure client ID}" +: "${AZURE_TENANT_ID:?Set the existing Azure tenant ID}" +: "${ACTIONS_ID_TOKEN_REQUEST_URL:?The job needs id-token: write}" +: "${ACTIONS_ID_TOKEN_REQUEST_TOKEN:?The job needs id-token: write}" + +token_response=$(curl --fail --silent --show-error \ + --header "Authorization: bearer ${ACTIONS_ID_TOKEN_REQUEST_TOKEN}" \ + "${ACTIONS_ID_TOKEN_REQUEST_URL}&audience=api%3A%2F%2FAzureADTokenExchange") +federated_token=$(jq --raw-output '.value // empty' <<< "$token_response") +unset token_response +test -n "$federated_token" +if [[ "${GITHUB_ACTIONS:-}" == true ]]; then + echo "::add-mask::$federated_token" +fi +az login \ + --service-principal \ + --username "$AZURE_CLIENT_ID" \ + --tenant "$AZURE_TENANT_ID" \ + --federated-token "$federated_token" \ + --output none +unset federated_token diff --git a/.github/azure-live-tests/azure-pipelines.sh b/.github/azure-live-tests/azure-pipelines.sh index ce1d02d8..c8a7d7b3 100644 --- a/.github/azure-live-tests/azure-pipelines.sh +++ b/.github/azure-live-tests/azure-pipelines.sh @@ -18,6 +18,7 @@ # Keep the fixture alive while Azure Pipelines uses its read-only identity. set -euo pipefail +script_dir=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd) run_id="" completed=false @@ -46,14 +47,26 @@ read_run_status() { done } +# A delayed cleanup can outlive the GitHub OIDC assertion used by the initial +# login. Retry once through a fresh login as the same service principal. +get_devops_access_token() { + local token + token=$(az account get-access-token \ + --resource https://app.vssps.visualstudio.com/ \ + --query accessToken \ + --output tsv) && { printf '%s\n' "$token"; return 0; } + bash "$script_dir/azure-oidc-login.sh" || return + az account get-access-token \ + --resource https://app.vssps.visualstudio.com/ \ + --query accessToken \ + --output tsv +} + # Called by the EXIT handler. # shellcheck disable=SC2329 cancel_unfinished_run() { local response - access_token=$(az account get-access-token \ - --resource https://app.vssps.visualstudio.com/ \ - --query accessToken \ - --output tsv) || return + access_token=$(get_devops_access_token) || return response=$(read_run_status "$access_token") || return $? state=$(jq --raw-output '.state' <<< "$response") || return if [[ "$state" != "completed" ]]; then @@ -127,10 +140,7 @@ echo "Queued Azure Pipelines run: $run_url" deadline=$((SECONDS + 3300)) while (( SECONDS < deadline )); do - access_token=$(az account get-access-token \ - --resource https://app.vssps.visualstudio.com/ \ - --query accessToken \ - --output tsv) + access_token=$(get_devops_access_token) || exit $? response=$(read_run_status "$access_token") || exit $? state=$(jq --raw-output '.state' <<< "$response") result=$(jq --raw-output '.result // empty' <<< "$response") diff --git a/.github/azure-live-tests/test_scripts.py b/.github/azure-live-tests/test_scripts.py index 64352957..1b52a3ca 100644 --- a/.github/azure-live-tests/test_scripts.py +++ b/.github/azure-live-tests/test_scripts.py @@ -31,9 +31,25 @@ root = pathlib.Path(os.environ['FAKE_AZ_STATE']) def option(name, default=''): return args[args.index(name)+1] if name in args else default +def calls_with(prefix): + return [json.loads(line) for line in (root / 'calls.jsonl').read_text().splitlines() + if json.loads(line)[:len(prefix)] == prefix] +def logged_in(): + return bool(calls_with(['login'])) with (root / 'calls.jsonl').open('a') as f: f.write(json.dumps(args) + '\n') -if args[:2] == ['account', 'get-access-token']: +if args[:1] == ['login']: + assert '--service-principal' in args + assert option('--username') == 'fake-client' + assert option('--tenant') == 'fake-tenant' + assert option('--federated-token') == 'fake-federated-token' +elif args[:2] == ['account', 'get-access-token']: + expire_after = int(os.getenv('EXPIRE_DEVOPS_AUTH_AFTER', '0')) + expired = (expire_after and + len(calls_with(['account', 'get-access-token'])) > expire_after) + if expired and not logged_in(): + print('AADSTS700... assertion expired', file=sys.stderr) + sys.exit(1) print('fake-devops-token') elif args[:3] == ['storage', 'blob', 'upload']: if os.getenv('FAIL_UPLOAD'): sys.exit(1) @@ -46,6 +62,9 @@ def option(name, default=''): print('fake-read-only-token') elif args[:3] == ['storage', 'blob', 'delete']: if os.getenv('FAIL_DELETE'): sys.exit(1) + if os.getenv('EXPIRE_BLOB_AUTH') and not logged_in(): + print('AADSTS700... assertion expired', file=sys.stderr) + sys.exit(1) (root / option('--name')).unlink() elif args[:2] == ['vm', 'create']: name = option('--name') @@ -79,6 +98,13 @@ def option(name, default=''): def option(name, default=''): return args[args.index(name)+1] if name in args else default method = option('--request', 'GET') +url = args[-1] +if url.startswith('https://oidc.example.test/token'): + if os.getenv('FAIL_OIDC'): sys.exit(35) + assert 'audience=api%3A%2F%2FAzureADTokenExchange' in url + assert option('--header') == 'Authorization: bearer fake-oidc-request-token' + print(json.dumps({'value': 'fake-federated-token'})) + sys.exit(0) # Both waiting and cancellation must finish before fixture cleanup. assert list(root.glob('ci-probe-*')) with (root / 'curl-calls.jsonl').open('a') as f: @@ -143,6 +169,10 @@ def setUp(self): 'AZURE_DEVOPS_ORGANIZATION': 'fixture', 'AZURE_DEVOPS_PROJECT': 'tests', 'AZURE_DEVOPS_PIPELINE_ID': '1', + 'AZURE_CLIENT_ID': 'fake-client', + 'AZURE_TENANT_ID': 'fake-tenant', + 'ACTIONS_ID_TOKEN_REQUEST_URL': 'https://oidc.example.test/token?api-version=2.0', + 'ACTIONS_ID_TOKEN_REQUEST_TOKEN': 'fake-oidc-request-token', 'AZURE_RUNTIME_RESOURCE_GROUP': 'fixture-rg', 'AZURE_IMDS_IDENTITY_ID': '/fake/identity', 'RUNNER_TEMP': str(self.root), @@ -241,6 +271,7 @@ def test_pipeline_reads_temporary_probe_until_completed(self): self.assertEqual(result.returncode, 0, result.stderr) self.assertTrue((self.state / 'queued.json').exists()) self.assertFalse((self.state / 'cancelled').exists()) + self.assertEqual([c for c in self.calls() if c[:1] == ['login']], []) self.assertEqual(list(self.state.glob('ci-probe-*')), []) def test_failed_pipeline_fails_job_and_cleans_probe(self): @@ -294,6 +325,36 @@ def test_cleanup_status_reset_is_retried_and_cancels_once(self): patches = [c for c in calls if c['method'] == 'PATCH'] self.assertEqual(len(patches), 1) self.assertTrue(all('/runs/23?api-version=7.1' in c['url'] for c in gets)) + + def test_pipeline_cleanup_reauthenticates_after_assertion_expiry(self): + self.env.update(EXPIRE_DEVOPS_AUTH_AFTER='2', + FAIL_PIPELINE_POLL='1') + result = self.run_script('with-probe.sh', 'bash', str(SCRIPTS / 'azure-pipelines.sh')) + self.assertEqual(result.returncode, 22, result.stderr) + logins = [c for c in self.calls() if c[:1] == ['login']] + self.assertEqual(len(logins), 1) + self.assertEqual(logins[0][logins[0].index('--username') + 1], 'fake-client') + self.assertTrue((self.state / 'cancelled').exists()) + self.assertNotIn('fake-federated-token', result.stdout + result.stderr) + self.assertNotIn('fake-devops-token', result.stdout + result.stderr) + self.assertEqual(list(self.state.glob('ci-probe-*')), []) + + def test_blob_cleanup_reauthenticates_after_assertion_expiry(self): + self.env['EXPIRE_BLOB_AUTH'] = '1' + result = self.run_script('with-probe.sh', 'true') + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(len([c for c in self.calls() if c[:1] == ['login']]), 1) + self.assertEqual(self.resources(), []) + self.assertNotIn('fake-federated-token', result.stdout + result.stderr) + + def test_failed_reauth_keeps_original_exit_and_cleans_probe(self): + self.env.update(EXPIRE_DEVOPS_AUTH_AFTER='2', + FAIL_PIPELINE_POLL='1', FAIL_OIDC='1') + result = self.run_script('with-probe.sh', 'bash', str(SCRIPTS / 'azure-pipelines.sh')) + self.assertEqual(result.returncode, 22, result.stderr) + self.assertFalse((self.state / 'cancelled').exists()) + self.assertEqual([c for c in self.calls() if c[:1] == ['login']], []) + self.assertIn('Failed to cancel', result.stderr) self.assertEqual(list(self.state.glob('ci-probe-*')), []) diff --git a/.github/azure-live-tests/with-probe.sh b/.github/azure-live-tests/with-probe.sh index a3e572bb..8b408eaf 100644 --- a/.github/azure-live-tests/with-probe.sh +++ b/.github/azure-live-tests/with-probe.sh @@ -24,6 +24,8 @@ set -euo pipefail : "${AZURE_STORAGE_CONTAINER:?Set the fixture container}" (( $# > 0 )) || { echo "Usage: with-probe.sh command [args...]" >&2; exit 2; } +script_dir=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd) + auth_mode=${AZURE_STORAGE_AUTH_MODE:-login} if [[ "$auth_mode" == key ]]; then export AZURE_STORAGE_KEY="${REQSIGN_AZURE_STORAGE_ACCOUNT_KEY:?Set the fixture account key}" @@ -33,11 +35,22 @@ storage_args=(--auth-mode "$auth_mode" --account-name "$AZURE_STORAGE_ACCOUNT" blob_name="ci-probe-$(python3 -c 'import uuid; print(uuid.uuid4())').txt" probe_dir=$(mktemp -d) created=false +delete_probe() { + az storage blob delete "${storage_args[@]}" --name "$blob_name" --output none && return 0 + # The initial federated login can outlive its GitHub OIDC assertion. Renew + # through the same job permission, then retry this probe's delete once. + if [[ "$auth_mode" == login ]]; then + bash "$script_dir/azure-oidc-login.sh" && + az storage blob delete "${storage_args[@]}" --name "$blob_name" --output none + return $? + fi + return 1 +} cleanup() { local result=$? trap - EXIT if [[ "$created" == true ]]; then - if ! az storage blob delete "${storage_args[@]}" --name "$blob_name" --output none; then + if ! delete_probe; then echo "Failed to delete the live-test probe" >&2 if (( result == 0 )); then result=1; fi fi From 75bc750b75516fcec6684786c93a700f728f1079 Mon Sep 17 00:00:00 2001 From: Xuanwo Date: Thu, 19 Nov 2026 17:31:04 +0100 Subject: [PATCH 2/3] fix(ci): keep Azure OIDC mask out of the captured token The login helper ::add-mask:: command was captured by command substitution together with the refreshed DevOps token. In Actions mode the Authorization header then contained the mask command and a newline, so cleanup status reads failed and the run was not cancelled. Set the global access_token directly and call the helper outside command substitution so the workflow command reaches the runner on stdout. Add Actions-mode expiry coverage with exact DevOps bearer checks, and restore the #959 cleanup probe-deletion assertion. --- .github/azure-live-tests/azure-pipelines.sh | 20 +++++++++------- .github/azure-live-tests/test_scripts.py | 26 +++++++++++++++++++++ 2 files changed, 38 insertions(+), 8 deletions(-) diff --git a/.github/azure-live-tests/azure-pipelines.sh b/.github/azure-live-tests/azure-pipelines.sh index c8a7d7b3..ca47a396 100644 --- a/.github/azure-live-tests/azure-pipelines.sh +++ b/.github/azure-live-tests/azure-pipelines.sh @@ -49,24 +49,28 @@ read_run_status() { # A delayed cleanup can outlive the GitHub OIDC assertion used by the initial # login. Retry once through a fresh login as the same service principal. -get_devops_access_token() { - local token - token=$(az account get-access-token \ +# Set the global access_token directly instead of returning it: the login +# helper must write its ::add-mask:: command to the runner's stdout, not into +# the captured token. +load_devops_access_token() { + if access_token=$(az account get-access-token \ --resource https://app.vssps.visualstudio.com/ \ --query accessToken \ - --output tsv) && { printf '%s\n' "$token"; return 0; } + --output tsv); then + return 0 + fi bash "$script_dir/azure-oidc-login.sh" || return - az account get-access-token \ + access_token=$(az account get-access-token \ --resource https://app.vssps.visualstudio.com/ \ --query accessToken \ - --output tsv + --output tsv) } # Called by the EXIT handler. # shellcheck disable=SC2329 cancel_unfinished_run() { local response - access_token=$(get_devops_access_token) || return + load_devops_access_token || return response=$(read_run_status "$access_token") || return $? state=$(jq --raw-output '.state' <<< "$response") || return if [[ "$state" != "completed" ]]; then @@ -140,7 +144,7 @@ echo "Queued Azure Pipelines run: $run_url" deadline=$((SECONDS + 3300)) while (( SECONDS < deadline )); do - access_token=$(get_devops_access_token) || exit $? + load_devops_access_token || exit $? response=$(read_run_status "$access_token") || exit $? state=$(jq --raw-output '.state' <<< "$response") result=$(jq --raw-output '.result // empty' <<< "$response") diff --git a/.github/azure-live-tests/test_scripts.py b/.github/azure-live-tests/test_scripts.py index 1b52a3ca..ba268911 100644 --- a/.github/azure-live-tests/test_scripts.py +++ b/.github/azure-live-tests/test_scripts.py @@ -105,6 +105,9 @@ def option(name, default=''): assert option('--header') == 'Authorization: bearer fake-oidc-request-token' print(json.dumps({'value': 'fake-federated-token'})) sys.exit(0) +if url.startswith('https://dev.azure.com/'): + assert option('--header') == 'Authorization: Bearer fake-devops-token', \ + 'unexpected DevOps authorization header' # Both waiting and cancellation must finish before fixture cleanup. assert list(root.glob('ci-probe-*')) with (root / 'curl-calls.jsonl').open('a') as f: @@ -325,6 +328,7 @@ def test_cleanup_status_reset_is_retried_and_cancels_once(self): patches = [c for c in calls if c['method'] == 'PATCH'] self.assertEqual(len(patches), 1) self.assertTrue(all('/runs/23?api-version=7.1' in c['url'] for c in gets)) + self.assertEqual(list(self.state.glob('ci-probe-*')), []) def test_pipeline_cleanup_reauthenticates_after_assertion_expiry(self): self.env.update(EXPIRE_DEVOPS_AUTH_AFTER='2', @@ -339,6 +343,28 @@ def test_pipeline_cleanup_reauthenticates_after_assertion_expiry(self): self.assertNotIn('fake-devops-token', result.stdout + result.stderr) self.assertEqual(list(self.state.glob('ci-probe-*')), []) + def test_pipeline_cleanup_reauthenticates_in_actions_mode(self): + self.env.update(EXPIRE_DEVOPS_AUTH_AFTER='2', + FAIL_PIPELINE_POLL='1', GITHUB_ACTIONS='true') + result = self.run_script('with-probe.sh', 'bash', str(SCRIPTS / 'azure-pipelines.sh')) + self.assertEqual(result.returncode, 22, result.stderr) + calls = self.curl_calls() + gets = [c for c in calls if c['method'] == 'GET'] + self.assertEqual(len(gets), 2) + self.assertEqual(len([c for c in calls if c['method'] == 'POST']), 1) + self.assertEqual(len([c for c in calls if c['method'] == 'PATCH']), 1) + self.assertTrue((self.state / 'cancelled').exists()) + masks = [line for line in result.stdout.splitlines() + if line.startswith('::add-mask::')] + self.assertEqual(masks, ['::add-mask::fake-devops-token', + '::add-mask::fake-federated-token']) + self.assertNotIn('::add-mask::', result.stderr) + unmasked_stdout = '\n'.join(line for line in result.stdout.splitlines() + if not line.startswith('::add-mask::')) + self.assertNotIn('fake-federated-token', unmasked_stdout + result.stderr) + self.assertNotIn('fake-devops-token', unmasked_stdout + result.stderr) + self.assertEqual(list(self.state.glob('ci-probe-*')), []) + def test_blob_cleanup_reauthenticates_after_assertion_expiry(self): self.env['EXPIRE_BLOB_AUTH'] = '1' result = self.run_script('with-probe.sh', 'true') From 24a596a8da026d899d515327ea901560a8ce7104 Mon Sep 17 00:00:00 2001 From: Xuanwo Date: Thu, 19 Nov 2026 19:16:32 +0100 Subject: [PATCH 3/3] ci(azure): bound probe queue wait and execution separately The probe had one 55-minute deadline that started when the run was queued, so a run stuck behind the single pool agent held the GitHub job for the whole deadline and the failure never said whether the probe or the queue was slow. Read the build timeline and bound the wait for an agent at 20 minutes and the execution from the observed agent start at 15 minutes. Only a Job record with a workerName counts as an agent start: a stage record starts before an agent is assigned, and a build's own startTime comes from the orchestration. A build that is still queued may have no timeline yet, which is not an execution start and not a read failure. The bounds and poll interval can be shortened for the local fixture, which covers the queue bound, the execution bound measured from a past agent start, a missing timeline, a timeline list, and a run that starts and finishes inside both bounds. The queue request, ownership check, reauthentication and cleanup are unchanged. --- .github/azure-live-tests/azure-pipelines.sh | 101 ++++++++++++++++---- .github/azure-live-tests/test_scripts.py | 88 ++++++++++++++++- 2 files changed, 169 insertions(+), 20 deletions(-) diff --git a/.github/azure-live-tests/azure-pipelines.sh b/.github/azure-live-tests/azure-pipelines.sh index c0bab449..f6cd5de5 100644 --- a/.github/azure-live-tests/azure-pipelines.sh +++ b/.github/azure-live-tests/azure-pipelines.sh @@ -22,44 +22,87 @@ script_dir=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd) run_id="" completed=false -# A status read is idempotent. Retry the observed TLS send reset (curl 35) and +# A run that is still waiting for the pool agent has not exercised the probe, +# so one shared deadline cannot tell a queue stall from a slow probe. Bound the +# wait for an agent and the execution after it starts separately; the defaults +# keep the worst case plus cleanup inside the GitHub job's 60-minute timeout +# and inside the lifetime of an access token, which cleanup can renew. +queue_wait_limit=${REQSIGN_AZURE_PIPELINES_QUEUE_LIMIT:-1200} +execution_limit=${REQSIGN_AZURE_PIPELINES_EXECUTION_LIMIT:-900} +# Local fixtures shorten these so the bounds are testable without waiting. +poll_interval=${REQSIGN_AZURE_PIPELINES_POLL_INTERVAL:-15} + +# A DevOps read is idempotent. Retry the observed TLS send reset (curl 35) and # HTTP 503 (curl 22) within a poll so one transient failure does not fail the # job. Other HTTP failures are permanent for this job. -status_read_attempts=3 -status_read_retry_delay=1 +devops_read_attempts=3 +devops_read_retry_delay=1 status_response_file="$RUNNER_TEMP/azure-pipelines-status-response.json" +timeline_response_file="$RUNNER_TEMP/azure-pipelines-timeline-response.json" -# Called by the EXIT handler and the polling loop. -read_run_status() { - local access_token=$1 +read_devops_json() { + local access_token=$1 path=$2 response_file=$3 allow_missing=${4:-false} local attempt curl_status http_status response retryable - for (( attempt = 1; attempt <= status_read_attempts; attempt++ )); do + for (( attempt = 1; attempt <= devops_read_attempts; attempt++ )); do if http_status=$(curl --fail --silent --show-error \ - --output "$status_response_file" \ + --output "$response_file" \ --write-out '%{http_code}' \ --header "Authorization: Bearer $access_token" \ - "https://dev.azure.com/${AZURE_DEVOPS_ORGANIZATION}/${AZURE_DEVOPS_PROJECT}/_apis/pipelines/${AZURE_DEVOPS_PIPELINE_ID}/runs/${run_id}?api-version=7.1"); then - response=$(<"$status_response_file") - rm -f "$status_response_file" + "https://dev.azure.com/${AZURE_DEVOPS_ORGANIZATION}/${AZURE_DEVOPS_PROJECT}${path}"); then + response=$(<"$response_file") + rm -f "$response_file" printf '%s\n' "$response" return 0 else curl_status=$? fi - rm -f "$status_response_file" + rm -f "$response_file" + # A build that is still queued may not have a timeline yet. That is a + # missing record, not a failed read. + if [[ "$allow_missing" == true ]] && (( curl_status == 22 )) && [[ "$http_status" == "404" ]]; then + return 0 + fi retryable=false if (( curl_status == 35 )); then retryable=true elif (( curl_status == 22 )) && [[ "$http_status" == "503" ]]; then retryable=true fi - if [[ "$retryable" != true ]] || (( attempt == status_read_attempts )); then + if [[ "$retryable" != true ]] || (( attempt == devops_read_attempts )); then return "$curl_status" fi - sleep "$status_read_retry_delay" + sleep "$devops_read_retry_delay" done } +# Called by the EXIT handler and the polling loop. +read_run_status() { + read_devops_json "$1" \ + "/_apis/pipelines/${AZURE_DEVOPS_PIPELINE_ID}/runs/${run_id}?api-version=7.1" \ + "$status_response_file" +} + +# Azure DevOps reports the agent that runs a job on the build timeline. A Job +# record carries workerName once an agent picked it up, and startTime is when it +# started there. Stage records also start before any agent is assigned, and a +# build's own startTime comes from the orchestration, so only Job records are +# evidence that execution began on an agent. +read_agent_started_at() { + local access_token=$1 timeline job_start allow_missing=true + timeline=$(read_devops_json "$access_token" \ + "/_apis/build/builds/${run_id}/timeline?api-version=7.1" \ + "$timeline_response_file" "$allow_missing") || return + [[ -n "$timeline" ]] || return 0 + # The timeline may be one timeline object or a list of them. + job_start=$(jq --raw-output ' + (if type == "array" then map(.records // []) | add // [] else .records // [] end) + | [ .[] | select(.type == "Job") + | select((.workerName // "") != "" and (.startTime // "") != "") + | .startTime ] + | sort | .[0] // empty' <<< "$timeline") || return + printf '%s\n' "$job_start" +} + # A delayed cleanup can outlive the GitHub OIDC assertion used by the initial # login. Retry once through a fresh login as the same service principal. # Set the global access_token directly instead of returning it: the login @@ -154,9 +197,12 @@ run_url=$(jq --raw-output '._links.web.href // empty' <<< "$response") test -n "$run_id" test -n "$run_url" echo "Queued Azure Pipelines run: $run_url" +echo "Azure Pipelines probe limits: ${queue_wait_limit}s queue wait, ${execution_limit}s execution after agent start: $run_url" -deadline=$((SECONDS + 3300)) -while (( SECONDS < deadline )); do +queue_deadline=$(( $(date +%s) + queue_wait_limit )) +agent_started_at="" +agent_started_epoch="" +while true; do load_devops_access_token || exit $? response=$(read_run_status "$access_token") || exit $? state=$(jq --raw-output '.state' <<< "$response") @@ -167,7 +213,24 @@ while (( SECONDS < deadline )); do test "$result" = "succeeded" exit fi - sleep 15 + + now=$(date +%s) + if [[ -z "$agent_started_epoch" ]]; then + agent_started_at=$(read_agent_started_at "$access_token") || exit $? + if [[ -n "$agent_started_at" ]]; then + if ! agent_started_epoch=$(date --date="$agent_started_at" +%s); then + echo "Failed to parse the Azure Pipelines job start time '$agent_started_at': $run_url" >&2 + exit 1 + fi + echo "Azure Pipelines job started on an agent at $agent_started_at (execution limit ${execution_limit}s): $run_url" + elif (( now >= queue_deadline )); then + echo "Azure Pipelines run did not start on an agent within ${queue_wait_limit}s of queueing: $run_url" >&2 + exit 1 + fi + fi + if [[ -n "$agent_started_epoch" ]] && (( now >= agent_started_epoch + execution_limit )); then + echo "Azure Pipelines run did not complete within ${execution_limit}s after its job started on an agent at ${agent_started_at}: $run_url" >&2 + exit 1 + fi + sleep "$poll_interval" done -echo "Azure Pipelines run did not complete before the timeout: $run_url" >&2 -exit 1 diff --git a/.github/azure-live-tests/test_scripts.py b/.github/azure-live-tests/test_scripts.py index a9de6420..d5dcc92c 100644 --- a/.github/azure-live-tests/test_scripts.py +++ b/.github/azure-live-tests/test_scripts.py @@ -107,6 +107,18 @@ def respond(body, code='200'): print(code) else: print(body) +def agent_start(): + import datetime + return os.getenv('AGENT_START') or datetime.datetime.now( + datetime.timezone.utc).strftime('%Y-%m-%dT%H:%M:%SZ') +def timeline_records(): + # A stage also starts before an agent is assigned. Only a Job record with a + # workerName proves that an agent picked the run up. + started = agent_start() + job = {'type': 'Job', 'state': 'inProgress', 'startTime': started} + if os.getenv('AGENT_ASSIGNED', '1') == '1': + job['workerName'] = 'reqsign-azure-vmss-1' + return [{'type': 'Stage', 'state': 'inProgress', 'startTime': started}, job] method = option('--request', 'GET') url = args[-1] if url.startswith('https://oidc.example.test/token'): @@ -122,7 +134,13 @@ def respond(body, code='200'): assert list(root.glob('ci-probe-*')) with (root / 'curl-calls.jsonl').open('a') as f: f.write(json.dumps({'method': method, 'url': args[-1]}) + '\n') -if method == 'POST': +if method == 'GET' and url.endswith('/timeline?api-version=7.1'): + if os.getenv('MISSING_TIMELINE'): + respond(json.dumps({'message': 'missing timeline'}), '404') + sys.exit(22) + records = {'records': timeline_records()} + respond(json.dumps([records] if os.getenv('TIMELINE_ARRAY') else records)) +elif method == 'POST': request = json.loads(option('--data')) params = request['templateParameters'] assert params['githubSha'] == 'a' * 40 @@ -159,6 +177,8 @@ def respond(body, code='200'): if count == 1: sys.exit(22) respond(json.dumps({'state': 'inProgress'})) + elif count <= int(os.getenv('PIPELINE_INPROGRESS_POLLS', '0')): + respond(json.dumps({'state': 'inProgress'})) else: respond(json.dumps({'state': 'completed', 'result': os.getenv('PIPELINE_RESULT', 'succeeded')})) @@ -201,6 +221,7 @@ def setUp(self): 'GITHUB_RUN_ID': '123', 'GITHUB_RUN_ATTEMPT': '1', 'GITHUB_OUTPUT': str(self.root / 'outputs'), + 'REQSIGN_AZURE_PIPELINES_POLL_INTERVAL': '0', } def run_script(self, script, *args): @@ -215,6 +236,10 @@ def curl_calls(self): return [json.loads(line) for line in (self.state / 'curl-calls.jsonl').read_text().splitlines()] + def timeline_calls(self): + return [c for c in self.curl_calls() + if c['url'].endswith('/builds/23/timeline?api-version=7.1')] + def resources(self): return [p.name for p in self.state.iterdir() if p.name not in {'calls.jsonl', 'curl-calls.jsonl', 'get-count'}] @@ -444,6 +469,67 @@ def test_permanent_pipeline_status_error_is_not_retried(self): self.assertNotIn('fake-devops-token', result.stdout + result.stderr) self.assertEqual(list(self.state.glob('ci-probe-*')), []) + def test_queue_wait_ends_before_an_agent_starts(self): + # A stage record starts before any agent is assigned. Only a Job record + # with a workerName is agent start, so this run is still queued. + self.env.update(PIPELINE_INPROGRESS_POLLS='1000', + AGENT_ASSIGNED='0', + AGENT_START='2020-01-01T00:00:00Z', + REQSIGN_AZURE_PIPELINES_QUEUE_LIMIT='0') + result = self.run_script('with-probe.sh', 'bash', str(SCRIPTS / 'azure-pipelines.sh')) + self.assertEqual(result.returncode, 1, result.stderr) + self.assertIn('did not start on an agent within 0s of queueing', result.stderr) + self.assertNotIn('job started on an agent', result.stdout) + self.assertEqual(len(self.timeline_calls()), 1) + self.assertTrue((self.state / 'cancelled').exists()) + self.assertEqual(list(self.state.glob('ci-probe-*')), []) + + def test_execution_bound_is_measured_from_the_agent_start(self): + # The default queue limit is an hour still to come, so only the + # execution bound measured from the observed agent start can fail this. + self.env.update(PIPELINE_INPROGRESS_POLLS='1000', + AGENT_START='2020-01-01T00:00:00Z', + REQSIGN_AZURE_PIPELINES_EXECUTION_LIMIT='60') + result = self.run_script('with-probe.sh', 'bash', str(SCRIPTS / 'azure-pipelines.sh')) + self.assertEqual(result.returncode, 1, result.stderr) + self.assertIn('job started on an agent at 2020-01-01T00:00:00Z', result.stdout) + self.assertIn('did not complete within 60s after its job started on an agent', + result.stderr) + self.assertNotIn('did not start on an agent', result.stderr) + self.assertTrue((self.state / 'cancelled').exists()) + self.assertEqual(list(self.state.glob('ci-probe-*')), []) + + def test_missing_timeline_keeps_waiting_for_the_agent(self): + self.env.update(PIPELINE_INPROGRESS_POLLS='1000', MISSING_TIMELINE='1', + REQSIGN_AZURE_PIPELINES_QUEUE_LIMIT='0') + result = self.run_script('with-probe.sh', 'bash', str(SCRIPTS / 'azure-pipelines.sh')) + self.assertEqual(result.returncode, 1, result.stderr) + self.assertIn('did not start on an agent within 0s of queueing', result.stderr) + self.assertNotEqual(result.returncode, 22) + self.assertTrue((self.state / 'cancelled').exists()) + self.assertEqual(list(self.state.glob('ci-probe-*')), []) + + def test_agent_start_is_read_from_a_timeline_list(self): + self.env.update(PIPELINE_INPROGRESS_POLLS='1000', TIMELINE_ARRAY='1', + AGENT_START='2020-01-01T00:00:00Z', + REQSIGN_AZURE_PIPELINES_EXECUTION_LIMIT='60') + result = self.run_script('with-probe.sh', 'bash', str(SCRIPTS / 'azure-pipelines.sh')) + self.assertEqual(result.returncode, 1, result.stderr) + self.assertIn('did not complete within 60s after its job started on an agent', + result.stderr) + self.assertTrue((self.state / 'cancelled').exists()) + self.assertEqual(list(self.state.glob('ci-probe-*')), []) + + def test_started_probe_completes_within_its_bounds(self): + self.env['PIPELINE_INPROGRESS_POLLS'] = '1' + result = self.run_script('with-probe.sh', 'bash', str(SCRIPTS / 'azure-pipelines.sh')) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertIn('job started on an agent', result.stdout) + self.assertFalse((self.state / 'cancelled').exists()) + self.assertEqual([c for c in self.curl_calls() if c['method'] == 'PATCH'], []) + self.assertEqual(len(self.timeline_calls()), 1) + self.assertEqual(list(self.state.glob('ci-probe-*')), []) + if __name__ == '__main__': unittest.main()