From 8e9d4b18ce0cd488579bcd94824873386d2f3219 Mon Sep 17 00:00:00 2001 From: Xuanwo Date: Thu, 19 Nov 2026 14:13:26 +0100 Subject: [PATCH] fix(ci): retry transient Azure pipeline status reads A single curl exit 35 connection reset while reading the state of an already queued run failed the Azure live-test job before the probe reported anything. Retry GET status reads up to three times with a short delay only for curl exit 35. The cleanup status read uses the same helper. Queue POST remains single-shot, and cancel PATCH remains at most once. Use --fail so error response bodies are not parsed or exposed, and preserve the final curl exit status. Cover one reset then success, exhausted retries, and a cleanup reset that still cancels once. --- .github/azure-live-tests/azure-pipelines.sh | 36 +++++++++-- .github/azure-live-tests/test_scripts.py | 71 ++++++++++++++++++--- 2 files changed, 92 insertions(+), 15 deletions(-) diff --git a/.github/azure-live-tests/azure-pipelines.sh b/.github/azure-live-tests/azure-pipelines.sh index e7d45001..ce1d02d8 100644 --- a/.github/azure-live-tests/azure-pipelines.sh +++ b/.github/azure-live-tests/azure-pipelines.sh @@ -21,17 +21,41 @@ set -euo pipefail run_id="" completed=false +# A status read is idempotent. Retry the observed TLS send reset (curl 35) +# within a poll so one transient reset does not fail the job. +status_read_attempts=3 +status_read_retry_delay=1 + +# Called by the EXIT handler and the polling loop. +read_run_status() { + local access_token=$1 + local attempt status response + for (( attempt = 1; attempt <= status_read_attempts; attempt++ )); do + if response=$(curl --fail --silent --show-error \ + --header "Authorization: Bearer $access_token" \ + "https://dev.azure.com/${AZURE_DEVOPS_ORGANIZATION}/${AZURE_DEVOPS_PROJECT}/_apis/pipelines/${AZURE_DEVOPS_PIPELINE_ID}/runs/${run_id}?api-version=7.1"); then + printf '%s\n' "$response" + return 0 + else + status=$? + fi + if (( status != 35 || attempt == status_read_attempts )); then + return "$status" + fi + sleep "$status_read_retry_delay" + done +} + # Called by the EXIT handler. # shellcheck disable=SC2329 cancel_unfinished_run() { + local response access_token=$(az account get-access-token \ --resource https://app.vssps.visualstudio.com/ \ --query accessToken \ --output tsv) || return - state=$(curl --fail-with-body --silent --show-error \ - --header "Authorization: Bearer $access_token" \ - "https://dev.azure.com/${AZURE_DEVOPS_ORGANIZATION}/${AZURE_DEVOPS_PROJECT}/_apis/pipelines/${AZURE_DEVOPS_PIPELINE_ID}/runs/${run_id}?api-version=7.1" | - jq --raw-output '.state') || return + response=$(read_run_status "$access_token") || return $? + state=$(jq --raw-output '.state' <<< "$response") || return if [[ "$state" != "completed" ]]; then curl --fail-with-body --silent --show-error \ --request PATCH \ @@ -107,9 +131,7 @@ while (( SECONDS < deadline )); do --resource https://app.vssps.visualstudio.com/ \ --query accessToken \ --output tsv) - response=$(curl --fail-with-body --silent --show-error \ - --header "Authorization: Bearer $access_token" \ - "https://dev.azure.com/${AZURE_DEVOPS_ORGANIZATION}/${AZURE_DEVOPS_PROJECT}/_apis/pipelines/${AZURE_DEVOPS_PIPELINE_ID}/runs/${run_id}?api-version=7.1") + response=$(read_run_status "$access_token") || exit $? state=$(jq --raw-output '.state' <<< "$response") result=$(jq --raw-output '.result // empty' <<< "$response") if [[ "$state" == "completed" ]]; then diff --git a/.github/azure-live-tests/test_scripts.py b/.github/azure-live-tests/test_scripts.py index c6c4e9ba..64352957 100644 --- a/.github/azure-live-tests/test_scripts.py +++ b/.github/azure-live-tests/test_scripts.py @@ -81,6 +81,8 @@ def option(name, default=''): method = option('--request', 'GET') # Both waiting and cancellation must finish before fixture cleanup. assert list(root.glob('ci-probe-*')) +with (root / 'curl-calls.jsonl').open('a') as f: + f.write(json.dumps({'method': method, 'url': args[-1]}) + '\n') if method == 'POST': request = json.loads(option('--data')) params = request['templateParameters'] @@ -95,14 +97,23 @@ def option(name, default=''): elif method == 'PATCH': assert json.loads(option('--data')) == {'status': 'cancelling'} (root / 'cancelled').touch() -elif os.getenv('FAIL_PIPELINE_POLL'): - if not (root / 'poll-failed').exists(): - (root / 'poll-failed').touch() - sys.exit(22) - print(json.dumps({'state': 'inProgress'})) else: - print(json.dumps({'state': 'completed', - 'result': os.getenv('PIPELINE_RESULT', 'succeeded')})) + counter = root / 'get-count' + count = int(counter.read_text()) + 1 if counter.exists() else 1 + counter.write_text(str(count)) + if os.getenv('RESET_PIPELINE_POLL'): + sys.exit(35) + if os.getenv('RESET_PIPELINE_POLL_ONCE') and count == 1: + sys.exit(35) + if os.getenv('RESET_CLEANUP_ONCE') and count == 2: + sys.exit(35) + if os.getenv('FAIL_PIPELINE_POLL'): + if count == 1: + sys.exit(22) + print(json.dumps({'state': 'inProgress'})) + else: + print(json.dumps({'state': 'completed', + 'result': os.getenv('PIPELINE_RESULT', 'succeeded')})) """ @@ -148,8 +159,13 @@ def calls(self): return [json.loads(line) for line in (self.state / 'calls.jsonl').read_text().splitlines()] + def curl_calls(self): + return [json.loads(line) for line in + (self.state / 'curl-calls.jsonl').read_text().splitlines()] + def resources(self): - return [p.name for p in self.state.iterdir() if p.name != 'calls.jsonl'] + return [p.name for p in self.state.iterdir() + if p.name not in {'calls.jsonl', 'curl-calls.jsonl', 'get-count'}] def test_probe_body_sas_and_cleanup(self): self.env.update(AZURE_STORAGE_AUTH_MODE='key', @@ -241,6 +257,45 @@ def test_poll_failure_cancels_pipeline_before_probe_cleanup(self): self.assertTrue((self.state / 'cancelled').exists()) self.assertEqual(list(self.state.glob('ci-probe-*')), []) + def test_pipeline_status_reset_is_retried(self): + self.env['RESET_PIPELINE_POLL_ONCE'] = '1' + result = self.run_script('with-probe.sh', 'bash', str(SCRIPTS / 'azure-pipelines.sh')) + self.assertEqual(result.returncode, 0, result.stderr) + calls = self.curl_calls() + gets = [c for c in calls if c['method'] == 'GET'] + self.assertEqual(len(gets), 2) + self.assertEqual(len([c for c in calls if c['method'] == 'POST']), 1) + self.assertEqual([c for c in calls if c['method'] == 'PATCH'], []) + self.assertTrue(all('/runs/23?api-version=7.1' in c['url'] for c in gets)) + self.assertNotIn('fake-devops-token', result.stdout + result.stderr) + self.assertEqual(list(self.state.glob('ci-probe-*')), []) + + def test_pipeline_status_reset_exhausts_retries(self): + self.env['RESET_PIPELINE_POLL'] = '1' + result = self.run_script('with-probe.sh', 'bash', str(SCRIPTS / 'azure-pipelines.sh')) + self.assertEqual(result.returncode, 35, result.stderr) + calls = self.curl_calls() + gets = [c for c in calls if c['method'] == 'GET'] + self.assertEqual(len(gets), 6) + self.assertEqual(len([c for c in calls if c['method'] == 'POST']), 1) + self.assertEqual([c for c in calls if c['method'] == 'PATCH'], []) + self.assertTrue(all('/runs/23?api-version=7.1' in c['url'] for c in gets)) + self.assertNotIn('fake-devops-token', result.stdout + result.stderr) + self.assertEqual(list(self.state.glob('ci-probe-*')), []) + + def test_cleanup_status_reset_is_retried_and_cancels_once(self): + self.env.update(FAIL_PIPELINE_POLL='1', RESET_CLEANUP_ONCE='1') + result = self.run_script('with-probe.sh', 'bash', str(SCRIPTS / 'azure-pipelines.sh')) + self.assertEqual(result.returncode, 22, result.stderr) + calls = self.curl_calls() + gets = [c for c in calls if c['method'] == 'GET'] + self.assertEqual(len(gets), 3) + self.assertEqual(len([c for c in calls if c['method'] == 'POST']), 1) + patches = [c for c in calls if c['method'] == 'PATCH'] + self.assertEqual(len(patches), 1) + self.assertTrue(all('/runs/23?api-version=7.1' in c['url'] for c in gets)) + self.assertEqual(list(self.state.glob('ci-probe-*')), []) + if __name__ == '__main__': unittest.main()