From 24021f111d1670cbd29404b28288ba8aa159b977 Mon Sep 17 00:00:00 2001 From: Xuanwo Date: Thu, 19 Nov 2026 16:33:28 +0100 Subject: [PATCH] fix(ci): retry HTTP 503 on Azure pipeline status reads The Azure live-test job 135585811335 in run 44531403273 failed on its first attempt with `curl: (22) The requested URL returned error: 503` while polling queued run 116. The job passed on attempt 2 without a code change, so the 503 was transient. Retry status GETs on HTTP 503 within the existing three-attempt, one-second budget from #959, while keeping the curl 35 retry. Capture the HTTP code with curl's write-out and discard the response body in a temporary file on failure, so failed or partial bodies are never parsed or exposed. Other HTTP failures return immediately, the queue POST still runs once, and the cancel PATCH still runs at most once. Validation: `python3 .github/azure-live-tests/test_scripts.py` passes 17 tests, including 503 then success, exhausted 503 retaining exit 22, and a 403 that is not retried. Shell syntax and Python compilation checks pass. No live Azure credentials or endpoints were used. --- .github/azure-live-tests/azure-pipelines.sh | 27 ++++++--- .github/azure-live-tests/test_scripts.py | 67 ++++++++++++++++++++- 2 files changed, 84 insertions(+), 10 deletions(-) diff --git a/.github/azure-live-tests/azure-pipelines.sh b/.github/azure-live-tests/azure-pipelines.sh index ce1d02d8..78911bda 100644 --- a/.github/azure-live-tests/azure-pipelines.sh +++ b/.github/azure-live-tests/azure-pipelines.sh @@ -21,26 +21,39 @@ set -euo pipefail run_id="" completed=false -# A status read is idempotent. Retry the observed TLS send reset (curl 35) -# within a poll so one transient reset does not fail the job. +# A status read is idempotent. Retry the observed TLS send reset (curl 35) and +# HTTP 503 (curl 22) within a poll so one transient failure does not fail the +# job. Other HTTP failures are permanent for this job. status_read_attempts=3 status_read_retry_delay=1 +status_response_file="$RUNNER_TEMP/azure-pipelines-status-response.json" # Called by the EXIT handler and the polling loop. read_run_status() { local access_token=$1 - local attempt status response + local attempt curl_status http_status response retryable for (( attempt = 1; attempt <= status_read_attempts; attempt++ )); do - if response=$(curl --fail --silent --show-error \ + if http_status=$(curl --fail --silent --show-error \ + --output "$status_response_file" \ + --write-out '%{http_code}' \ --header "Authorization: Bearer $access_token" \ "https://dev.azure.com/${AZURE_DEVOPS_ORGANIZATION}/${AZURE_DEVOPS_PROJECT}/_apis/pipelines/${AZURE_DEVOPS_PIPELINE_ID}/runs/${run_id}?api-version=7.1"); then + response=$(<"$status_response_file") + rm -f "$status_response_file" printf '%s\n' "$response" return 0 else - status=$? + curl_status=$? fi - if (( status != 35 || attempt == status_read_attempts )); then - return "$status" + rm -f "$status_response_file" + retryable=false + if (( curl_status == 35 )); then + retryable=true + elif (( curl_status == 22 )) && [[ "$http_status" == "503" ]]; then + retryable=true + fi + if [[ "$retryable" != true ]] || (( attempt == status_read_attempts )); then + return "$curl_status" fi sleep "$status_read_retry_delay" done diff --git a/.github/azure-live-tests/test_scripts.py b/.github/azure-live-tests/test_scripts.py index 64352957..cba9fee3 100644 --- a/.github/azure-live-tests/test_scripts.py +++ b/.github/azure-live-tests/test_scripts.py @@ -78,6 +78,16 @@ def option(name, default=''): root = pathlib.Path(os.environ['FAKE_AZ_STATE']) def option(name, default=''): return args[args.index(name)+1] if name in args else default +def respond(body, code='200'): + output = option('--output') + if output: + pathlib.Path(output).write_text(body) + write_out = option('--write-out') + if write_out: + assert write_out == '%{http_code}', write_out + print(code) + else: + print(body) method = option('--request', 'GET') # Both waiting and cancellation must finish before fixture cleanup. assert list(root.glob('ci-probe-*')) @@ -107,13 +117,22 @@ def option(name, default=''): sys.exit(35) if os.getenv('RESET_CLEANUP_ONCE') and count == 2: sys.exit(35) + if os.getenv('HTTP_503_PIPELINE_POLL'): + respond('sensitive-503-response-body', '503') + sys.exit(22) + if os.getenv('HTTP_503_PIPELINE_POLL_ONCE') and count == 1: + respond('sensitive-503-response-body', '503') + sys.exit(22) + if os.getenv('HTTP_403_PIPELINE_POLL'): + respond('sensitive-403-response-body', '403') + sys.exit(22) if os.getenv('FAIL_PIPELINE_POLL'): if count == 1: sys.exit(22) - print(json.dumps({'state': 'inProgress'})) + respond(json.dumps({'state': 'inProgress'})) else: - print(json.dumps({'state': 'completed', - 'result': os.getenv('PIPELINE_RESULT', 'succeeded')})) + respond(json.dumps({'state': 'completed', + 'result': os.getenv('PIPELINE_RESULT', 'succeeded')})) """ @@ -296,6 +315,48 @@ def test_cleanup_status_reset_is_retried_and_cancels_once(self): self.assertTrue(all('/runs/23?api-version=7.1' in c['url'] for c in gets)) self.assertEqual(list(self.state.glob('ci-probe-*')), []) + def test_pipeline_status_503_is_retried(self): + self.env['HTTP_503_PIPELINE_POLL_ONCE'] = '1' + result = self.run_script('with-probe.sh', 'bash', str(SCRIPTS / 'azure-pipelines.sh')) + self.assertEqual(result.returncode, 0, result.stderr) + calls = self.curl_calls() + gets = [c for c in calls if c['method'] == 'GET'] + self.assertEqual(len(gets), 2) + self.assertEqual(len([c for c in calls if c['method'] == 'POST']), 1) + self.assertEqual([c for c in calls if c['method'] == 'PATCH'], []) + self.assertTrue(all('/runs/23?api-version=7.1' in c['url'] for c in gets)) + self.assertNotIn('sensitive-503-response-body', result.stdout + result.stderr) + self.assertNotIn('fake-devops-token', result.stdout + result.stderr) + self.assertEqual(list(self.state.glob('ci-probe-*')), []) + + def test_pipeline_status_503_exhausts_retries(self): + self.env['HTTP_503_PIPELINE_POLL'] = '1' + result = self.run_script('with-probe.sh', 'bash', str(SCRIPTS / 'azure-pipelines.sh')) + self.assertEqual(result.returncode, 22, result.stderr) + calls = self.curl_calls() + gets = [c for c in calls if c['method'] == 'GET'] + self.assertEqual(len(gets), 6) + self.assertEqual(len([c for c in calls if c['method'] == 'POST']), 1) + self.assertEqual([c for c in calls if c['method'] == 'PATCH'], []) + self.assertTrue(all('/runs/23?api-version=7.1' in c['url'] for c in gets)) + self.assertNotIn('sensitive-503-response-body', result.stdout + result.stderr) + self.assertNotIn('fake-devops-token', result.stdout + result.stderr) + self.assertEqual(list(self.state.glob('ci-probe-*')), []) + + def test_permanent_pipeline_status_error_is_not_retried(self): + self.env['HTTP_403_PIPELINE_POLL'] = '1' + result = self.run_script('with-probe.sh', 'bash', str(SCRIPTS / 'azure-pipelines.sh')) + self.assertEqual(result.returncode, 22, result.stderr) + calls = self.curl_calls() + gets = [c for c in calls if c['method'] == 'GET'] + self.assertEqual(len(gets), 2) + self.assertEqual(len([c for c in calls if c['method'] == 'POST']), 1) + self.assertEqual([c for c in calls if c['method'] == 'PATCH'], []) + self.assertTrue(all('/runs/23?api-version=7.1' in c['url'] for c in gets)) + self.assertNotIn('sensitive-403-response-body', result.stdout + result.stderr) + self.assertNotIn('fake-devops-token', result.stdout + result.stderr) + self.assertEqual(list(self.state.glob('ci-probe-*')), []) + if __name__ == '__main__': unittest.main()