diff --git a/.github/azure-live-tests/azure-pipelines.sh b/.github/azure-live-tests/azure-pipelines.sh index ce1d02d8..78911bda 100644 --- a/.github/azure-live-tests/azure-pipelines.sh +++ b/.github/azure-live-tests/azure-pipelines.sh @@ -21,26 +21,39 @@ set -euo pipefail run_id="" completed=false -# A status read is idempotent. Retry the observed TLS send reset (curl 35) -# within a poll so one transient reset does not fail the job. +# A status read is idempotent. Retry the observed TLS send reset (curl 35) and +# HTTP 503 (curl 22) within a poll so one transient failure does not fail the +# job. Other HTTP failures are permanent for this job. status_read_attempts=3 status_read_retry_delay=1 +status_response_file="$RUNNER_TEMP/azure-pipelines-status-response.json" # Called by the EXIT handler and the polling loop. read_run_status() { local access_token=$1 - local attempt status response + local attempt curl_status http_status response retryable for (( attempt = 1; attempt <= status_read_attempts; attempt++ )); do - if response=$(curl --fail --silent --show-error \ + if http_status=$(curl --fail --silent --show-error \ + --output "$status_response_file" \ + --write-out '%{http_code}' \ --header "Authorization: Bearer $access_token" \ "https://dev.azure.com/${AZURE_DEVOPS_ORGANIZATION}/${AZURE_DEVOPS_PROJECT}/_apis/pipelines/${AZURE_DEVOPS_PIPELINE_ID}/runs/${run_id}?api-version=7.1"); then + response=$(<"$status_response_file") + rm -f "$status_response_file" printf '%s\n' "$response" return 0 else - status=$? + curl_status=$? fi - if (( status != 35 || attempt == status_read_attempts )); then - return "$status" + rm -f "$status_response_file" + retryable=false + if (( curl_status == 35 )); then + retryable=true + elif (( curl_status == 22 )) && [[ "$http_status" == "503" ]]; then + retryable=true + fi + if [[ "$retryable" != true ]] || (( attempt == status_read_attempts )); then + return "$curl_status" fi sleep "$status_read_retry_delay" done diff --git a/.github/azure-live-tests/test_scripts.py b/.github/azure-live-tests/test_scripts.py index 64352957..cba9fee3 100644 --- a/.github/azure-live-tests/test_scripts.py +++ b/.github/azure-live-tests/test_scripts.py @@ -78,6 +78,16 @@ def option(name, default=''): root = pathlib.Path(os.environ['FAKE_AZ_STATE']) def option(name, default=''): return args[args.index(name)+1] if name in args else default +def respond(body, code='200'): + output = option('--output') + if output: + pathlib.Path(output).write_text(body) + write_out = option('--write-out') + if write_out: + assert write_out == '%{http_code}', write_out + print(code) + else: + print(body) method = option('--request', 'GET') # Both waiting and cancellation must finish before fixture cleanup. assert list(root.glob('ci-probe-*')) @@ -107,13 +117,22 @@ def option(name, default=''): sys.exit(35) if os.getenv('RESET_CLEANUP_ONCE') and count == 2: sys.exit(35) + if os.getenv('HTTP_503_PIPELINE_POLL'): + respond('sensitive-503-response-body', '503') + sys.exit(22) + if os.getenv('HTTP_503_PIPELINE_POLL_ONCE') and count == 1: + respond('sensitive-503-response-body', '503') + sys.exit(22) + if os.getenv('HTTP_403_PIPELINE_POLL'): + respond('sensitive-403-response-body', '403') + sys.exit(22) if os.getenv('FAIL_PIPELINE_POLL'): if count == 1: sys.exit(22) - print(json.dumps({'state': 'inProgress'})) + respond(json.dumps({'state': 'inProgress'})) else: - print(json.dumps({'state': 'completed', - 'result': os.getenv('PIPELINE_RESULT', 'succeeded')})) + respond(json.dumps({'state': 'completed', + 'result': os.getenv('PIPELINE_RESULT', 'succeeded')})) """ @@ -296,6 +315,48 @@ def test_cleanup_status_reset_is_retried_and_cancels_once(self): self.assertTrue(all('/runs/23?api-version=7.1' in c['url'] for c in gets)) self.assertEqual(list(self.state.glob('ci-probe-*')), []) + def test_pipeline_status_503_is_retried(self): + self.env['HTTP_503_PIPELINE_POLL_ONCE'] = '1' + result = self.run_script('with-probe.sh', 'bash', str(SCRIPTS / 'azure-pipelines.sh')) + self.assertEqual(result.returncode, 0, result.stderr) + calls = self.curl_calls() + gets = [c for c in calls if c['method'] == 'GET'] + self.assertEqual(len(gets), 2) + self.assertEqual(len([c for c in calls if c['method'] == 'POST']), 1) + self.assertEqual([c for c in calls if c['method'] == 'PATCH'], []) + self.assertTrue(all('/runs/23?api-version=7.1' in c['url'] for c in gets)) + self.assertNotIn('sensitive-503-response-body', result.stdout + result.stderr) + self.assertNotIn('fake-devops-token', result.stdout + result.stderr) + self.assertEqual(list(self.state.glob('ci-probe-*')), []) + + def test_pipeline_status_503_exhausts_retries(self): + self.env['HTTP_503_PIPELINE_POLL'] = '1' + result = self.run_script('with-probe.sh', 'bash', str(SCRIPTS / 'azure-pipelines.sh')) + self.assertEqual(result.returncode, 22, result.stderr) + calls = self.curl_calls() + gets = [c for c in calls if c['method'] == 'GET'] + self.assertEqual(len(gets), 6) + self.assertEqual(len([c for c in calls if c['method'] == 'POST']), 1) + self.assertEqual([c for c in calls if c['method'] == 'PATCH'], []) + self.assertTrue(all('/runs/23?api-version=7.1' in c['url'] for c in gets)) + self.assertNotIn('sensitive-503-response-body', result.stdout + result.stderr) + self.assertNotIn('fake-devops-token', result.stdout + result.stderr) + self.assertEqual(list(self.state.glob('ci-probe-*')), []) + + def test_permanent_pipeline_status_error_is_not_retried(self): + self.env['HTTP_403_PIPELINE_POLL'] = '1' + result = self.run_script('with-probe.sh', 'bash', str(SCRIPTS / 'azure-pipelines.sh')) + self.assertEqual(result.returncode, 22, result.stderr) + calls = self.curl_calls() + gets = [c for c in calls if c['method'] == 'GET'] + self.assertEqual(len(gets), 2) + self.assertEqual(len([c for c in calls if c['method'] == 'POST']), 1) + self.assertEqual([c for c in calls if c['method'] == 'PATCH'], []) + self.assertTrue(all('/runs/23?api-version=7.1' in c['url'] for c in gets)) + self.assertNotIn('sensitive-403-response-body', result.stdout + result.stderr) + self.assertNotIn('fake-devops-token', result.stdout + result.stderr) + self.assertEqual(list(self.state.glob('ci-probe-*')), []) + if __name__ == '__main__': unittest.main()