Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
27 changes: 20 additions & 7 deletions .github/azure-live-tests/azure-pipelines.sh
Original file line number Diff line number Diff line change
Expand Up @@ -21,26 +21,39 @@ set -euo pipefail
run_id=""
completed=false

# A status read is idempotent. Retry the observed TLS send reset (curl 35)
# within a poll so one transient reset does not fail the job.
# A status read is idempotent. Retry the observed TLS send reset (curl 35) and
# HTTP 503 (curl 22) within a poll so one transient failure does not fail the
# job. Other HTTP failures are permanent for this job.
status_read_attempts=3
status_read_retry_delay=1
status_response_file="$RUNNER_TEMP/azure-pipelines-status-response.json"

# Called by the EXIT handler and the polling loop.
read_run_status() {
local access_token=$1
local attempt status response
local attempt curl_status http_status response retryable
for (( attempt = 1; attempt <= status_read_attempts; attempt++ )); do
if response=$(curl --fail --silent --show-error \
if http_status=$(curl --fail --silent --show-error \
--output "$status_response_file" \
--write-out '%{http_code}' \
--header "Authorization: Bearer $access_token" \
"https://dev.azure.com/${AZURE_DEVOPS_ORGANIZATION}/${AZURE_DEVOPS_PROJECT}/_apis/pipelines/${AZURE_DEVOPS_PIPELINE_ID}/runs/${run_id}?api-version=7.1"); then
response=$(<"$status_response_file")
rm -f "$status_response_file"
printf '%s\n' "$response"
return 0
else
status=$?
curl_status=$?
fi
if (( status != 35 || attempt == status_read_attempts )); then
return "$status"
rm -f "$status_response_file"
retryable=false
if (( curl_status == 35 )); then
retryable=true
elif (( curl_status == 22 )) && [[ "$http_status" == "503" ]]; then
retryable=true
fi
if [[ "$retryable" != true ]] || (( attempt == status_read_attempts )); then
return "$curl_status"
fi
sleep "$status_read_retry_delay"
done
Expand Down
67 changes: 64 additions & 3 deletions .github/azure-live-tests/test_scripts.py
Original file line number Diff line number Diff line change
Expand Up @@ -78,6 +78,16 @@ def option(name, default=''):
root = pathlib.Path(os.environ['FAKE_AZ_STATE'])
def option(name, default=''):
return args[args.index(name)+1] if name in args else default
def respond(body, code='200'):
output = option('--output')
if output:
pathlib.Path(output).write_text(body)
write_out = option('--write-out')
if write_out:
assert write_out == '%{http_code}', write_out
print(code)
else:
print(body)
method = option('--request', 'GET')
# Both waiting and cancellation must finish before fixture cleanup.
assert list(root.glob('ci-probe-*'))
Expand Down Expand Up @@ -107,13 +117,22 @@ def option(name, default=''):
sys.exit(35)
if os.getenv('RESET_CLEANUP_ONCE') and count == 2:
sys.exit(35)
if os.getenv('HTTP_503_PIPELINE_POLL'):
respond('sensitive-503-response-body', '503')
sys.exit(22)
if os.getenv('HTTP_503_PIPELINE_POLL_ONCE') and count == 1:
respond('sensitive-503-response-body', '503')
sys.exit(22)
if os.getenv('HTTP_403_PIPELINE_POLL'):
respond('sensitive-403-response-body', '403')
sys.exit(22)
if os.getenv('FAIL_PIPELINE_POLL'):
if count == 1:
sys.exit(22)
print(json.dumps({'state': 'inProgress'}))
respond(json.dumps({'state': 'inProgress'}))
else:
print(json.dumps({'state': 'completed',
'result': os.getenv('PIPELINE_RESULT', 'succeeded')}))
respond(json.dumps({'state': 'completed',
'result': os.getenv('PIPELINE_RESULT', 'succeeded')}))
"""


Expand Down Expand Up @@ -296,6 +315,48 @@ def test_cleanup_status_reset_is_retried_and_cancels_once(self):
self.assertTrue(all('/runs/23?api-version=7.1' in c['url'] for c in gets))
self.assertEqual(list(self.state.glob('ci-probe-*')), [])

def test_pipeline_status_503_is_retried(self):
self.env['HTTP_503_PIPELINE_POLL_ONCE'] = '1'
result = self.run_script('with-probe.sh', 'bash', str(SCRIPTS / 'azure-pipelines.sh'))
self.assertEqual(result.returncode, 0, result.stderr)
calls = self.curl_calls()
gets = [c for c in calls if c['method'] == 'GET']
self.assertEqual(len(gets), 2)
self.assertEqual(len([c for c in calls if c['method'] == 'POST']), 1)
self.assertEqual([c for c in calls if c['method'] == 'PATCH'], [])
self.assertTrue(all('/runs/23?api-version=7.1' in c['url'] for c in gets))
self.assertNotIn('sensitive-503-response-body', result.stdout + result.stderr)
self.assertNotIn('fake-devops-token', result.stdout + result.stderr)
self.assertEqual(list(self.state.glob('ci-probe-*')), [])

def test_pipeline_status_503_exhausts_retries(self):
self.env['HTTP_503_PIPELINE_POLL'] = '1'
result = self.run_script('with-probe.sh', 'bash', str(SCRIPTS / 'azure-pipelines.sh'))
self.assertEqual(result.returncode, 22, result.stderr)
calls = self.curl_calls()
gets = [c for c in calls if c['method'] == 'GET']
self.assertEqual(len(gets), 6)
self.assertEqual(len([c for c in calls if c['method'] == 'POST']), 1)
self.assertEqual([c for c in calls if c['method'] == 'PATCH'], [])
self.assertTrue(all('/runs/23?api-version=7.1' in c['url'] for c in gets))
self.assertNotIn('sensitive-503-response-body', result.stdout + result.stderr)
self.assertNotIn('fake-devops-token', result.stdout + result.stderr)
self.assertEqual(list(self.state.glob('ci-probe-*')), [])

def test_permanent_pipeline_status_error_is_not_retried(self):
self.env['HTTP_403_PIPELINE_POLL'] = '1'
result = self.run_script('with-probe.sh', 'bash', str(SCRIPTS / 'azure-pipelines.sh'))
self.assertEqual(result.returncode, 22, result.stderr)
calls = self.curl_calls()
gets = [c for c in calls if c['method'] == 'GET']
self.assertEqual(len(gets), 2)
self.assertEqual(len([c for c in calls if c['method'] == 'POST']), 1)
self.assertEqual([c for c in calls if c['method'] == 'PATCH'], [])
self.assertTrue(all('/runs/23?api-version=7.1' in c['url'] for c in gets))
self.assertNotIn('sensitive-403-response-body', result.stdout + result.stderr)
self.assertNotIn('fake-devops-token', result.stdout + result.stderr)
self.assertEqual(list(self.state.glob('ci-probe-*')), [])


if __name__ == '__main__':
unittest.main()
Loading