Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
36 changes: 29 additions & 7 deletions .github/azure-live-tests/azure-pipelines.sh
Original file line number Diff line number Diff line change
Expand Up @@ -21,17 +21,41 @@ set -euo pipefail
run_id=""
completed=false

# A status read is idempotent. Retry the observed TLS send reset (curl 35)
# within a poll so one transient reset does not fail the job.
status_read_attempts=3
status_read_retry_delay=1

# Called by the EXIT handler and the polling loop.
read_run_status() {
local access_token=$1
local attempt status response
for (( attempt = 1; attempt <= status_read_attempts; attempt++ )); do
if response=$(curl --fail --silent --show-error \
--header "Authorization: Bearer $access_token" \
"https://dev.azure.com/${AZURE_DEVOPS_ORGANIZATION}/${AZURE_DEVOPS_PROJECT}/_apis/pipelines/${AZURE_DEVOPS_PIPELINE_ID}/runs/${run_id}?api-version=7.1"); then
printf '%s\n' "$response"
return 0
else
status=$?
fi
if (( status != 35 || attempt == status_read_attempts )); then
return "$status"
fi
sleep "$status_read_retry_delay"
done
}

# Called by the EXIT handler.
# shellcheck disable=SC2329
cancel_unfinished_run() {
local response
access_token=$(az account get-access-token \
--resource https://app.vssps.visualstudio.com/ \
--query accessToken \
--output tsv) || return
state=$(curl --fail-with-body --silent --show-error \
--header "Authorization: Bearer $access_token" \
"https://dev.azure.com/${AZURE_DEVOPS_ORGANIZATION}/${AZURE_DEVOPS_PROJECT}/_apis/pipelines/${AZURE_DEVOPS_PIPELINE_ID}/runs/${run_id}?api-version=7.1" |
jq --raw-output '.state') || return
response=$(read_run_status "$access_token") || return $?
state=$(jq --raw-output '.state' <<< "$response") || return
if [[ "$state" != "completed" ]]; then
curl --fail-with-body --silent --show-error \
--request PATCH \
Expand Down Expand Up @@ -107,9 +131,7 @@ while (( SECONDS < deadline )); do
--resource https://app.vssps.visualstudio.com/ \
--query accessToken \
--output tsv)
response=$(curl --fail-with-body --silent --show-error \
--header "Authorization: Bearer $access_token" \
"https://dev.azure.com/${AZURE_DEVOPS_ORGANIZATION}/${AZURE_DEVOPS_PROJECT}/_apis/pipelines/${AZURE_DEVOPS_PIPELINE_ID}/runs/${run_id}?api-version=7.1")
response=$(read_run_status "$access_token") || exit $?
state=$(jq --raw-output '.state' <<< "$response")
result=$(jq --raw-output '.result // empty' <<< "$response")
if [[ "$state" == "completed" ]]; then
Expand Down
71 changes: 63 additions & 8 deletions .github/azure-live-tests/test_scripts.py
Original file line number Diff line number Diff line change
Expand Up @@ -81,6 +81,8 @@ def option(name, default=''):
method = option('--request', 'GET')
# Both waiting and cancellation must finish before fixture cleanup.
assert list(root.glob('ci-probe-*'))
with (root / 'curl-calls.jsonl').open('a') as f:
f.write(json.dumps({'method': method, 'url': args[-1]}) + '\n')
if method == 'POST':
request = json.loads(option('--data'))
params = request['templateParameters']
Expand All @@ -95,14 +97,23 @@ def option(name, default=''):
elif method == 'PATCH':
assert json.loads(option('--data')) == {'status': 'cancelling'}
(root / 'cancelled').touch()
elif os.getenv('FAIL_PIPELINE_POLL'):
if not (root / 'poll-failed').exists():
(root / 'poll-failed').touch()
sys.exit(22)
print(json.dumps({'state': 'inProgress'}))
else:
print(json.dumps({'state': 'completed',
'result': os.getenv('PIPELINE_RESULT', 'succeeded')}))
counter = root / 'get-count'
count = int(counter.read_text()) + 1 if counter.exists() else 1
counter.write_text(str(count))
if os.getenv('RESET_PIPELINE_POLL'):
sys.exit(35)
if os.getenv('RESET_PIPELINE_POLL_ONCE') and count == 1:
sys.exit(35)
if os.getenv('RESET_CLEANUP_ONCE') and count == 2:
sys.exit(35)
if os.getenv('FAIL_PIPELINE_POLL'):
if count == 1:
sys.exit(22)
print(json.dumps({'state': 'inProgress'}))
else:
print(json.dumps({'state': 'completed',
'result': os.getenv('PIPELINE_RESULT', 'succeeded')}))
"""


Expand Down Expand Up @@ -148,8 +159,13 @@ def calls(self):
return [json.loads(line) for line in
(self.state / 'calls.jsonl').read_text().splitlines()]

def curl_calls(self):
return [json.loads(line) for line in
(self.state / 'curl-calls.jsonl').read_text().splitlines()]

def resources(self):
return [p.name for p in self.state.iterdir() if p.name != 'calls.jsonl']
return [p.name for p in self.state.iterdir()
if p.name not in {'calls.jsonl', 'curl-calls.jsonl', 'get-count'}]

def test_probe_body_sas_and_cleanup(self):
self.env.update(AZURE_STORAGE_AUTH_MODE='key',
Expand Down Expand Up @@ -241,6 +257,45 @@ def test_poll_failure_cancels_pipeline_before_probe_cleanup(self):
self.assertTrue((self.state / 'cancelled').exists())
self.assertEqual(list(self.state.glob('ci-probe-*')), [])

def test_pipeline_status_reset_is_retried(self):
self.env['RESET_PIPELINE_POLL_ONCE'] = '1'
result = self.run_script('with-probe.sh', 'bash', str(SCRIPTS / 'azure-pipelines.sh'))
self.assertEqual(result.returncode, 0, result.stderr)
calls = self.curl_calls()
gets = [c for c in calls if c['method'] == 'GET']
self.assertEqual(len(gets), 2)
self.assertEqual(len([c for c in calls if c['method'] == 'POST']), 1)
self.assertEqual([c for c in calls if c['method'] == 'PATCH'], [])
self.assertTrue(all('/runs/23?api-version=7.1' in c['url'] for c in gets))
self.assertNotIn('fake-devops-token', result.stdout + result.stderr)
self.assertEqual(list(self.state.glob('ci-probe-*')), [])

def test_pipeline_status_reset_exhausts_retries(self):
self.env['RESET_PIPELINE_POLL'] = '1'
result = self.run_script('with-probe.sh', 'bash', str(SCRIPTS / 'azure-pipelines.sh'))
self.assertEqual(result.returncode, 35, result.stderr)
calls = self.curl_calls()
gets = [c for c in calls if c['method'] == 'GET']
self.assertEqual(len(gets), 6)
self.assertEqual(len([c for c in calls if c['method'] == 'POST']), 1)
self.assertEqual([c for c in calls if c['method'] == 'PATCH'], [])
self.assertTrue(all('/runs/23?api-version=7.1' in c['url'] for c in gets))
self.assertNotIn('fake-devops-token', result.stdout + result.stderr)
self.assertEqual(list(self.state.glob('ci-probe-*')), [])

def test_cleanup_status_reset_is_retried_and_cancels_once(self):
self.env.update(FAIL_PIPELINE_POLL='1', RESET_CLEANUP_ONCE='1')
result = self.run_script('with-probe.sh', 'bash', str(SCRIPTS / 'azure-pipelines.sh'))
self.assertEqual(result.returncode, 22, result.stderr)
calls = self.curl_calls()
gets = [c for c in calls if c['method'] == 'GET']
self.assertEqual(len(gets), 3)
self.assertEqual(len([c for c in calls if c['method'] == 'POST']), 1)
patches = [c for c in calls if c['method'] == 'PATCH']
self.assertEqual(len(patches), 1)
self.assertTrue(all('/runs/23?api-version=7.1' in c['url'] for c in gets))
self.assertEqual(list(self.state.glob('ci-probe-*')), [])


if __name__ == '__main__':
unittest.main()
Loading