Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
101 changes: 82 additions & 19 deletions .github/azure-live-tests/azure-pipelines.sh
Original file line number Diff line number Diff line change
Expand Up @@ -22,44 +22,87 @@ script_dir=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)
run_id=""
completed=false

# A status read is idempotent. Retry the observed TLS send reset (curl 35) and
# A run that is still waiting for the pool agent has not exercised the probe,
# so one shared deadline cannot tell a queue stall from a slow probe. Bound the
# wait for an agent and the execution after it starts separately; the defaults
# keep the worst case plus cleanup inside the GitHub job's 60-minute timeout
# and inside the lifetime of an access token, which cleanup can renew.
queue_wait_limit=${REQSIGN_AZURE_PIPELINES_QUEUE_LIMIT:-1200}
execution_limit=${REQSIGN_AZURE_PIPELINES_EXECUTION_LIMIT:-900}
# Local fixtures shorten these so the bounds are testable without waiting.
poll_interval=${REQSIGN_AZURE_PIPELINES_POLL_INTERVAL:-15}

# A DevOps read is idempotent. Retry the observed TLS send reset (curl 35) and
# HTTP 503 (curl 22) within a poll so one transient failure does not fail the
# job. Other HTTP failures are permanent for this job.
status_read_attempts=3
status_read_retry_delay=1
devops_read_attempts=3
devops_read_retry_delay=1
status_response_file="$RUNNER_TEMP/azure-pipelines-status-response.json"
timeline_response_file="$RUNNER_TEMP/azure-pipelines-timeline-response.json"

# Called by the EXIT handler and the polling loop.
read_run_status() {
local access_token=$1
read_devops_json() {
local access_token=$1 path=$2 response_file=$3 allow_missing=${4:-false}
local attempt curl_status http_status response retryable
for (( attempt = 1; attempt <= status_read_attempts; attempt++ )); do
for (( attempt = 1; attempt <= devops_read_attempts; attempt++ )); do
if http_status=$(curl --fail --silent --show-error \
--output "$status_response_file" \
--output "$response_file" \
--write-out '%{http_code}' \
--header "Authorization: Bearer $access_token" \
"https://dev.azure.com/${AZURE_DEVOPS_ORGANIZATION}/${AZURE_DEVOPS_PROJECT}/_apis/pipelines/${AZURE_DEVOPS_PIPELINE_ID}/runs/${run_id}?api-version=7.1"); then
response=$(<"$status_response_file")
rm -f "$status_response_file"
"https://dev.azure.com/${AZURE_DEVOPS_ORGANIZATION}/${AZURE_DEVOPS_PROJECT}${path}"); then
response=$(<"$response_file")
rm -f "$response_file"
printf '%s\n' "$response"
return 0
else
curl_status=$?
fi
rm -f "$status_response_file"
rm -f "$response_file"
# A build that is still queued may not have a timeline yet. That is a
# missing record, not a failed read.
if [[ "$allow_missing" == true ]] && (( curl_status == 22 )) && [[ "$http_status" == "404" ]]; then
return 0
fi
retryable=false
if (( curl_status == 35 )); then
retryable=true
elif (( curl_status == 22 )) && [[ "$http_status" == "503" ]]; then
retryable=true
fi
if [[ "$retryable" != true ]] || (( attempt == status_read_attempts )); then
if [[ "$retryable" != true ]] || (( attempt == devops_read_attempts )); then
return "$curl_status"
fi
sleep "$status_read_retry_delay"
sleep "$devops_read_retry_delay"
done
}

# Called by the EXIT handler and the polling loop.
read_run_status() {
read_devops_json "$1" \
"/_apis/pipelines/${AZURE_DEVOPS_PIPELINE_ID}/runs/${run_id}?api-version=7.1" \
"$status_response_file"
}

# Azure DevOps reports the agent that runs a job on the build timeline. A Job
# record carries workerName once an agent picked it up, and startTime is when it
# started there. Stage records also start before any agent is assigned, and a
# build's own startTime comes from the orchestration, so only Job records are
# evidence that execution began on an agent.
read_agent_started_at() {
local access_token=$1 timeline job_start allow_missing=true
timeline=$(read_devops_json "$access_token" \
"/_apis/build/builds/${run_id}/timeline?api-version=7.1" \
"$timeline_response_file" "$allow_missing") || return
[[ -n "$timeline" ]] || return 0
# The timeline may be one timeline object or a list of them.
job_start=$(jq --raw-output '
(if type == "array" then map(.records // []) | add // [] else .records // [] end)
| [ .[] | select(.type == "Job")
| select((.workerName // "") != "" and (.startTime // "") != "")
| .startTime ]
| sort | .[0] // empty' <<< "$timeline") || return
printf '%s\n' "$job_start"
}

# A delayed cleanup can outlive the GitHub OIDC assertion used by the initial
# login. Retry once through a fresh login as the same service principal.
# Set the global access_token directly instead of returning it: the login
Expand Down Expand Up @@ -154,9 +197,12 @@ run_url=$(jq --raw-output '._links.web.href // empty' <<< "$response")
test -n "$run_id"
test -n "$run_url"
echo "Queued Azure Pipelines run: $run_url"
echo "Azure Pipelines probe limits: ${queue_wait_limit}s queue wait, ${execution_limit}s execution after agent start: $run_url"

deadline=$((SECONDS + 3300))
while (( SECONDS < deadline )); do
queue_deadline=$(( $(date +%s) + queue_wait_limit ))
agent_started_at=""
agent_started_epoch=""
while true; do
load_devops_access_token || exit $?
response=$(read_run_status "$access_token") || exit $?
state=$(jq --raw-output '.state' <<< "$response")
Expand All @@ -167,7 +213,24 @@ while (( SECONDS < deadline )); do
test "$result" = "succeeded"
exit
fi
sleep 15

now=$(date +%s)
if [[ -z "$agent_started_epoch" ]]; then
agent_started_at=$(read_agent_started_at "$access_token") || exit $?
if [[ -n "$agent_started_at" ]]; then
if ! agent_started_epoch=$(date --date="$agent_started_at" +%s); then
echo "Failed to parse the Azure Pipelines job start time '$agent_started_at': $run_url" >&2
exit 1
fi
echo "Azure Pipelines job started on an agent at $agent_started_at (execution limit ${execution_limit}s): $run_url"
elif (( now >= queue_deadline )); then
echo "Azure Pipelines run did not start on an agent within ${queue_wait_limit}s of queueing: $run_url" >&2
exit 1
fi
fi
if [[ -n "$agent_started_epoch" ]] && (( now >= agent_started_epoch + execution_limit )); then
echo "Azure Pipelines run did not complete within ${execution_limit}s after its job started on an agent at ${agent_started_at}: $run_url" >&2
exit 1
fi
sleep "$poll_interval"
done
echo "Azure Pipelines run did not complete before the timeout: $run_url" >&2
exit 1
88 changes: 87 additions & 1 deletion .github/azure-live-tests/test_scripts.py
Original file line number Diff line number Diff line change
Expand Up @@ -107,6 +107,18 @@ def respond(body, code='200'):
print(code)
else:
print(body)
def agent_start():
import datetime
return os.getenv('AGENT_START') or datetime.datetime.now(
datetime.timezone.utc).strftime('%Y-%m-%dT%H:%M:%SZ')
def timeline_records():
# A stage also starts before an agent is assigned. Only a Job record with a
# workerName proves that an agent picked the run up.
started = agent_start()
job = {'type': 'Job', 'state': 'inProgress', 'startTime': started}
if os.getenv('AGENT_ASSIGNED', '1') == '1':
job['workerName'] = 'reqsign-azure-vmss-1'
return [{'type': 'Stage', 'state': 'inProgress', 'startTime': started}, job]
method = option('--request', 'GET')
url = args[-1]
if url.startswith('https://oidc.example.test/token'):
Expand All @@ -122,7 +134,13 @@ def respond(body, code='200'):
assert list(root.glob('ci-probe-*'))
with (root / 'curl-calls.jsonl').open('a') as f:
f.write(json.dumps({'method': method, 'url': args[-1]}) + '\n')
if method == 'POST':
if method == 'GET' and url.endswith('/timeline?api-version=7.1'):
if os.getenv('MISSING_TIMELINE'):
respond(json.dumps({'message': 'missing timeline'}), '404')
sys.exit(22)
records = {'records': timeline_records()}
respond(json.dumps([records] if os.getenv('TIMELINE_ARRAY') else records))
elif method == 'POST':
request = json.loads(option('--data'))
params = request['templateParameters']
assert params['githubSha'] == 'a' * 40
Expand Down Expand Up @@ -159,6 +177,8 @@ def respond(body, code='200'):
if count == 1:
sys.exit(22)
respond(json.dumps({'state': 'inProgress'}))
elif count <= int(os.getenv('PIPELINE_INPROGRESS_POLLS', '0')):
respond(json.dumps({'state': 'inProgress'}))
else:
respond(json.dumps({'state': 'completed',
'result': os.getenv('PIPELINE_RESULT', 'succeeded')}))
Expand Down Expand Up @@ -201,6 +221,7 @@ def setUp(self):
'GITHUB_RUN_ID': '123',
'GITHUB_RUN_ATTEMPT': '1',
'GITHUB_OUTPUT': str(self.root / 'outputs'),
'REQSIGN_AZURE_PIPELINES_POLL_INTERVAL': '0',
}

def run_script(self, script, *args):
Expand All @@ -215,6 +236,10 @@ def curl_calls(self):
return [json.loads(line) for line in
(self.state / 'curl-calls.jsonl').read_text().splitlines()]

def timeline_calls(self):
return [c for c in self.curl_calls()
if c['url'].endswith('/builds/23/timeline?api-version=7.1')]

def resources(self):
return [p.name for p in self.state.iterdir()
if p.name not in {'calls.jsonl', 'curl-calls.jsonl', 'get-count'}]
Expand Down Expand Up @@ -444,6 +469,67 @@ def test_permanent_pipeline_status_error_is_not_retried(self):
self.assertNotIn('fake-devops-token', result.stdout + result.stderr)
self.assertEqual(list(self.state.glob('ci-probe-*')), [])

def test_queue_wait_ends_before_an_agent_starts(self):
# A stage record starts before any agent is assigned. Only a Job record
# with a workerName is agent start, so this run is still queued.
self.env.update(PIPELINE_INPROGRESS_POLLS='1000',
AGENT_ASSIGNED='0',
AGENT_START='2020-01-01T00:00:00Z',
REQSIGN_AZURE_PIPELINES_QUEUE_LIMIT='0')
result = self.run_script('with-probe.sh', 'bash', str(SCRIPTS / 'azure-pipelines.sh'))
self.assertEqual(result.returncode, 1, result.stderr)
self.assertIn('did not start on an agent within 0s of queueing', result.stderr)
self.assertNotIn('job started on an agent', result.stdout)
self.assertEqual(len(self.timeline_calls()), 1)
self.assertTrue((self.state / 'cancelled').exists())
self.assertEqual(list(self.state.glob('ci-probe-*')), [])

def test_execution_bound_is_measured_from_the_agent_start(self):
# The default queue limit is an hour still to come, so only the
# execution bound measured from the observed agent start can fail this.
self.env.update(PIPELINE_INPROGRESS_POLLS='1000',
AGENT_START='2020-01-01T00:00:00Z',
REQSIGN_AZURE_PIPELINES_EXECUTION_LIMIT='60')
result = self.run_script('with-probe.sh', 'bash', str(SCRIPTS / 'azure-pipelines.sh'))
self.assertEqual(result.returncode, 1, result.stderr)
self.assertIn('job started on an agent at 2020-01-01T00:00:00Z', result.stdout)
self.assertIn('did not complete within 60s after its job started on an agent',
result.stderr)
self.assertNotIn('did not start on an agent', result.stderr)
self.assertTrue((self.state / 'cancelled').exists())
self.assertEqual(list(self.state.glob('ci-probe-*')), [])

def test_missing_timeline_keeps_waiting_for_the_agent(self):
self.env.update(PIPELINE_INPROGRESS_POLLS='1000', MISSING_TIMELINE='1',
REQSIGN_AZURE_PIPELINES_QUEUE_LIMIT='0')
result = self.run_script('with-probe.sh', 'bash', str(SCRIPTS / 'azure-pipelines.sh'))
self.assertEqual(result.returncode, 1, result.stderr)
self.assertIn('did not start on an agent within 0s of queueing', result.stderr)
self.assertNotEqual(result.returncode, 22)
self.assertTrue((self.state / 'cancelled').exists())
self.assertEqual(list(self.state.glob('ci-probe-*')), [])

def test_agent_start_is_read_from_a_timeline_list(self):
self.env.update(PIPELINE_INPROGRESS_POLLS='1000', TIMELINE_ARRAY='1',
AGENT_START='2020-01-01T00:00:00Z',
REQSIGN_AZURE_PIPELINES_EXECUTION_LIMIT='60')
result = self.run_script('with-probe.sh', 'bash', str(SCRIPTS / 'azure-pipelines.sh'))
self.assertEqual(result.returncode, 1, result.stderr)
self.assertIn('did not complete within 60s after its job started on an agent',
result.stderr)
self.assertTrue((self.state / 'cancelled').exists())
self.assertEqual(list(self.state.glob('ci-probe-*')), [])

def test_started_probe_completes_within_its_bounds(self):
self.env['PIPELINE_INPROGRESS_POLLS'] = '1'
result = self.run_script('with-probe.sh', 'bash', str(SCRIPTS / 'azure-pipelines.sh'))
self.assertEqual(result.returncode, 0, result.stderr)
self.assertIn('job started on an agent', result.stdout)
self.assertFalse((self.state / 'cancelled').exists())
self.assertEqual([c for c in self.curl_calls() if c['method'] == 'PATCH'], [])
self.assertEqual(len(self.timeline_calls()), 1)
self.assertEqual(list(self.state.glob('ci-probe-*')), [])


if __name__ == '__main__':
unittest.main()
Loading