diff --git a/.github/azure-live-tests/azure-pipelines.sh b/.github/azure-live-tests/azure-pipelines.sh index c0bab449..f6cd5de5 100644 --- a/.github/azure-live-tests/azure-pipelines.sh +++ b/.github/azure-live-tests/azure-pipelines.sh @@ -22,44 +22,87 @@ script_dir=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd) run_id="" completed=false -# A status read is idempotent. Retry the observed TLS send reset (curl 35) and +# A run that is still waiting for the pool agent has not exercised the probe, +# so one shared deadline cannot tell a queue stall from a slow probe. Bound the +# wait for an agent and the execution after it starts separately; the defaults +# keep the worst case plus cleanup inside the GitHub job's 60-minute timeout +# and inside the lifetime of an access token, which cleanup can renew. +queue_wait_limit=${REQSIGN_AZURE_PIPELINES_QUEUE_LIMIT:-1200} +execution_limit=${REQSIGN_AZURE_PIPELINES_EXECUTION_LIMIT:-900} +# Local fixtures shorten these so the bounds are testable without waiting. +poll_interval=${REQSIGN_AZURE_PIPELINES_POLL_INTERVAL:-15} + +# A DevOps read is idempotent. Retry the observed TLS send reset (curl 35) and # HTTP 503 (curl 22) within a poll so one transient failure does not fail the # job. Other HTTP failures are permanent for this job. -status_read_attempts=3 -status_read_retry_delay=1 +devops_read_attempts=3 +devops_read_retry_delay=1 status_response_file="$RUNNER_TEMP/azure-pipelines-status-response.json" +timeline_response_file="$RUNNER_TEMP/azure-pipelines-timeline-response.json" -# Called by the EXIT handler and the polling loop. -read_run_status() { - local access_token=$1 +read_devops_json() { + local access_token=$1 path=$2 response_file=$3 allow_missing=${4:-false} local attempt curl_status http_status response retryable - for (( attempt = 1; attempt <= status_read_attempts; attempt++ )); do + for (( attempt = 1; attempt <= devops_read_attempts; attempt++ )); do if http_status=$(curl --fail --silent --show-error \ - --output "$status_response_file" \ + --output "$response_file" \ --write-out '%{http_code}' \ --header "Authorization: Bearer $access_token" \ - "https://dev.azure.com/${AZURE_DEVOPS_ORGANIZATION}/${AZURE_DEVOPS_PROJECT}/_apis/pipelines/${AZURE_DEVOPS_PIPELINE_ID}/runs/${run_id}?api-version=7.1"); then - response=$(<"$status_response_file") - rm -f "$status_response_file" + "https://dev.azure.com/${AZURE_DEVOPS_ORGANIZATION}/${AZURE_DEVOPS_PROJECT}${path}"); then + response=$(<"$response_file") + rm -f "$response_file" printf '%s\n' "$response" return 0 else curl_status=$? fi - rm -f "$status_response_file" + rm -f "$response_file" + # A build that is still queued may not have a timeline yet. That is a + # missing record, not a failed read. + if [[ "$allow_missing" == true ]] && (( curl_status == 22 )) && [[ "$http_status" == "404" ]]; then + return 0 + fi retryable=false if (( curl_status == 35 )); then retryable=true elif (( curl_status == 22 )) && [[ "$http_status" == "503" ]]; then retryable=true fi - if [[ "$retryable" != true ]] || (( attempt == status_read_attempts )); then + if [[ "$retryable" != true ]] || (( attempt == devops_read_attempts )); then return "$curl_status" fi - sleep "$status_read_retry_delay" + sleep "$devops_read_retry_delay" done } +# Called by the EXIT handler and the polling loop. +read_run_status() { + read_devops_json "$1" \ + "/_apis/pipelines/${AZURE_DEVOPS_PIPELINE_ID}/runs/${run_id}?api-version=7.1" \ + "$status_response_file" +} + +# Azure DevOps reports the agent that runs a job on the build timeline. A Job +# record carries workerName once an agent picked it up, and startTime is when it +# started there. Stage records also start before any agent is assigned, and a +# build's own startTime comes from the orchestration, so only Job records are +# evidence that execution began on an agent. +read_agent_started_at() { + local access_token=$1 timeline job_start allow_missing=true + timeline=$(read_devops_json "$access_token" \ + "/_apis/build/builds/${run_id}/timeline?api-version=7.1" \ + "$timeline_response_file" "$allow_missing") || return + [[ -n "$timeline" ]] || return 0 + # The timeline may be one timeline object or a list of them. + job_start=$(jq --raw-output ' + (if type == "array" then map(.records // []) | add // [] else .records // [] end) + | [ .[] | select(.type == "Job") + | select((.workerName // "") != "" and (.startTime // "") != "") + | .startTime ] + | sort | .[0] // empty' <<< "$timeline") || return + printf '%s\n' "$job_start" +} + # A delayed cleanup can outlive the GitHub OIDC assertion used by the initial # login. Retry once through a fresh login as the same service principal. # Set the global access_token directly instead of returning it: the login @@ -154,9 +197,12 @@ run_url=$(jq --raw-output '._links.web.href // empty' <<< "$response") test -n "$run_id" test -n "$run_url" echo "Queued Azure Pipelines run: $run_url" +echo "Azure Pipelines probe limits: ${queue_wait_limit}s queue wait, ${execution_limit}s execution after agent start: $run_url" -deadline=$((SECONDS + 3300)) -while (( SECONDS < deadline )); do +queue_deadline=$(( $(date +%s) + queue_wait_limit )) +agent_started_at="" +agent_started_epoch="" +while true; do load_devops_access_token || exit $? response=$(read_run_status "$access_token") || exit $? state=$(jq --raw-output '.state' <<< "$response") @@ -167,7 +213,24 @@ while (( SECONDS < deadline )); do test "$result" = "succeeded" exit fi - sleep 15 + + now=$(date +%s) + if [[ -z "$agent_started_epoch" ]]; then + agent_started_at=$(read_agent_started_at "$access_token") || exit $? + if [[ -n "$agent_started_at" ]]; then + if ! agent_started_epoch=$(date --date="$agent_started_at" +%s); then + echo "Failed to parse the Azure Pipelines job start time '$agent_started_at': $run_url" >&2 + exit 1 + fi + echo "Azure Pipelines job started on an agent at $agent_started_at (execution limit ${execution_limit}s): $run_url" + elif (( now >= queue_deadline )); then + echo "Azure Pipelines run did not start on an agent within ${queue_wait_limit}s of queueing: $run_url" >&2 + exit 1 + fi + fi + if [[ -n "$agent_started_epoch" ]] && (( now >= agent_started_epoch + execution_limit )); then + echo "Azure Pipelines run did not complete within ${execution_limit}s after its job started on an agent at ${agent_started_at}: $run_url" >&2 + exit 1 + fi + sleep "$poll_interval" done -echo "Azure Pipelines run did not complete before the timeout: $run_url" >&2 -exit 1 diff --git a/.github/azure-live-tests/test_scripts.py b/.github/azure-live-tests/test_scripts.py index a9de6420..d5dcc92c 100644 --- a/.github/azure-live-tests/test_scripts.py +++ b/.github/azure-live-tests/test_scripts.py @@ -107,6 +107,18 @@ def respond(body, code='200'): print(code) else: print(body) +def agent_start(): + import datetime + return os.getenv('AGENT_START') or datetime.datetime.now( + datetime.timezone.utc).strftime('%Y-%m-%dT%H:%M:%SZ') +def timeline_records(): + # A stage also starts before an agent is assigned. Only a Job record with a + # workerName proves that an agent picked the run up. + started = agent_start() + job = {'type': 'Job', 'state': 'inProgress', 'startTime': started} + if os.getenv('AGENT_ASSIGNED', '1') == '1': + job['workerName'] = 'reqsign-azure-vmss-1' + return [{'type': 'Stage', 'state': 'inProgress', 'startTime': started}, job] method = option('--request', 'GET') url = args[-1] if url.startswith('https://oidc.example.test/token'): @@ -122,7 +134,13 @@ def respond(body, code='200'): assert list(root.glob('ci-probe-*')) with (root / 'curl-calls.jsonl').open('a') as f: f.write(json.dumps({'method': method, 'url': args[-1]}) + '\n') -if method == 'POST': +if method == 'GET' and url.endswith('/timeline?api-version=7.1'): + if os.getenv('MISSING_TIMELINE'): + respond(json.dumps({'message': 'missing timeline'}), '404') + sys.exit(22) + records = {'records': timeline_records()} + respond(json.dumps([records] if os.getenv('TIMELINE_ARRAY') else records)) +elif method == 'POST': request = json.loads(option('--data')) params = request['templateParameters'] assert params['githubSha'] == 'a' * 40 @@ -159,6 +177,8 @@ def respond(body, code='200'): if count == 1: sys.exit(22) respond(json.dumps({'state': 'inProgress'})) + elif count <= int(os.getenv('PIPELINE_INPROGRESS_POLLS', '0')): + respond(json.dumps({'state': 'inProgress'})) else: respond(json.dumps({'state': 'completed', 'result': os.getenv('PIPELINE_RESULT', 'succeeded')})) @@ -201,6 +221,7 @@ def setUp(self): 'GITHUB_RUN_ID': '123', 'GITHUB_RUN_ATTEMPT': '1', 'GITHUB_OUTPUT': str(self.root / 'outputs'), + 'REQSIGN_AZURE_PIPELINES_POLL_INTERVAL': '0', } def run_script(self, script, *args): @@ -215,6 +236,10 @@ def curl_calls(self): return [json.loads(line) for line in (self.state / 'curl-calls.jsonl').read_text().splitlines()] + def timeline_calls(self): + return [c for c in self.curl_calls() + if c['url'].endswith('/builds/23/timeline?api-version=7.1')] + def resources(self): return [p.name for p in self.state.iterdir() if p.name not in {'calls.jsonl', 'curl-calls.jsonl', 'get-count'}] @@ -444,6 +469,67 @@ def test_permanent_pipeline_status_error_is_not_retried(self): self.assertNotIn('fake-devops-token', result.stdout + result.stderr) self.assertEqual(list(self.state.glob('ci-probe-*')), []) + def test_queue_wait_ends_before_an_agent_starts(self): + # A stage record starts before any agent is assigned. Only a Job record + # with a workerName is agent start, so this run is still queued. + self.env.update(PIPELINE_INPROGRESS_POLLS='1000', + AGENT_ASSIGNED='0', + AGENT_START='2020-01-01T00:00:00Z', + REQSIGN_AZURE_PIPELINES_QUEUE_LIMIT='0') + result = self.run_script('with-probe.sh', 'bash', str(SCRIPTS / 'azure-pipelines.sh')) + self.assertEqual(result.returncode, 1, result.stderr) + self.assertIn('did not start on an agent within 0s of queueing', result.stderr) + self.assertNotIn('job started on an agent', result.stdout) + self.assertEqual(len(self.timeline_calls()), 1) + self.assertTrue((self.state / 'cancelled').exists()) + self.assertEqual(list(self.state.glob('ci-probe-*')), []) + + def test_execution_bound_is_measured_from_the_agent_start(self): + # The default queue limit is an hour still to come, so only the + # execution bound measured from the observed agent start can fail this. + self.env.update(PIPELINE_INPROGRESS_POLLS='1000', + AGENT_START='2020-01-01T00:00:00Z', + REQSIGN_AZURE_PIPELINES_EXECUTION_LIMIT='60') + result = self.run_script('with-probe.sh', 'bash', str(SCRIPTS / 'azure-pipelines.sh')) + self.assertEqual(result.returncode, 1, result.stderr) + self.assertIn('job started on an agent at 2020-01-01T00:00:00Z', result.stdout) + self.assertIn('did not complete within 60s after its job started on an agent', + result.stderr) + self.assertNotIn('did not start on an agent', result.stderr) + self.assertTrue((self.state / 'cancelled').exists()) + self.assertEqual(list(self.state.glob('ci-probe-*')), []) + + def test_missing_timeline_keeps_waiting_for_the_agent(self): + self.env.update(PIPELINE_INPROGRESS_POLLS='1000', MISSING_TIMELINE='1', + REQSIGN_AZURE_PIPELINES_QUEUE_LIMIT='0') + result = self.run_script('with-probe.sh', 'bash', str(SCRIPTS / 'azure-pipelines.sh')) + self.assertEqual(result.returncode, 1, result.stderr) + self.assertIn('did not start on an agent within 0s of queueing', result.stderr) + self.assertNotEqual(result.returncode, 22) + self.assertTrue((self.state / 'cancelled').exists()) + self.assertEqual(list(self.state.glob('ci-probe-*')), []) + + def test_agent_start_is_read_from_a_timeline_list(self): + self.env.update(PIPELINE_INPROGRESS_POLLS='1000', TIMELINE_ARRAY='1', + AGENT_START='2020-01-01T00:00:00Z', + REQSIGN_AZURE_PIPELINES_EXECUTION_LIMIT='60') + result = self.run_script('with-probe.sh', 'bash', str(SCRIPTS / 'azure-pipelines.sh')) + self.assertEqual(result.returncode, 1, result.stderr) + self.assertIn('did not complete within 60s after its job started on an agent', + result.stderr) + self.assertTrue((self.state / 'cancelled').exists()) + self.assertEqual(list(self.state.glob('ci-probe-*')), []) + + def test_started_probe_completes_within_its_bounds(self): + self.env['PIPELINE_INPROGRESS_POLLS'] = '1' + result = self.run_script('with-probe.sh', 'bash', str(SCRIPTS / 'azure-pipelines.sh')) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertIn('job started on an agent', result.stdout) + self.assertFalse((self.state / 'cancelled').exists()) + self.assertEqual([c for c in self.curl_calls() if c['method'] == 'PATCH'], []) + self.assertEqual(len(self.timeline_calls()), 1) + self.assertEqual(list(self.state.glob('ci-probe-*')), []) + if __name__ == '__main__': unittest.main()