Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
45 changes: 45 additions & 0 deletions .github/azure-live-tests/azure-oidc-login.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,45 @@
#!/usr/bin/env bash
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.

# Renew the existing Azure CLI login through the job's id-token permission.
# The initial azure/login assertion expires long before a delayed cleanup.
# Requesting a fresh assertion with the same audience and logging in as the
# same service principal adds no credential source or permission.
set -euo pipefail

: "${AZURE_CLIENT_ID:?Set the existing Azure client ID}"
: "${AZURE_TENANT_ID:?Set the existing Azure tenant ID}"
: "${ACTIONS_ID_TOKEN_REQUEST_URL:?The job needs id-token: write}"
: "${ACTIONS_ID_TOKEN_REQUEST_TOKEN:?The job needs id-token: write}"

token_response=$(curl --fail --silent --show-error \
--header "Authorization: bearer ${ACTIONS_ID_TOKEN_REQUEST_TOKEN}" \
"${ACTIONS_ID_TOKEN_REQUEST_URL}&audience=api%3A%2F%2FAzureADTokenExchange")
federated_token=$(jq --raw-output '.value // empty' <<< "$token_response")
unset token_response
test -n "$federated_token"
if [[ "${GITHUB_ACTIONS:-}" == true ]]; then
echo "::add-mask::$federated_token"
fi
az login \
--service-principal \
--username "$AZURE_CLIENT_ID" \
--tenant "$AZURE_TENANT_ID" \
--federated-token "$federated_token" \
--output none
unset federated_token
30 changes: 22 additions & 8 deletions .github/azure-live-tests/azure-pipelines.sh
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@

# Keep the fixture alive while Azure Pipelines uses its read-only identity.
set -euo pipefail
script_dir=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)
run_id=""
completed=false

Expand Down Expand Up @@ -59,14 +60,30 @@ read_run_status() {
done
}

# A delayed cleanup can outlive the GitHub OIDC assertion used by the initial
# login. Retry once through a fresh login as the same service principal.
# Set the global access_token directly instead of returning it: the login
# helper must write its ::add-mask:: command to the runner's stdout, not into
# the captured token.
load_devops_access_token() {
if access_token=$(az account get-access-token \
--resource https://app.vssps.visualstudio.com/ \
--query accessToken \
--output tsv); then
return 0
fi
bash "$script_dir/azure-oidc-login.sh" || return
access_token=$(az account get-access-token \
--resource https://app.vssps.visualstudio.com/ \
--query accessToken \
--output tsv)
}

# Called by the EXIT handler.
# shellcheck disable=SC2329
cancel_unfinished_run() {
local response
access_token=$(az account get-access-token \
--resource https://app.vssps.visualstudio.com/ \
--query accessToken \
--output tsv) || return
load_devops_access_token || return
response=$(read_run_status "$access_token") || return $?
state=$(jq --raw-output '.state' <<< "$response") || return
if [[ "$state" != "completed" ]]; then
Expand Down Expand Up @@ -140,10 +157,7 @@ echo "Queued Azure Pipelines run: $run_url"

deadline=$((SECONDS + 3300))
while (( SECONDS < deadline )); do
access_token=$(az account get-access-token \
--resource https://app.vssps.visualstudio.com/ \
--query accessToken \
--output tsv)
load_devops_access_token || exit $?
response=$(read_run_status "$access_token") || exit $?
state=$(jq --raw-output '.state' <<< "$response")
result=$(jq --raw-output '.result // empty' <<< "$response")
Expand Down
89 changes: 88 additions & 1 deletion .github/azure-live-tests/test_scripts.py
Original file line number Diff line number Diff line change
Expand Up @@ -31,9 +31,25 @@
root = pathlib.Path(os.environ['FAKE_AZ_STATE'])
def option(name, default=''):
return args[args.index(name)+1] if name in args else default
def calls_with(prefix):
return [json.loads(line) for line in (root / 'calls.jsonl').read_text().splitlines()
if json.loads(line)[:len(prefix)] == prefix]
def logged_in():
return bool(calls_with(['login']))
with (root / 'calls.jsonl').open('a') as f:
f.write(json.dumps(args) + '\n')
if args[:2] == ['account', 'get-access-token']:
if args[:1] == ['login']:
assert '--service-principal' in args
assert option('--username') == 'fake-client'
assert option('--tenant') == 'fake-tenant'
assert option('--federated-token') == 'fake-federated-token'
elif args[:2] == ['account', 'get-access-token']:
expire_after = int(os.getenv('EXPIRE_DEVOPS_AUTH_AFTER', '0'))
expired = (expire_after and
len(calls_with(['account', 'get-access-token'])) > expire_after)
if expired and not logged_in():
print('AADSTS700... assertion expired', file=sys.stderr)
sys.exit(1)
print('fake-devops-token')
elif args[:3] == ['storage', 'blob', 'upload']:
if os.getenv('FAIL_UPLOAD'): sys.exit(1)
Expand All @@ -46,6 +62,9 @@ def option(name, default=''):
print('fake-read-only-token')
elif args[:3] == ['storage', 'blob', 'delete']:
if os.getenv('FAIL_DELETE'): sys.exit(1)
if os.getenv('EXPIRE_BLOB_AUTH') and not logged_in():
print('AADSTS700... assertion expired', file=sys.stderr)
sys.exit(1)
(root / option('--name')).unlink()
elif args[:2] == ['vm', 'create']:
name = option('--name')
Expand Down Expand Up @@ -89,6 +108,16 @@ def respond(body, code='200'):
else:
print(body)
method = option('--request', 'GET')
url = args[-1]
if url.startswith('https://oidc.example.test/token'):
if os.getenv('FAIL_OIDC'): sys.exit(35)
assert 'audience=api%3A%2F%2FAzureADTokenExchange' in url
assert option('--header') == 'Authorization: bearer fake-oidc-request-token'
print(json.dumps({'value': 'fake-federated-token'}))
sys.exit(0)
if url.startswith('https://dev.azure.com/'):
assert option('--header') == 'Authorization: Bearer fake-devops-token', \
'unexpected DevOps authorization header'
# Both waiting and cancellation must finish before fixture cleanup.
assert list(root.glob('ci-probe-*'))
with (root / 'curl-calls.jsonl').open('a') as f:
Expand Down Expand Up @@ -162,6 +191,10 @@ def setUp(self):
'AZURE_DEVOPS_ORGANIZATION': 'fixture',
'AZURE_DEVOPS_PROJECT': 'tests',
'AZURE_DEVOPS_PIPELINE_ID': '1',
'AZURE_CLIENT_ID': 'fake-client',
'AZURE_TENANT_ID': 'fake-tenant',
'ACTIONS_ID_TOKEN_REQUEST_URL': 'https://oidc.example.test/token?api-version=2.0',
'ACTIONS_ID_TOKEN_REQUEST_TOKEN': 'fake-oidc-request-token',
'AZURE_RUNTIME_RESOURCE_GROUP': 'fixture-rg',
'AZURE_IMDS_IDENTITY_ID': '/fake/identity',
'RUNNER_TEMP': str(self.root),
Expand Down Expand Up @@ -260,6 +293,7 @@ def test_pipeline_reads_temporary_probe_until_completed(self):
self.assertEqual(result.returncode, 0, result.stderr)
self.assertTrue((self.state / 'queued.json').exists())
self.assertFalse((self.state / 'cancelled').exists())
self.assertEqual([c for c in self.calls() if c[:1] == ['login']], [])
self.assertEqual(list(self.state.glob('ci-probe-*')), [])

def test_failed_pipeline_fails_job_and_cleans_probe(self):
Expand Down Expand Up @@ -315,6 +349,59 @@ def test_cleanup_status_reset_is_retried_and_cancels_once(self):
self.assertTrue(all('/runs/23?api-version=7.1' in c['url'] for c in gets))
self.assertEqual(list(self.state.glob('ci-probe-*')), [])

def test_pipeline_cleanup_reauthenticates_after_assertion_expiry(self):
self.env.update(EXPIRE_DEVOPS_AUTH_AFTER='2',
FAIL_PIPELINE_POLL='1')
result = self.run_script('with-probe.sh', 'bash', str(SCRIPTS / 'azure-pipelines.sh'))
self.assertEqual(result.returncode, 22, result.stderr)
logins = [c for c in self.calls() if c[:1] == ['login']]
self.assertEqual(len(logins), 1)
self.assertEqual(logins[0][logins[0].index('--username') + 1], 'fake-client')
self.assertTrue((self.state / 'cancelled').exists())
self.assertNotIn('fake-federated-token', result.stdout + result.stderr)
self.assertNotIn('fake-devops-token', result.stdout + result.stderr)
self.assertEqual(list(self.state.glob('ci-probe-*')), [])

def test_pipeline_cleanup_reauthenticates_in_actions_mode(self):
self.env.update(EXPIRE_DEVOPS_AUTH_AFTER='2',
FAIL_PIPELINE_POLL='1', GITHUB_ACTIONS='true')
result = self.run_script('with-probe.sh', 'bash', str(SCRIPTS / 'azure-pipelines.sh'))
self.assertEqual(result.returncode, 22, result.stderr)
calls = self.curl_calls()
gets = [c for c in calls if c['method'] == 'GET']
self.assertEqual(len(gets), 2)
self.assertEqual(len([c for c in calls if c['method'] == 'POST']), 1)
self.assertEqual(len([c for c in calls if c['method'] == 'PATCH']), 1)
self.assertTrue((self.state / 'cancelled').exists())
masks = [line for line in result.stdout.splitlines()
if line.startswith('::add-mask::')]
self.assertEqual(masks, ['::add-mask::fake-devops-token',
'::add-mask::fake-federated-token'])
self.assertNotIn('::add-mask::', result.stderr)
unmasked_stdout = '\n'.join(line for line in result.stdout.splitlines()
if not line.startswith('::add-mask::'))
self.assertNotIn('fake-federated-token', unmasked_stdout + result.stderr)
self.assertNotIn('fake-devops-token', unmasked_stdout + result.stderr)
self.assertEqual(list(self.state.glob('ci-probe-*')), [])

def test_blob_cleanup_reauthenticates_after_assertion_expiry(self):
self.env['EXPIRE_BLOB_AUTH'] = '1'
result = self.run_script('with-probe.sh', 'true')
self.assertEqual(result.returncode, 0, result.stderr)
self.assertEqual(len([c for c in self.calls() if c[:1] == ['login']]), 1)
self.assertEqual(self.resources(), [])
self.assertNotIn('fake-federated-token', result.stdout + result.stderr)

def test_failed_reauth_keeps_original_exit_and_cleans_probe(self):
self.env.update(EXPIRE_DEVOPS_AUTH_AFTER='2',
FAIL_PIPELINE_POLL='1', FAIL_OIDC='1')
result = self.run_script('with-probe.sh', 'bash', str(SCRIPTS / 'azure-pipelines.sh'))
self.assertEqual(result.returncode, 22, result.stderr)
self.assertFalse((self.state / 'cancelled').exists())
self.assertEqual([c for c in self.calls() if c[:1] == ['login']], [])
self.assertIn('Failed to cancel', result.stderr)
self.assertEqual(list(self.state.glob('ci-probe-*')), [])

def test_pipeline_status_503_is_retried(self):
self.env['HTTP_503_PIPELINE_POLL_ONCE'] = '1'
result = self.run_script('with-probe.sh', 'bash', str(SCRIPTS / 'azure-pipelines.sh'))
Expand Down
15 changes: 14 additions & 1 deletion .github/azure-live-tests/with-probe.sh
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,8 @@ set -euo pipefail
: "${AZURE_STORAGE_CONTAINER:?Set the fixture container}"
(( $# > 0 )) || { echo "Usage: with-probe.sh command [args...]" >&2; exit 2; }

script_dir=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)

auth_mode=${AZURE_STORAGE_AUTH_MODE:-login}
if [[ "$auth_mode" == key ]]; then
export AZURE_STORAGE_KEY="${REQSIGN_AZURE_STORAGE_ACCOUNT_KEY:?Set the fixture account key}"
Expand All @@ -33,11 +35,22 @@ storage_args=(--auth-mode "$auth_mode" --account-name "$AZURE_STORAGE_ACCOUNT"
blob_name="ci-probe-$(python3 -c 'import uuid; print(uuid.uuid4())').txt"
probe_dir=$(mktemp -d)
created=false
delete_probe() {
az storage blob delete "${storage_args[@]}" --name "$blob_name" --output none && return 0
# The initial federated login can outlive its GitHub OIDC assertion. Renew
# through the same job permission, then retry this probe's delete once.
if [[ "$auth_mode" == login ]]; then
bash "$script_dir/azure-oidc-login.sh" &&
az storage blob delete "${storage_args[@]}" --name "$blob_name" --output none
return $?
fi
return 1
}
cleanup() {
local result=$?
trap - EXIT
if [[ "$created" == true ]]; then
if ! az storage blob delete "${storage_args[@]}" --name "$blob_name" --output none; then
if ! delete_probe; then
echo "Failed to delete the live-test probe" >&2
if (( result == 0 )); then result=1; fi
fi
Expand Down
Loading