diff --git a/.github/azure-live-tests/superseded-probe.sh b/.github/azure-live-tests/superseded-probe.sh new file mode 100644 index 00000000..5416a596 --- /dev/null +++ b/.github/azure-live-tests/superseded-probe.sh @@ -0,0 +1,80 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# Decide whether a newer run of this workflow on the same branch and event will +# cover this live probe. The probe group keeps pending entries, so a probe can +# start long after its own commit was superseded. +# +# Skip only when a newer run exists that already got past the workflow's path +# filters: a change that filters the workflow out creates no run, so an older +# queued probe stays the only live validation for its commit. Any missing or +# unexpected answer keeps the probe; a wasted probe costs a runner, a missing +# live validation costs coverage. +set -euo pipefail + +workflow=${1:?Usage: superseded-probe.sh } +: "${GITHUB_OUTPUT:?Set by Actions}" + +superseded=false +replacement="" +branch="" +case "${GITHUB_EVENT_NAME:-}" in + pull_request|pull_request_target) branch=${GITHUB_HEAD_REF:-} ;; + push) branch=${GITHUB_REF_NAME:-} ;; +esac + +if [[ -z "${GITHUB_TOKEN:-}" ]]; then + echo "Cannot check for a newer run without a token; running the probe" +elif [[ -z "${GITHUB_REPOSITORY:-}" || -z "${GITHUB_RUN_ID:-}" || -z "$branch" ]]; then + echo "Cannot check for a newer run without the run context; running the probe" +else + response_file=$(mktemp) + trap 'rm -f "$response_file"' EXIT + if ! http_status=$(curl --get --silent --show-error \ + --output "$response_file" \ + --write-out '%{http_code}' \ + --header "Authorization: Bearer $GITHUB_TOKEN" \ + --header "Accept: application/vnd.github+json" \ + --data-urlencode "event=$GITHUB_EVENT_NAME" \ + --data-urlencode "branch=$branch" \ + --data-urlencode "per_page=50" \ + "https://api.github.com/repos/${GITHUB_REPOSITORY}/actions/workflows/${workflow}/runs"); then + echo "Could not list workflow runs; running the probe" >&2 + elif [[ "$http_status" != "200" ]]; then + echo "Could not list workflow runs (HTTP $http_status); running the probe" >&2 + else + replacement=$(jq --raw-output \ + --argjson run_id "$GITHUB_RUN_ID" \ + --arg event "$GITHUB_EVENT_NAME" \ + --arg branch "$branch" ' + [ .workflow_runs[]? + | select(.id > $run_id) + | select(.event == $event) + | select(.head_branch == $branch) + | select(.status != "completed") + | .id ] | sort | first // empty' "$response_file" 2>/dev/null) || replacement="" + [[ -z "$replacement" ]] || superseded=true + fi +fi + +echo "superseded=$superseded" >> "$GITHUB_OUTPUT" +if [[ "$superseded" == true ]]; then + echo "Skipping the live probe: newer run $replacement on $branch will cover it" +else + echo "Running the live probe: no newer run of $workflow on $branch is in flight" +fi diff --git a/.github/azure-live-tests/test_scripts.py b/.github/azure-live-tests/test_scripts.py index d5dcc92c..0285a1fc 100644 --- a/.github/azure-live-tests/test_scripts.py +++ b/.github/azure-live-tests/test_scripts.py @@ -25,6 +25,7 @@ import unittest SCRIPTS = Path(__file__).resolve().parent +FIXTURE_GITHUB_TOKEN = 'gh-fixture-token' FAKE_AZ = r"""#!/usr/bin/env python3 import json, os, pathlib, sys args = sys.argv[1:] @@ -92,7 +93,7 @@ def logged_in(): FAKE_CURL = r"""#!/usr/bin/env python3 -import json, os, pathlib, sys +import json, os, pathlib, sys, urllib.parse args = sys.argv[1:] root = pathlib.Path(os.environ['FAKE_AZ_STATE']) def option(name, default=''): @@ -130,6 +131,35 @@ def timeline_records(): if url.startswith('https://dev.azure.com/'): assert option('--header') == 'Authorization: Bearer fake-devops-token', \ 'unexpected DevOps authorization header' +if url.startswith('https://api.github.com/'): + assert option('--header') == 'Authorization: Bearer __GH_TOKEN__', \ + 'unexpected GitHub authorization header' + assert method == 'GET' + if os.getenv('RUNS_API_FAIL'): + sys.exit(35) + query = [] + for i, arg in enumerate(args): + if arg == '--data-urlencode': + name, _, value = args[i + 1].partition('=') + # curl percent-encodes the value, which matters for branch names. + query.append(name + '=' + urllib.parse.quote(value, safe='')) + if query: + url = url + '?' + '&'.join(query) + with (root / 'curl-calls.jsonl').open('a') as f: + f.write(json.dumps({'method': method, 'url': url}) + '\n') + if os.getenv('RUNS_API_BROKEN'): + respond('upstream returned a page, not JSON') + elif os.getenv('RUNS_API_STATUS', '200') != '200': + respond(json.dumps({'message': 'server error'}), os.getenv('RUNS_API_STATUS')) + else: + # The fixture answers with the run under test unless a test supplies + # the workflow runs the check should see. + respond(os.getenv('RUNS_API_BODY') or json.dumps({ + 'total_count': 1, + 'workflow_runs': [{'id': 100, 'event': 'pull_request', + 'head_branch': 'xuanwo/feature', + 'status': 'completed', 'conclusion': 'success'}]})) + sys.exit(0) # Both waiting and cancellation must finish before fixture cleanup. assert list(root.glob('ci-probe-*')) with (root / 'curl-calls.jsonl').open('a') as f: @@ -196,7 +226,7 @@ def setUp(self): binary.write_text(FAKE_AZ) binary.chmod(0o755) curl = self.root / 'curl' - curl.write_text(FAKE_CURL) + curl.write_text(FAKE_CURL.replace('__GH_TOKEN__', FIXTURE_GITHUB_TOKEN)) curl.chmod(0o755) self.env = { **os.environ, @@ -221,6 +251,10 @@ def setUp(self): 'GITHUB_RUN_ID': '123', 'GITHUB_RUN_ATTEMPT': '1', 'GITHUB_OUTPUT': str(self.root / 'outputs'), + 'GITHUB_TOKEN': FIXTURE_GITHUB_TOKEN, + 'GITHUB_REPOSITORY': 'apache/reqsign', + 'GITHUB_EVENT_NAME': 'pull_request', + 'GITHUB_HEAD_REF': 'xuanwo/feature', 'REQSIGN_AZURE_PIPELINES_POLL_INTERVAL': '0', } @@ -531,5 +565,244 @@ def test_started_probe_completes_within_its_bounds(self): self.assertEqual(list(self.state.glob('ci-probe-*')), []) + def superseded(self): + return (self.root / 'outputs').read_text().strip() + + def runs_api_urls(self): + path = self.state / 'curl-calls.jsonl' + if not path.exists(): + return [] + return [c['url'] for c in self.curl_calls() + if c['url'].startswith('https://api.github.com/')] + + def check_probe(self): + return self.run_script('superseded-probe.sh', 'azure_storage.yml') + + def newer_run(self, **fields): + run = {'id': 200, 'event': 'pull_request', 'head_branch': 'xuanwo/feature', + 'status': 'queued'} + run.update(fields) + return json.dumps({'total_count': 1, 'workflow_runs': [run]}) + + def test_probe_skipped_when_a_newer_run_is_in_flight(self): + self.env['RUNS_API_BODY'] = self.newer_run() + result = self.check_probe() + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(self.superseded(), 'superseded=true') + self.assertIn('Skipping the live probe: newer run 200', result.stdout) + self.assertNotIn(FIXTURE_GITHUB_TOKEN, result.stdout + result.stderr) + urls = self.runs_api_urls() + self.assertEqual(len(urls), 1) + self.assertIn('/repos/apache/reqsign/actions/workflows/azure_storage.yml/runs?', urls[0]) + self.assertIn('event=pull_request', urls[0]) + self.assertIn('branch=xuanwo%2Ffeature', urls[0]) + self.assertIn('per_page=50', urls[0]) + + def test_probe_kept_when_no_newer_run_is_in_flight(self): + self.env['RUNS_API_BODY'] = json.dumps({'workflow_runs': [ + {'id': 50, 'event': 'pull_request', 'head_branch': 'xuanwo/feature', + 'status': 'queued'}, + {'id': 123, 'event': 'pull_request', 'head_branch': 'xuanwo/feature', + 'status': 'completed'}]}) + result = self.check_probe() + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(self.superseded(), 'superseded=false') + self.assertIn('Running the live probe', result.stdout) + + def test_probe_kept_when_the_newer_run_already_completed(self): + # A finished newer run is not an in-flight replacement; an older probe + # that still runs is cheaper than a missing live validation. + self.env['RUNS_API_BODY'] = self.newer_run(status='completed', + conclusion='cancelled') + result = self.check_probe() + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(self.superseded(), 'superseded=false') + + def test_probe_kept_when_the_newer_run_is_for_another_branch(self): + self.env['RUNS_API_BODY'] = self.newer_run(head_branch='xuanwo/other') + result = self.check_probe() + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(self.superseded(), 'superseded=false') + + def test_probe_skipped_for_a_newer_push_on_main(self): + del self.env['GITHUB_HEAD_REF'] + self.env.update(GITHUB_EVENT_NAME='push', GITHUB_REF_NAME='main', + RUNS_API_BODY=self.newer_run(event='push', head_branch='main', + status='in_progress')) + result = self.check_probe() + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(self.superseded(), 'superseded=true') + self.assertIn('event=push', self.runs_api_urls()[0]) + self.assertIn('branch=main', self.runs_api_urls()[0]) + + def test_probe_kept_when_the_runs_api_fails(self): + self.env['RUNS_API_STATUS'] = '500' + result = self.check_probe() + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(self.superseded(), 'superseded=false') + self.assertIn('HTTP 500', result.stderr) + + def test_probe_kept_when_the_runs_api_is_unreadable(self): + self.env['RUNS_API_BROKEN'] = '1' + result = self.check_probe() + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(self.superseded(), 'superseded=false') + + def test_probe_kept_when_the_runs_api_is_unreachable(self): + self.env['RUNS_API_FAIL'] = '1' + result = self.check_probe() + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(self.superseded(), 'superseded=false') + self.assertIn('Could not list workflow runs; running the probe', result.stderr) + + def test_probe_kept_without_a_token(self): + del self.env['GITHUB_TOKEN'] + result = self.check_probe() + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(self.superseded(), 'superseded=false') + self.assertEqual(self.runs_api_urls(), []) + + +ROOT = SCRIPTS.parent.parent + + +def pipeline_checkout_script(): + """Return the inline shell of the pipeline's checkout step.""" + text = (ROOT / 'azure-pipelines.yml').read_text() + end = text.index(' displayName: Check out the GitHub Actions revision') + start = text.rindex(' - bash: |', 0, end) + lines = text[start:end].splitlines()[1:] + return '\n'.join(line[6:] if line.startswith(' ') else line for line in lines) + + +class ProbeWiringTests(unittest.TestCase): + """The workflow has to skip every remote step for a superseded probe.""" + + def probe_job_text(self): + text = (ROOT / '.github/workflows/azure_storage.yml').read_text() + job = text[text.index(' test_azure_pipelines_provider:'):] + return job[:job.index('\n signing_test:')] + + def test_skip_check_runs_before_any_remote_step(self): + job = self.probe_job_text() + check = job.index('superseded-probe.sh') + for later in ('Configure 1Password Connect', 'Log in to Azure with GitHub OIDC', + 'with-probe.sh'): + self.assertLess(check, job.index(later), later) + + def test_every_remote_step_waits_for_the_skip_result(self): + job = self.probe_job_text() + guard = "if: steps.current.outputs.superseded != 'true'" + self.assertEqual(job.count(guard), 4) + self.assertIn('id: current', job) + self.assertIn('superseded: ${{ steps.current.outputs.superseded }}', job) + self.assertIn('actions: read', job) + self.assertIn("group: ${{ github.workflow }}-azure-pipelines-probe", job) + self.assertIn('queue: max', job) + self.assertIn('cancel-in-progress: false', job) + + def test_summary_reports_a_superseded_probe_instead_of_a_pass(self): + text = (ROOT / '.github/workflows/azure_storage.yml').read_text() + self.assertIn('AZURE_PIPELINES_SUPERSEDED', text) + self.assertIn('skipped (superseded by a newer run)', text) + + +class ProbePipelineTests(unittest.TestCase): + """The pipeline must check out the revision that triggered the probe.""" + + def setUp(self): + self.temp = tempfile.TemporaryDirectory() + self.addCleanup(self.temp.cleanup) + root = Path(self.temp.name) + self.remote = root / 'remote.git' + self.seed = root / 'seed' + self.checkout = root / 'checkout' + self.git_env = {**os.environ, + 'GIT_AUTHOR_NAME': 'fixture', + 'GIT_AUTHOR_EMAIL': 'fixture@example.test', + 'GIT_COMMITTER_NAME': 'fixture', + 'GIT_COMMITTER_EMAIL': 'fixture@example.test'} + self.git('init', '-q', '--bare', str(self.remote)) + # GitHub served commit-by-SHA fetches for both branch commits and + # displaced pull request merge commits when this was checked read-only. + self.git('-C', str(self.remote), 'config', + 'uploadpack.allowReachableSHA1InWant', 'true') + self.git('-C', str(self.remote), 'config', + 'uploadpack.allowAnySHA1InWant', 'true') + self.git('init', '-q', '-b', 'main', str(self.seed)) + self.queued = self.commit('main.txt', 'queued revision') + self.git('-C', str(self.seed), 'push', '-q', str(self.remote), 'main') + self.newer = self.commit('main.txt', 'docs-only successor') + self.git('-C', str(self.seed), 'push', '-q', str(self.remote), 'main') + self.git('init', '-q', '-b', 'main', str(self.checkout)) + + def git(self, *args): + return subprocess.run(['git', *args], check=True, capture_output=True, + text=True, env=self.git_env) + + def commit(self, name, message): + (self.seed / name).write_text(message) + self.git('-C', str(self.seed), 'add', name) + self.git('-C', str(self.seed), 'commit', '-q', '-m', message) + return self.git('-C', str(self.seed), 'rev-parse', 'HEAD').stdout.strip() + + def head(self): + result = subprocess.run(['git', '-C', str(self.checkout), 'rev-parse', 'HEAD'], + capture_output=True, text=True) + return result.stdout.strip() if result.returncode == 0 else '' + + def run_checkout(self, sha, ref): + script = pipeline_checkout_script() + self.assertIn('https://github.com/apache/reqsign.git', script) + script = script.replace('https://github.com/apache/reqsign.git', str(self.remote)) + env = {**os.environ, 'GITHUB_SHA': sha, 'GITHUB_REF': ref} + return subprocess.run(['bash', '-c', script], cwd=self.checkout, env=env, + capture_output=True, text=True) + + def test_checks_out_the_queued_commit_after_the_branch_advanced(self): + result = self.run_checkout(self.queued, 'refs/heads/main') + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(self.head(), self.queued) + self.assertNotEqual(self.head(), self.newer) + self.assertEqual((self.checkout / 'main.txt').read_text(), 'queued revision') + + def test_checks_out_the_queued_merge_commit_after_the_merge_ref_moved(self): + self.git('-C', str(self.seed), 'checkout', '-q', '-B', 'pr', self.queued) + self.commit('pr.txt', 'pull request head') + self.git('-C', str(self.seed), 'merge', '-q', '--no-ff', '-m', 'merge one', 'main') + first_merge = self.git('-C', str(self.seed), 'rev-parse', 'HEAD').stdout.strip() + self.git('-C', str(self.seed), 'push', '-q', str(self.remote), + 'HEAD:refs/pull/1/merge') + self.commit('pr.txt', 'pull request head again') + self.git('-C', str(self.seed), 'merge', '-q', '--no-ff', '-m', 'merge two', 'main') + second_merge = self.git('-C', str(self.seed), 'rev-parse', 'HEAD').stdout.strip() + self.git('-C', str(self.seed), 'push', '-q', '--force', str(self.remote), + 'HEAD:refs/pull/1/merge') + + result = self.run_checkout(first_merge, 'refs/pull/1/merge') + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(self.head(), first_merge) + self.assertNotEqual(self.head(), second_merge) + + def test_refuses_to_check_out_a_ref_that_moved(self): + # The queued commit is gone, so only newer code is fetchable. The step + # has to stop instead of testing a revision that never triggered it. + result = self.run_checkout('0' * 40, 'refs/heads/main') + self.assertNotEqual(result.returncode, 0) + self.assertIn('instead of the triggering commit', result.stderr) + self.assertEqual(self.head(), '') + + def test_rejects_a_ref_the_workflow_never_runs_on(self): + result = self.run_checkout(self.queued, 'refs/remotes/origin/main') + self.assertNotEqual(result.returncode, 0) + self.assertIn('Unsupported GitHub ref', result.stderr) + self.assertEqual(self.head(), '') + + def test_rejects_a_malformed_commit(self): + result = self.run_checkout('not-a-commit', 'refs/heads/main') + self.assertNotEqual(result.returncode, 0) + self.assertEqual(self.head(), '') + + if __name__ == '__main__': unittest.main() diff --git a/.github/workflows/aliyun_oss.yml b/.github/workflows/aliyun_oss.yml index beaa5b32..bc29e1d3 100644 --- a/.github/workflows/aliyun_oss.yml +++ b/.github/workflows/aliyun_oss.yml @@ -21,9 +21,29 @@ on: push: branches: - main + # Live coverage needs this service and the shared workspace crates. Keep + # unrelated changes out of the hosted runner queue and the cloud quota. + paths: + - "Cargo.toml" + - "Cargo.lock" + - "rust-toolchain.toml" + - "core/**" + - "context/**" + - "reqsign/**" + - "services/aliyun-oss/**" + - ".github/workflows/aliyun_oss.yml" pull_request: branches: - main + paths: + - "Cargo.toml" + - "Cargo.lock" + - "rust-toolchain.toml" + - "core/**" + - "context/**" + - "reqsign/**" + - "services/aliyun-oss/**" + - ".github/workflows/aliyun_oss.yml" concurrency: group: ${{ github.workflow }}-${{ github.ref }}-${{ github.event_name }} diff --git a/.github/workflows/aws_v4.yml b/.github/workflows/aws_v4.yml index 349b07aa..cb317ded 100644 --- a/.github/workflows/aws_v4.yml +++ b/.github/workflows/aws_v4.yml @@ -21,9 +21,33 @@ on: push: branches: - main + # Live coverage needs this service and the shared workspace crates. Keep + # unrelated changes out of the hosted runner queue and the cloud quota. + paths: + - "Cargo.toml" + - "Cargo.lock" + - "rust-toolchain.toml" + - "core/**" + - "context/**" + - "reqsign/**" + - "services/aws-core/**" + - "services/aws-v4/**" + - ".github/aws-live-tests/**" + - ".github/workflows/aws_v4.yml" pull_request: branches: - main + paths: + - "Cargo.toml" + - "Cargo.lock" + - "rust-toolchain.toml" + - "core/**" + - "context/**" + - "reqsign/**" + - "services/aws-core/**" + - "services/aws-v4/**" + - ".github/aws-live-tests/**" + - ".github/workflows/aws_v4.yml" concurrency: group: ${{ github.workflow }}-${{ github.ref }}-${{ github.event_name }} diff --git a/.github/workflows/azure_storage.yml b/.github/workflows/azure_storage.yml index eee68bae..30f3066c 100644 --- a/.github/workflows/azure_storage.yml +++ b/.github/workflows/azure_storage.yml @@ -21,13 +21,37 @@ on: push: branches: - main + # Live coverage needs this service and the shared workspace crates. Keep + # unrelated changes out of the hosted runner queue and the cloud quota. + paths: + - "Cargo.toml" + - "Cargo.lock" + - "rust-toolchain.toml" + - "core/**" + - "context/**" + - "reqsign/**" + - "services/azure-storage/**" + - "azure-pipelines.yml" + - ".github/azure-live-tests/**" + - ".github/workflows/azure_storage.yml" pull_request: branches: - main + paths: + - "Cargo.toml" + - "Cargo.lock" + - "rust-toolchain.toml" + - "core/**" + - "context/**" + - "reqsign/**" + - "services/azure-storage/**" + - "azure-pipelines.yml" + - ".github/azure-live-tests/**" + - ".github/workflows/azure_storage.yml" -concurrency: - group: ${{ github.workflow }}-${{ github.ref }}-${{ github.event_name }} - cancel-in-progress: true +# Cancellation is per job so that a superseded run stops its own work without +# cancelling the Azure Pipelines probe, whose cleanup has to finish. A +# workflow-level group would cancel that probe with the rest of the run. env: RUST_BACKTRACE: "1" @@ -40,6 +64,9 @@ permissions: {} jobs: unit_test: runs-on: ubuntu-latest + concurrency: + group: ${{ github.workflow }}-unit-test-${{ github.ref }}-${{ github.event_name }} + cancel-in-progress: true steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: @@ -53,6 +80,9 @@ jobs: check_secrets: runs-on: ubuntu-latest + concurrency: + group: ${{ github.workflow }}-check-secrets-${{ github.ref }}-${{ github.event_name }} + cancel-in-progress: true outputs: trusted: ${{ steps.check.outputs.trusted }} steps: @@ -73,6 +103,9 @@ jobs: needs: check_secrets if: needs.check_secrets.outputs.trusted == 'true' runs-on: ubuntu-latest + concurrency: + group: ${{ github.workflow }}-static-provider-${{ github.ref }}-${{ github.event_name }} + cancel-in-progress: true permissions: contents: read id-token: write @@ -120,6 +153,9 @@ jobs: needs: check_secrets if: needs.check_secrets.outputs.trusted == 'true' runs-on: ubuntu-latest + concurrency: + group: ${{ github.workflow }}-env-provider-${{ github.ref }}-${{ github.event_name }} + cancel-in-progress: true permissions: contents: read id-token: write @@ -167,6 +203,9 @@ jobs: needs: check_secrets if: needs.check_secrets.outputs.trusted == 'true' runs-on: ubuntu-latest + concurrency: + group: ${{ github.workflow }}-client-secret-provider-${{ github.ref }}-${{ github.event_name }} + cancel-in-progress: true permissions: contents: read id-token: write @@ -203,6 +242,9 @@ jobs: needs: check_secrets if: needs.check_secrets.outputs.trusted == 'true' runs-on: ubuntu-latest + concurrency: + group: ${{ github.workflow }}-client-certificate-provider-${{ github.ref }}-${{ github.event_name }} + cancel-in-progress: true permissions: contents: read id-token: write @@ -246,6 +288,9 @@ jobs: needs: check_secrets if: needs.check_secrets.outputs.trusted == 'true' runs-on: ubuntu-latest + concurrency: + group: ${{ github.workflow }}-azure-cli-provider-${{ github.ref }}-${{ github.event_name }} + cancel-in-progress: true permissions: contents: read id-token: write @@ -281,6 +326,9 @@ jobs: needs: check_secrets if: needs.check_secrets.outputs.trusted == 'true' runs-on: ubuntu-latest + concurrency: + group: ${{ github.workflow }}-workload-identity-provider-${{ github.ref }}-${{ github.event_name }} + cancel-in-progress: true permissions: contents: read id-token: write @@ -328,6 +376,9 @@ jobs: needs: check_secrets if: needs.check_secrets.outputs.trusted == 'true' runs-on: ubuntu-latest + concurrency: + group: ${{ github.workflow }}-default-provider-${{ github.ref }}-${{ github.event_name }} + cancel-in-progress: true permissions: contents: read id-token: write @@ -363,6 +414,9 @@ jobs: needs: check_secrets if: needs.check_secrets.outputs.trusted == 'true' runs-on: ubuntu-latest + concurrency: + group: ${{ github.workflow }}-imds-provider-${{ github.ref }}-${{ github.event_name }} + cancel-in-progress: true env: AZURE_IMDS_CLIENT_ID: c4544d03-a76a-4908-b5c0-047b8c7e432c AZURE_IMDS_IDENTITY_ID: /subscriptions/9e6b5234-646f-4f48-b5cf-9761a56f0917/resourceGroups/opendal/providers/Microsoft.ManagedIdentity/userAssignedIdentities/reqsign-test @@ -466,6 +520,16 @@ jobs: needs: check_secrets if: needs.check_secrets.outputs.trusted == 'true' runs-on: ubuntu-latest + # The pool behind this probe has a single agent. Serialize probes across + # pull requests, main and reruns, and never cancel one in progress: a + # superseded run still has to cancel its Azure run and delete its probe + # blob. queue: max keeps pending entries instead of replacing them, so a + # newer run on another ref cannot drop live coverage; a probe that a newer + # run on its own ref really replaces is skipped by the first step below. + concurrency: + group: ${{ github.workflow }}-azure-pipelines-probe + cancel-in-progress: false + queue: max timeout-minutes: 60 env: AZURE_DEVOPS_ORGANIZATION: opendal @@ -473,17 +537,29 @@ jobs: AZURE_DEVOPS_PIPELINE_ID: "1" permissions: contents: read + # List this workflow's runs for the branch so a superseded probe can + # leave the single pool agent to its replacement. + actions: read id-token: write + outputs: + superseded: ${{ steps.current.outputs.superseded }} steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false + - name: Check whether a newer run replaces this probe + id: current + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: bash .github/azure-live-tests/superseded-probe.sh azure_storage.yml - name: Configure 1Password Connect + if: steps.current.outputs.superseded != 'true' uses: 1Password/load-secrets-action/configure@70062d7a876d3eb6334754fa26efd2fbd90c32f2 # v5.0.1 with: connect-host: ${{ secrets.OP_CONNECT_HOST }} connect-token: ${{ secrets.OP_CONNECT_TOKEN }} - name: Load Azure control-plane configuration + if: steps.current.outputs.superseded != 'true' uses: 1Password/load-secrets-action@70062d7a876d3eb6334754fa26efd2fbd90c32f2 # v5.0.1 with: export-env: true @@ -491,18 +567,23 @@ jobs: AZURE_TENANT_ID: op://reqsign/azure-storage/tenant_id AZURE_CLIENT_ID: op://reqsign/azure-storage/client_id - name: Log in to Azure with GitHub OIDC + if: steps.current.outputs.superseded != 'true' uses: azure/login@a641126d1b8aa4d1fa005f4f92df94a3a4c4c906 # v3.1.0 with: client-id: ${{ env.AZURE_CLIENT_ID }} tenant-id: ${{ env.AZURE_TENANT_ID }} subscription-id: ${{ env.AZURE_SUBSCRIPTION_ID }} - name: Test AzurePipelinesCredentialProvider with an ephemeral probe + if: steps.current.outputs.superseded != 'true' run: bash .github/azure-live-tests/with-probe.sh bash .github/azure-live-tests/azure-pipelines.sh signing_test: needs: check_secrets if: needs.check_secrets.outputs.trusted == 'true' runs-on: ubuntu-latest + concurrency: + group: ${{ github.workflow }}-signing-test-${{ github.ref }}-${{ github.event_name }} + cancel-in-progress: true steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: @@ -543,6 +624,9 @@ jobs: - signing_test if: always() && !cancelled() runs-on: ubuntu-latest + concurrency: + group: ${{ github.workflow }}-summary-${{ github.ref }}-${{ github.event_name }} + cancel-in-progress: true steps: - name: Summarize Azure provider coverage env: @@ -557,6 +641,7 @@ jobs: DEFAULT_RESULT: ${{ needs.test_default_provider.result }} IMDS_RESULT: ${{ needs.test_imds_provider.result }} AZURE_PIPELINES_RESULT: ${{ needs.test_azure_pipelines_provider.result }} + AZURE_PIPELINES_SUPERSEDED: ${{ needs.test_azure_pipelines_provider.outputs.superseded }} SIGNING_RESULT: ${{ needs.signing_test.result }} run: | { @@ -574,7 +659,11 @@ jobs: echo "| WorkloadIdentityCredentialProvider | GitHub OIDC, Microsoft Entra ID, and Azure Blob Storage | $WORKLOAD_IDENTITY_RESULT |" echo "| DefaultCredentialProvider | Azure CLI through the default chain and Azure Blob Storage | $DEFAULT_RESULT |" echo "| ImdsCredentialProvider | Ephemeral Azure VM IMDS and Azure Blob Storage | $IMDS_RESULT |" - echo "| AzurePipelinesCredentialProvider | GitHub-queued Azure Pipelines service connection and Azure Blob Storage | $AZURE_PIPELINES_RESULT |" + if [[ "$AZURE_PIPELINES_SUPERSEDED" == "true" ]]; then + echo "| AzurePipelinesCredentialProvider | GitHub-queued Azure Pipelines service connection and Azure Blob Storage | skipped (superseded by a newer run) |" + else + echo "| AzurePipelinesCredentialProvider | GitHub-queued Azure Pipelines service connection and Azure Blob Storage | ${AZURE_PIPELINES_RESULT} |" + fi echo "| Azure Shared Key and SAS signing | Azure Blob Storage | $SIGNING_RESULT |" else echo "| Live service integration tests | Skipped for an untrusted change | skipped |" diff --git a/.github/workflows/google.yml b/.github/workflows/google.yml index 66682b23..ed6035c8 100644 --- a/.github/workflows/google.yml +++ b/.github/workflows/google.yml @@ -21,9 +21,33 @@ on: push: branches: - main + # Live coverage needs this service and the shared workspace crates. Keep + # unrelated changes out of the hosted runner queue and the cloud quota. + paths: + - "Cargo.toml" + - "Cargo.lock" + - "rust-toolchain.toml" + - "core/**" + - "context/**" + - "reqsign/**" + - "services/aws-core/**" + - "services/aws-v4/**" + - "services/google/**" + - ".github/workflows/google.yml" pull_request: branches: - main + paths: + - "Cargo.toml" + - "Cargo.lock" + - "rust-toolchain.toml" + - "core/**" + - "context/**" + - "reqsign/**" + - "services/aws-core/**" + - "services/aws-v4/**" + - "services/google/**" + - ".github/workflows/google.yml" concurrency: group: ${{ github.workflow }}-${{ github.ref }}-${{ github.event_name }} diff --git a/.github/workflows/tencent_cos_live.yml b/.github/workflows/tencent_cos_live.yml index e1855c04..dfdb1708 100644 --- a/.github/workflows/tencent_cos_live.yml +++ b/.github/workflows/tencent_cos_live.yml @@ -25,6 +25,17 @@ on: push: branches: - main + # Live coverage needs this service and the shared workspace crates. Keep + # unrelated changes out of the hosted runner queue and the cloud quota. + paths: + - "Cargo.toml" + - "Cargo.lock" + - "rust-toolchain.toml" + - "core/**" + - "context/**" + - "reqsign/**" + - "services/tencent-cos/**" + - ".github/workflows/tencent_cos_live.yml" workflow_dispatch: concurrency: diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 82f605d3..289edefb 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -44,11 +44,21 @@ steps: ;; esac - git fetch --no-tags --depth=1 \ - https://github.com/apache/reqsign.git \ - "$GITHUB_REF" + # Fetch the commit the GitHub workflow ran instead of the ref it named. + # A ref can advance while the run waits for this agent, and a merge ref + # is replaced by the next push, so fetching the ref alone would fail on + # the mismatch or check out code that never triggered the probe. + repository=https://github.com/apache/reqsign.git + if ! git fetch --no-tags --depth=1 "$repository" "$GITHUB_SHA"; then + # Keep fetching the ref as a fallback; the check below still requires + # the commit to be the one that triggered this run. + git fetch --no-tags --depth=1 "$repository" "$GITHUB_REF" + fi fetched_sha=$(git rev-parse 'FETCH_HEAD^{commit}') - test "$fetched_sha" = "$GITHUB_SHA" + if [[ "$fetched_sha" != "$GITHUB_SHA" ]]; then + echo "Fetched $fetched_sha instead of the triggering commit $GITHUB_SHA" >&2 + exit 1 + fi git checkout --detach "$fetched_sha" displayName: Check out the GitHub Actions revision env: