From bff978bd1af645d9782f995e03b76a9516fb7d83 Mon Sep 17 00:00:00 2001 From: Xuanwo Date: Thu, 19 Nov 2026 19:11:45 +0100 Subject: [PATCH 1/4] ci(azure): serialize the Pipelines probe and scope cancellation per job The Azure Pipelines probe queues a run on a pool that has a single agent. A workflow-level concurrency group cancels that probe together with the rest of a superseded run, so the queued or running Azure run keeps the agent and the probe's cancel/delete cleanup never gets to run. Give every job its own ref-scoped group so a newer run still supersedes an older run's work, and serialize the probe in one group shared by pull requests, main and reruns. The probe is never cancelled in progress, and queue: max keeps pending probes instead of replacing them, so a newer run cannot drop live coverage that is already waiting for the agent. --- .github/workflows/azure_storage.yml | 51 +++++++++++++++++++++++++++-- 1 file changed, 48 insertions(+), 3 deletions(-) diff --git a/.github/workflows/azure_storage.yml b/.github/workflows/azure_storage.yml index eee68bae..3cdf49fe 100644 --- a/.github/workflows/azure_storage.yml +++ b/.github/workflows/azure_storage.yml @@ -25,9 +25,9 @@ on: branches: - main -concurrency: - group: ${{ github.workflow }}-${{ github.ref }}-${{ github.event_name }} - cancel-in-progress: true +# Cancellation is per job so that a superseded run stops its own work without +# cancelling the Azure Pipelines probe, whose cleanup has to finish. A +# workflow-level group would cancel that probe with the rest of the run. env: RUST_BACKTRACE: "1" @@ -40,6 +40,9 @@ permissions: {} jobs: unit_test: runs-on: ubuntu-latest + concurrency: + group: ${{ github.workflow }}-unit-test-${{ github.ref }}-${{ github.event_name }} + cancel-in-progress: true steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: @@ -53,6 +56,9 @@ jobs: check_secrets: runs-on: ubuntu-latest + concurrency: + group: ${{ github.workflow }}-check-secrets-${{ github.ref }}-${{ github.event_name }} + cancel-in-progress: true outputs: trusted: ${{ steps.check.outputs.trusted }} steps: @@ -73,6 +79,9 @@ jobs: needs: check_secrets if: needs.check_secrets.outputs.trusted == 'true' runs-on: ubuntu-latest + concurrency: + group: ${{ github.workflow }}-static-provider-${{ github.ref }}-${{ github.event_name }} + cancel-in-progress: true permissions: contents: read id-token: write @@ -120,6 +129,9 @@ jobs: needs: check_secrets if: needs.check_secrets.outputs.trusted == 'true' runs-on: ubuntu-latest + concurrency: + group: ${{ github.workflow }}-env-provider-${{ github.ref }}-${{ github.event_name }} + cancel-in-progress: true permissions: contents: read id-token: write @@ -167,6 +179,9 @@ jobs: needs: check_secrets if: needs.check_secrets.outputs.trusted == 'true' runs-on: ubuntu-latest + concurrency: + group: ${{ github.workflow }}-client-secret-provider-${{ github.ref }}-${{ github.event_name }} + cancel-in-progress: true permissions: contents: read id-token: write @@ -203,6 +218,9 @@ jobs: needs: check_secrets if: needs.check_secrets.outputs.trusted == 'true' runs-on: ubuntu-latest + concurrency: + group: ${{ github.workflow }}-client-certificate-provider-${{ github.ref }}-${{ github.event_name }} + cancel-in-progress: true permissions: contents: read id-token: write @@ -246,6 +264,9 @@ jobs: needs: check_secrets if: needs.check_secrets.outputs.trusted == 'true' runs-on: ubuntu-latest + concurrency: + group: ${{ github.workflow }}-azure-cli-provider-${{ github.ref }}-${{ github.event_name }} + cancel-in-progress: true permissions: contents: read id-token: write @@ -281,6 +302,9 @@ jobs: needs: check_secrets if: needs.check_secrets.outputs.trusted == 'true' runs-on: ubuntu-latest + concurrency: + group: ${{ github.workflow }}-workload-identity-provider-${{ github.ref }}-${{ github.event_name }} + cancel-in-progress: true permissions: contents: read id-token: write @@ -328,6 +352,9 @@ jobs: needs: check_secrets if: needs.check_secrets.outputs.trusted == 'true' runs-on: ubuntu-latest + concurrency: + group: ${{ github.workflow }}-default-provider-${{ github.ref }}-${{ github.event_name }} + cancel-in-progress: true permissions: contents: read id-token: write @@ -363,6 +390,9 @@ jobs: needs: check_secrets if: needs.check_secrets.outputs.trusted == 'true' runs-on: ubuntu-latest + concurrency: + group: ${{ github.workflow }}-imds-provider-${{ github.ref }}-${{ github.event_name }} + cancel-in-progress: true env: AZURE_IMDS_CLIENT_ID: c4544d03-a76a-4908-b5c0-047b8c7e432c AZURE_IMDS_IDENTITY_ID: /subscriptions/9e6b5234-646f-4f48-b5cf-9761a56f0917/resourceGroups/opendal/providers/Microsoft.ManagedIdentity/userAssignedIdentities/reqsign-test @@ -466,6 +496,15 @@ jobs: needs: check_secrets if: needs.check_secrets.outputs.trusted == 'true' runs-on: ubuntu-latest + # The pool behind this probe has a single agent. Serialize probes across + # pull requests, main and reruns, and never cancel one in progress: a + # superseded run still has to cancel its Azure run and delete its probe + # blob. queue: max keeps pending probes instead of replacing them, so a + # newer run cannot drop live coverage already waiting for the agent. + concurrency: + group: ${{ github.workflow }}-azure-pipelines-probe + cancel-in-progress: false + queue: max timeout-minutes: 60 env: AZURE_DEVOPS_ORGANIZATION: opendal @@ -503,6 +542,9 @@ jobs: needs: check_secrets if: needs.check_secrets.outputs.trusted == 'true' runs-on: ubuntu-latest + concurrency: + group: ${{ github.workflow }}-signing-test-${{ github.ref }}-${{ github.event_name }} + cancel-in-progress: true steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: @@ -543,6 +585,9 @@ jobs: - signing_test if: always() && !cancelled() runs-on: ubuntu-latest + concurrency: + group: ${{ github.workflow }}-summary-${{ github.ref }}-${{ github.event_name }} + cancel-in-progress: true steps: - name: Summarize Azure provider coverage env: From 7da0d761a1e7e917ddfa2fdcc21e96c8f127ec7e Mon Sep 17 00:00:00 2001 From: Xuanwo Date: Thu, 19 Nov 2026 19:11:52 +0100 Subject: [PATCH 2/4] ci: run live cloud tests only for paths that can change them Every push and pull request queued all thirteen live workflows, including changes that cannot affect a provider. Gate the live cloud workflows on their service, the shared workspace crates, the workspace manifest and toolchain, and the workflow and probe scripts the service uses. Offline-only workflows keep their current triggers, and the workspace build, lint and unit tests in ci.yml still cover every crate, so coverage is only dropped where the provider code cannot have changed. --- .github/workflows/aliyun_oss.yml | 20 ++++++++++++++++++++ .github/workflows/aws_v4.yml | 24 ++++++++++++++++++++++++ .github/workflows/azure_storage.yml | 24 ++++++++++++++++++++++++ .github/workflows/google.yml | 24 ++++++++++++++++++++++++ .github/workflows/tencent_cos_live.yml | 11 +++++++++++ 5 files changed, 103 insertions(+) diff --git a/.github/workflows/aliyun_oss.yml b/.github/workflows/aliyun_oss.yml index beaa5b32..bc29e1d3 100644 --- a/.github/workflows/aliyun_oss.yml +++ b/.github/workflows/aliyun_oss.yml @@ -21,9 +21,29 @@ on: push: branches: - main + # Live coverage needs this service and the shared workspace crates. Keep + # unrelated changes out of the hosted runner queue and the cloud quota. + paths: + - "Cargo.toml" + - "Cargo.lock" + - "rust-toolchain.toml" + - "core/**" + - "context/**" + - "reqsign/**" + - "services/aliyun-oss/**" + - ".github/workflows/aliyun_oss.yml" pull_request: branches: - main + paths: + - "Cargo.toml" + - "Cargo.lock" + - "rust-toolchain.toml" + - "core/**" + - "context/**" + - "reqsign/**" + - "services/aliyun-oss/**" + - ".github/workflows/aliyun_oss.yml" concurrency: group: ${{ github.workflow }}-${{ github.ref }}-${{ github.event_name }} diff --git a/.github/workflows/aws_v4.yml b/.github/workflows/aws_v4.yml index 349b07aa..cb317ded 100644 --- a/.github/workflows/aws_v4.yml +++ b/.github/workflows/aws_v4.yml @@ -21,9 +21,33 @@ on: push: branches: - main + # Live coverage needs this service and the shared workspace crates. Keep + # unrelated changes out of the hosted runner queue and the cloud quota. + paths: + - "Cargo.toml" + - "Cargo.lock" + - "rust-toolchain.toml" + - "core/**" + - "context/**" + - "reqsign/**" + - "services/aws-core/**" + - "services/aws-v4/**" + - ".github/aws-live-tests/**" + - ".github/workflows/aws_v4.yml" pull_request: branches: - main + paths: + - "Cargo.toml" + - "Cargo.lock" + - "rust-toolchain.toml" + - "core/**" + - "context/**" + - "reqsign/**" + - "services/aws-core/**" + - "services/aws-v4/**" + - ".github/aws-live-tests/**" + - ".github/workflows/aws_v4.yml" concurrency: group: ${{ github.workflow }}-${{ github.ref }}-${{ github.event_name }} diff --git a/.github/workflows/azure_storage.yml b/.github/workflows/azure_storage.yml index 3cdf49fe..6c72ddc5 100644 --- a/.github/workflows/azure_storage.yml +++ b/.github/workflows/azure_storage.yml @@ -21,9 +21,33 @@ on: push: branches: - main + # Live coverage needs this service and the shared workspace crates. Keep + # unrelated changes out of the hosted runner queue and the cloud quota. + paths: + - "Cargo.toml" + - "Cargo.lock" + - "rust-toolchain.toml" + - "core/**" + - "context/**" + - "reqsign/**" + - "services/azure-storage/**" + - "azure-pipelines.yml" + - ".github/azure-live-tests/**" + - ".github/workflows/azure_storage.yml" pull_request: branches: - main + paths: + - "Cargo.toml" + - "Cargo.lock" + - "rust-toolchain.toml" + - "core/**" + - "context/**" + - "reqsign/**" + - "services/azure-storage/**" + - "azure-pipelines.yml" + - ".github/azure-live-tests/**" + - ".github/workflows/azure_storage.yml" # Cancellation is per job so that a superseded run stops its own work without # cancelling the Azure Pipelines probe, whose cleanup has to finish. A diff --git a/.github/workflows/google.yml b/.github/workflows/google.yml index 66682b23..ed6035c8 100644 --- a/.github/workflows/google.yml +++ b/.github/workflows/google.yml @@ -21,9 +21,33 @@ on: push: branches: - main + # Live coverage needs this service and the shared workspace crates. Keep + # unrelated changes out of the hosted runner queue and the cloud quota. + paths: + - "Cargo.toml" + - "Cargo.lock" + - "rust-toolchain.toml" + - "core/**" + - "context/**" + - "reqsign/**" + - "services/aws-core/**" + - "services/aws-v4/**" + - "services/google/**" + - ".github/workflows/google.yml" pull_request: branches: - main + paths: + - "Cargo.toml" + - "Cargo.lock" + - "rust-toolchain.toml" + - "core/**" + - "context/**" + - "reqsign/**" + - "services/aws-core/**" + - "services/aws-v4/**" + - "services/google/**" + - ".github/workflows/google.yml" concurrency: group: ${{ github.workflow }}-${{ github.ref }}-${{ github.event_name }} diff --git a/.github/workflows/tencent_cos_live.yml b/.github/workflows/tencent_cos_live.yml index e1855c04..dfdb1708 100644 --- a/.github/workflows/tencent_cos_live.yml +++ b/.github/workflows/tencent_cos_live.yml @@ -25,6 +25,17 @@ on: push: branches: - main + # Live coverage needs this service and the shared workspace crates. Keep + # unrelated changes out of the hosted runner queue and the cloud quota. + paths: + - "Cargo.toml" + - "Cargo.lock" + - "rust-toolchain.toml" + - "core/**" + - "context/**" + - "reqsign/**" + - "services/tencent-cos/**" + - ".github/workflows/tencent_cos_live.yml" workflow_dispatch: concurrency: From dc527e039b7db8b15d3ad8b342d1655f060f45f1 Mon Sep 17 00:00:00 2001 From: Xuanwo Date: Thu, 19 Nov 2026 20:06:42 +0100 Subject: [PATCH 3/4] ci(azure): skip a probe that a newer run already replaces The probe group keeps pending entries so a newer run on another ref cannot drop live coverage, but that also lets an older entry take the single pool agent for a revision a newer run already replaced. Before any 1Password, OIDC, blob or Azure work, list this workflow's runs for the branch and skip the probe only when a newer run for the same event is still in flight. A change the path filters drop creates no run, and an unrelated ref or an unclear API answer keeps the probe, so the check cannot remove the only live validation. The skip costs one read-only API call and nothing else; the summary reports it as skipped instead of passed. --- .github/azure-live-tests/superseded-probe.sh | 80 +++++++++ .github/azure-live-tests/test_scripts.py | 171 ++++++++++++++++++- .github/workflows/azure_storage.yml | 26 ++- 3 files changed, 272 insertions(+), 5 deletions(-) create mode 100644 .github/azure-live-tests/superseded-probe.sh diff --git a/.github/azure-live-tests/superseded-probe.sh b/.github/azure-live-tests/superseded-probe.sh new file mode 100644 index 00000000..5416a596 --- /dev/null +++ b/.github/azure-live-tests/superseded-probe.sh @@ -0,0 +1,80 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# Decide whether a newer run of this workflow on the same branch and event will +# cover this live probe. The probe group keeps pending entries, so a probe can +# start long after its own commit was superseded. +# +# Skip only when a newer run exists that already got past the workflow's path +# filters: a change that filters the workflow out creates no run, so an older +# queued probe stays the only live validation for its commit. Any missing or +# unexpected answer keeps the probe; a wasted probe costs a runner, a missing +# live validation costs coverage. +set -euo pipefail + +workflow=${1:?Usage: superseded-probe.sh } +: "${GITHUB_OUTPUT:?Set by Actions}" + +superseded=false +replacement="" +branch="" +case "${GITHUB_EVENT_NAME:-}" in + pull_request|pull_request_target) branch=${GITHUB_HEAD_REF:-} ;; + push) branch=${GITHUB_REF_NAME:-} ;; +esac + +if [[ -z "${GITHUB_TOKEN:-}" ]]; then + echo "Cannot check for a newer run without a token; running the probe" +elif [[ -z "${GITHUB_REPOSITORY:-}" || -z "${GITHUB_RUN_ID:-}" || -z "$branch" ]]; then + echo "Cannot check for a newer run without the run context; running the probe" +else + response_file=$(mktemp) + trap 'rm -f "$response_file"' EXIT + if ! http_status=$(curl --get --silent --show-error \ + --output "$response_file" \ + --write-out '%{http_code}' \ + --header "Authorization: Bearer $GITHUB_TOKEN" \ + --header "Accept: application/vnd.github+json" \ + --data-urlencode "event=$GITHUB_EVENT_NAME" \ + --data-urlencode "branch=$branch" \ + --data-urlencode "per_page=50" \ + "https://api.github.com/repos/${GITHUB_REPOSITORY}/actions/workflows/${workflow}/runs"); then + echo "Could not list workflow runs; running the probe" >&2 + elif [[ "$http_status" != "200" ]]; then + echo "Could not list workflow runs (HTTP $http_status); running the probe" >&2 + else + replacement=$(jq --raw-output \ + --argjson run_id "$GITHUB_RUN_ID" \ + --arg event "$GITHUB_EVENT_NAME" \ + --arg branch "$branch" ' + [ .workflow_runs[]? + | select(.id > $run_id) + | select(.event == $event) + | select(.head_branch == $branch) + | select(.status != "completed") + | .id ] | sort | first // empty' "$response_file" 2>/dev/null) || replacement="" + [[ -z "$replacement" ]] || superseded=true + fi +fi + +echo "superseded=$superseded" >> "$GITHUB_OUTPUT" +if [[ "$superseded" == true ]]; then + echo "Skipping the live probe: newer run $replacement on $branch will cover it" +else + echo "Running the live probe: no newer run of $workflow on $branch is in flight" +fi diff --git a/.github/azure-live-tests/test_scripts.py b/.github/azure-live-tests/test_scripts.py index a9de6420..e70804d1 100644 --- a/.github/azure-live-tests/test_scripts.py +++ b/.github/azure-live-tests/test_scripts.py @@ -25,6 +25,7 @@ import unittest SCRIPTS = Path(__file__).resolve().parent +FIXTURE_GITHUB_TOKEN = 'gh-fixture-token' FAKE_AZ = r"""#!/usr/bin/env python3 import json, os, pathlib, sys args = sys.argv[1:] @@ -92,7 +93,7 @@ def logged_in(): FAKE_CURL = r"""#!/usr/bin/env python3 -import json, os, pathlib, sys +import json, os, pathlib, sys, urllib.parse args = sys.argv[1:] root = pathlib.Path(os.environ['FAKE_AZ_STATE']) def option(name, default=''): @@ -118,6 +119,35 @@ def respond(body, code='200'): if url.startswith('https://dev.azure.com/'): assert option('--header') == 'Authorization: Bearer fake-devops-token', \ 'unexpected DevOps authorization header' +if url.startswith('https://api.github.com/'): + assert option('--header') == 'Authorization: Bearer __GH_TOKEN__', \ + 'unexpected GitHub authorization header' + assert method == 'GET' + if os.getenv('RUNS_API_FAIL'): + sys.exit(35) + query = [] + for i, arg in enumerate(args): + if arg == '--data-urlencode': + name, _, value = args[i + 1].partition('=') + # curl percent-encodes the value, which matters for branch names. + query.append(name + '=' + urllib.parse.quote(value, safe='')) + if query: + url = url + '?' + '&'.join(query) + with (root / 'curl-calls.jsonl').open('a') as f: + f.write(json.dumps({'method': method, 'url': url}) + '\n') + if os.getenv('RUNS_API_BROKEN'): + respond('upstream returned a page, not JSON') + elif os.getenv('RUNS_API_STATUS', '200') != '200': + respond(json.dumps({'message': 'server error'}), os.getenv('RUNS_API_STATUS')) + else: + # The fixture answers with the run under test unless a test supplies + # the workflow runs the check should see. + respond(os.getenv('RUNS_API_BODY') or json.dumps({ + 'total_count': 1, + 'workflow_runs': [{'id': 100, 'event': 'pull_request', + 'head_branch': 'xuanwo/feature', + 'status': 'completed', 'conclusion': 'success'}]})) + sys.exit(0) # Both waiting and cancellation must finish before fixture cleanup. assert list(root.glob('ci-probe-*')) with (root / 'curl-calls.jsonl').open('a') as f: @@ -176,7 +206,7 @@ def setUp(self): binary.write_text(FAKE_AZ) binary.chmod(0o755) curl = self.root / 'curl' - curl.write_text(FAKE_CURL) + curl.write_text(FAKE_CURL.replace('__GH_TOKEN__', FIXTURE_GITHUB_TOKEN)) curl.chmod(0o755) self.env = { **os.environ, @@ -201,6 +231,10 @@ def setUp(self): 'GITHUB_RUN_ID': '123', 'GITHUB_RUN_ATTEMPT': '1', 'GITHUB_OUTPUT': str(self.root / 'outputs'), + 'GITHUB_TOKEN': FIXTURE_GITHUB_TOKEN, + 'GITHUB_REPOSITORY': 'apache/reqsign', + 'GITHUB_EVENT_NAME': 'pull_request', + 'GITHUB_HEAD_REF': 'xuanwo/feature', } def run_script(self, script, *args): @@ -445,5 +479,138 @@ def test_permanent_pipeline_status_error_is_not_retried(self): self.assertEqual(list(self.state.glob('ci-probe-*')), []) + def superseded(self): + return (self.root / 'outputs').read_text().strip() + + def runs_api_urls(self): + path = self.state / 'curl-calls.jsonl' + if not path.exists(): + return [] + return [c['url'] for c in self.curl_calls() + if c['url'].startswith('https://api.github.com/')] + + def check_probe(self): + return self.run_script('superseded-probe.sh', 'azure_storage.yml') + + def newer_run(self, **fields): + run = {'id': 200, 'event': 'pull_request', 'head_branch': 'xuanwo/feature', + 'status': 'queued'} + run.update(fields) + return json.dumps({'total_count': 1, 'workflow_runs': [run]}) + + def test_probe_skipped_when_a_newer_run_is_in_flight(self): + self.env['RUNS_API_BODY'] = self.newer_run() + result = self.check_probe() + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(self.superseded(), 'superseded=true') + self.assertIn('Skipping the live probe: newer run 200', result.stdout) + self.assertNotIn(FIXTURE_GITHUB_TOKEN, result.stdout + result.stderr) + urls = self.runs_api_urls() + self.assertEqual(len(urls), 1) + self.assertIn('/repos/apache/reqsign/actions/workflows/azure_storage.yml/runs?', urls[0]) + self.assertIn('event=pull_request', urls[0]) + self.assertIn('branch=xuanwo%2Ffeature', urls[0]) + self.assertIn('per_page=50', urls[0]) + + def test_probe_kept_when_no_newer_run_is_in_flight(self): + self.env['RUNS_API_BODY'] = json.dumps({'workflow_runs': [ + {'id': 50, 'event': 'pull_request', 'head_branch': 'xuanwo/feature', + 'status': 'queued'}, + {'id': 123, 'event': 'pull_request', 'head_branch': 'xuanwo/feature', + 'status': 'completed'}]}) + result = self.check_probe() + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(self.superseded(), 'superseded=false') + self.assertIn('Running the live probe', result.stdout) + + def test_probe_kept_when_the_newer_run_already_completed(self): + # A finished newer run is not an in-flight replacement; an older probe + # that still runs is cheaper than a missing live validation. + self.env['RUNS_API_BODY'] = self.newer_run(status='completed', + conclusion='cancelled') + result = self.check_probe() + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(self.superseded(), 'superseded=false') + + def test_probe_kept_when_the_newer_run_is_for_another_branch(self): + self.env['RUNS_API_BODY'] = self.newer_run(head_branch='xuanwo/other') + result = self.check_probe() + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(self.superseded(), 'superseded=false') + + def test_probe_skipped_for_a_newer_push_on_main(self): + del self.env['GITHUB_HEAD_REF'] + self.env.update(GITHUB_EVENT_NAME='push', GITHUB_REF_NAME='main', + RUNS_API_BODY=self.newer_run(event='push', head_branch='main', + status='in_progress')) + result = self.check_probe() + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(self.superseded(), 'superseded=true') + self.assertIn('event=push', self.runs_api_urls()[0]) + self.assertIn('branch=main', self.runs_api_urls()[0]) + + def test_probe_kept_when_the_runs_api_fails(self): + self.env['RUNS_API_STATUS'] = '500' + result = self.check_probe() + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(self.superseded(), 'superseded=false') + self.assertIn('HTTP 500', result.stderr) + + def test_probe_kept_when_the_runs_api_is_unreadable(self): + self.env['RUNS_API_BROKEN'] = '1' + result = self.check_probe() + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(self.superseded(), 'superseded=false') + + def test_probe_kept_when_the_runs_api_is_unreachable(self): + self.env['RUNS_API_FAIL'] = '1' + result = self.check_probe() + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(self.superseded(), 'superseded=false') + self.assertIn('Could not list workflow runs; running the probe', result.stderr) + + def test_probe_kept_without_a_token(self): + del self.env['GITHUB_TOKEN'] + result = self.check_probe() + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(self.superseded(), 'superseded=false') + self.assertEqual(self.runs_api_urls(), []) + + +ROOT = SCRIPTS.parent.parent + + +class ProbeWiringTests(unittest.TestCase): + """The workflow has to skip every remote step for a superseded probe.""" + + def probe_job_text(self): + text = (ROOT / '.github/workflows/azure_storage.yml').read_text() + job = text[text.index(' test_azure_pipelines_provider:'):] + return job[:job.index('\n signing_test:')] + + def test_skip_check_runs_before_any_remote_step(self): + job = self.probe_job_text() + check = job.index('superseded-probe.sh') + for later in ('Configure 1Password Connect', 'Log in to Azure with GitHub OIDC', + 'with-probe.sh'): + self.assertLess(check, job.index(later), later) + + def test_every_remote_step_waits_for_the_skip_result(self): + job = self.probe_job_text() + guard = "if: steps.current.outputs.superseded != 'true'" + self.assertEqual(job.count(guard), 4) + self.assertIn('id: current', job) + self.assertIn('superseded: ${{ steps.current.outputs.superseded }}', job) + self.assertIn('actions: read', job) + self.assertIn("group: ${{ github.workflow }}-azure-pipelines-probe", job) + self.assertIn('queue: max', job) + self.assertIn('cancel-in-progress: false', job) + + def test_summary_reports_a_superseded_probe_instead_of_a_pass(self): + text = (ROOT / '.github/workflows/azure_storage.yml').read_text() + self.assertIn('AZURE_PIPELINES_SUPERSEDED', text) + self.assertIn('skipped (superseded by a newer run)', text) + + if __name__ == '__main__': unittest.main() diff --git a/.github/workflows/azure_storage.yml b/.github/workflows/azure_storage.yml index 6c72ddc5..30f3066c 100644 --- a/.github/workflows/azure_storage.yml +++ b/.github/workflows/azure_storage.yml @@ -523,8 +523,9 @@ jobs: # The pool behind this probe has a single agent. Serialize probes across # pull requests, main and reruns, and never cancel one in progress: a # superseded run still has to cancel its Azure run and delete its probe - # blob. queue: max keeps pending probes instead of replacing them, so a - # newer run cannot drop live coverage already waiting for the agent. + # blob. queue: max keeps pending entries instead of replacing them, so a + # newer run on another ref cannot drop live coverage; a probe that a newer + # run on its own ref really replaces is skipped by the first step below. concurrency: group: ${{ github.workflow }}-azure-pipelines-probe cancel-in-progress: false @@ -536,17 +537,29 @@ jobs: AZURE_DEVOPS_PIPELINE_ID: "1" permissions: contents: read + # List this workflow's runs for the branch so a superseded probe can + # leave the single pool agent to its replacement. + actions: read id-token: write + outputs: + superseded: ${{ steps.current.outputs.superseded }} steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false + - name: Check whether a newer run replaces this probe + id: current + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: bash .github/azure-live-tests/superseded-probe.sh azure_storage.yml - name: Configure 1Password Connect + if: steps.current.outputs.superseded != 'true' uses: 1Password/load-secrets-action/configure@70062d7a876d3eb6334754fa26efd2fbd90c32f2 # v5.0.1 with: connect-host: ${{ secrets.OP_CONNECT_HOST }} connect-token: ${{ secrets.OP_CONNECT_TOKEN }} - name: Load Azure control-plane configuration + if: steps.current.outputs.superseded != 'true' uses: 1Password/load-secrets-action@70062d7a876d3eb6334754fa26efd2fbd90c32f2 # v5.0.1 with: export-env: true @@ -554,12 +567,14 @@ jobs: AZURE_TENANT_ID: op://reqsign/azure-storage/tenant_id AZURE_CLIENT_ID: op://reqsign/azure-storage/client_id - name: Log in to Azure with GitHub OIDC + if: steps.current.outputs.superseded != 'true' uses: azure/login@a641126d1b8aa4d1fa005f4f92df94a3a4c4c906 # v3.1.0 with: client-id: ${{ env.AZURE_CLIENT_ID }} tenant-id: ${{ env.AZURE_TENANT_ID }} subscription-id: ${{ env.AZURE_SUBSCRIPTION_ID }} - name: Test AzurePipelinesCredentialProvider with an ephemeral probe + if: steps.current.outputs.superseded != 'true' run: bash .github/azure-live-tests/with-probe.sh bash .github/azure-live-tests/azure-pipelines.sh signing_test: @@ -626,6 +641,7 @@ jobs: DEFAULT_RESULT: ${{ needs.test_default_provider.result }} IMDS_RESULT: ${{ needs.test_imds_provider.result }} AZURE_PIPELINES_RESULT: ${{ needs.test_azure_pipelines_provider.result }} + AZURE_PIPELINES_SUPERSEDED: ${{ needs.test_azure_pipelines_provider.outputs.superseded }} SIGNING_RESULT: ${{ needs.signing_test.result }} run: | { @@ -643,7 +659,11 @@ jobs: echo "| WorkloadIdentityCredentialProvider | GitHub OIDC, Microsoft Entra ID, and Azure Blob Storage | $WORKLOAD_IDENTITY_RESULT |" echo "| DefaultCredentialProvider | Azure CLI through the default chain and Azure Blob Storage | $DEFAULT_RESULT |" echo "| ImdsCredentialProvider | Ephemeral Azure VM IMDS and Azure Blob Storage | $IMDS_RESULT |" - echo "| AzurePipelinesCredentialProvider | GitHub-queued Azure Pipelines service connection and Azure Blob Storage | $AZURE_PIPELINES_RESULT |" + if [[ "$AZURE_PIPELINES_SUPERSEDED" == "true" ]]; then + echo "| AzurePipelinesCredentialProvider | GitHub-queued Azure Pipelines service connection and Azure Blob Storage | skipped (superseded by a newer run) |" + else + echo "| AzurePipelinesCredentialProvider | GitHub-queued Azure Pipelines service connection and Azure Blob Storage | ${AZURE_PIPELINES_RESULT} |" + fi echo "| Azure Shared Key and SAS signing | Azure Blob Storage | $SIGNING_RESULT |" else echo "| Live service integration tests | Skipped for an untrusted change | skipped |" From 33e1a9318130dfd5d0e576d07b1823b4b335479c Mon Sep 17 00:00:00 2001 From: Xuanwo Date: Thu, 19 Nov 2026 20:07:02 +0100 Subject: [PATCH 4/4] ci(azure): check out the commit that triggered the probe The pipeline read the mutable ref it was triggered from. A push that advanced the ref made the checkout abort on the SHA mismatch, and a merge ref replaced by the next push could only resolve newer code, so a queued probe could neither validate its own revision nor prove it had not moved. Fetch the commit the GitHub workflow ran and require the resolved commit to match it before checking out detached. The ref stays only as a fetch fallback and still has to resolve to that commit, and the ref/SHA shape checks are unchanged. The tests drive the step against a fixture remote where the branch and a pull request merge ref advance after queueing. --- .github/azure-live-tests/test_scripts.py | 106 +++++++++++++++++++++++ azure-pipelines.yml | 18 +++- 2 files changed, 120 insertions(+), 4 deletions(-) diff --git a/.github/azure-live-tests/test_scripts.py b/.github/azure-live-tests/test_scripts.py index e70804d1..8aa454cc 100644 --- a/.github/azure-live-tests/test_scripts.py +++ b/.github/azure-live-tests/test_scripts.py @@ -580,6 +580,15 @@ def test_probe_kept_without_a_token(self): ROOT = SCRIPTS.parent.parent +def pipeline_checkout_script(): + """Return the inline shell of the pipeline's checkout step.""" + text = (ROOT / 'azure-pipelines.yml').read_text() + end = text.index(' displayName: Check out the GitHub Actions revision') + start = text.rindex(' - bash: |', 0, end) + lines = text[start:end].splitlines()[1:] + return '\n'.join(line[6:] if line.startswith(' ') else line for line in lines) + + class ProbeWiringTests(unittest.TestCase): """The workflow has to skip every remote step for a superseded probe.""" @@ -612,5 +621,102 @@ def test_summary_reports_a_superseded_probe_instead_of_a_pass(self): self.assertIn('skipped (superseded by a newer run)', text) +class ProbePipelineTests(unittest.TestCase): + """The pipeline must check out the revision that triggered the probe.""" + + def setUp(self): + self.temp = tempfile.TemporaryDirectory() + self.addCleanup(self.temp.cleanup) + root = Path(self.temp.name) + self.remote = root / 'remote.git' + self.seed = root / 'seed' + self.checkout = root / 'checkout' + self.git_env = {**os.environ, + 'GIT_AUTHOR_NAME': 'fixture', + 'GIT_AUTHOR_EMAIL': 'fixture@example.test', + 'GIT_COMMITTER_NAME': 'fixture', + 'GIT_COMMITTER_EMAIL': 'fixture@example.test'} + self.git('init', '-q', '--bare', str(self.remote)) + # GitHub served commit-by-SHA fetches for both branch commits and + # displaced pull request merge commits when this was checked read-only. + self.git('-C', str(self.remote), 'config', + 'uploadpack.allowReachableSHA1InWant', 'true') + self.git('-C', str(self.remote), 'config', + 'uploadpack.allowAnySHA1InWant', 'true') + self.git('init', '-q', '-b', 'main', str(self.seed)) + self.queued = self.commit('main.txt', 'queued revision') + self.git('-C', str(self.seed), 'push', '-q', str(self.remote), 'main') + self.newer = self.commit('main.txt', 'docs-only successor') + self.git('-C', str(self.seed), 'push', '-q', str(self.remote), 'main') + self.git('init', '-q', '-b', 'main', str(self.checkout)) + + def git(self, *args): + return subprocess.run(['git', *args], check=True, capture_output=True, + text=True, env=self.git_env) + + def commit(self, name, message): + (self.seed / name).write_text(message) + self.git('-C', str(self.seed), 'add', name) + self.git('-C', str(self.seed), 'commit', '-q', '-m', message) + return self.git('-C', str(self.seed), 'rev-parse', 'HEAD').stdout.strip() + + def head(self): + result = subprocess.run(['git', '-C', str(self.checkout), 'rev-parse', 'HEAD'], + capture_output=True, text=True) + return result.stdout.strip() if result.returncode == 0 else '' + + def run_checkout(self, sha, ref): + script = pipeline_checkout_script() + self.assertIn('https://github.com/apache/reqsign.git', script) + script = script.replace('https://github.com/apache/reqsign.git', str(self.remote)) + env = {**os.environ, 'GITHUB_SHA': sha, 'GITHUB_REF': ref} + return subprocess.run(['bash', '-c', script], cwd=self.checkout, env=env, + capture_output=True, text=True) + + def test_checks_out_the_queued_commit_after_the_branch_advanced(self): + result = self.run_checkout(self.queued, 'refs/heads/main') + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(self.head(), self.queued) + self.assertNotEqual(self.head(), self.newer) + self.assertEqual((self.checkout / 'main.txt').read_text(), 'queued revision') + + def test_checks_out_the_queued_merge_commit_after_the_merge_ref_moved(self): + self.git('-C', str(self.seed), 'checkout', '-q', '-B', 'pr', self.queued) + self.commit('pr.txt', 'pull request head') + self.git('-C', str(self.seed), 'merge', '-q', '--no-ff', '-m', 'merge one', 'main') + first_merge = self.git('-C', str(self.seed), 'rev-parse', 'HEAD').stdout.strip() + self.git('-C', str(self.seed), 'push', '-q', str(self.remote), + 'HEAD:refs/pull/1/merge') + self.commit('pr.txt', 'pull request head again') + self.git('-C', str(self.seed), 'merge', '-q', '--no-ff', '-m', 'merge two', 'main') + second_merge = self.git('-C', str(self.seed), 'rev-parse', 'HEAD').stdout.strip() + self.git('-C', str(self.seed), 'push', '-q', '--force', str(self.remote), + 'HEAD:refs/pull/1/merge') + + result = self.run_checkout(first_merge, 'refs/pull/1/merge') + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(self.head(), first_merge) + self.assertNotEqual(self.head(), second_merge) + + def test_refuses_to_check_out_a_ref_that_moved(self): + # The queued commit is gone, so only newer code is fetchable. The step + # has to stop instead of testing a revision that never triggered it. + result = self.run_checkout('0' * 40, 'refs/heads/main') + self.assertNotEqual(result.returncode, 0) + self.assertIn('instead of the triggering commit', result.stderr) + self.assertEqual(self.head(), '') + + def test_rejects_a_ref_the_workflow_never_runs_on(self): + result = self.run_checkout(self.queued, 'refs/remotes/origin/main') + self.assertNotEqual(result.returncode, 0) + self.assertIn('Unsupported GitHub ref', result.stderr) + self.assertEqual(self.head(), '') + + def test_rejects_a_malformed_commit(self): + result = self.run_checkout('not-a-commit', 'refs/heads/main') + self.assertNotEqual(result.returncode, 0) + self.assertEqual(self.head(), '') + + if __name__ == '__main__': unittest.main() diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 82f605d3..289edefb 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -44,11 +44,21 @@ steps: ;; esac - git fetch --no-tags --depth=1 \ - https://github.com/apache/reqsign.git \ - "$GITHUB_REF" + # Fetch the commit the GitHub workflow ran instead of the ref it named. + # A ref can advance while the run waits for this agent, and a merge ref + # is replaced by the next push, so fetching the ref alone would fail on + # the mismatch or check out code that never triggered the probe. + repository=https://github.com/apache/reqsign.git + if ! git fetch --no-tags --depth=1 "$repository" "$GITHUB_SHA"; then + # Keep fetching the ref as a fallback; the check below still requires + # the commit to be the one that triggered this run. + git fetch --no-tags --depth=1 "$repository" "$GITHUB_REF" + fi fetched_sha=$(git rev-parse 'FETCH_HEAD^{commit}') - test "$fetched_sha" = "$GITHUB_SHA" + if [[ "$fetched_sha" != "$GITHUB_SHA" ]]; then + echo "Fetched $fetched_sha instead of the triggering commit $GITHUB_SHA" >&2 + exit 1 + fi git checkout --detach "$fetched_sha" displayName: Check out the GitHub Actions revision env: