From baea7de964ca6387c38ea8945a85cf713b7c5c00 Mon Sep 17 00:00:00 2001 From: lr90 Date: Fri, 23 Oct 2026 21:25:09 +0800 Subject: [PATCH 01/11] ci: migrate trusted macOS lanes to self-hosted ARM runners --- .github/actionlint.yaml | 11 + .github/workflows/macos-results.yml | 40 +++ .github/workflows/macos-trusted.yml | 213 ++++++++++++ .github/workflows/test.yml | 97 +++--- CONTRIBUTING.md | 5 + docs/guides/macos-ci-runner.md | 234 ++++++++++++++ docs/guides/repository-automation.md | 8 + scripts/ci/macos-job-started.sh | 15 + scripts/ci/macos-runner-admit | 12 + scripts/ci/macos-runner-preflight.sh | 12 + scripts/ci/macos_runner.py | 462 +++++++++++++++++++++++++++ scripts/ci/run-macos-lanes.sh | 45 +++ scripts/ci/test_ci_build_coverage.py | 43 +-- scripts/ci/test_macos_runner.py | 388 ++++++++++++++++++++++ 14 files changed, 1500 insertions(+), 85 deletions(-) create mode 100644 .github/actionlint.yaml create mode 100644 .github/workflows/macos-results.yml create mode 100644 .github/workflows/macos-trusted.yml create mode 100644 docs/guides/macos-ci-runner.md create mode 100755 scripts/ci/macos-job-started.sh create mode 100755 scripts/ci/macos-runner-admit create mode 100755 scripts/ci/macos-runner-preflight.sh create mode 100755 scripts/ci/macos_runner.py create mode 100755 scripts/ci/run-macos-lanes.sh create mode 100755 scripts/ci/test_macos_runner.py diff --git a/.github/actionlint.yaml b/.github/actionlint.yaml new file mode 100644 index 0000000000..02d48fffcf --- /dev/null +++ b/.github/actionlint.yaml @@ -0,0 +1,11 @@ +self-hosted-runner: + labels: [mac-arm64-mini, astra-macos-admitted] +paths: + .github/workflows/macos-trusted.yml: + ignore: + # GitHub supports cache-mode since 2026-09-10; actionlint 1.7.12 does not. + # Remove only this narrow compatibility filter when its parser catches up. + - '^unexpected key "cache-mode" for "workflow" section\.' + .github/workflows/macos-results.yml: + ignore: + - '^unexpected key "cache-mode" for "workflow" section\.' diff --git a/.github/workflows/macos-results.yml b/.github/workflows/macos-results.yml new file mode 100644 index 0000000000..8d0d995a36 --- /dev/null +++ b/.github/workflows/macos-results.yml @@ -0,0 +1,40 @@ +name: Mac CI hosted result observer +on: + workflow_run: + workflows: [Test Suite] + types: [completed] +permissions: + contents: read +cache-mode: none +jobs: + observe: + if: github.repository == 'matrixorigin/Astra' && vars.ASTRA_MACOS_MIGRATION_ENABLED == 'true' + runs-on: ubuntu-latest + environment: astra-macos-control + timeout-minutes: 5 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + with: + ref: main + persist-credentials: false + - uses: actions/create-github-app-token@64ab729a6294000f5aa4a4bc0b73d1f32bc45510 # v3 + id: app + with: + app-id: ${{ vars.ASTRA_MACOS_APP_ID }} + private-key: ${{ secrets.ASTRA_MACOS_APP_PRIVATE_KEY }} + owner: matrixorigin + repositories: Astra + permission-members: read + permission-organization-self-hosted-runners: read + permission-administration: read + permission-actions: read + permission-contents: read + permission-pull-requests: read + permission-checks: write + permission-environments: read + # No PR checkout, downloaded artifacts, code execution or cached scripts. + - name: Observe ordinary hosted results + env: + GH_TOKEN: ${{ steps.app.outputs.token }} + ASTRA_MACOS_APP_ID: ${{ vars.ASTRA_MACOS_APP_ID }} + run: python3 scripts/ci/macos_runner.py observe diff --git a/.github/workflows/macos-trusted.yml b/.github/workflows/macos-trusted.yml new file mode 100644 index 0000000000..77077c239b --- /dev/null +++ b/.github/workflows/macos-trusted.yml @@ -0,0 +1,213 @@ +name: Mac CI + +on: + pull_request_target: + branches: [main] + types: [opened, synchronize, reopened, ready_for_review] + push: + branches: [main] + workflow_dispatch: + inputs: + pr_number: + description: Member PR to re-authorize and execute on hosted macOS + required: true + type: string + +permissions: + contents: read +cache-mode: read +concurrency: + group: mac-ci-${{ github.event.pull_request.number || inputs.pr_number || github.ref }} + cancel-in-progress: true + +env: + CARGO_INCREMENTAL: "0" + CARGO_TERM_COLOR: always + ASTRA_MACOS_APP_ID: ${{ vars.ASTRA_MACOS_APP_ID }} + +jobs: + plan: + if: >- + github.repository == 'matrixorigin/Astra' && + vars.ASTRA_MACOS_MIGRATION_ENABLED == 'true' && + (github.event_name != 'workflow_dispatch' || github.ref == 'refs/heads/main') + runs-on: ubuntu-latest + environment: astra-macos-control + timeout-minutes: 5 + outputs: + plan: ${{ steps.plan.outputs.plan }} + route: ${{ steps.plan.outputs.route }} + merge: ${{ steps.plan.outputs.merge }} + kind: ${{ steps.plan.outputs.kind }} + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + with: + ref: main + persist-credentials: false + - uses: actions/create-github-app-token@64ab729a6294000f5aa4a4bc0b73d1f32bc45510 # v3 + id: app + with: + app-id: ${{ vars.ASTRA_MACOS_APP_ID }} + private-key: ${{ secrets.ASTRA_MACOS_APP_PRIVATE_KEY }} + owner: matrixorigin + repositories: Astra + permission-members: read + permission-organization-self-hosted-runners: read + permission-administration: read + permission-actions: read + permission-contents: read + permission-pull-requests: read + permission-checks: write + permission-environments: read + - id: plan + name: Verify source, membership and enforcement + env: + GH_TOKEN: ${{ steps.app.outputs.token }} + ASTRA_MACOS_GROUP_ID: ${{ vars.ASTRA_MACOS_GROUP_ID }} + ASTRA_MACOS_MAIN_GROUP_ID: ${{ vars.ASTRA_MACOS_MAIN_GROUP_ID }} + ASTRA_MACOS_MAIN_ISOLATED: ${{ vars.ASTRA_MACOS_MAIN_ISOLATED }} + run: python3 scripts/ci/macos_runner.py plan + + mini: + name: Migrated macOS lanes + needs: plan + if: needs.plan.outputs.route == 'self-hosted' + runs-on: + group: astra-macos-${{ needs.plan.outputs.kind }} + labels: [self-hosted, macOS, ARM64, mac-arm64-mini, astra-macos-admitted] + timeout-minutes: 90 + permissions: + contents: read + steps: + # The root-owned job-started hook runs before even this step, including + # individual/failed-only reruns. Its separate credential owner is mandatory. + - name: Verify admitted platform and capacity + timeout-minutes: 2 + shell: bash + run: /opt/astra-ci/bin/preflight + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + with: + ref: main + path: _mac_control + persist-credentials: false + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + with: + ref: ${{ needs.plan.outputs.merge }} + path: _mac_source + persist-credentials: false + - uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7 + with: + node-version-file: _mac_source/.nvmrc + - name: Run all three macOS lanes + shell: bash + working-directory: _mac_source + env: + CARGO_TARGET_DIR: ${{ runner.temp }}/astra-macos-${{ github.run_id }}-${{ github.run_attempt }}/target + npm_config_cache: ${{ runner.temp }}/astra-macos-${{ github.run_id }}-${{ github.run_attempt }}/npm + run: bash ../_mac_control/scripts/ci/run-macos-lanes.sh all + - name: Remove job artifacts + shell: bash + if: always() + run: rm -rf "${RUNNER_TEMP}/astra-macos-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" + + hosted-admission: + needs: [plan, mini] + if: >- + !cancelled() && needs.plan.result == 'success' && + (needs.plan.outputs.route == 'hosted' || + (needs.plan.outputs.route == 'self-hosted' && needs.mini.result == 'failure')) + runs-on: ubuntu-latest + environment: astra-macos-control + timeout-minutes: 5 + outputs: + allowed: ${{ steps.admission.outputs.allowed }} + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + with: + ref: main + persist-credentials: false + - uses: actions/create-github-app-token@64ab729a6294000f5aa4a4bc0b73d1f32bc45510 # v3 + id: app + with: + app-id: ${{ vars.ASTRA_MACOS_APP_ID }} + private-key: ${{ secrets.ASTRA_MACOS_APP_PRIVATE_KEY }} + owner: matrixorigin + repositories: Astra + permission-members: read + permission-organization-self-hosted-runners: read + permission-administration: read + permission-actions: read + permission-contents: read + permission-pull-requests: read + permission-checks: write + permission-environments: read + - id: admission + name: Fresh authorization for hosted handoff + env: + GH_TOKEN: ${{ steps.app.outputs.token }} + PLAN: ${{ needs.plan.outputs.plan }} + run: python3 scripts/ci/macos_runner.py refresh --plan "$PLAN" + + hosted: + name: Fallback macOS lanes + needs: [plan, hosted-admission] + if: >- + !cancelled() && needs.hosted-admission.result == 'success' && + needs.hosted-admission.outputs.allowed == 'true' + runs-on: macos-15 + timeout-minutes: 90 + permissions: + contents: read + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + with: + ref: main + path: _mac_control + persist-credentials: false + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + with: + ref: ${{ needs.plan.outputs.merge }} + path: _mac_source + persist-credentials: false + - uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7 + with: + node-version-file: _mac_source/.nvmrc + - name: Run all three macOS lanes + shell: bash + working-directory: _mac_source + env: + CARGO_TARGET_DIR: ${{ runner.temp }}/astra-macos-${{ github.run_id }}-${{ github.run_attempt }}/target + npm_config_cache: ${{ runner.temp }}/astra-macos-${{ github.run_id }}-${{ github.run_attempt }}/npm + run: bash ../_mac_control/scripts/ci/run-macos-lanes.sh all + + publish: + needs: [plan, mini, hosted-admission, hosted] + if: always() && needs.plan.result == 'success' + runs-on: ubuntu-latest + environment: astra-macos-control + timeout-minutes: 5 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + with: + ref: main + persist-credentials: false + - uses: actions/create-github-app-token@64ab729a6294000f5aa4a4bc0b73d1f32bc45510 # v3 + id: app + with: + app-id: ${{ vars.ASTRA_MACOS_APP_ID }} + private-key: ${{ secrets.ASTRA_MACOS_APP_PRIVATE_KEY }} + owner: matrixorigin + repositories: Astra + permission-members: read + permission-organization-self-hosted-runners: read + permission-administration: read + permission-actions: read + permission-contents: read + permission-pull-requests: read + permission-checks: write + permission-environments: read + - name: Publish verified App-owned results + env: + GH_TOKEN: ${{ steps.app.outputs.token }} + PLAN: ${{ needs.plan.outputs.plan }} + run: python3 scripts/ci/macos_runner.py finish --plan "$PLAN" diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 88250cc5a0..34af12c60c 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -2,6 +2,8 @@ name: Test Suite permissions: contents: read + checks: read + pull-requests: read on: push: @@ -23,6 +25,7 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 5 outputs: + macos_delegated: ${{ steps.mac.outputs.delegated }} test_cli: ${{ steps.scope.outputs.test_cli }} test_runtime: ${{ steps.scope.outputs.test_runtime }} test_services: ${{ steps.scope.outputs.test_services }} @@ -39,6 +42,15 @@ jobs: - name: Classify changed paths id: scope run: python3 scripts/ci/ci_scope.py + - name: Read trusted macOS execution hint + id: mac + if: vars.ASTRA_MACOS_MIGRATION_ENABLED == 'true' + continue-on-error: true + env: + GH_TOKEN: ${{ github.token }} + ASTRA_MACOS_APP_ID: ${{ vars.ASTRA_MACOS_APP_ID }} + # A missing/forged hint cannot satisfy the App-owned required results. + run: python3 scripts/ci/macos_runner.py hint # Package groups remain parallel. CLI builds and executes its full inventory # once; stable protected labels require that same completed test result. @@ -240,7 +252,9 @@ jobs: name: "Terminal: ${{ matrix.segment }} (${{ matrix.os }})" needs: scope if: ${{ !cancelled() && (needs.scope.result != 'success' || needs.scope.outputs.test_cli == 'true' || needs.scope.outputs.test_core == 'true') }} - runs-on: ${{ matrix.os }} + runs-on: ${{ matrix.os == 'macos-15' && matrix.segment != 'dev-tty-reader' && needs.scope.outputs.macos_delegated == 'true' && 'ubuntu-latest' || matrix.os }} + env: + RUN_LANE: ${{ !(matrix.os == 'macos-15' && matrix.segment != 'dev-tty-reader' && needs.scope.outputs.macos_delegated == 'true') }} timeout-minutes: 45 strategy: fail-fast: false @@ -248,50 +262,32 @@ jobs: os: [ubuntu-latest, macos-15] segment: [default-reader, dev-tty-reader, reflow] steps: + - name: Delegated to App-verified macOS execution + if: env.RUN_LANE != 'true' + run: echo 'Protected Mac CI checks are independently required.' - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + if: env.RUN_LANE == 'true' with: persist-credentials: false - uses: ./.github/actions/rust-setup + if: env.RUN_LANE == 'true' with: shared-key: ci-terminal-pty - # One writer per OS populates the shared dependency cache on pushes. save-cache: ${{ github.event_name == 'push' && matrix.segment == 'default-reader' }} install-nextest: false - # Separate runners keep dev-tty feature unification out of the default - # reader and avoid serializing two lib builds and the reflow binary build. - - name: Test default terminal reader through PTYs - if: matrix.segment == 'default-reader' - run: | - cargo test --locked -p astra-cli --lib tui::terminal_startup -- --test-threads=2 - - name: Initialize system Git for macOS CLI observation - if: matrix.os == 'macos-15' && matrix.segment == 'default-reader' - timeout-minutes: 1 - run: | - time /usr/bin/env -i PATH=/usr/bin:/bin HOME=/nonexistent LANG=C LC_ALL=C \ - GIT_CONFIG_NOSYSTEM=1 GIT_CONFIG_GLOBAL=/dev/null GIT_CONFIG_SYSTEM=/dev/null \ - GIT_OPTIONAL_LOCKS=0 GIT_TERMINAL_PROMPT=0 \ - /usr/bin/git -C "$GITHUB_WORKSPACE" -c core.fsmonitor=false \ - -c core.untrackedCache=false rev-parse --show-toplevel - - name: Test macOS CLI Bash execution - if: matrix.os == 'macos-15' && matrix.segment == 'default-reader' - # Reuse the default-feature CLI test executable already built above. - run: >- - cargo test --locked --manifest-path Cargo.toml -p astra-cli - --lib edge_tools::shell::tests::bash_echo_returns_output -- --exact - - name: Test dev-tty terminal reader through PTYs - if: matrix.segment == 'dev-tty-reader' - run: | - cargo test --locked -p astra-cli --lib --features crossterm/use-dev-tty tui::terminal_startup -- --test-threads=2 - uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7 - if: matrix.segment == 'reflow' + if: env.RUN_LANE == 'true' && matrix.segment == 'reflow' with: node-version-file: .nvmrc + - name: Test default terminal reader and macOS CLI Bash + if: env.RUN_LANE == 'true' && matrix.segment == 'default-reader' + run: bash scripts/ci/run-macos-lanes.sh default-reader + - name: Test dev-tty terminal reader through PTYs + if: matrix.segment == 'dev-tty-reader' + run: cargo test --locked -p astra-cli --lib --features crossterm/use-dev-tty tui::terminal_startup -- --test-threads=2 - name: Test inline terminal resize with xterm reflow - if: matrix.segment == 'reflow' - run: | - cargo build --locked -p astra-cli --bin astra - npm ci --ignore-scripts --prefix scripts/tui-reflow - ASTRA_TEST_BINARY="$PWD/target/debug/astra" npm test --prefix scripts/tui-reflow + if: env.RUN_LANE == 'true' && matrix.segment == 'reflow' + run: bash scripts/ci/run-macos-lanes.sh reflow macos-workspace-coordination: name: "Test: macOS workspace coordination" @@ -320,42 +316,27 @@ jobs: name: "macOS workspace coordination contracts" needs: scope if: ${{ !cancelled() && (needs.scope.result != 'success' || needs.scope.outputs.test_core == 'true' || needs.scope.outputs.test_cli == 'true') }} - runs-on: macos-15 + runs-on: ${{ needs.scope.outputs.macos_delegated == 'true' && 'ubuntu-latest' || 'macos-15' }} + env: + RUN_LANE: ${{ needs.scope.outputs.macos_delegated != 'true' }} timeout-minutes: 30 steps: + - name: Delegated to App-verified macOS execution + if: env.RUN_LANE != 'true' + run: echo 'Protected Mac CI checks are independently required.' - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + if: env.RUN_LANE == 'true' with: persist-credentials: false - uses: ./.github/actions/rust-setup + if: env.RUN_LANE == 'true' with: - # Reuse the CLI dependency cache; default-reader is its only writer. shared-key: ci-terminal-pty save-cache: "false" install-nextest: "false" - - name: Lint macOS workspace coordination - run: >- - cargo clippy --locked --manifest-path Cargo.toml -p astra-tools - --all-targets -- -D warnings - - name: Initialize system Git for bounded observation probes - timeout-minutes: 1 - # Checkout uses Homebrew Git, but the hardened observer selects - # /usr/bin/git with a cleared environment. Initialize that executable - # before its first use under the production two-second probe deadline. - # Keep the timing visible so runner startup failures are diagnosable. - run: | - time /usr/bin/env -i PATH=/usr/bin:/bin HOME=/nonexistent LANG=C LC_ALL=C \ - GIT_CONFIG_NOSYSTEM=1 GIT_CONFIG_GLOBAL=/dev/null GIT_CONFIG_SYSTEM=/dev/null \ - GIT_OPTIONAL_LOCKS=0 GIT_TERMINAL_PROMPT=0 \ - /usr/bin/git -C "$GITHUB_WORKSPACE" -c core.fsmonitor=false \ - -c core.untrackedCache=false rev-parse --show-toplevel - name: Test macOS workspace coordination invariants - # Several invariants launch re-exec lock contenders while fingerprint - # probes deliberately have a short wall-clock deadline. Keep the module - # serial on shared CI hosts; the contender test supplies explicit - # cross-process concurrency without starving unrelated probes. - run: >- - cargo test --locked --manifest-path Cargo.toml -p astra-tools - workspace_observation::tests:: -- --test-threads=1 + if: env.RUN_LANE == 'true' + run: bash scripts/ci/run-macos-lanes.sh coordination # Preserve the required core check while its package groups run separately. # The aggregate requires all selected online jobs, including integration. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 3b884b8b31..c3e8261768 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -118,6 +118,11 @@ On macOS, the default terminal-reader lane also runs the CLI Bash regression using its existing default-feature test executable. Tools lint and serial workspace-coordination tests remain parallel; the protected coordination check requires both paths to succeed. Feature-specific terminal tests remain separate. +The opt-in [self-hosted macOS migration](docs/guides/macos-ci-runner.md) runs +three lanes together for verified organization-member PRs. Its required results +come from a dedicated App; ordinary hosted delegation hints cannot satisfy them. +Non-members and unknown identities retain ordinary hosted PR approval. Main stays +hosted until an independently isolated main runner is accepted. The lint gate retains restored artifacts; age-based cleanup is not a lint prerequisite. For fork pull requests, an update may require maintainer approval before the diff --git a/docs/guides/macos-ci-runner.md b/docs/guides/macos-ci-runner.md new file mode 100644 index 0000000000..52bc0b719d --- /dev/null +++ b/docs/guides/macos-ci-runner.md @@ -0,0 +1,234 @@ +# Self-hosted macOS CI + +This guide owns the rollout of the three Mac CI lanes tracked in +[issue #1106](https://github.com/matrixorigin/Astra/issues/1106). +The implementation is opt-in. A merged workflow is not evidence that the Mac +is configured, admitted, compatible, or faster than hosted runners. + +## Workload and routing + +The migrated unit runs default-reader PTYs and the CLI Bash regression, reflow, +and workspace coordination. It runs sequentially on one runner and reuses its +build directory. This avoids scheduling three simultaneous Rust builds on a +single small machine. The canonical commands live in +`scripts/ci/run-macos-lanes.sh` and are shared with ordinary hosted CI. + +| Event | Execution | +| --- | --- | +| Confirmed matrixorigin member PR, current verified merge, admitted idle Mac | Self-hosted ARM Mac | +| Confirmed member, Mac unavailable or admission/platform failure before tests | Protected hosted macOS fallback | +| Non-member, membership API failure, or unknown identity | Ordinary `pull_request` hosted workflow | +| Main push before isolated main account is accepted | Ordinary hosted workflow | +| Main push with accepted separate main account/group | Admitted main Mac runner; protected hosted fallback | +| Other branches/events | Existing hosted routing | + +macOS dev-tty and vendored terminal units remain hosted, as do Linux and release +builds. Path selection calls the existing `ci_scope.classify` on the verified +base-to-merge comparison. Unknown/truncated comparisons select the lanes. + +A PR author is identified by the actual GitHub PR record, including user ID. +Only `GET /orgs/matrixorigin/members/{author}` returning 204 qualifies. The +credential must see private membership. Fork ownership, actor, association and +public profiles are not substituted. Qualifying members vouch for their PR's +contents, including collaborators' changes; this does not prove every writer is +a member. + +## Control and required results + +`macos-trusted.yml` is base-owned `pull_request_target` control. Only confirmed +members execute PR code through that event. The controller independently reads +the current base B, head H and merge M and requires exactly two ordered parents +`M^1=B` and `M^2=H`. It checks out immutable M. Null/stale merges, force-pushes, +base advancement or changed repository identity cannot silently select another +revision. + +A dedicated GitHub App publishes three contexts on H: + +- `Mac CI: default reader` +- `Mac CI: reflow` +- `Mac CI: coordination` + +All three must be required with their source bound to that App. Control checks +this source binding before dispatch. Its private key and installation token +remain on GitHub-hosted Ubuntu control jobs, never on PR execution machines. +Store the private key only in the `astra-macos-control` GitHub Environment, +restricted by a custom branch rule to branch `main` alone (no tag rule). +Do not duplicate it as a repository/organization secret: same-repository PRs +could otherwise request it. PR refs and feature-branch dispatches must be denied +by the environment policy. Control verifies that exact policy before dispatch. +The publisher reads actual GitHub run/job/step results; it does not accept PR +artifacts or shell-supplied success flags. A successful combined job satisfies +all three contexts; any actual test failure fails them conservatively. + +The ordinary `pull_request` workflow reads App-owned authorization as a +best-effort deduplication hint. Delegated copies use cheap Ubuntu notification +jobs. The hint cannot publish or replace the three required App checks. Missing +hints preserve hosted execution, so a control race can cause duplicate work, +not missing coverage. A forged hint leaves the member's trusted checks pending. + +Non-members retain ordinary fork approval. `macos-results.yml` observes the +completed Test Suite through `workflow_run`, reads GitHub metadata only, and +publishes the ordinary path's actual three hosted results. It never downloads +artifacts or executes PR code. Approval-pending, missing, stale, cancelled or +failed execution cannot become a successful skip. This retains the pre-existing +PR-modifiable test-definition limitation for non-member hosted tests. + +Both privileged-event workflows explicitly restrict remote cache tokens with +`cache-mode: read` or `none`. Disabling a client upload is not the authorization +boundary. Actionlint 1.7.12 does not parse this GitHub-supported key: the config +filters only that exact unsupported-key diagnostic for these two files. Other +workflow errors remain checked. Remove that narrow filter when actionlint gains +support. See [GitHub cache permissions](https://docs.github.com/en/actions/reference/workflows-and-actions/workflow-syntax#cache-mode). + +## Mandatory runner-side admission + +Use a dedicated `astra-macos-pr` group, selecting only Astra, allowing its public +repository, restricted to exactly: + +```text +matrixorigin/Astra/.github/workflows/macos-trusted.yml@refs/heads/main +``` + +Remove every unrestricted registration/path to this same execution identity. +Labels and repository restrictions alone do not protect against PR YAML changing +`runs-on`. The group must enforce the workflow and protected ref. If the actual +GitHub plan cannot enforce that restriction, do not enable this design. + +Runner labels are `self-hosted`, `macOS`, `ARM64`, `mac-arm64-mini` and +`astra-macos-admitted`. Add the last label only after admission and acceptance +have actually passed. The runner's physical name/location is not part of the +public configuration. + +Install a root-owned job-started hook before accepting jobs. The PR runner must +be a non-admin account named `astra-ci-pr` without general sudo rights. A +separate non-login `astra-ci-admission` account owns a read-only API credential +with private Members read, organization runners/groups read, Actions read, +contents/pull-request/checks read. It must not publish checks or write contents. +No credential is readable by the runner account. + +Install these reviewed files outside all runner-writable paths: + +| Source | Root-owned destination | +| --- | --- | +| `scripts/ci/macos_runner.py`, `scripts/ci/ci_scope.py` | `/opt/astra-ci/lib/` | +| `scripts/ci/macos-runner-admit` | `/opt/astra-ci/bin/admit` | +| `scripts/ci/macos-job-started.sh` | `/opt/astra-ci/bin/job-started.sh` | +| `scripts/ci/macos-runner-preflight.sh` | `/opt/astra-ci/bin/preflight` | + +Use a root-managed Python 3.11+ installation at `/opt/astra-ci/python`, with no +runner-writable executable, library or parent directory. The small hook itself +uses system Python for JSON only; the policy helper uses isolated Python mode. +The fixed sudo target accepts no command arguments and selects a root-owned +policy by sudo's verified invoking user, not by JSON supplied by a PR. Configure +sudoers for exactly this command with no arguments and no environment override: + +```text +astra-ci-pr ALL=(astra-ci-admission) NOPASSWD: /opt/astra-ci/bin/admit "" +``` + +No general shell, Python, environment-preserving sudo, or arbitrary config-path +invocation is permitted. Protect every installed parent directory and config. +Set `ACTIONS_RUNNER_HOOK_JOB_STARTED=/opt/astra-ci/bin/job-started.sh` in the runner's +service environment, then restart that runner through its reviewed operations +process. Changing repository YAML does not install this hook. + +Example `/etc/astra-ci/pr.json` (root-owned, readable only by admission owner): + +```json +{ + "app_id": 12345, + "kind": "pr", + "group_id": 123, + "runner_name": "", + "token_file": "/etc/astra-ci/admission-token" +} +``` + +The helper has a 90-second deadline; the hook client also bounds the privileged +subprocess at 100 seconds. The `.sh` extension on the installed hook is required +by GitHub's hook launcher. + +The helper reads the publisher App's authorization from GitHub, validates the +actual workflow/repository/event/current attempt and active job on the expected +runner/group, rechecks the current tuple and private membership, and only then +admits. This runs for individual and failed-only reruns even when the previous +hosted router job was successful. Caller-controlled JSON cannot supply policy, +credentials, an API host or an arbitrary command. Hook rejection fails the job; +the workflow separately authorizes hosted handoff. + +## Host state and platform acceptance + +Keep main hosted by default. PR job artifacts/npm cache use a private per-run +scratch directory and are removed afterward. Persistent user state, including +HOME, Cargo/rustup, tool binaries, Git config/helpers, keychains, launchd agents, +background processes and writable shares, is still under the PR account's trust. +A clean checkout or scratch target does not erase those effects. + +Prefer disposable VMs per job. To enable main migration, use an independently +installed runner under a different non-admin OS account `astra-ci-main`, separate +home/toolchain/caches, and group `astra-macos-main` with the same workflow/ref +restriction. Provide `/etc/astra-ci/main.json` with kind `main`, the main runner +name/group, and add only the same fixed sudo command for that account. Two runner +registrations under the same OS account do not qualify: the fixed helper maps +the PR identity only to PR policy. Validate filesystem/process isolation and +harmless pollution sentinels for all state above before accepting main. Separate +accounts retain shared-kernel/host compromise risk; they are not disposable VMs. + +The installed preflight requires macOS 15+, native ARM64 and at least 30 GiB +free on the workspace filesystem. It records OS, CPU count, RAM and free disk +without host identity. A machine name containing `macos-13` is not proof of its +current OS. Upgrade/replace an older installation through its normal reviewed +process before assigning the admission label. Preinstall the repository's pinned +Rust toolchain, Node/action runtime dependencies and working system Git. The +runner must support the current `cache-mode` token enforcement. + +Before rollout measure cold/warm full-lane runs, queue delay, peak RSS/disk and +competing work. Start with one registration and serial combined execution; +do not multiply runner processes to hide capacity issues. Record acceptable +queue p50/p95 and job duration limits against observed PR arrival bursts. These +measurements have not been performed by the application PR. + +An online/idle API observation is not a reservation. A runner can disappear +between dispatch and start; queued self-hosted work remains pending. An operator +can cancel the stale run and use `workflow_dispatch` with its PR number for a +fresh protected hosted run. That dispatch always selects hosted execution. +Failed-only hosted retries are refused when they reuse an older successful +admission job; rerun all jobs for new hosted authorization. Membership revocation +or a stale tuple blocks privileged-event execution; a fresh plan can choose the +ordinary approved hosted path. Genuine test failures are never hidden by the +infrastructure fallback. + +## Enable, verify and roll back + +1. Merge the reviewed implementation while migration remains disabled. +2. Configure the dedicated publisher App, groups, isolated credential owner, + root-owned hook/helper and platform described above. Validate direct-label + bypass, fork/member/unknown routing, revoked-member reruns, tuple changes, + cache write denial, pollution isolation and capacity on disposable fixtures. +3. Create GitHub Environment `astra-macos-control`, allowing only branch `main` + via its custom deployment branch policy, with no allowed tags or wildcards. + After verifying that PR refs and feature dispatches cannot access it, store + `ASTRA_MACOS_APP_PRIVATE_KEY` only as that environment's secret. Set repository + variable `ASTRA_MACOS_APP_ID` and `ASTRA_MACOS_GROUP_ID` to the App/PR group IDs. + The App needs repository Actions/contents/PR read, administration read, + checks write and environments read; organization Members and self-hosted + runners/groups read. +4. Make the three contexts required with source pinned to the App. Preserve the + existing protected checks. The migration control refuses dispatch until this + binding is visible through the rules/protection APIs. +5. Set `ASTRA_MACOS_MIGRATION_ENABLED=true` only after the preceding acceptance. + For main, separately set `ASTRA_MACOS_MAIN_GROUP_ID` and + `ASTRA_MACOS_MAIN_ISOLATED=true` after its isolation acceptance. +6. Verify actual job runner names/groups and reduced hosted Mac jobs. Record + hosted displacement, Mac execution and total elapsed time independently. + +Activation needs organization/repository administration and runner-host access. +The application PR does not modify live groups, secrets, branch protection, +remote clusters or runner services. Current caller API queries for runners and +groups returned 403; none of those settings is claimed verified or enabled. + +For rollback, keep required result publication working until in-flight PRs are +settled. Remove the three new App-required contexts through reviewed policy +change, disable `ASTRA_MACOS_MIGRATION_ENABLED`, then remove/disable the admitted +runner registration as needed. Ordinary hosted execution is retained throughout +when no valid delegation hint exists. Do not turn missing App results into pass. diff --git a/docs/guides/repository-automation.md b/docs/guides/repository-automation.md index 46fa750ed6..adb2f59313 100644 --- a/docs/guides/repository-automation.md +++ b/docs/guides/repository-automation.md @@ -119,3 +119,11 @@ before rerunning CI: This separation prevents permission failures from being treated as flaky CI and keeps external-fork behavior part of the repository's delivery contract. + +## Self-hosted macOS migration + +The three-lane member/main migration, dedicated result source, runner-side +admission, isolation requirements and explicit activation/rollback steps are +documented in [Self-hosted macOS CI](macos-ci-runner.md). Migration is opt-in; +ordinary hosted execution remains the default until administration and host +acceptance are complete. diff --git a/scripts/ci/macos-job-started.sh b/scripts/ci/macos-job-started.sh new file mode 100755 index 0000000000..ca6dc8818a --- /dev/null +++ b/scripts/ci/macos-job-started.sh @@ -0,0 +1,15 @@ +#!/bin/bash +# Install root-owned outside every runner-writable path. No sourced PR code. +set -euo pipefail +/usr/bin/python3 -I -c ' +import json, os, subprocess +from pathlib import Path +event = json.loads(Path(os.environ["GITHUB_EVENT_PATH"]).read_text()) +head = event.get("pull_request", {}).get("head", {}).get("sha") or event.get("after") +if not head: + raise SystemExit("Use a fresh pull_request_target event for self-hosted admission") +request = {"run_id": int(os.environ["GITHUB_RUN_ID"]), + "attempt": int(os.environ["GITHUB_RUN_ATTEMPT"]), "head": head} +subprocess.run(["/usr/bin/sudo", "-n", "-u", "astra-ci-admission", "/opt/astra-ci/bin/admit"], + input=json.dumps(request), text=True, check=True, timeout=100) +' diff --git a/scripts/ci/macos-runner-admit b/scripts/ci/macos-runner-admit new file mode 100755 index 0000000000..df3eb60407 --- /dev/null +++ b/scripts/ci/macos-runner-admit @@ -0,0 +1,12 @@ +#!/bin/bash +# Root-owned fixed sudo target; no command arguments or caller paths allowed. +set -euo pipefail +[ "$#" -eq 0 ] +case "${SUDO_USER:-}" in + astra-ci-pr) policy=/etc/astra-ci/pr.json ;; + astra-ci-main) policy=/etc/astra-ci/main.json ;; + *) echo 'Unrecognized runner identity' >&2; exit 1 ;; +esac +exec /usr/bin/env -i PATH=/usr/bin:/bin \ + /opt/astra-ci/python/bin/python3 -I /opt/astra-ci/lib/macos_runner.py \ + admit --config "$policy" diff --git a/scripts/ci/macos-runner-preflight.sh b/scripts/ci/macos-runner-preflight.sh new file mode 100755 index 0000000000..3362832337 --- /dev/null +++ b/scripts/ci/macos-runner-preflight.sh @@ -0,0 +1,12 @@ +#!/usr/bin/env bash +set -euo pipefail +[ "$(uname -s)" = Darwin ] +[ "$(uname -m)" = arm64 ] +version="$(sw_vers -productVersion)" +[ "${version%%.*}" -ge 15 ] +available="$(df -Pk "$GITHUB_WORKSPACE" | awk 'NR==2 {print $4}')" +[[ "$available" =~ ^[0-9]+$ ]] && [ "$available" -ge 31457280 ] +# Record capacity without host identity, network information or credentials. +printf 'macOS=%s arch=arm64 free_disk_KiB=%s\n' "$version" "$available" +sysctl -n hw.ncpu hw.memsize +rustc --version diff --git a/scripts/ci/macos_runner.py b/scripts/ci/macos_runner.py new file mode 100755 index 0000000000..1638905f04 --- /dev/null +++ b/scripts/ci/macos_runner.py @@ -0,0 +1,462 @@ +#!/usr/bin/env python3 +"""Trusted macOS routing, admission and result publication (stdlib only). + +No PR artifacts or shell-provided success results are used as evidence. The +same policy is installed read-only outside the runner account for job hooks. +""" +from __future__ import annotations + +import argparse +import json +import os +from pathlib import Path +import re +import signal +import sys +import time +import urllib.error +import urllib.request + +sys.path.insert(0, str(Path(__file__).resolve().parent)) +from ci_scope import classify + +REPO = "matrixorigin/Astra" +WORKFLOW = ".github/workflows/macos-trusted.yml" +WORKFLOW_REF = f"{REPO}/{WORKFLOW}@refs/heads/main" +CONTEXTS = ("Mac CI: default reader", "Mac CI: reflow", "Mac CI: coordination") +SHA = re.compile(r"^[0-9a-f]{40}$") + + +class Unavailable(Exception): + pass + + +class NoRedirect(urllib.request.HTTPRedirectHandler): + def redirect_request(self, *args, **kwargs): + return None + + +class API: + def __init__(self, token): + self.token = token + self.opener = urllib.request.build_opener(NoRedirect) + + def request(self, path, data=None, method=None): + if not path.startswith("/") or path.startswith("//"): + raise Unavailable("invalid API path") + req = urllib.request.Request("https://api.github.com" + path, + data=None if data is None else json.dumps(data).encode(), method=method, + headers={"Authorization": f"Bearer {self.token}", + "Accept": "application/vnd.github+json", + "X-GitHub-Api-Version": "2022-11-28", + "Content-Type": "application/json"}) + try: + with self.opener.open(req, timeout=20) as response: + raw = response.read() + return response.status, json.loads(raw) if raw else None + except urllib.error.HTTPError as error: + return error.code, None # Never print provider responses or credentials. + except (OSError, ValueError): + raise Unavailable("GitHub API unavailable") from None + + def get(self, path): + status, value = self.request(path) + if status != 200: + raise Unavailable(f"GitHub API returned {status}") + return value + + def pages(self, path, key=None): + result = [] + for page in range(1, 101): + value = self.get(path + ("&" if "?" in path else "?") + f"per_page=100&page={page}") + rows = value[key] if key else value + result.extend(rows) + if len(rows) < 100: + return result + raise Unavailable("API pagination limit") + + +def member(api, login): + if not re.fullmatch(r"[A-Za-z0-9-]{1,39}", login): + return False + # A configured Members:read credential is required, including private members. + try: + return api.request(f"/orgs/matrixorigin/members/{login}")[0] == 204 + except Unavailable: + return False + + +def source(api, number): + pr = api.get(f"/repos/{REPO}/pulls/{int(number)}") + if (pr["state"] != "open" or pr["base"]["repo"]["full_name"] != REPO + or pr["base"]["ref"] != "main"): + raise Unavailable("PR is not an open main pull request") + base = api.get(f"/repos/{REPO}/git/ref/heads/main")["object"]["sha"] + head, merge = pr["head"]["sha"], pr.get("merge_commit_sha") + if not all(isinstance(s, str) and SHA.fullmatch(s) for s in (base, head, merge)): + raise Unavailable("merge revision unresolved") + commit = api.get(f"/repos/{REPO}/git/commits/{merge}") + if [p["sha"] for p in commit["parents"]] != [base, head]: + raise Unavailable("merge parents do not match current base/head") + return {"kind": "pr", "pr": int(number), "base": base, "head": head, + "merge": merge, "head_repository_id": pr["head"]["repo"]["id"], + "author": pr["user"]["login"], "author_id": pr["user"]["id"]} + + +def same_source(a, b): + return all(a.get(k) == b.get(k) for k in + ("kind", "pr", "base", "head", "merge", "head_repository_id", "author_id")) + + +def selected(api, plan): + try: + diff = api.get(f"/repos/{REPO}/compare/{plan['base']}...{plan['merge']}") + files = diff.get("files") + if files is None or len(files) >= 300: + return True # GitHub truncates compare's file inventory at 300. + paths = [name for f in files for name in (f["filename"], f.get("previous_filename")) if name] + scope = classify(paths) + return scope["test_cli"] or scope["test_core"] + except (Unavailable, KeyError, TypeError): + return True + + +def require_check_sources(api, app_id): + environment = api.get(f"/repos/{REPO}/environments/astra-macos-control") + policy = environment.get("deployment_branch_policy") or {} + if policy != {"protected_branches": False, "custom_branch_policies": True}: + raise Unavailable("control environment must select only the main branch") + branches = api.pages(f"/repos/{REPO}/environments/astra-macos-control/deployment-branch-policies", "branch_policies") + if len(branches) != 1 or branches[0].get("name") != "main" or branches[0].get("type") != "branch": + raise Unavailable("control environment allows an untrusted branch or tag") + required = {} + for rule in api.get(f"/repos/{REPO}/rules/branches/main"): + if rule["type"] == "required_status_checks": + for check in rule["parameters"]["required_status_checks"]: + required[check["context"]] = check.get("integration_id") + status, protection = api.request(f"/repos/{REPO}/branches/main/protection") + if status == 200: + for check in (protection.get("required_status_checks") or {}).get("checks", []): + required[check["context"]] = check.get("app_id") + elif status != 404: + raise Unavailable("cannot verify required-check source binding") + if any(required.get(name) != app_id for name in CONTEXTS): + raise Unavailable("three Mac CI checks must be required from the dedicated App") + + +def group_ready(api, group_id, kind): + group = api.get(f"/orgs/matrixorigin/actions/runner-groups/{int(group_id)}") + if (group["name"] != f"astra-macos-{kind}" or not group.get("restricted_to_workflows") + or group.get("selected_workflows") != [WORKFLOW_REF] + or group.get("visibility") != "selected" or not group.get("allows_public_repositories")): + raise Unavailable("runner group is not restricted to the protected workflow") + repos = api.pages(f"/orgs/matrixorigin/actions/runner-groups/{group_id}/repositories", "repositories") + if [r["full_name"] for r in repos] != [REPO]: + raise Unavailable("runner group must select only Astra") + runners = api.pages(f"/orgs/matrixorigin/actions/runner-groups/{group_id}/runners", "runners") + labels = {"self-hosted", "macOS", "ARM64", "mac-arm64-mini", "astra-macos-admitted"} + return any(r["status"] == "online" and not r["busy"] and + labels <= {label["name"] for label in r["labels"]} for r in runners) + + +def checks(api, head): + return api.pages(f"/repos/{REPO}/commits/{head}/check-runs", "check_runs") + + +def save_checks(api, plan, conclusion=None, summary="Awaiting verified execution"): + existing = {name: [] for name in CONTEXTS} + for check in checks(api, plan["head"]): + if check["name"] in existing and check.get("app", {}).get("id") == plan["app_id"]: + existing[check["name"]].append(check) + for name in CONTEXTS: + data = {"name": name, "head_sha": plan["head"], + "external_id": f"astra-macos-v1:{plan['run_id']}:{plan['head']}", + "status": "completed" if conclusion else "in_progress", + "output": {"title": summary, "summary": summary, "text": json.dumps(plan, sort_keys=True)}} + if conclusion: + data["conclusion"] = conclusion + previous = max(existing[name], key=lambda c: c["id"], default=None) + if previous: + try: + old = json.loads(previous["output"]["text"]) + if (old["run_id"], old["attempt"]) > (plan["run_id"], plan["attempt"]): + continue # A late publisher cannot overwrite newer control. + reuse = old["run_id"] == plan["run_id"] and old["attempt"] == plan["attempt"] + except (ValueError, KeyError, TypeError): + reuse = False + else: + reuse = False + if reuse: + data.pop("head_sha") + status, _ = api.request(f"/repos/{REPO}/check-runs/{previous['id']}", data, "PATCH") + expected = 200 + else: + status, _ = api.request(f"/repos/{REPO}/check-runs", data) + expected = 201 + if status != expected: + raise Unavailable("could not publish dedicated App checks") + + +def authorization(api, head, run_id, app_id): + found = {} + for check in checks(api, head): + if check["name"] not in CONTEXTS or check.get("app", {}).get("id") != app_id: + continue + try: + plan = json.loads(check["output"]["text"]) + except (KeyError, TypeError, ValueError): + continue + if plan.get("schema") == 1 and plan.get("run_id") == run_id and plan.get("head") == head: + if check["name"] not in found or found[check["name"]][0] < check["id"]: + found[check["name"]] = (check["id"], plan) + if len(found) != len(CONTEXTS): + raise Unavailable("no complete App authorization") + plans = [p for _, p in found.values()] + if any(p != plans[0] for p in plans): + raise Unavailable("inconsistent App authorization") + return plans[0] + + +def validate_run(api, plan, attempt=None): + run = api.get(f"/repos/{REPO}/actions/runs/{int(plan['run_id'])}") + expected_events = ("pull_request_target", "workflow_dispatch") if plan["kind"] == "pr" else ("push",) + if (run["event"] not in expected_events or run["path"] != WORKFLOW + or run["repository"]["full_name"] != REPO + or (attempt is not None and run["run_attempt"] != attempt)): + raise Unavailable("unexpected workflow, repository, event or attempt") + return run + + +def refresh(api, plan): + if plan["kind"] == "pr": + current = source(api, plan["pr"]) + if not same_source(current, plan) or not member(api, current["author"]): + raise Unavailable("source changed or membership is not confirmed") + else: + head = api.get(f"/repos/{REPO}/git/ref/heads/main")["object"]["sha"] + if plan["head"] != head: + raise Unavailable("main has advanced") + + +def admit(api, request, config): + # Invoked by a root-owned job-started hook under a distinct credential owner. + # No caller-supplied token, path, repository, API URL or command is accepted. + if set(request) != {"run_id", "attempt", "head"} or not SHA.fullmatch(request["head"]): + raise Unavailable("invalid admission request") + plan = authorization(api, request["head"], int(request["run_id"]), int(config["app_id"])) + if plan["route"] != "self-hosted" or plan["kind"] != config["kind"]: + raise Unavailable("authorization is for another execution boundary") + if int(config["group_id"]) != plan["group_id"]: + raise Unavailable("runner group differs from the authorized group") + group_ready(api, int(config["group_id"]), config["kind"]) + validate_run(api, plan, int(request["attempt"])) + refresh(api, plan) # Fresh private-membership check on every actual attempt. + jobs = api.pages(f"/repos/{REPO}/actions/runs/{plan['run_id']}/attempts/{request['attempt']}/jobs", "jobs") + if not any(j["name"] == "Migrated macOS lanes" and j["status"] == "in_progress" + and j["runner_name"] == config["runner_name"] + and j["runner_group_id"] == int(config["group_id"]) for j in jobs): + raise Unavailable("no matching active job on the admitted runner") + return plan + + +def verified_job(api, plan, name): + run = validate_run(api, plan) + jobs = api.pages(f"/repos/{REPO}/actions/runs/{run['id']}/jobs?filter=latest", "jobs") + matching = [j for j in jobs if j["name"] == name] + if len(matching) != 1: + return None + job = matching[0] + if job["conclusion"] != "success": + return job["conclusion"] if job["status"] == "completed" else None + steps = {s["name"]: s["conclusion"] for s in job["steps"]} + if steps.get("Run all three macOS lanes") != "success": + return "failure" + if name == "Migrated macOS lanes" and job["runner_group_id"] != plan["group_id"]: + return "failure" + return "success" + + +def can_fallback(api, plan): + if plan["route"] == "hosted": + return True + jobs = api.pages(f"/repos/{REPO}/actions/runs/{plan['run_id']}/jobs?filter=latest", "jobs") + primary = [j for j in jobs if j["name"] == "Migrated macOS lanes"] + return (len(primary) == 1 and primary[0]["conclusion"] == "failure" and not any( + s["name"] == "Run all three macOS lanes" and s["conclusion"] not in (None, "skipped") + for s in primary[0]["steps"])) + + +def hosted_result(api, plan): + runs = api.pages(f"/repos/{REPO}/actions/workflows/test.yml/runs?event=pull_request&head_sha={plan['head']}", "workflow_runs") + for run in sorted(runs, key=lambda r: r["id"], reverse=True): + if run["head_repository"]["id"] != plan["head_repository_id"]: + continue + refs = run.get("pull_requests", []) + if not any(p["number"] == plan["pr"] and p["head"]["sha"] == plan["head"] + and p["base"]["sha"] == plan["base"] for p in refs): + continue + jobs = api.pages(f"/repos/{REPO}/actions/runs/{run['id']}/jobs?filter=latest", "jobs") + wanted = {"Terminal: default-reader (macos-15)", "Terminal: reflow (macos-15)", + "macOS workspace coordination contracts"} + actual = {j["name"]: j for j in jobs if j["name"] in wanted} + if set(actual) != wanted or any(j["status"] != "completed" for j in actual.values()): + return None + return "success" if all(j["conclusion"] == "success" and "macos-15" in j["labels"] + for j in actual.values()) else "failure" + return None + + +def outputs(values): + with open(os.environ["GITHUB_OUTPUT"], "a") as out: + for key, value in values.items(): + out.write(f"{key}={json.dumps(value, separators=(',', ':')) if isinstance(value, dict) else value}\n") + + +def plan_command(api, event, app_id): + if os.environ["GITHUB_REPOSITORY"] != REPO: + raise Unavailable("wrong repository") + require_check_sources(api, app_id) + kind = "main" if os.environ["GITHUB_EVENT_NAME"] == "push" else "pr" + if kind == "pr": + number = event.get("number") or event.get("inputs", {}).get("pr_number") + plan = source(api, number) + trusted = member(api, plan["author"]) + else: + head = event["after"] + if event["ref"] != "refs/heads/main" or not SHA.fullmatch(head): + raise Unavailable("unexpected push") + plan = {"kind": "main", "pr": 0, "base": event["before"], "head": head, "merge": head} + trusted = os.environ.get("ASTRA_MACOS_MAIN_ISOLATED") == "true" + plan.update(schema=1, run_id=int(os.environ["GITHUB_RUN_ID"]), app_id=app_id, + attempt=int(os.environ["GITHUB_RUN_ATTEMPT"]), route="ordinary", group_id=0) + if not selected(api, plan): + plan["route"] = "unaffected" + save_checks(api, plan, "success", "Canonical scope does not select these lanes") + elif trusted: + plan["route"] = "hosted" + group_id = os.environ.get("ASTRA_MACOS_GROUP_ID" if kind == "pr" else "ASTRA_MACOS_MAIN_GROUP_ID", "") + if os.environ["GITHUB_EVENT_NAME"] != "workflow_dispatch" and group_id.isdecimal(): + try: + if group_ready(api, int(group_id), kind): + plan.update(route="self-hosted", group_id=int(group_id)) + except Unavailable: + pass # A member still receives protected hosted execution. + save_checks(api, plan) + elif kind == "pr": + save_checks(api, plan) + else: + plan["route"] = "disabled" # Keep main hosted until isolation is accepted. + outputs({"plan": plan, "route": plan["route"], "merge": plan["merge"], "kind": kind}) + + +def hint(api, event, app_id): + if event.get("pull_request"): + if event["pull_request"].get("base", {}).get("ref") != "main": + return False + elif event.get("ref") != "refs/heads/main": + return False + head = event.get("pull_request", {}).get("head", {}).get("sha") or event.get("after") + number = event.get("number", 0) + if not isinstance(head, str) or not SHA.fullmatch(head): + return False + for _ in range(10): + try: + rows = checks(api, head) + run_ids = {int(c["external_id"].split(":")[1]) for c in rows + if c.get("app", {}).get("id") == app_id + and c.get("external_id", "").startswith("astra-macos-v1:")} + for run_id in sorted(run_ids, reverse=True): + plan = authorization(api, head, run_id, app_id) + if plan["pr"] != number or plan["route"] not in ("self-hosted", "hosted"): + continue + validate_run(api, plan) + if number and not same_source(plan, source(api, number)): + continue + return True + except (Unavailable, ValueError, KeyError): + pass + time.sleep(2) + return False + + +def finish(api, plan): + if plan["route"] in ("disabled", "unaffected"): + return + if plan["route"] == "ordinary": + if plan["kind"] != "pr" or not same_source(plan, source(api, plan["pr"])): + raise Unavailable("hosted result is stale") + if member(api, plan["author"]): + raise Unavailable("author now qualifies for protected execution; refresh routing") + result = hosted_result(api, plan) + else: + refresh(api, plan) + fallback = verified_job(api, plan, "Fallback macOS lanes") + primary = verified_job(api, plan, "Migrated macOS lanes") if plan["route"] == "self-hosted" else None + result = primary if primary == "success" or not can_fallback(api, plan) else fallback or primary + if result is not None: + observed = {**plan, "attempt": validate_run(api, plan)["run_attempt"]} + save_checks(api, observed, "success" if result == "success" else "failure", "Verified macOS execution: " + result) + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("command", choices=("plan", "hint", "finish", "refresh", "admit", "observe")) + parser.add_argument("--plan") + parser.add_argument("--config") + args = parser.parse_args() + try: + if args.command == "admit": + signal.alarm(90) # Hooks themselves have no GitHub timeout. + config = json.loads(Path(args.config).read_text()) + api = API(Path(config["token_file"]).read_text().strip()) + request = sys.stdin.read(4097) + if len(request) > 4096: + raise Unavailable("admission request too large") + admit(api, json.loads(request), config) + print("Runner admission accepted") + return 0 + api = API(os.environ["GH_TOKEN"]) + event = json.loads(Path(os.environ["GITHUB_EVENT_PATH"]).read_text()) + app_id = int(os.environ["ASTRA_MACOS_APP_ID"]) + if args.command == "plan": + plan_command(api, event, app_id) + elif args.command == "hint": + outputs({"delegated": str(hint(api, event, app_id)).lower()}) + elif args.command == "observe": + run = api.get(f"/repos/{REPO}/actions/runs/{int(event['workflow_run']['id'])}") + if run["event"] != "pull_request" or run["path"] != ".github/workflows/test.yml": + return 0 + for pr in api.get(f"/repos/{REPO}/commits/{run['head_sha']}/pulls"): + if pr["head"]["sha"] != run["head_sha"]: + continue + current = source(api, pr["number"]) + rows = checks(api, current["head"]) + plans = [] + for row in rows: + if row.get("app", {}).get("id") == app_id and row["name"] == CONTEXTS[0]: + try: + p = json.loads(row["output"]["text"]) + if p["route"] == "ordinary" and same_source(p, current): + plans.append((row["id"], p)) + except (KeyError, TypeError, ValueError): + pass + if plans: + finish(api, max(plans, key=lambda p: p[0])[1]) + else: + plan = json.loads(args.plan) + if args.command == "finish": + finish(api, plan) + else: + refresh(api, plan) + # Failed-only reruns cannot reuse a hosted authorization job. + if int(os.environ["GITHUB_RUN_ATTEMPT"]) != plan["attempt"]: + raise Unavailable("rerun all jobs for a fresh hosted admission") + outputs({"allowed": str(can_fallback(api, plan)).lower()}) + return 0 + except (Unavailable, KeyError, ValueError, TypeError, OSError): + print("::error::Mac CI admission/result unavailable; retain hosted checks or rerun trusted control. No success published.") + return 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/ci/run-macos-lanes.sh b/scripts/ci/run-macos-lanes.sh new file mode 100755 index 0000000000..03364fb0e8 --- /dev/null +++ b/scripts/ci/run-macos-lanes.sh @@ -0,0 +1,45 @@ +#!/usr/bin/env bash +# Canonical commands shared by hosted jobs and the protected migration workflow. +set -euo pipefail +lane="${1:?lane required}" +lane_workspace="$PWD" +observe_git() { + python3 - "$lane_workspace" <<'PYTHON' +import os, subprocess, sys, time +started = time.monotonic() +subprocess.run(["/usr/bin/git", "-C", sys.argv[1], "-c", "core.fsmonitor=false", + "-c", "core.untrackedCache=false", "rev-parse", "--show-toplevel"], + env={"PATH": "/usr/bin:/bin", "HOME": "/nonexistent", "LANG": "C", "LC_ALL": "C", + "GIT_CONFIG_NOSYSTEM": "1", "GIT_CONFIG_GLOBAL": os.devnull, + "GIT_CONFIG_SYSTEM": os.devnull, "GIT_OPTIONAL_LOCKS": "0", + "GIT_TERMINAL_PROMPT": "0"}, check=True, timeout=60) +print(f"System Git observation initialized in {time.monotonic() - started:.3f}s") +PYTHON +} +case "$lane" in + default-reader) + cargo test --locked -p astra-cli --lib tui::terminal_startup -- --test-threads=2 + if [ "$(uname -s)" = Darwin ]; then + observe_git + cargo test --locked --manifest-path Cargo.toml -p astra-cli \ + --lib edge_tools::shell::tests::bash_echo_returns_output -- --exact + fi + ;; + reflow) + cargo build --locked -p astra-cli --bin astra + npm ci --ignore-scripts --prefix scripts/tui-reflow + ASTRA_TEST_BINARY="${CARGO_TARGET_DIR:-$PWD/target}/debug/astra" npm test --prefix scripts/tui-reflow + ;; + coordination) + cargo clippy --locked --manifest-path Cargo.toml -p astra-tools --all-targets -- -D warnings + observe_git + cargo test --locked --manifest-path Cargo.toml -p astra-tools \ + workspace_observation::tests:: -- --test-threads=1 + ;; + all) + bash "$0" default-reader + bash "$0" reflow + bash "$0" coordination + ;; + *) echo "Unknown macOS lane" >&2; exit 2 ;; +esac diff --git a/scripts/ci/test_ci_build_coverage.py b/scripts/ci/test_ci_build_coverage.py index 7affe0952d..f43d087270 100755 --- a/scripts/ci/test_ci_build_coverage.py +++ b/scripts/ci/test_ci_build_coverage.py @@ -169,39 +169,28 @@ def test_vendored_terminal_units_are_locked_and_pty_tests_run_both_readers(self) pty = workflow.split("\n terminal-pty:\n", 1)[1].split("\n macos-workspace-coordination:", 1)[0] self.assertIn("os: [ubuntu-latest, macos-15]", pty) self.assertIn("segment: [default-reader, dev-tty-reader, reflow]", pty) - for segment, step, features in ( - ("default-reader", "Test default terminal reader through PTYs", ""), - ("dev-tty-reader", "Test dev-tty terminal reader through PTYs", "--features crossterm/use-dev-tty "), - ): - with self.subTest(segment=segment): - commands = workflow_run_script(".github/workflows/test.yml", step).strip() - self.assertEqual(commands, - f"cargo test --locked -p astra-cli --lib {features}tui::terminal_startup -- --test-threads=2") - block = pty.split(f" - name: {step}\n", 1)[1].split(" - ", 1)[0] - self.assertIn(f"if: matrix.segment == '{segment}'", block) + self.assertIn("--features crossterm/use-dev-tty tui::terminal_startup -- --test-threads=2", pty) + self.assertIn("bash scripts/ci/run-macos-lanes.sh default-reader", pty) def test_terminal_reflow_uses_real_binary_and_locked_emulator(self): - commands = workflow_run_script(".github/workflows/test.yml", "Test inline terminal resize with xterm reflow") + commands = (ROOT / "scripts/ci/run-macos-lanes.sh").read_text() self.assertIn("cargo build --locked -p astra-cli --bin astra", commands) self.assertIn("npm ci --ignore-scripts --prefix scripts/tui-reflow", commands) - self.assertIn('ASTRA_TEST_BINARY="$PWD/target/debug/astra" npm test --prefix scripts/tui-reflow', commands) + self.assertIn('ASTRA_TEST_BINARY="${CARGO_TARGET_DIR:-$PWD/target}/debug/astra" npm test --prefix scripts/tui-reflow', commands) def test_macos_cli_bash_reuses_default_reader_without_losing_coordination_contracts(self): - workflow = (ROOT / ".github/workflows/test.yml").read_text() - terminal = workflow.split("\n terminal-pty-shards:\n", 1)[1].split("\n macos-workspace-coordination:\n", 1)[0] - self.assertEqual(workflow.count("- name: Test macOS CLI Bash execution"), 1) - bash = terminal.split(" - name: Test macOS CLI Bash execution\n", 1)[1].split(" - ", 1)[0] - self.assertIn("if: matrix.os == 'macos-15' && matrix.segment == 'default-reader'", bash) - self.assertIn("--lib edge_tools::shell::tests::bash_echo_returns_output -- --exact", bash) - self.assertLess(terminal.index("Test default terminal reader through PTYs"), terminal.index("Initialize system Git for macOS CLI observation")) - self.assertLess(terminal.index("Initialize system Git for macOS CLI observation"), terminal.index("Test macOS CLI Bash execution")) - contracts = workflow.split("\n macos-workspace-coordination-tests:\n", 1)[1].split("\n test-online-core:", 1)[0] - self.assertIn('install-nextest: "false"', contracts) - self.assertIn("-p astra-tools", contracts) - self.assertIn("--all-targets -- -D warnings", contracts) - self.assertIn("workspace_observation::tests:: -- --test-threads=1", contracts) - self.assertIn("/usr/bin/git", contracts) - self.assertNotIn("-p astra-cli", contracts, "Do not compile the same CLI test executable twice on macOS") + commands = (ROOT / "scripts/ci/run-macos-lanes.sh").read_text() + reader = commands.split(" default-reader)\n", 1)[1].split(" ;;", 1)[0] + self.assertLess(reader.index("tui::terminal_startup"), reader.index("observe_git")) + self.assertLess(reader.index("observe_git"), reader.index("bash_echo_returns_output")) + self.assertIn("--lib edge_tools::shell::tests::bash_echo_returns_output -- --exact", reader) + self.assertNotIn("--features", reader) + self.assertNotIn("cargo build", reader, "Reuse the default CLI library test executable") + contracts = commands.split(" coordination)\n", 1)[1].split(" ;;", 1)[0] + for protected in ("-p astra-tools", "--all-targets -- -D warnings", + "workspace_observation::tests:: -- --test-threads=1", "observe_git"): + self.assertIn(protected, contracts) + self.assertNotIn("-p astra-cli", contracts) def test_ci_resolves_the_actual_builder_base_not_only_the_planner(self): dockerfile = (ROOT / "Dockerfile").read_text() diff --git a/scripts/ci/test_macos_runner.py b/scripts/ci/test_macos_runner.py new file mode 100755 index 0000000000..a1d2c832e8 --- /dev/null +++ b/scripts/ci/test_macos_runner.py @@ -0,0 +1,388 @@ +#!/usr/bin/env python3 +"""Exercise routing/admission against GitHub-shaped API data and real shell dispatch.""" +import copy +import json +import os +from pathlib import Path +import subprocess +import sys +import tempfile +import unittest +from unittest.mock import patch + +import macos_runner as m + +ROOT = Path(__file__).resolve().parents[2] +B, H, M = "b" * 40, "a" * 40, "c" * 40 + + +class FakeAPI(m.API): + def __init__(self): + self.data = {} + self.status = {} + self.calls = [] + self.writes = [] + self.checks = [] + + def request(self, path, data=None, method=None): + self.calls.append((path, method)) + if data is not None: + self.writes.append(copy.deepcopy(data)) + if method == "PATCH": + row = next(c for c in self.checks if c["id"] == int(path.rsplit("/", 1)[1])) + row.update(copy.deepcopy(data)) + return 200, row + row = {**copy.deepcopy(data), "id": len(self.checks) + 1, "app": {"id": 5}} + self.checks.append(row) + return 201, row + if path.endswith("/check-runs"): + return 200, {"check_runs": copy.deepcopy(self.checks)} + return self.status.get(path, 200 if path in self.data else 404), copy.deepcopy(self.data.get(path)) + + def pages(self, path, key=None): + value = self.get(path) + return value[key] if key else value + + +class PolicyTests(unittest.TestCase): + def setUp(self): + self.api = a = FakeAPI() + self.pr_path = f"/repos/{m.REPO}/pulls/42" + a.data[self.pr_path] = {"state": "open", "number": 42, + "base": {"repo": {"full_name": m.REPO}, "ref": "main"}, + "head": {"sha": H, "repo": {"id": 2}}, "merge_commit_sha": M, + "user": {"login": "fixture-member", "id": 7}} + a.data[f"/repos/{m.REPO}/git/ref/heads/main"] = {"object": {"sha": B}} + a.data[f"/repos/{m.REPO}/git/commits/{M}"] = {"parents": [{"sha": B}, {"sha": H}]} + a.data[f"/repos/{m.REPO}/compare/{B}...{M}"] = {"files": [{"filename": "crates/core/src/lib.rs"}]} + a.status["/orgs/matrixorigin/members/fixture-member"] = 204 + self.group_path = "/orgs/matrixorigin/actions/runner-groups/9" + a.data[self.group_path] = {"name": "astra-macos-pr", "restricted_to_workflows": True, + "selected_workflows": [m.WORKFLOW_REF], "visibility": "selected", "allows_public_repositories": True} + a.data[self.group_path + "/repositories"] = {"repositories": [{"full_name": m.REPO}]} + a.data[self.group_path + "/runners"] = {"runners": [{"name": "fixture-arm", "status": "online", "busy": False, + "labels": [{"name": n} for n in ("self-hosted", "macOS", "ARM64", "mac-arm64-mini", "astra-macos-admitted")]}]} + a.data[f"/repos/{m.REPO}/rules/branches/main"] = [{"type": "required_status_checks", "parameters": { + "required_status_checks": [{"context": c, "integration_id": 5} for c in m.CONTEXTS]}}] + a.data[f"/repos/{m.REPO}/environments/astra-macos-control"] = { + "deployment_branch_policy": {"protected_branches": False, "custom_branch_policies": True}} + a.data[f"/repos/{m.REPO}/environments/astra-macos-control/deployment-branch-policies"] = { + "branch_policies": [{"name": "main", "type": "branch"}]} + self.run_path = f"/repos/{m.REPO}/actions/runs/100" + a.data[self.run_path] = {"id": 100, "repository": {"full_name": m.REPO}, + "event": "pull_request_target", "path": m.WORKFLOW, "run_attempt": 1} + self.active = {"name": "Migrated macOS lanes", "status": "in_progress", "conclusion": None, + "runner_name": "fixture-arm", "runner_group_id": 9, "steps": []} + a.data[self.run_path + "/attempts/1/jobs"] = {"jobs": [self.active]} + a.data[self.run_path + "/jobs?filter=latest"] = {"jobs": [self.active]} + self.plan = {**m.source(a, 42), "schema": 1, "app_id": 5, "run_id": 100, + "attempt": 1, "route": "self-hosted", "group_id": 9} + self.config = {"app_id": 5, "kind": "pr", "group_id": 9, "runner_name": "fixture-arm"} + self.request = {"run_id": 100, "attempt": 1, "head": H} + + def authorize(self): + m.save_checks(self.api, self.plan) + + def test_merge_requires_current_base_head_and_exact_ordered_parents(self): + for parents in ([H, B], [B], [B, H, M], [B, "d" * 40]): + with self.subTest(parents=parents): + self.api.data[f"/repos/{m.REPO}/git/commits/{M}"]["parents"] = [{"sha": p} for p in parents] + with self.assertRaises(m.Unavailable): + m.source(self.api, 42) + self.api.data[self.pr_path]["merge_commit_sha"] = None + with self.assertRaises(m.Unavailable): + m.source(self.api, 42) + + def test_wrong_repository_branch_closed_pr_and_unknown_head_fail(self): + for mutate in (lambda p: p.update(state="closed"), + lambda p: p["base"].update(ref="develop"), + lambda p: p["base"]["repo"].update(full_name="fixture/other"), + lambda p: p["head"].update(sha="invalid")): + with self.subTest(mutate=mutate): + before = copy.deepcopy(self.api.data[self.pr_path]) + mutate(self.api.data[self.pr_path]) + with self.assertRaises(m.Unavailable): + m.source(self.api, 42) + self.api.data[self.pr_path] = before + + def test_membership_redirect_and_failures_never_authorize(self): + for status in (204, 302, 401, 403, 404, 429, 500): + with self.subTest(status=status): + self.api.status["/orgs/matrixorigin/members/fixture-member"] = status + self.assertEqual(m.member(self.api, "fixture-member"), status == 204) + self.assertFalse(m.member(self.api, "../fixture")) + self.assertIsNone(m.NoRedirect().redirect_request(None, None, 302, None, None, None)) + + def test_public_or_repo_only_runner_policy_cannot_authorize(self): + for key, value in (("name", "Default"), ("restricted_to_workflows", False), + ("selected_workflows", [m.WORKFLOW_REF, "fixture/other@main"]), + ("visibility", "all"), ("allows_public_repositories", False)): + with self.subTest(key=key): + before = copy.deepcopy(self.api.data[self.group_path]) + self.api.data[self.group_path][key] = value + with self.assertRaises(m.Unavailable): + m.group_ready(self.api, 9, "pr") + self.api.data[self.group_path] = before + self.api.data[self.group_path + "/repositories"]["repositories"].append({"full_name": "fixture/other"}) + with self.assertRaises(m.Unavailable): + m.group_ready(self.api, 9, "pr") + + def test_offline_busy_or_unadmitted_runner_uses_hosted_route(self): + runner = self.api.data[self.group_path + "/runners"]["runners"][0] + for change in ({"status": "offline"}, {"busy": True}, {"labels": []}): + with self.subTest(change=change): + before = copy.deepcopy(runner) + runner.update(change) + self.assertFalse(m.group_ready(self.api, 9, "pr")) + runner.update(before) + + def test_publisher_key_environment_cannot_allow_pr_refs_wildcards_or_tags(self): + path = f"/repos/{m.REPO}/environments/astra-macos-control/deployment-branch-policies" + for policies in ([], [{"name": "*", "type": "branch"}], [{"name": "main", "type": "tag"}], + [{"name": "main", "type": "branch"}, {"name": "develop", "type": "branch"}]): + with self.subTest(policies=policies): + self.api.data[path]["branch_policies"] = policies + with self.assertRaises(m.Unavailable): + m.require_check_sources(self.api, 5) + + def test_required_contexts_must_bind_to_the_publisher_app(self): + m.require_check_sources(self.api, 5) + for source in (None, 0, 6): + with self.subTest(source=source): + self.api.data[f"/repos/{m.REPO}/rules/branches/main"][0]["parameters"]["required_status_checks"][0]["integration_id"] = source + with self.assertRaises(m.Unavailable): + m.require_check_sources(self.api, 5) + + def test_private_member_personal_fork_is_admitted_and_each_attempt_rechecks(self): + self.authorize() + self.assertEqual(m.admit(self.api, self.request, self.config), self.plan) + self.api.data[self.run_path]["run_attempt"] = 2 + self.api.data[self.run_path + "/attempts/2/jobs"] = {"jobs": [self.active]} + self.request["attempt"] = 2 + self.assertEqual(m.admit(self.api, self.request, self.config), self.plan) + self.api.status["/orgs/matrixorigin/members/fixture-member"] = 404 + with self.assertRaises(m.Unavailable): + m.admit(self.api, self.request, self.config) + self.assertGreaterEqual(sum(p.endswith("members/fixture-member") for p, _ in self.api.calls), 3) + + def test_forged_app_workflow_attempt_runner_and_event_are_rejected(self): + self.authorize() + cases = [lambda: self.api.checks[0]["app"].update(id=9), + lambda: self.api.data[self.run_path].update(path=".github/workflows/test.yml"), + lambda: self.api.data[self.run_path].update(event="pull_request"), + lambda: self.api.data[self.run_path].update(run_attempt=2), + lambda: self.active.update(runner_name="other"), + lambda: self.active.update(runner_group_id=1)] + for mutate in cases: + with self.subTest(mutate=mutate): + snapshot = copy.deepcopy((self.api.data, self.api.checks)) + mutate() + with self.assertRaises(m.Unavailable): + m.admit(self.api, self.request, self.config) + self.api.data, self.api.checks = snapshot + self.active = self.api.data[self.run_path + "/attempts/1/jobs"]["jobs"][0] + + def test_admission_cannot_cross_main_account_or_use_stale_source(self): + self.authorize() + with self.assertRaises(m.Unavailable): + m.admit(self.api, self.request, {**self.config, "kind": "main"}) + self.api.data[self.pr_path]["user"]["id"] = 8 + with self.assertRaises(m.Unavailable): + m.admit(self.api, self.request, self.config) + self.api.data[self.pr_path]["user"]["id"] = 7 + self.api.data[f"/repos/{m.REPO}/git/ref/heads/main"]["object"]["sha"] = "d" * 40 + with self.assertRaises(m.Unavailable): + m.admit(self.api, self.request, self.config) + + def test_scope_uses_canonical_classifier_and_truncation_is_full_scope(self): + path = f"/repos/{m.REPO}/compare/{B}...{M}" + self.api.data[path] = {"files": [{"filename": "docs/guide.md"}]} + self.assertFalse(m.selected(self.api, self.plan)) + self.api.data[path] = {"files": [{"filename": "docs/guide.md", "previous_filename": "crates/core/src/lib.rs"}]} + self.assertTrue(m.selected(self.api, self.plan)) + self.api.data[path] = {"files": [{"filename": "docs/a.md"}] * 300} + self.assertTrue(m.selected(self.api, self.plan)) + self.api.status[path] = 403 + self.assertTrue(m.selected(self.api, self.plan)) + + def test_ordinary_member_lookup_failure_and_resource_failure_choose_distinct_routes(self): + with tempfile.TemporaryDirectory() as directory, patch.dict(os.environ, { + "GITHUB_REPOSITORY": m.REPO, "GITHUB_EVENT_NAME": "pull_request_target", + "GITHUB_RUN_ID": "100", "GITHUB_RUN_ATTEMPT": "1", + "GITHUB_OUTPUT": str(Path(directory) / "outputs"), "ASTRA_MACOS_GROUP_ID": "9"}): + for status, busy, expected in ((204, False, "self-hosted"), (204, True, "hosted"), + (404, False, "ordinary"), (403, False, "ordinary"), (302, False, "ordinary")): + with self.subTest(status=status, busy=busy): + self.api.status["/orgs/matrixorigin/members/fixture-member"] = status + self.api.data[self.group_path + "/runners"]["runners"][0]["busy"] = busy + m.plan_command(self.api, {"number": 42}, 5) + plan = json.loads(self.api.checks[-1]["output"]["text"]) + self.assertEqual(plan["route"], expected) + + def test_real_test_failure_cannot_be_masked_by_a_successful_fallback(self): + self.authorize() + failed = {**self.active, "status": "completed", "conclusion": "failure", + "steps": [{"name": "Run all three macOS lanes", "conclusion": "failure"}]} + fallback = {**failed, "name": "Fallback macOS lanes", "conclusion": "success", + "steps": [{"name": "Run all three macOS lanes", "conclusion": "success"}]} + self.api.data[self.run_path + "/jobs?filter=latest"] = {"jobs": [failed, fallback]} + self.assertFalse(m.can_fallback(self.api, self.plan)) + m.finish(self.api, self.plan) + self.assertTrue(all(c["conclusion"] == "failure" for c in self.api.checks)) + failed["steps"] = [] # Admission failed before executing PR tests. + self.assertTrue(m.can_fallback(self.api, self.plan)) + m.finish(self.api, self.plan) + self.assertTrue(all(c["conclusion"] == "success" for c in self.api.checks)) + + def test_success_requires_actual_test_step_and_expected_runner_group(self): + self.authorize() + self.active.update(status="completed", conclusion="success") + self.assertEqual(m.verified_job(self.api, self.plan, "Migrated macOS lanes"), "failure") + self.active["steps"] = [{"name": "Run all three macOS lanes", "conclusion": "success"}] + self.assertEqual(m.verified_job(self.api, self.plan, "Migrated macOS lanes"), "success") + self.active["runner_group_id"] = 1 + self.assertEqual(m.verified_job(self.api, self.plan, "Migrated macOS lanes"), "failure") + + def test_untrusted_ordinary_success_does_not_satisfy_member_results(self): + plan = {**self.plan, "route": "ordinary"} + self.authorize() + with self.assertRaises(m.Unavailable): + m.finish(self.api, plan) + self.assertFalse(any(c.get("conclusion") == "success" for c in self.api.checks)) + + def test_delayed_publication_cannot_overwrite_newer_attempt(self): + self.authorize() + newer = {**self.plan, "attempt": 2} + m.save_checks(self.api, newer) + count = len(self.api.writes) + m.save_checks(self.api, self.plan, "success") + self.assertEqual(len(self.api.writes), count) + self.assertFalse(any(c.get("conclusion") == "success" for c in self.api.checks)) + + def test_missing_hint_keeps_hosted_execution_and_app_hint_is_not_a_result(self): + with patch.object(m.time, "sleep"): + self.assertFalse(m.hint(self.api, {"number": 42, "pull_request": {"head": {"sha": H}, "base": {"ref": "main"}}}, 5)) + self.authorize() + self.assertTrue(m.hint(self.api, {"number": 42, "pull_request": {"head": {"sha": H}, "base": {"ref": "main"}}}, 5)) + self.assertTrue(all(c["status"] == "in_progress" for c in self.api.checks)) + + def test_ordinary_hosted_results_require_matching_source_all_three_jobs_and_real_macos(self): + runs_path = f"/repos/{m.REPO}/actions/workflows/test.yml/runs?event=pull_request&head_sha={H}" + self.api.data[runs_path] = {"workflow_runs": [{"id": 200, "head_repository": {"id": 2}, + "pull_requests": [{"number": 42, "head": {"sha": H}, "base": {"sha": B}}]}]} + names = ("Terminal: default-reader (macos-15)", "Terminal: reflow (macos-15)", + "macOS workspace coordination contracts") + jobs = [{"name": name, "status": "completed", "conclusion": "success", "labels": ["macos-15"]} for name in names] + self.api.data[f"/repos/{m.REPO}/actions/runs/200/jobs?filter=latest"] = {"jobs": jobs} + self.assertEqual(m.hosted_result(self.api, self.plan), "success") + for key, value in (("status", "in_progress"), ("conclusion", "cancelled"), ("labels", ["ubuntu-latest"])): + before = copy.deepcopy(jobs[0]) + jobs[0][key] = value + self.assertNotEqual(m.hosted_result(self.api, self.plan), "success") + jobs[0] = before + jobs.pop() + self.assertIsNone(m.hosted_result(self.api, self.plan)) + self.api.data[runs_path]["workflow_runs"][0]["pull_requests"][0]["base"]["sha"] = "d" * 40 + self.assertIsNone(m.hosted_result(self.api, self.plan)) + + def test_main_stays_hosted_without_explicit_isolation_acceptance(self): + with tempfile.TemporaryDirectory() as directory, patch.dict(os.environ, { + "GITHUB_REPOSITORY": m.REPO, "GITHUB_EVENT_NAME": "push", + "GITHUB_RUN_ID": "100", "GITHUB_RUN_ATTEMPT": "1", + "GITHUB_OUTPUT": str(Path(directory) / "outputs"), "ASTRA_MACOS_MAIN_ISOLATED": "false"}): + m.plan_command(self.api, {"ref": "refs/heads/main", "before": B, "after": H}, 5) + out = dict(line.split("=", 1) for line in Path(os.environ["GITHUB_OUTPUT"]).read_text().splitlines()) + self.assertEqual(out["route"], "disabled") + self.assertEqual(self.api.writes, []) + + def test_hosted_failed_only_rerun_requires_new_admission_not_cached_router(self): + with tempfile.TemporaryDirectory() as directory: + event = Path(directory) / "event.json" + event.write_text("{}") + output = Path(directory) / "outputs" + with patch.dict(os.environ, {"GH_TOKEN": "fixture", "ASTRA_MACOS_APP_ID": "5", + "GITHUB_EVENT_PATH": str(event), "GITHUB_OUTPUT": str(output), "GITHUB_RUN_ATTEMPT": "2"}), \ + patch.object(m, "API", return_value=self.api), \ + patch.object(sys, "argv", ["macos_runner.py", "refresh", "--plan", json.dumps(self.plan)]): + self.assertEqual(m.main(), 1) + self.assertFalse(output.exists()) + + def test_api_pagination_collects_all_pages_and_errors_are_fail_closed(self): + api = m.API("fixture") + pages = [(200, {"jobs": [{"id": i} for i in range(100)]}), (200, {"jobs": [{"id": 100}]})] + with patch.object(api, "request", side_effect=pages) as request: + self.assertEqual(len(api.pages("/fixture/jobs", "jobs")), 101) + self.assertIn("page=2", request.call_args.args[0]) + with patch.object(api, "request", return_value=(403, None)): + with self.assertRaises(m.Unavailable): + api.pages("/fixture/jobs", "jobs") + with self.assertRaises(m.Unavailable): + api.request("https://fixture.invalid/token") + + + +class LaneShellTests(unittest.TestCase): + def test_canonical_commands_and_all_failures_propagate(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + for name in ("cargo", "npm"): + path = root / name + path.write_text(f"#!{sys.executable}\n" + ''' +import json, os, sys +with open(os.environ["CALLS"], "a") as out: out.write(json.dumps([os.path.basename(sys.argv[0]), *sys.argv[1:]]) + "\\n") +failed = os.environ.get("FAIL") and os.environ["FAIL"] in " ".join(sys.argv[1:]) +if not failed and os.path.basename(sys.argv[0]) == "cargo" and sys.argv[1] == "build": + from pathlib import Path + binary = Path(os.environ["CARGO_TARGET_DIR"]) / "debug/astra" + binary.parent.mkdir(parents=True, exist_ok=True) + binary.write_text("fixture binary") +if not failed and os.path.basename(sys.argv[0]) == "npm" and sys.argv[1] == "test": + failed = not os.path.isfile(os.environ["ASTRA_TEST_BINARY"]) +sys.exit(17 if failed else 0) +''') + path.chmod(0o755) + # Keep the real system Git observation, with a harmless synthetic repo. + subprocess.run(["git", "init", str(root)], check=True, capture_output=True) + for lane, failures in (("default-reader", ("", "tui::terminal_startup")), + ("reflow", ("", "build", "ci", "test")), + ("coordination", ("", "clippy", "workspace_observation::tests::"))): + for fail in failures: + with self.subTest(lane=lane, fail=fail): + calls = root / "calls" + calls.unlink(missing_ok=True) + result = subprocess.run(["bash", str(ROOT / "scripts/ci/run-macos-lanes.sh"), lane], + cwd=root, env={**os.environ, "PATH": f"{root}:{os.environ['PATH']}", + "CALLS": str(calls), "FAIL": fail, "GITHUB_WORKSPACE": str(root), + "CARGO_TARGET_DIR": str(root / "scratch-target")}, + capture_output=True, text=True) + self.assertEqual(result.returncode, 17 if fail else 0, result.stderr) + commands = [json.loads(line) for line in calls.read_text().splitlines()] + if not fail and lane == "coordination": + self.assertEqual(len(commands), 2) + self.assertIn("--test-threads=1", commands[-1]) + if not fail and lane == "reflow": + self.assertEqual([c[:2] for c in commands], [["cargo", "build"], ["npm", "ci"], ["npm", "test"]]) + + def test_credential_owner_and_cache_boundary_are_outside_pr_execution(self): + workflow = (ROOT / ".github/workflows/macos-trusted.yml").read_text() + for job, end in (("mini", "hosted-admission"), ("hosted", "publish")): + execution = workflow.split(f"\n {job}:\n", 1)[1].split(f"\n {end}:\n", 1)[0] + self.assertNotIn("secrets.", execution) + self.assertNotIn("create-github-app-token", execution) + self.assertNotIn("GH_TOKEN", execution) + self.assertNotIn("environment:", execution) + self.assertIn("cache-mode: read", workflow) + self.assertNotIn("cache-mode: write", workflow) + self.assertNotIn("workflow_call:", workflow) + observer = (ROOT / ".github/workflows/macos-results.yml").read_text() + self.assertNotIn("download-artifact", observer) + self.assertNotIn("_mac_source", observer) + self.assertIn("cache-mode: none", observer) + hook = (ROOT / "scripts/ci/macos-runner-admit").read_text() + self.assertIn('[ "$#" -eq 0 ]', hook) + self.assertIn("SUDO_USER", hook) + self.assertNotIn("eval", hook) + + +if __name__ == "__main__": + unittest.main() From 8ab8874eebf0cbd45a67a00bad6b45126a51008f Mon Sep 17 00:00:00 2001 From: lr90 Date: Fri, 23 Oct 2026 22:19:28 +0800 Subject: [PATCH 02/11] fix(ci): bound macOS routing and reconcile missing results --- .github/workflows/macos-results.yml | 12 ++- .github/workflows/macos-trusted.yml | 2 +- .github/workflows/test.yml | 20 +++- docs/guides/macos-ci-runner.md | 54 ++++++++--- scripts/ci/macos_runner.py | 98 +++++++++++++++++-- scripts/ci/test_macos_runner.py | 141 +++++++++++++++++++++++++++- 6 files changed, 297 insertions(+), 30 deletions(-) diff --git a/.github/workflows/macos-results.yml b/.github/workflows/macos-results.yml index 8d0d995a36..7980e1563a 100644 --- a/.github/workflows/macos-results.yml +++ b/.github/workflows/macos-results.yml @@ -1,8 +1,11 @@ -name: Mac CI hosted result observer +name: Mac CI result observer on: workflow_run: workflows: [Test Suite] types: [completed] + schedule: + - cron: '*/15 * * * *' + workflow_dispatch: permissions: contents: read cache-mode: none @@ -34,7 +37,14 @@ jobs: permission-environments: read # No PR checkout, downloaded artifacts, code execution or cached scripts. - name: Observe ordinary hosted results + if: github.event_name == 'workflow_run' env: GH_TOKEN: ${{ steps.app.outputs.token }} ASTRA_MACOS_APP_ID: ${{ vars.ASTRA_MACOS_APP_ID }} run: python3 scripts/ci/macos_runner.py observe + - name: Reconcile missing and expired results + if: github.event_name != 'workflow_run' + env: + GH_TOKEN: ${{ steps.app.outputs.token }} + ASTRA_MACOS_APP_ID: ${{ vars.ASTRA_MACOS_APP_ID }} + run: python3 scripts/ci/macos_runner.py reconcile diff --git a/.github/workflows/macos-trusted.yml b/.github/workflows/macos-trusted.yml index 77077c239b..2df6bf56d4 100644 --- a/.github/workflows/macos-trusted.yml +++ b/.github/workflows/macos-trusted.yml @@ -9,7 +9,7 @@ on: workflow_dispatch: inputs: pr_number: - description: Member PR to re-authorize and execute on hosted macOS + description: PR to re-authorize with hosted routing (non-members keep ordinary PR approval) required: true type: string diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 34af12c60c..6ef3db5834 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -2,8 +2,6 @@ name: Test Suite permissions: contents: read - checks: read - pull-requests: read on: push: @@ -22,6 +20,11 @@ env: jobs: scope: name: Determine test scope + permissions: + contents: read + checks: read + pull-requests: read + actions: read runs-on: ubuntu-latest timeout-minutes: 5 outputs: @@ -44,7 +47,10 @@ jobs: run: python3 scripts/ci/ci_scope.py - name: Read trusted macOS execution hint id: mac - if: vars.ASTRA_MACOS_MIGRATION_ENABLED == 'true' + if: >- + vars.ASTRA_MACOS_MIGRATION_ENABLED == 'true' && + (steps.scope.outputs.test_cli == 'true' || steps.scope.outputs.test_core == 'true') + timeout-minutes: 2 continue-on-error: true env: GH_TOKEN: ${{ github.token }} @@ -264,7 +270,9 @@ jobs: steps: - name: Delegated to App-verified macOS execution if: env.RUN_LANE != 'true' - run: echo 'Protected Mac CI checks are independently required.' + run: | + echo 'Requested macOS lane delegated to protected Mac CI; this Ubuntu job only reports delegation.' + echo 'Execution results are the three App-owned Mac CI checks. The platform in this legacy check name is the requested platform.' >> "$GITHUB_STEP_SUMMARY" - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 if: env.RUN_LANE == 'true' with: @@ -323,7 +331,9 @@ jobs: steps: - name: Delegated to App-verified macOS execution if: env.RUN_LANE != 'true' - run: echo 'Protected Mac CI checks are independently required.' + run: | + echo 'Requested macOS lane delegated to protected Mac CI; this Ubuntu job only reports delegation.' + echo 'Execution results are the three App-owned Mac CI checks. The platform in this legacy check name is the requested platform.' >> "$GITHUB_STEP_SUMMARY" - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 if: env.RUN_LANE == 'true' with: diff --git a/docs/guides/macos-ci-runner.md b/docs/guides/macos-ci-runner.md index 52bc0b719d..e96e6888ab 100644 --- a/docs/guides/macos-ci-runner.md +++ b/docs/guides/macos-ci-runner.md @@ -48,8 +48,11 @@ A dedicated GitHub App publishes three contexts on H: - `Mac CI: reflow` - `Mac CI: coordination` -All three must be required with their source bound to that App. Control checks -this source binding before dispatch. Its private key and installation token +All three must be required with their source bound to that App, together with +strict up-to-date branch checks. Source binding prevents impersonation; the +up-to-date rule prevents a historical check on H from satisfying a different +base tuple after main advances. Control verifies both requirements before +dispatch. Its private key and installation token remain on GitHub-hosted Ubuntu control jobs, never on PR execution machines. Store the private key only in the `astra-macos-control` GitHub Environment, restricted by a custom branch rule to branch `main` alone (no tag rule). @@ -65,13 +68,32 @@ best-effort deduplication hint. Delegated copies use cheap Ubuntu notification jobs. The hint cannot publish or replace the three required App checks. Missing hints preserve hosted execution, so a control race can cause duplicate work, not missing coverage. A forged hint leaves the member's trusted checks pending. +Only the scope job receives the extra Actions/checks/PR read permissions. The +hint waits up to a 90-second polling budget for delayed control, bounded by a +two-minute step timeout; a known ordinary route or permission denial returns +immediately. Separate event queues can still outlast that budget. Such runs keep +hosted execution and can duplicate work; this is not claimed eliminated. The +activation pilot must demonstrate `delegated=true` with an actual fork PR's +`GITHUB_TOKEN`, not only a fixture or a personal token. Non-members retain ordinary fork approval. `macos-results.yml` observes the completed Test Suite through `workflow_run`, reads GitHub metadata only, and publishes the ordinary path's actual three hosted results. It never downloads artifacts or executes PR code. Approval-pending, missing, stale, cancelled or -failed execution cannot become a successful skip. This retains the pre-existing -PR-modifiable test-definition limitation for non-member hosted tests. +failed execution cannot become a successful skip. A completed ordinary run with +missing/skipped selected lanes fails explicitly, including scope disagreement, +instead of staying pending forever. This retains the pre-existing PR-modifiable +test-definition limitation for non-member hosted tests. + +The same metadata-only observer reconciles pending contexts every 15 minutes +(and supports manual dispatch), recovering lost completion notifications. +Authorization expires three hours after planning. At expiry, pending contexts +are failed with a recovery instruction; expiry never means tests passed. Active +API/Actions outages or delayed cron execution can delay reconciliation, so this +is not a platform availability guarantee. Expired authorization is rejected by +the runner hook and result publisher. After resolving approval/unavailability, +start fresh trusted control and rerun the applicable tests as required. The +reconciler never executes PR code, auto-approves forks or relaxes result checks. Both privileged-event workflows explicitly restrict remote cache tokens with `cache-mode: read` or `none`. Disabling a client upload is not the authorization @@ -213,8 +235,8 @@ infrastructure fallback. The App needs repository Actions/contents/PR read, administration read, checks write and environments read; organization Members and self-hosted runners/groups read. -4. Make the three contexts required with source pinned to the App. Preserve the - existing protected checks. The migration control refuses dispatch until this +4. Make the three contexts required with source pinned to the App, and enforce + strict up-to-date branch checks. Preserve the existing protected checks. The migration control refuses dispatch until this binding is visible through the rules/protection APIs. 5. Set `ASTRA_MACOS_MIGRATION_ENABLED=true` only after the preceding acceptance. For main, separately set `ASTRA_MACOS_MAIN_GROUP_ID` and @@ -227,8 +249,18 @@ The application PR does not modify live groups, secrets, branch protection, remote clusters or runner services. Current caller API queries for runners and groups returned 403; none of those settings is claimed verified or enabled. -For rollback, keep required result publication working until in-flight PRs are -settled. Remove the three new App-required contexts through reviewed policy -change, disable `ASTRA_MACOS_MIGRATION_ENABLED`, then remove/disable the admitted -runner registration as needed. Ordinary hosted execution is retained throughout -when no valid delegation hint exists. Do not turn missing App results into pass. +For rollback, pause merges and cancel in-flight Mac CI work/disable admitted +registrations through the reviewed operations process. Set +`ASTRA_MACOS_MIGRATION_ENABLED=false` first. Rerun ordinary Test Suite for +in-flight PRs and verify actual hosted macOS execution of the three lanes; an +older delegated echo is not proof. Only then remove the three App-required +contexts through reviewed protection configuration, and resume merges. Keep the +App contexts blocking during this transition. Disabling the flag does not revoke +already-issued runner authorization by itself. Never remove the App-required +contexts while delegated legacy checks alone could permit merging. + +Legacy job names retain the requested platform for protected-check compatibility. +A delegated job's summary explicitly identifies its Ubuntu notification role; +actual execution is attested by the three App-owned contexts and their run/job +metadata. Main push concurrency intentionally keeps only the newest generation, +as ordinary Test Suite already does; cancelled results do not become passes. diff --git a/scripts/ci/macos_runner.py b/scripts/ci/macos_runner.py index 1638905f04..511b715711 100755 --- a/scripts/ci/macos_runner.py +++ b/scripts/ci/macos_runner.py @@ -25,12 +25,18 @@ WORKFLOW_REF = f"{REPO}/{WORKFLOW}@refs/heads/main" CONTEXTS = ("Mac CI: default reader", "Mac CI: reflow", "Mac CI: coordination") SHA = re.compile(r"^[0-9a-f]{40}$") +AUTHORIZATION_SECONDS = 3 * 60 * 60 +HINT_WAIT_SECONDS = 90 class Unavailable(Exception): pass +class PermissionDenied(Unavailable): + pass + + class NoRedirect(urllib.request.HTTPRedirectHandler): def redirect_request(self, *args, **kwargs): return None @@ -61,6 +67,8 @@ def request(self, path, data=None, method=None): def get(self, path): status, value = self.request(path) + if status in (401, 403): + raise PermissionDenied("API permission unavailable") if status != 200: raise Unavailable(f"GitHub API returned {status}") return value @@ -130,16 +138,22 @@ def require_check_sources(api, app_id): if len(branches) != 1 or branches[0].get("name") != "main" or branches[0].get("type") != "branch": raise Unavailable("control environment allows an untrusted branch or tag") required = {} + up_to_date = False for rule in api.get(f"/repos/{REPO}/rules/branches/main"): if rule["type"] == "required_status_checks": + up_to_date |= rule["parameters"].get("strict_required_status_checks_policy") is True for check in rule["parameters"]["required_status_checks"]: required[check["context"]] = check.get("integration_id") status, protection = api.request(f"/repos/{REPO}/branches/main/protection") if status == 200: - for check in (protection.get("required_status_checks") or {}).get("checks", []): + protected_checks = protection.get("required_status_checks") or {} + up_to_date |= protected_checks.get("strict") is True + for check in protected_checks.get("checks", []): required[check["context"]] = check.get("app_id") elif status != 404: raise Unavailable("cannot verify required-check source binding") + if not up_to_date: + raise Unavailable("main must require checks on an up-to-date branch") if any(required.get(name) != app_id for name in CONTEXTS): raise Unavailable("three Mac CI checks must be required from the dedicated App") @@ -163,7 +177,7 @@ def checks(api, head): return api.pages(f"/repos/{REPO}/commits/{head}/check-runs", "check_runs") -def save_checks(api, plan, conclusion=None, summary="Awaiting verified execution"): +def save_checks(api, plan, conclusion=None, summary="Awaiting verified execution", *, only_pending=False): existing = {name: [] for name in CONTEXTS} for check in checks(api, plan["head"]): if check["name"] in existing and check.get("app", {}).get("id") == plan["app_id"]: @@ -177,6 +191,8 @@ def save_checks(api, plan, conclusion=None, summary="Awaiting verified execution data["conclusion"] = conclusion previous = max(existing[name], key=lambda c: c["id"], default=None) if previous: + if only_pending and previous["status"] == "completed": + continue try: old = json.loads(previous["output"]["text"]) if (old["run_id"], old["attempt"]) > (plan["run_id"], plan["attempt"]): @@ -227,7 +243,18 @@ def validate_run(api, plan, attempt=None): return run +def expired(plan): + deadline = plan.get("expires_at") + return not isinstance(deadline, int) or time.time() >= deadline + + +def require_unexpired(plan): + if expired(plan): + raise Unavailable("authorization expired; start fresh trusted control") + + def refresh(api, plan): + require_unexpired(plan) if plan["kind"] == "pr": current = source(api, plan["pr"]) if not same_source(current, plan) or not member(api, current["author"]): @@ -300,7 +327,8 @@ def hosted_result(api, plan): "macOS workspace coordination contracts"} actual = {j["name"]: j for j in jobs if j["name"] in wanted} if set(actual) != wanted or any(j["status"] != "completed" for j in actual.values()): - return None + # A completed run cannot later gain skipped/missing lane execution. + return "failure" if run["status"] == "completed" else None return "success" if all(j["conclusion"] == "success" and "macos-15" in j["labels"] for j in actual.values()) else "failure" return None @@ -328,7 +356,8 @@ def plan_command(api, event, app_id): plan = {"kind": "main", "pr": 0, "base": event["before"], "head": head, "merge": head} trusted = os.environ.get("ASTRA_MACOS_MAIN_ISOLATED") == "true" plan.update(schema=1, run_id=int(os.environ["GITHUB_RUN_ID"]), app_id=app_id, - attempt=int(os.environ["GITHUB_RUN_ATTEMPT"]), route="ordinary", group_id=0) + attempt=int(os.environ["GITHUB_RUN_ATTEMPT"]), route="ordinary", group_id=0, + expires_at=int(time.time()) + AUTHORIZATION_SECONDS) if not selected(api, plan): plan["route"] = "unaffected" save_checks(api, plan, "success", "Canonical scope does not select these lanes") @@ -359,7 +388,8 @@ def hint(api, event, app_id): number = event.get("number", 0) if not isinstance(head, str) or not SHA.fullmatch(head): return False - for _ in range(10): + deadline = time.monotonic() + HINT_WAIT_SECONDS + for _ in range(HINT_WAIT_SECONDS // 2 + 1): try: rows = checks(api, head) run_ids = {int(c["external_id"].split(":")[1]) for c in rows @@ -367,21 +397,35 @@ def hint(api, event, app_id): and c.get("external_id", "").startswith("astra-macos-v1:")} for run_id in sorted(run_ids, reverse=True): plan = authorization(api, head, run_id, app_id) - if plan["pr"] != number or plan["route"] not in ("self-hosted", "hosted"): + if plan["pr"] != number: continue + require_unexpired(plan) validate_run(api, plan) if number and not same_source(plan, source(api, number)): continue - return True + # Once trusted control has chosen ordinary hosted execution, + # there is no reason to keep delaying a non-member's scope job. + if plan["route"] in ("ordinary", "unaffected", "disabled"): + return False + if plan["route"] in ("self-hosted", "hosted"): + return True + except PermissionDenied: + print("::notice::Mac CI hint lacks API access; retaining hosted execution.") + return False except (Unavailable, ValueError, KeyError): pass - time.sleep(2) + remaining = deadline - time.monotonic() + if remaining <= 0: + break + time.sleep(min(2, remaining)) + print("::notice::Mac CI routing hint unavailable within budget; retaining hosted execution.") return False def finish(api, plan): if plan["route"] in ("disabled", "unaffected"): - return + return True + require_unexpired(plan) if plan["route"] == "ordinary": if plan["kind"] != "pr" or not same_source(plan, source(api, plan["pr"])): raise Unavailable("hosted result is stale") @@ -396,11 +440,43 @@ def finish(api, plan): if result is not None: observed = {**plan, "attempt": validate_run(api, plan)["run_attempt"]} save_checks(api, observed, "success" if result == "success" else "failure", "Verified macOS execution: " + result) + return True + return False + + +def reconcile(api, app_id): + """Repair lost completion notifications and fail expired pending checks.""" + require_check_sources(api, app_id) + prs = api.pages(f"/repos/{REPO}/pulls?state=open&base=main") + for pr in prs: + try: + rows = checks(api, pr["head"]["sha"]) + latest = {} + for row in rows: + if row["name"] in CONTEXTS and row.get("app", {}).get("id") == app_id: + if row["name"] not in latest or row["id"] > latest[row["name"]]["id"]: + latest[row["name"]] = row + if not latest or (len(latest) == len(CONTEXTS) and + all(c["status"] == "completed" for c in latest.values())): + continue + plan = json.loads(max(latest.values(), key=lambda c: c["id"])["output"]["text"]) + if (plan["schema"] != 1 or plan["app_id"] != app_id or plan["kind"] != "pr" + or plan["head"] != pr["head"]["sha"] or plan["pr"] != pr["number"]): + continue + if expired(plan): + save_checks(api, plan, "failure", + "Mac CI authorization expired; resolve approval/runner state and start fresh Mac CI control", + only_pending=True) + else: + finish(api, plan) + except (Unavailable, KeyError, TypeError, ValueError): + # Other PRs still get reconciliation during a transient per-PR failure. + print("::notice::A pending Mac CI result could not be reconciled; no success published.") def main(): parser = argparse.ArgumentParser() - parser.add_argument("command", choices=("plan", "hint", "finish", "refresh", "admit", "observe")) + parser.add_argument("command", choices=("plan", "hint", "finish", "refresh", "admit", "observe", "reconcile")) parser.add_argument("--plan") parser.add_argument("--config") args = parser.parse_args() @@ -422,6 +498,8 @@ def main(): plan_command(api, event, app_id) elif args.command == "hint": outputs({"delegated": str(hint(api, event, app_id)).lower()}) + elif args.command == "reconcile": + reconcile(api, app_id) elif args.command == "observe": run = api.get(f"/repos/{REPO}/actions/runs/{int(event['workflow_run']['id'])}") if run["event"] != "pull_request" or run["path"] != ".github/workflows/test.yml": diff --git a/scripts/ci/test_macos_runner.py b/scripts/ci/test_macos_runner.py index a1d2c832e8..1a89ecc3c4 100755 --- a/scripts/ci/test_macos_runner.py +++ b/scripts/ci/test_macos_runner.py @@ -7,6 +7,7 @@ import subprocess import sys import tempfile +import time import unittest from unittest.mock import patch @@ -63,6 +64,7 @@ def setUp(self): a.data[self.group_path + "/runners"] = {"runners": [{"name": "fixture-arm", "status": "online", "busy": False, "labels": [{"name": n} for n in ("self-hosted", "macOS", "ARM64", "mac-arm64-mini", "astra-macos-admitted")]}]} a.data[f"/repos/{m.REPO}/rules/branches/main"] = [{"type": "required_status_checks", "parameters": { + "strict_required_status_checks_policy": True, "required_status_checks": [{"context": c, "integration_id": 5} for c in m.CONTEXTS]}}] a.data[f"/repos/{m.REPO}/environments/astra-macos-control"] = { "deployment_branch_policy": {"protected_branches": False, "custom_branch_policies": True}} @@ -76,7 +78,8 @@ def setUp(self): a.data[self.run_path + "/attempts/1/jobs"] = {"jobs": [self.active]} a.data[self.run_path + "/jobs?filter=latest"] = {"jobs": [self.active]} self.plan = {**m.source(a, 42), "schema": 1, "app_id": 5, "run_id": 100, - "attempt": 1, "route": "self-hosted", "group_id": 9} + "attempt": 1, "route": "self-hosted", "group_id": 9, + "expires_at": int(time.time()) + m.AUTHORIZATION_SECONDS} self.config = {"app_id": 5, "kind": "pr", "group_id": 9, "runner_name": "fixture-arm"} self.request = {"run_id": 100, "attempt": 1, "head": H} @@ -153,6 +156,11 @@ def test_required_contexts_must_bind_to_the_publisher_app(self): with self.assertRaises(m.Unavailable): m.require_check_sources(self.api, 5) + def test_up_to_date_branch_enforcement_is_required_for_tuple_bound_checks(self): + self.api.data[f"/repos/{m.REPO}/rules/branches/main"][0]["parameters"]["strict_required_status_checks_policy"] = False + with self.assertRaises(m.Unavailable): + m.require_check_sources(self.api, 5) + def test_private_member_personal_fork_is_admitted_and_each_attempt_rechecks(self): self.authorize() self.assertEqual(m.admit(self.api, self.request, self.config), self.plan) @@ -268,7 +276,7 @@ def test_missing_hint_keeps_hosted_execution_and_app_hint_is_not_a_result(self): def test_ordinary_hosted_results_require_matching_source_all_three_jobs_and_real_macos(self): runs_path = f"/repos/{m.REPO}/actions/workflows/test.yml/runs?event=pull_request&head_sha={H}" - self.api.data[runs_path] = {"workflow_runs": [{"id": 200, "head_repository": {"id": 2}, + self.api.data[runs_path] = {"workflow_runs": [{"id": 200, "status": "completed", "head_repository": {"id": 2}, "pull_requests": [{"number": 42, "head": {"sha": H}, "base": {"sha": B}}]}]} names = ("Terminal: default-reader (macos-15)", "Terminal: reflow (macos-15)", "macOS workspace coordination contracts") @@ -281,6 +289,8 @@ def test_ordinary_hosted_results_require_matching_source_all_three_jobs_and_real self.assertNotEqual(m.hosted_result(self.api, self.plan), "success") jobs[0] = before jobs.pop() + self.assertEqual(m.hosted_result(self.api, self.plan), "failure") + self.api.data[runs_path]["workflow_runs"][0]["status"] = "in_progress" self.assertIsNone(m.hosted_result(self.api, self.plan)) self.api.data[runs_path]["workflow_runs"][0]["pull_requests"][0]["base"]["sha"] = "d" * 40 self.assertIsNone(m.hosted_result(self.api, self.plan)) @@ -319,6 +329,119 @@ def test_api_pagination_collects_all_pages_and_errors_are_fail_closed(self): with self.assertRaises(m.Unavailable): api.request("https://fixture.invalid/token") + def test_ordinary_hint_returns_without_sleep_and_permission_denial_is_immediate(self): + event = {"number": 42, "pull_request": {"head": {"sha": H}, "base": {"ref": "main"}}} + m.save_checks(self.api, {**self.plan, "route": "ordinary"}) + with patch.object(m.time, "sleep") as sleep: + self.assertFalse(m.hint(self.api, event, 5)) + sleep.assert_not_called() + self.api.checks.clear() + self.authorize() + self.api.status[self.run_path] = 403 + with patch.object(m.time, "sleep") as sleep: + self.assertFalse(m.hint(self.api, event, 5)) + sleep.assert_not_called() + + def test_hint_waits_for_delayed_plan_and_exhausted_budget_keeps_hosted(self): + event = {"number": 42, "pull_request": {"head": {"sha": H}, "base": {"ref": "main"}}} + self.authorize() + records = copy.deepcopy(self.api.checks) + self.api.checks.clear() + def publish_after_first_sleep(_seconds): + self.api.checks = records + with patch.object(m.time, "sleep", side_effect=publish_after_first_sleep) as sleep: + self.assertTrue(m.hint(self.api, event, 5)) + self.assertEqual(sleep.call_count, 1) + self.api.checks.clear() + with patch.object(m.time, "sleep") as sleep, patch.object(m.time, "monotonic", side_effect=[0, 91]): + self.assertFalse(m.hint(self.api, event, 5)) + sleep.assert_not_called() + + def test_expired_authorization_rejects_actual_rerun_and_publication(self): + self.plan["expires_at"] = 1 + self.authorize() + with self.assertRaises(m.Unavailable): + m.admit(self.api, self.request, self.config) + with self.assertRaises(m.Unavailable): + m.finish(self.api, self.plan) + self.assertTrue(all(c["status"] == "in_progress" for c in self.api.checks)) + + def prepare_ordinary_reconciliation(self): + self.api.status["/orgs/matrixorigin/members/fixture-member"] = 404 + self.plan["route"] = "ordinary" + self.authorize() + self.api.data[f"/repos/{m.REPO}/pulls?state=open&base=main"] = [ + {"number": 42, "head": {"sha": H}}] + self.api.data[f"/repos/{m.REPO}/actions/workflows/test.yml/runs?event=pull_request&head_sha={H}"] = { + "workflow_runs": [{"id": 200, "status": "completed", "head_repository": {"id": 2}, + "pull_requests": [{"number": 42, "head": {"sha": H}, "base": {"sha": B}}]}]} + self.jobs200 = f"/repos/{m.REPO}/actions/runs/200/jobs?filter=latest" + self.api.data[self.jobs200] = {"jobs": [ + {"name": name, "status": "completed", "conclusion": "success", "labels": ["macos-15"]} + for name in ("Terminal: default-reader (macos-15)", "Terminal: reflow (macos-15)", + "macOS workspace coordination contracts")]} + + def test_reconciliation_recovers_a_lost_completion_notification(self): + self.prepare_ordinary_reconciliation() + m.reconcile(self.api, 5) + self.assertTrue(all(c["status"] == "completed" and c["conclusion"] == "success" for c in self.api.checks)) + writes = len(self.api.writes) + m.reconcile(self.api, 5) + self.assertEqual(len(self.api.writes), writes) + + def test_completed_run_with_missing_or_skipped_lanes_fails_instead_of_pending(self): + for mode in ("missing", "skipped"): + with self.subTest(mode=mode): + self.api.checks.clear() + self.prepare_ordinary_reconciliation() + if mode == "missing": + self.api.data[self.jobs200]["jobs"].pop() + else: + self.api.data[self.jobs200]["jobs"][0]["conclusion"] = "skipped" + m.reconcile(self.api, 5) + self.assertTrue(all(c["status"] == "completed" and c["conclusion"] == "failure" for c in self.api.checks)) + + def test_approval_pending_expires_and_does_not_become_a_successful_skip(self): + self.prepare_ordinary_reconciliation() + self.api.data[f"/repos/{m.REPO}/actions/workflows/test.yml/runs?event=pull_request&head_sha={H}"]["workflow_runs"] = [] + m.reconcile(self.api, 5) + self.assertTrue(all(c["status"] == "in_progress" for c in self.api.checks)) + with patch.object(m.time, "time", return_value=self.plan["expires_at"] + 1): + m.reconcile(self.api, 5) + self.assertTrue(all(c["status"] == "completed" and c["conclusion"] == "failure" for c in self.api.checks)) + self.assertTrue(all("expired" in c["output"]["summary"] for c in self.api.checks)) + + def test_legacy_authorization_without_deadline_fails_closed_instead_of_staying_pending(self): + self.prepare_ordinary_reconciliation() + del self.plan["expires_at"] + m.save_checks(self.api, self.plan) + m.reconcile(self.api, 5) + self.assertTrue(all(c["conclusion"] == "failure" for c in self.api.checks)) + with self.assertRaises(m.Unavailable): + m.require_unexpired(self.plan) + + def test_one_unavailable_pr_does_not_starve_reconciliation_of_other_prs(self): + self.prepare_ordinary_reconciliation() + self.api.data[f"/repos/{m.REPO}/pulls?state=open&base=main"].insert(0, + {"number": 43, "head": {"sha": "d" * 40}}) + pages = self.api.pages + def flaky_pages(path, key=None): + if path == f"/repos/{m.REPO}/commits/{'d' * 40}/check-runs": + raise m.Unavailable("fixture unavailable") + return pages(path, key) + with patch.object(self.api, "pages", side_effect=flaky_pages): + m.reconcile(self.api, 5) + self.assertTrue(all(c["conclusion"] == "success" for c in self.api.checks)) + + def test_expiry_reconciliation_does_not_downgrade_already_successful_results(self): + self.prepare_ordinary_reconciliation() + m.save_checks(self.api, self.plan, "success") + with patch.object(m.time, "time", return_value=self.plan["expires_at"] + 1): + m.reconcile(self.api, 5) + m.save_checks(self.api, self.plan, "failure", only_pending=True) + self.assertTrue(all(c["conclusion"] == "success" for c in self.api.checks)) + + class LaneShellTests(unittest.TestCase): @@ -363,6 +486,20 @@ def test_canonical_commands_and_all_failures_propagate(self): if not fail and lane == "reflow": self.assertEqual([c[:2] for c in commands], [["cargo", "build"], ["npm", "ci"], ["npm", "test"]]) + def test_hint_permissions_are_scoped_and_reconciliation_stays_metadata_only(self): + workflow = (ROOT / ".github/workflows/test.yml").read_text() + top = workflow.split("\non:", 1)[0] + self.assertNotIn("actions: read", top) + self.assertNotIn("checks: read", top) + scope = workflow.split("\n scope:\n", 1)[1].split("\n cli-test-build:\n", 1)[0] + for permission in ("actions: read", "checks: read", "pull-requests: read", "contents: read"): + self.assertIn(permission, scope) + observer = (ROOT / ".github/workflows/macos-results.yml").read_text() + self.assertIn("cron: '*/15 * * * *'", observer) + self.assertIn("macos_runner.py reconcile", observer) + self.assertNotIn("permission-actions: write", observer) + self.assertNotIn("_mac_source", observer) + def test_credential_owner_and_cache_boundary_are_outside_pr_execution(self): workflow = (ROOT / ".github/workflows/macos-trusted.yml").read_text() for job, end in (("mini", "hosted-admission"), ("hosted", "publish")): From 3485fa4e5f19d2851c253ab0487e74f9c97e5633 Mon Sep 17 00:00:00 2001 From: lr90 Date: Fri, 23 Oct 2026 23:47:16 +0800 Subject: [PATCH 03/11] fix(ci): recover missing Mac checks and parallelize hosted fallback --- .github/workflows/macos-results.yml | 6 +- .github/workflows/macos-trusted.yml | 36 +++++- .github/workflows/test.yml | 38 ++++-- docs/guides/macos-ci-runner.md | 71 ++++++++--- scripts/ci/macos_runner.py | 176 ++++++++++++++++++++++------ scripts/ci/test_ci_parallelism.py | 7 +- scripts/ci/test_macos_runner.py | 163 ++++++++++++++++++++++++-- 7 files changed, 420 insertions(+), 77 deletions(-) diff --git a/.github/workflows/macos-results.yml b/.github/workflows/macos-results.yml index 7980e1563a..4e1316b265 100644 --- a/.github/workflows/macos-results.yml +++ b/.github/workflows/macos-results.yml @@ -9,6 +9,9 @@ on: permissions: contents: read cache-mode: none +concurrency: + group: mac-ci-result-controller + cancel-in-progress: false jobs: observe: if: github.repository == 'matrixorigin/Astra' && vars.ASTRA_MACOS_MIGRATION_ENABLED == 'true' @@ -30,7 +33,8 @@ jobs: permission-members: read permission-organization-self-hosted-runners: read permission-administration: read - permission-actions: read + # Only metadata control can dispatch the fixed main workflow. + permission-actions: write permission-contents: read permission-pull-requests: read permission-checks: write diff --git a/.github/workflows/macos-trusted.yml b/.github/workflows/macos-trusted.yml index 2df6bf56d4..d0d35790c5 100644 --- a/.github/workflows/macos-trusted.yml +++ b/.github/workflows/macos-trusted.yml @@ -149,13 +149,17 @@ jobs: run: python3 scripts/ci/macos_runner.py refresh --plan "$PLAN" hosted: - name: Fallback macOS lanes + name: "Fallback macOS: ${{ matrix.lane }}" needs: [plan, hosted-admission] if: >- !cancelled() && needs.hosted-admission.result == 'success' && needs.hosted-admission.outputs.allowed == 'true' runs-on: macos-15 - timeout-minutes: 90 + timeout-minutes: 45 + strategy: + fail-fast: false + matrix: + lane: [default-reader, reflow, coordination] permissions: contents: read steps: @@ -169,16 +173,38 @@ jobs: ref: ${{ needs.plan.outputs.merge }} path: _mac_source persist-credentials: false + - name: Select candidate Rust toolchain + working-directory: _mac_source + shell: bash + run: | + rustc --version + echo "RUSTUP_TOOLCHAIN=$(rustup show active-toolchain | cut -d ' ' -f 1)" >> "$GITHUB_ENV" + - uses: mozilla-actions/sccache-action@9e7fa8a12102821edf02ca5dbea1acd0f89a2696 # v0.0.10 + id: sccache + continue-on-error: true + - name: Configure read-only compiler cache + if: steps.sccache.outcome == 'success' + working-directory: _mac_source + shell: bash + run: bash ../_mac_control/scripts/ci/configure-sccache.sh + # Restore only. Candidate code cannot write the base branch cache. + - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2 + continue-on-error: true + with: + workspaces: _mac_source -> target + shared-key: ci-terminal-pty-main + save-if: "false" - uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7 + if: matrix.lane == 'reflow' with: node-version-file: _mac_source/.nvmrc - - name: Run all three macOS lanes + - name: Run macOS lane shell: bash working-directory: _mac_source env: - CARGO_TARGET_DIR: ${{ runner.temp }}/astra-macos-${{ github.run_id }}-${{ github.run_attempt }}/target npm_config_cache: ${{ runner.temp }}/astra-macos-${{ github.run_id }}-${{ github.run_attempt }}/npm - run: bash ../_mac_control/scripts/ci/run-macos-lanes.sh all + MACOS_LANE: ${{ matrix.lane }} + run: bash ../_mac_control/scripts/ci/run-macos-lanes.sh "$MACOS_LANE" publish: needs: [plan, mini, hosted-admission, hosted] diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 6ef3db5834..272bba2769 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -22,13 +22,9 @@ jobs: name: Determine test scope permissions: contents: read - checks: read - pull-requests: read - actions: read runs-on: ubuntu-latest timeout-minutes: 5 outputs: - macos_delegated: ${{ steps.mac.outputs.delegated }} test_cli: ${{ steps.scope.outputs.test_cli }} test_runtime: ${{ steps.scope.outputs.test_runtime }} test_services: ${{ steps.scope.outputs.test_services }} @@ -45,11 +41,31 @@ jobs: - name: Classify changed paths id: scope run: python3 scripts/ci/ci_scope.py + + macos-routing: + name: Read macOS delegation + needs: scope + if: >- + !cancelled() && vars.ASTRA_MACOS_MIGRATION_ENABLED == 'true' && + (needs.scope.outputs.test_cli == 'true' || needs.scope.outputs.test_core == 'true') + runs-on: ubuntu-latest + timeout-minutes: 3 + permissions: + contents: read + checks: read + pull-requests: read + actions: read + outputs: + macos_delegated: ${{ steps.mac.outputs.delegated }} + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + with: + persist-credentials: false - name: Read trusted macOS execution hint id: mac if: >- vars.ASTRA_MACOS_MIGRATION_ENABLED == 'true' && - (steps.scope.outputs.test_cli == 'true' || steps.scope.outputs.test_core == 'true') + (needs.scope.outputs.test_cli == 'true' || needs.scope.outputs.test_core == 'true') timeout-minutes: 2 continue-on-error: true env: @@ -256,11 +272,11 @@ jobs: terminal-pty-shards: name: "Terminal: ${{ matrix.segment }} (${{ matrix.os }})" - needs: scope + needs: [scope, macos-routing] if: ${{ !cancelled() && (needs.scope.result != 'success' || needs.scope.outputs.test_cli == 'true' || needs.scope.outputs.test_core == 'true') }} - runs-on: ${{ matrix.os == 'macos-15' && matrix.segment != 'dev-tty-reader' && needs.scope.outputs.macos_delegated == 'true' && 'ubuntu-latest' || matrix.os }} + runs-on: ${{ matrix.os == 'macos-15' && matrix.segment != 'dev-tty-reader' && needs.macos-routing.outputs.macos_delegated == 'true' && 'ubuntu-latest' || matrix.os }} env: - RUN_LANE: ${{ !(matrix.os == 'macos-15' && matrix.segment != 'dev-tty-reader' && needs.scope.outputs.macos_delegated == 'true') }} + RUN_LANE: ${{ !(matrix.os == 'macos-15' && matrix.segment != 'dev-tty-reader' && needs.macos-routing.outputs.macos_delegated == 'true') }} timeout-minutes: 45 strategy: fail-fast: false @@ -322,11 +338,11 @@ jobs: macos-workspace-coordination-tests: name: "macOS workspace coordination contracts" - needs: scope + needs: [scope, macos-routing] if: ${{ !cancelled() && (needs.scope.result != 'success' || needs.scope.outputs.test_core == 'true' || needs.scope.outputs.test_cli == 'true') }} - runs-on: ${{ needs.scope.outputs.macos_delegated == 'true' && 'ubuntu-latest' || 'macos-15' }} + runs-on: ${{ needs.macos-routing.outputs.macos_delegated == 'true' && 'ubuntu-latest' || 'macos-15' }} env: - RUN_LANE: ${{ needs.scope.outputs.macos_delegated != 'true' }} + RUN_LANE: ${{ needs.macos-routing.outputs.macos_delegated != 'true' }} timeout-minutes: 30 steps: - name: Delegated to App-verified macOS execution diff --git a/docs/guides/macos-ci-runner.md b/docs/guides/macos-ci-runner.md index e96e6888ab..ecaccdba9d 100644 --- a/docs/guides/macos-ci-runner.md +++ b/docs/guides/macos-ci-runner.md @@ -8,10 +8,14 @@ is configured, admitted, compatible, or faster than hosted runners. ## Workload and routing The migrated unit runs default-reader PTYs and the CLI Bash regression, reflow, -and workspace coordination. It runs sequentially on one runner and reuses its +and workspace coordination. On the mini it runs sequentially and reuses its build directory. This avoids scheduling three simultaneous Rust builds on a single small machine. The canonical commands live in `scripts/ci/run-macos-lanes.sh` and are shared with ordinary hosted CI. +Protected hosted fallback runs three parallel jobs, with read-only Rust artifact +and compiler caches. It does not serialize three hosted Macs onto one job. +Build reuse on the mini and hosted cache hit rates still require measurement; +adding the durations of the previous independent jobs is not a latency forecast. | Event | Execution | | --- | --- | @@ -40,7 +44,11 @@ members execute PR code through that event. The controller independently reads the current base B, head H and merge M and requires exactly two ordered parents `M^1=B` and `M^2=H`. It checks out immutable M. Null/stale merges, force-pushes, base advancement or changed repository identity cannot silently select another -revision. +revision. Planning retries a not-yet-ready merge up to 12 times (two seconds +between attempts), re-reading the full tuple and retaining the event's head. +Exhaustion creates three explicit failed, retryable App checks instead of +silently leaving the required contexts absent. A later head is never substituted +into an earlier event's plan. A dedicated GitHub App publishes three contexts on H: @@ -61,15 +69,20 @@ could otherwise request it. PR refs and feature-branch dispatches must be denied by the environment policy. Control verifies that exact policy before dispatch. The publisher reads actual GitHub run/job/step results; it does not accept PR artifacts or shell-supplied success flags. A successful combined job satisfies -all three contexts; any actual test failure fails them conservatively. +all three contexts; hosted fallback must prove all three parallel macOS jobs +and their execution steps succeeded. Any actual test failure fails the contexts +conservatively and cannot be masked by fallback. The ordinary `pull_request` workflow reads App-owned authorization as a best-effort deduplication hint. Delegated copies use cheap Ubuntu notification jobs. The hint cannot publish or replace the three required App checks. Missing hints preserve hosted execution, so a control race can cause duplicate work, not missing coverage. A forged hint leaves the member's trusted checks pending. -Only the scope job receives the extra Actions/checks/PR read permissions. The -hint waits up to a 90-second polling budget for delayed control, bounded by a +Only the separate `macos-routing` job receives the extra Actions/checks/PR read +permissions. Linux package groups and vendored tests depend directly on scope +and do not wait for this hint; the mixed Linux/macOS terminal matrix still waits +for routing. The hint waits up to a 90-second polling budget for delayed +control, bounded by a two-minute step timeout; a known ordinary route or permission denial returns immediately. Separate event queues can still outlast that budget. Such runs keep hosted execution and can duplicate work; this is not claimed eliminated. The @@ -85,15 +98,35 @@ missing/skipped selected lanes fails explicitly, including scope disagreement, instead of staying pending forever. This retains the pre-existing PR-modifiable test-definition limitation for non-member hosted tests. -The same metadata-only observer reconciles pending contexts every 15 minutes -(and supports manual dispatch), recovering lost completion notifications. -Authorization expires three hours after planning. At expiry, pending contexts -are failed with a recovery instruction; expiry never means tests passed. Active -API/Actions outages or delayed cron execution can delay reconciliation, so this -is not a platform availability guarantee. Expired authorization is rejected by -the runner hook and result publisher. After resolving approval/unavailability, -start fresh trusted control and rerun the applicable tests as required. The -reconciler never executes PR code, auto-approves forks or relaxes result checks. +The same metadata-only observer reconciles every 15 minutes (and supports +manual dispatch), recovering lost completion notifications and PRs with no App +checks at all. Missing or retryable stale plans dispatch only +`macos-trusted.yml` at `main`, with a numeric PR number; that workflow repeats +all source and membership checks. Recovery dispatches intentionally use protected +hosted routing for members; non-members still use ordinary `pull_request` +approval and execution. The controller never executes PR code or auto-approves +forks. Its App token alone requests Actions **write** for dispatch; plan, +admission, publisher and runner-side credentials retain Actions read. See +[GitHub's dispatch permission](https://docs.github.com/en/rest/actions/workflows#create-a-workflow-dispatch-event). +The observer is serialized and writes a non-authorizing recovery marker before +dispatch. Same-source requests are throttled for 30 minutes, including lost +responses or failed dispatches; later real control supersedes the marker. + +When base/head/merge or author trust changes during execution, publication +immediately fails the old contexts with an explicit stale-source explanation. +Reconciliation requests fresh control once a verified merge is available; old +results cannot overwrite a newer run/attempt. Successful checks and actual test +failures are not automatically retried. A missing merge remains an explicit +retryable failure until it resolves. + +Authorization expires three hours after planning. At expiry, pending execution +contexts are failed with a recovery instruction; expiry never means tests +passed. Active API/Actions outages or delayed cron execution can delay recovery, +so this is not a platform availability guarantee. Expired authorization is +rejected by the runner hook and result publisher. After resolving fork approval +or runner unavailability, start fresh trusted control and rerun applicable tests +as required. The observer's five-minute job timeout also bounds an individual +scan; prolonged API failures may require manual recovery. Both privileged-event workflows explicitly restrict remote cache tokens with `cache-mode: read` or `none`. Disabling a client upload is not the authorization @@ -227,14 +260,20 @@ infrastructure fallback. root-owned hook/helper and platform described above. Validate direct-label bypass, fork/member/unknown routing, revoked-member reruns, tuple changes, cache write denial, pollution isolation and capacity on disposable fixtures. + Include an initially null/stale merge, zero App checks, stale completion, + lost/forbidden recovery dispatch, recovery throttling and all three parallel + hosted results. Recovery must never execute a non-member under the trusted + event or treat a marker/timeout as a pass. 3. Create GitHub Environment `astra-macos-control`, allowing only branch `main` via its custom deployment branch policy, with no allowed tags or wildcards. After verifying that PR refs and feature dispatches cannot access it, store `ASTRA_MACOS_APP_PRIVATE_KEY` only as that environment's secret. Set repository variable `ASTRA_MACOS_APP_ID` and `ASTRA_MACOS_GROUP_ID` to the App/PR group IDs. - The App needs repository Actions/contents/PR read, administration read, + The App installation needs repository Actions write (only the metadata + observer requests a write token), contents/PR read, administration read, checks write and environments read; organization Members and self-hosted - runners/groups read. + runners/groups read. Other control jobs explicitly downscope Actions to read; + execution jobs receive no App token. 4. Make the three contexts required with source pinned to the App, and enforce strict up-to-date branch checks. Preserve the existing protected checks. The migration control refuses dispatch until this binding is visible through the rules/protection APIs. diff --git a/scripts/ci/macos_runner.py b/scripts/ci/macos_runner.py index 511b715711..749f2610ae 100755 --- a/scripts/ci/macos_runner.py +++ b/scripts/ci/macos_runner.py @@ -27,12 +27,23 @@ SHA = re.compile(r"^[0-9a-f]{40}$") AUTHORIZATION_SECONDS = 3 * 60 * 60 HINT_WAIT_SECONDS = 90 +SOURCE_ATTEMPTS = 12 +RECOVERY_COOLDOWN_SECONDS = 30 * 60 +HOSTED_LANES = ("default-reader", "reflow", "coordination") class Unavailable(Exception): pass +class MergePending(Unavailable): + pass + + +class StaleSource(Unavailable): + pass + + class PermissionDenied(Unavailable): pass @@ -94,23 +105,41 @@ def member(api, login): return False -def source(api, number): +def pr_identity(api, number, expected_head=None): pr = api.get(f"/repos/{REPO}/pulls/{int(number)}") if (pr["state"] != "open" or pr["base"]["repo"]["full_name"] != REPO or pr["base"]["ref"] != "main"): raise Unavailable("PR is not an open main pull request") base = api.get(f"/repos/{REPO}/git/ref/heads/main")["object"]["sha"] - head, merge = pr["head"]["sha"], pr.get("merge_commit_sha") - if not all(isinstance(s, str) and SHA.fullmatch(s) for s in (base, head, merge)): - raise Unavailable("merge revision unresolved") - commit = api.get(f"/repos/{REPO}/git/commits/{merge}") - if [p["sha"] for p in commit["parents"]] != [base, head]: - raise Unavailable("merge parents do not match current base/head") + head = pr["head"]["sha"] + if not all(isinstance(s, str) and SHA.fullmatch(s) for s in (base, head)): + raise Unavailable("invalid base/head revision") + if expected_head is not None and head != expected_head: + raise StaleSource("trigger head has been superseded") return {"kind": "pr", "pr": int(number), "base": base, "head": head, - "merge": merge, "head_repository_id": pr["head"]["repo"]["id"], + "merge": pr.get("merge_commit_sha"), "head_repository_id": pr["head"]["repo"]["id"], "author": pr["user"]["login"], "author_id": pr["user"]["id"]} +def source(api, number, *, attempts=1, expected_head=None): + # GitHub computes the test merge asynchronously. Re-read the entire tuple on + # each retry, never substitute a head from a newer synchronize event. + for attempt in range(attempts): + plan = pr_identity(api, number, expected_head) + merge = plan["merge"] + if isinstance(merge, str) and SHA.fullmatch(merge): + status, commit = api.request(f"/repos/{REPO}/git/commits/{merge}") + if status in (401, 403): + raise PermissionDenied("merge API permission unavailable") + if status == 200 and [p["sha"] for p in commit["parents"]] == [plan["base"], plan["head"]]: + return plan + if status not in (200, 404): + raise Unavailable("merge API unavailable") + if attempt + 1 < attempts: + time.sleep(2) + raise MergePending("merge revision not ready for current base/head") + + def same_source(a, b): return all(a.get(k) == b.get(k) for k in ("kind", "pr", "base", "head", "merge", "head_repository_id", "author_id")) @@ -256,13 +285,19 @@ def require_unexpired(plan): def refresh(api, plan): require_unexpired(plan) if plan["kind"] == "pr": - current = source(api, plan["pr"]) - if not same_source(current, plan) or not member(api, current["author"]): - raise Unavailable("source changed or membership is not confirmed") + identity = pr_identity(api, plan["pr"]) + if not same_source(identity, plan) or not member(api, identity["author"]): + raise StaleSource("source changed or membership is not confirmed") + try: + current = source(api, plan["pr"]) + except MergePending: + raise StaleSource("merge no longer matches current base/head") from None + if not same_source(current, plan): + raise StaleSource("source advanced during refresh") else: head = api.get(f"/repos/{REPO}/git/ref/heads/main")["object"]["sha"] if plan["head"] != head: - raise Unavailable("main has advanced") + raise StaleSource("main has advanced") def admit(api, request, config): @@ -291,12 +326,15 @@ def verified_job(api, plan, name): jobs = api.pages(f"/repos/{REPO}/actions/runs/{run['id']}/jobs?filter=latest", "jobs") matching = [j for j in jobs if j["name"] == name] if len(matching) != 1: - return None + return "failure" if run.get("status") == "completed" else None job = matching[0] if job["conclusion"] != "success": return job["conclusion"] if job["status"] == "completed" else None steps = {s["name"]: s["conclusion"] for s in job["steps"]} - if steps.get("Run all three macOS lanes") != "success": + expected_step = "Run all three macOS lanes" if name == "Migrated macOS lanes" else "Run macOS lane" + if steps.get(expected_step) != "success": + return "failure" + if name.startswith("Fallback macOS:") and "macos-15" not in job.get("labels", []): return "failure" if name == "Migrated macOS lanes" and job["runner_group_id"] != plan["group_id"]: return "failure" @@ -345,9 +383,16 @@ def plan_command(api, event, app_id): raise Unavailable("wrong repository") require_check_sources(api, app_id) kind = "main" if os.environ["GITHUB_EVENT_NAME"] == "push" else "pr" + unresolved = False if kind == "pr": number = event.get("number") or event.get("inputs", {}).get("pr_number") - plan = source(api, number) + expected_head = event.get("pull_request", {}).get("head", {}).get("sha") + try: + plan = source(api, number, attempts=SOURCE_ATTEMPTS, expected_head=expected_head) + except MergePending: + plan = pr_identity(api, number, expected_head) + plan["merge"] = "" + unresolved = True trusted = member(api, plan["author"]) else: head = event["after"] @@ -358,7 +403,10 @@ def plan_command(api, event, app_id): plan.update(schema=1, run_id=int(os.environ["GITHUB_RUN_ID"]), app_id=app_id, attempt=int(os.environ["GITHUB_RUN_ATTEMPT"]), route="ordinary", group_id=0, expires_at=int(time.time()) + AUTHORIZATION_SECONDS) - if not selected(api, plan): + if unresolved: + plan.update(route="unresolved", retryable=True) + save_checks(api, plan, "failure", "Merge revision not ready; trusted reconciliation will retry") + elif not selected(api, plan): plan["route"] = "unaffected" save_checks(api, plan, "success", "Canonical scope does not select these lanes") elif trusted: @@ -400,12 +448,14 @@ def hint(api, event, app_id): if plan["pr"] != number: continue require_unexpired(plan) + if plan["route"] in ("unresolved", "recovering"): + return False validate_run(api, plan) if number and not same_source(plan, source(api, number)): continue # Once trusted control has chosen ordinary hosted execution, # there is no reason to keep delaying a non-member's scope job. - if plan["route"] in ("ordinary", "unaffected", "disabled"): + if plan["route"] in ("ordinary", "unaffected", "disabled", "unresolved", "recovering"): return False if plan["route"] in ("self-hosted", "hosted"): return True @@ -423,20 +473,36 @@ def hint(api, event, app_id): def finish(api, plan): - if plan["route"] in ("disabled", "unaffected"): + if plan["route"] in ("disabled", "unaffected", "unresolved", "recovering"): return True require_unexpired(plan) - if plan["route"] == "ordinary": - if plan["kind"] != "pr" or not same_source(plan, source(api, plan["pr"])): - raise Unavailable("hosted result is stale") - if member(api, plan["author"]): - raise Unavailable("author now qualifies for protected execution; refresh routing") - result = hosted_result(api, plan) - else: - refresh(api, plan) - fallback = verified_job(api, plan, "Fallback macOS lanes") - primary = verified_job(api, plan, "Migrated macOS lanes") if plan["route"] == "self-hosted" else None - result = primary if primary == "success" or not can_fallback(api, plan) else fallback or primary + try: + if plan["route"] == "ordinary": + if plan["kind"] != "pr" or not same_source(plan, pr_identity(api, plan["pr"])): + raise StaleSource("hosted result is stale") + try: + current = source(api, plan["pr"]) + except MergePending: + raise StaleSource("hosted merge is stale") from None + if not same_source(current, plan): + raise StaleSource("hosted source advanced during refresh") + if member(api, plan["author"]): + raise StaleSource("author now qualifies for protected execution") + result = hosted_result(api, plan) + else: + refresh(api, plan) + primary = verified_job(api, plan, "Migrated macOS lanes") if plan["route"] == "self-hosted" else None + if primary == "success" or not can_fallback(api, plan): + result = primary + else: + lanes = [verified_job(api, plan, "Fallback macOS: " + lane) for lane in HOSTED_LANES] + result = (None if any(r is None for r in lanes) else + "success" if all(r == "success" for r in lanes) else "failure") + except StaleSource: + # Failure, rather than neutral/skipped, keeps required checks fail-closed. + save_checks(api, {**plan, "retryable": True}, "failure", + "Stale macOS source or trust; trusted reconciliation will request fresh control") + return True if result is not None: observed = {**plan, "attempt": validate_run(api, plan)["run_attempt"]} save_checks(api, observed, "success" if result == "success" else "failure", "Verified macOS execution: " + result) @@ -444,8 +510,48 @@ def finish(api, plan): return False +def recover(api, pr, app_id, previous=None): + """Only the protected metadata controller may dispatch fixed main control. + + A non-authorizing marker bounds duplicate dispatches (including lost API + responses). A later real plan supersedes it by run id. No execution evidence + or member authorization is manufactured here. + """ + try: + current = source(api, pr["number"], expected_head=pr["head"]["sha"]) + except MergePending: + current = pr_identity(api, pr["number"], pr["head"]["sha"]) + marker = {**current, "schema": 1, "app_id": app_id, "run_id": 0, + "attempt": 0, "route": "unresolved", "group_id": 0, + "retryable": True, "expires_at": int(time.time()) + AUTHORIZATION_SECONDS} + save_checks(api, marker, "failure", "Merge revision not ready; trusted reconciliation will retry") + return + if (previous and same_source(previous, current) and + time.time() - previous.get("recovery_requested_at", 0) < RECOVERY_COOLDOWN_SECONDS): + return + marker = {**current, "schema": 1, "app_id": app_id, + "run_id": previous["run_id"] if previous else 0, + "attempt": previous["attempt"] if previous else 0, + "route": "recovering", "group_id": 0, "retryable": True, + "recovery_requested_at": int(time.time()), + "expires_at": int(time.time()) + AUTHORIZATION_SECONDS} + save_checks(api, marker, summary="Requesting fresh protected Mac CI control") + # A real plan may have appeared since the scan. Do not dispatch over it. + latest = {} + for row in checks(api, current["head"]): + if row["name"] in CONTEXTS and row.get("app", {}).get("id") == app_id: + if row["name"] not in latest or row["id"] > latest[row["name"]]["id"]: + latest[row["name"]] = row + if any(json.loads(row["output"]["text"]) != marker for row in latest.values()): + return + status, _ = api.request(f"/repos/{REPO}/actions/workflows/macos-trusted.yml/dispatches", + {"ref": "main", "inputs": {"pr_number": str(pr["number"])}}) + if status not in (200, 204): + raise Unavailable("could not request fresh trusted control") + + def reconcile(api, app_id): - """Repair lost completion notifications and fail expired pending checks.""" + """Recover absent plans/stale sources; close expired or lost results.""" require_check_sources(api, app_id) prs = api.pages(f"/repos/{REPO}/pulls?state=open&base=main") for pr in prs: @@ -456,14 +562,18 @@ def reconcile(api, app_id): if row["name"] in CONTEXTS and row.get("app", {}).get("id") == app_id: if row["name"] not in latest or row["id"] > latest[row["name"]]["id"]: latest[row["name"]] = row - if not latest or (len(latest) == len(CONTEXTS) and - all(c["status"] == "completed" for c in latest.values())): + if not latest: + recover(api, pr, app_id) continue plan = json.loads(max(latest.values(), key=lambda c: c["id"])["output"]["text"]) if (plan["schema"] != 1 or plan["app_id"] != app_id or plan["kind"] != "pr" or plan["head"] != pr["head"]["sha"] or plan["pr"] != pr["number"]): continue - if expired(plan): + if plan.get("retryable"): + recover(api, pr, app_id, plan) + elif len(latest) == len(CONTEXTS) and all(c["status"] == "completed" for c in latest.values()): + continue + elif expired(plan): save_checks(api, plan, "failure", "Mac CI authorization expired; resolve approval/runner state and start fresh Mac CI control", only_pending=True) diff --git a/scripts/ci/test_ci_parallelism.py b/scripts/ci/test_ci_parallelism.py index ac5ae5961c..2043f8bbd8 100755 --- a/scripts/ci/test_ci_parallelism.py +++ b/scripts/ci/test_ci_parallelism.py @@ -143,11 +143,12 @@ def test_online_gate_requires_all_matrix_jobs_to_succeed(self): }, capture_output=True, text=True) self.assertEqual(result.returncode == 0, status == moi_status == "success") - def test_parallel_jobs_depend_only_on_scope_and_gates_depend_on_shards(self): + def test_package_jobs_avoid_macos_routing_and_gates_depend_on_execution(self): workflow = (ROOT / ".github/workflows/test.yml").read_text() for job in ("terminal-pty-shards", "test-online"): header = workflow.split(f"\n {job}:\n", 1)[1].split(" steps:\n", 1)[0] - self.assertIn(" needs: scope\n", header) + expected = "[scope, macos-routing]" if job == "terminal-pty-shards" else "scope" + self.assertIn(f" needs: {expected}\n", header) self.assertIn("!cancelled()", header) self.assertIn("fail-fast: false", header) terminal = workflow.split("\n terminal-pty:\n", 1)[1].split("\n terminal-pty-shards:", 1)[0] @@ -163,7 +164,7 @@ def test_parallel_jobs_depend_only_on_scope_and_gates_depend_on_shards(self): self.assertIn("needs.scope.result != 'success'", coordination) self.assertIn("if: ${{ !cancelled() }}", coordination) contracts = workflow.split("\n macos-workspace-coordination-tests:\n", 1)[1].split(" steps:\n", 1)[0] - self.assertIn("needs: scope", contracts) + self.assertIn("needs: [scope, macos-routing]", contracts) self.assertNotIn("terminal-pty-shards", contracts, "Tools contracts must not wait for the CLI build") online = workflow.split("\n test-online:\n", 1)[1] self.assertIn("lane: [core-runtime, core-turn-core, core-services, integration]", online) diff --git a/scripts/ci/test_macos_runner.py b/scripts/ci/test_macos_runner.py index 1a89ecc3c4..b1fd6b64e0 100755 --- a/scripts/ci/test_macos_runner.py +++ b/scripts/ci/test_macos_runner.py @@ -29,6 +29,8 @@ def request(self, path, data=None, method=None): self.calls.append((path, method)) if data is not None: self.writes.append(copy.deepcopy(data)) + if path.endswith("/dispatches"): + return self.status.get(path, 204), None if method == "PATCH": row = next(c for c in self.checks if c["id"] == int(path.rsplit("/", 1)[1])) row.update(copy.deepcopy(data)) @@ -227,13 +229,25 @@ def test_ordinary_member_lookup_failure_and_resource_failure_choose_distinct_rou plan = json.loads(self.api.checks[-1]["output"]["text"]) self.assertEqual(plan["route"], expected) + def test_recovery_dispatch_revalidates_members_and_uses_hosted_without_auto_approval(self): + with tempfile.TemporaryDirectory() as directory, patch.dict(os.environ, { + "GITHUB_REPOSITORY": m.REPO, "GITHUB_EVENT_NAME": "workflow_dispatch", + "GITHUB_RUN_ID": "100", "GITHUB_RUN_ATTEMPT": "1", + "GITHUB_OUTPUT": str(Path(directory) / "outputs"), "ASTRA_MACOS_GROUP_ID": "9"}): + for membership, route in ((204, "hosted"), (404, "ordinary"), (403, "ordinary")): + with self.subTest(membership=membership): + self.api.status["/orgs/matrixorigin/members/fixture-member"] = membership + m.plan_command(self.api, {"inputs": {"pr_number": "42"}}, 5) + self.assertEqual(json.loads(self.api.checks[-1]["output"]["text"])["route"], route) + def test_real_test_failure_cannot_be_masked_by_a_successful_fallback(self): self.authorize() failed = {**self.active, "status": "completed", "conclusion": "failure", "steps": [{"name": "Run all three macOS lanes", "conclusion": "failure"}]} - fallback = {**failed, "name": "Fallback macOS lanes", "conclusion": "success", - "steps": [{"name": "Run all three macOS lanes", "conclusion": "success"}]} - self.api.data[self.run_path + "/jobs?filter=latest"] = {"jobs": [failed, fallback]} + fallbacks = [{**failed, "name": "Fallback macOS: " + lane, "conclusion": "success", + "labels": ["macos-15"], + "steps": [{"name": "Run macOS lane", "conclusion": "success"}]} for lane in m.HOSTED_LANES] + self.api.data[self.run_path + "/jobs?filter=latest"] = {"jobs": [failed, *fallbacks]} self.assertFalse(m.can_fallback(self.api, self.plan)) m.finish(self.api, self.plan) self.assertTrue(all(c["conclusion"] == "failure" for c in self.api.checks)) @@ -254,9 +268,8 @@ def test_success_requires_actual_test_step_and_expected_runner_group(self): def test_untrusted_ordinary_success_does_not_satisfy_member_results(self): plan = {**self.plan, "route": "ordinary"} self.authorize() - with self.assertRaises(m.Unavailable): - m.finish(self.api, plan) - self.assertFalse(any(c.get("conclusion") == "success" for c in self.api.checks)) + self.assertTrue(m.finish(self.api, plan)) + self.assertTrue(all(c.get("conclusion") == "failure" for c in self.api.checks)) def test_delayed_publication_cannot_overwrite_newer_attempt(self): self.authorize() @@ -441,6 +454,136 @@ def test_expiry_reconciliation_does_not_downgrade_already_successful_results(sel m.save_checks(self.api, self.plan, "failure", only_pending=True) self.assertTrue(all(c["conclusion"] == "success" for c in self.api.checks)) + def test_merge_source_retries_null_and_old_parents_without_changing_trigger_head(self): + for mode in ("null", "old-parents"): + with self.subTest(mode=mode): + original = copy.deepcopy(self.api.data) + if mode == "null": + self.api.data[self.pr_path]["merge_commit_sha"] = None + else: + self.api.data[f"/repos/{m.REPO}/git/commits/{M}"]["parents"][0]["sha"] = "d" * 40 + with patch.object(m.time, "sleep", side_effect=lambda _: self.api.data.update(original)) as sleep: + self.assertEqual(m.source(self.api, 42, attempts=3, expected_head=H)["merge"], M) + self.assertEqual(sleep.call_count, 1) + self.api.data[self.pr_path]["head"]["sha"] = "e" * 40 + with patch.object(m.time, "sleep") as sleep, self.assertRaises(m.StaleSource): + m.source(self.api, 42, attempts=3, expected_head=H) + sleep.assert_not_called() + + def test_plan_exhaustion_publishes_retryable_failure_instead_of_missing_checks(self): + self.api.data[self.pr_path]["merge_commit_sha"] = None + with tempfile.TemporaryDirectory() as directory, patch.dict(os.environ, { + "GITHUB_REPOSITORY": m.REPO, "GITHUB_EVENT_NAME": "pull_request_target", + "GITHUB_RUN_ID": "100", "GITHUB_RUN_ATTEMPT": "1", + "GITHUB_OUTPUT": str(Path(directory) / "outputs")}), patch.object(m.time, "sleep"): + m.plan_command(self.api, {"number": 42, "pull_request": {"head": {"sha": H}}}, 5) + self.assertEqual(len(self.api.checks), 3) + self.assertTrue(all(c["conclusion"] == "failure" for c in self.api.checks)) + self.api.data[self.pr_path]["merge_commit_sha"] = M + self.api.data[f"/repos/{m.REPO}/pulls?state=open&base=main"] = [{"number": 42, "head": {"sha": H}}] + m.reconcile(self.api, 5) + self.assertEqual(self.api.writes[-1], {"ref": "main", "inputs": {"pr_number": "42"}}) + + def test_zero_checks_recovery_is_bounded_and_never_authorizes_execution(self): + self.api.data[f"/repos/{m.REPO}/pulls?state=open&base=main"] = [{"number": 42, "head": {"sha": H}}] + m.reconcile(self.api, 5) + self.assertEqual(sum(path.endswith("/dispatches") for path, _ in self.api.calls), 1) + marker = json.loads(self.api.checks[-1]["output"]["text"]) + self.assertEqual(marker["route"], "recovering") + with self.assertRaises(m.Unavailable): + m.admit(self.api, self.request, self.config) + m.reconcile(self.api, 5) + self.assertEqual(sum(path.endswith("/dispatches") for path, _ in self.api.calls), 1) + with patch.object(m.time, "time", return_value=marker["recovery_requested_at"] + m.RECOVERY_COOLDOWN_SECONDS): + m.reconcile(self.api, 5) + self.assertEqual(sum(path.endswith("/dispatches") for path, _ in self.api.calls), 2) + self.authorize() # Real plan replaces the non-authorizing marker. + self.assertEqual(m.authorization(self.api, H, 100, 5), self.plan) + + def test_recovery_does_not_dispatch_over_a_real_plan_or_retry_test_failures(self): + self.authorize() + m.recover(self.api, {"number": 42, "head": {"sha": H}}, 5) + self.assertFalse(any(path.endswith("/dispatches") for path, _ in self.api.calls)) + m.save_checks(self.api, self.plan, "failure", "Actual test failure") + self.api.data[f"/repos/{m.REPO}/pulls?state=open&base=main"] = [{"number": 42, "head": {"sha": H}}] + m.reconcile(self.api, 5) + self.assertFalse(any(path.endswith("/dispatches") for path, _ in self.api.calls)) + + def test_dispatch_permission_failure_is_throttled_and_recovers_after_cooldown(self): + pr = {"number": 42, "head": {"sha": H}} + path = f"/repos/{m.REPO}/actions/workflows/macos-trusted.yml/dispatches" + self.api.status[path] = 403 + with self.assertRaises(m.Unavailable): + m.recover(self.api, pr, 5) + marker = json.loads(self.api.checks[-1]["output"]["text"]) + m.recover(self.api, pr, 5, marker) + self.assertEqual(sum(p == path for p, _ in self.api.calls), 1) + self.api.status[path] = 204 + with patch.object(m.time, "time", return_value=marker["recovery_requested_at"] + m.RECOVERY_COOLDOWN_SECONDS): + m.recover(self.api, pr, 5, marker) + self.assertEqual(sum(p == path for p, _ in self.api.calls), 2) + + def test_refresh_rejects_a_source_changed_during_its_second_api_read(self): + with patch.object(m, "source", return_value={**self.plan, "head": "e" * 40}): + with self.assertRaises(m.StaleSource): + m.refresh(self.api, self.plan) + + def test_zero_checks_unresolved_merge_is_explicit_failure_and_does_not_dispatch(self): + self.api.data[self.pr_path]["merge_commit_sha"] = None + self.api.data[f"/repos/{m.REPO}/pulls?state=open&base=main"] = [{"number": 42, "head": {"sha": H}}] + m.reconcile(self.api, 5) + self.assertTrue(all(c["conclusion"] == "failure" for c in self.api.checks)) + self.assertFalse(any(path.endswith("/dispatches") for path, _ in self.api.calls)) + + def test_stale_source_closes_immediately_and_reconciliation_requests_fresh_control(self): + self.authorize() + newer_base = "d" * 40 + self.api.data[f"/repos/{m.REPO}/git/ref/heads/main"]["object"]["sha"] = newer_base + self.assertTrue(m.finish(self.api, self.plan)) + self.assertTrue(all(c["conclusion"] == "failure" and "Stale" in c["output"]["summary"] for c in self.api.checks)) + self.api.data[f"/repos/{m.REPO}/git/commits/{M}"]["parents"][0]["sha"] = newer_base + self.api.data[f"/repos/{m.REPO}/pulls?state=open&base=main"] = [{"number": 42, "head": {"sha": H}}] + m.reconcile(self.api, 5) + self.assertEqual(self.api.writes[-1], {"ref": "main", "inputs": {"pr_number": "42"}}) + + def test_stale_old_publisher_cannot_close_new_attempt(self): + self.authorize() + m.save_checks(self.api, {**self.plan, "attempt": 2}) + self.api.data[f"/repos/{m.REPO}/git/ref/heads/main"]["object"]["sha"] = "d" * 40 + writes = len(self.api.writes) + self.assertTrue(m.finish(self.api, self.plan)) + self.assertEqual(len(self.api.writes), writes) + + def test_parallel_hosted_requires_every_real_lane_and_retains_failures(self): + plan = {**self.plan, "route": "hosted"} + jobs = [{"name": "Fallback macOS: " + lane, "status": "completed", "conclusion": "success", + "labels": ["macos-15"], "steps": [{"name": "Run macOS lane", "conclusion": "success"}]} + for lane in m.HOSTED_LANES] + self.api.data[self.run_path + "/jobs?filter=latest"] = {"jobs": jobs} + m.save_checks(self.api, plan) + for mutation in ("missing", "failed", "echo", "ubuntu", "success"): + with self.subTest(mutation=mutation): + before = copy.deepcopy(jobs) + if mutation == "missing": + jobs.pop() + elif mutation == "failed": + jobs[0]["conclusion"] = "failure" + elif mutation == "echo": + jobs[0]["steps"] = [] + elif mutation == "ubuntu": + jobs[0]["labels"] = ["ubuntu-latest"] + result = m.finish(self.api, plan) + if mutation == "missing": + self.assertFalse(result) + self.api.data[self.run_path]["status"] = "completed" + self.assertTrue(m.finish(self.api, plan)) + self.assertTrue(all(c["conclusion"] == "failure" for c in self.api.checks)) + self.api.data[self.run_path].pop("status") + else: + self.assertTrue(result) + self.assertTrue(all(c["conclusion"] == ("success" if mutation == "success" else "failure") for c in self.api.checks)) + jobs[:] = before + @@ -491,13 +634,17 @@ def test_hint_permissions_are_scoped_and_reconciliation_stays_metadata_only(self top = workflow.split("\non:", 1)[0] self.assertNotIn("actions: read", top) self.assertNotIn("checks: read", top) - scope = workflow.split("\n scope:\n", 1)[1].split("\n cli-test-build:\n", 1)[0] + scope = workflow.split("\n macos-routing:\n", 1)[1].split("\n cli-test-build:\n", 1)[0] for permission in ("actions: read", "checks: read", "pull-requests: read", "contents: read"): self.assertIn(permission, scope) observer = (ROOT / ".github/workflows/macos-results.yml").read_text() self.assertIn("cron: '*/15 * * * *'", observer) self.assertIn("macos_runner.py reconcile", observer) - self.assertNotIn("permission-actions: write", observer) + self.assertIn("permission-actions: write", observer) + self.assertIn("cancel-in-progress: false", observer) + classifier = workflow.split("\n scope:\n", 1)[1].split("\n macos-routing:\n", 1)[0] + self.assertNotIn("macos_runner.py hint", classifier) + self.assertNotIn("actions: read", classifier) self.assertNotIn("_mac_source", observer) def test_credential_owner_and_cache_boundary_are_outside_pr_execution(self): From 50466af51b51591f30f75354a0eab031c1b19f89 Mon Sep 17 00:00:00 2001 From: lr90 Date: Sat, 24 Oct 2026 10:31:25 +0800 Subject: [PATCH 04/11] fix(ci): enforce hosted rerun admission and link fork results --- .github/workflows/macos-trusted.yml | 7 ++ .github/workflows/test.yml | 7 ++ docs/guides/macos-ci-runner.md | 38 +++++++- scripts/ci/macos_runner.py | 95 ++++++++++++++----- scripts/ci/test_macos_runner.py | 141 ++++++++++++++++++++++++++-- 5 files changed, 255 insertions(+), 33 deletions(-) diff --git a/.github/workflows/macos-trusted.yml b/.github/workflows/macos-trusted.yml index d0d35790c5..a619998fd8 100644 --- a/.github/workflows/macos-trusted.yml +++ b/.github/workflows/macos-trusted.yml @@ -168,6 +168,13 @@ jobs: ref: main path: _mac_control persist-credentials: false + - name: Enforce current hosted authorization + shell: bash + env: + PLAN: ${{ needs.plan.outputs.plan }} + ASTRA_MACOS_HOSTED_ADMITTED: ${{ needs.hosted-admission.outputs.allowed }} + run: | + python3 _mac_control/scripts/ci/macos_runner.py guard-hosted --plan "$PLAN" - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 with: ref: ${{ needs.plan.outputs.merge }} diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 272bba2769..17239db857 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -1,4 +1,11 @@ name: Test Suite +# API pull_requests and commit->PR links can both be empty for fork runs. +# Record the event tuple before checkout; the observer validates it against GitHub. +run-name: >- + ${{ github.event_name == 'pull_request' && + format('Test Suite PR {0} base={1} head={2} merge={3}', github.event.number, + github.event.pull_request.base.sha, github.event.pull_request.head.sha, github.sha) || + format('Test Suite {0}', github.sha) }} permissions: contents: read diff --git a/docs/guides/macos-ci-runner.md b/docs/guides/macos-ci-runner.md index ecaccdba9d..0e5210be59 100644 --- a/docs/guides/macos-ci-runner.md +++ b/docs/guides/macos-ci-runner.md @@ -98,6 +98,24 @@ missing/skipped selected lanes fails explicitly, including scope disagreement, instead of staying pending forever. This retains the pre-existing PR-modifiable test-definition limitation for non-member hosted tests. +Fork runs can have empty `pull_requests` arrays, and the commit-to-PR API can +also return no associations. Test Suite therefore sets `run-name` from GitHub's +PR number, event base/head and `github.sha` (the tested merge SHA), before any +checkout. Both observation and reconciliation read this tuple from authenticated +Actions metadata, fetch that PR directly, and verify repository/head identity, +current base and the immutable merge's exact ordered parents. Non-empty PR +associations must agree as well; an empty association alone never rejects a run. +No downloaded artifact, test stdout or current merge ref substitutes for the +recorded tested SHA. Completed historical runs without this tuple fail closed; +rerun the normal approved Test Suite after rollout. See GitHub's +[PR merge context](https://docs.github.com/en/actions/reference/workflows-and-actions/events-that-trigger-workflows#pull_request) +and [run-name expressions](https://docs.github.com/en/actions/reference/workflows-and-actions/workflow-syntax#run-name). + +This records the ordinary workflow's event context, not a signed attestation: +PR authors can modify that workflow and its tests, as before. It cannot grant +membership or authorize privileged execution. Member results still require the +protected path and fresh member/source checks. + The same metadata-only observer reconciles every 15 minutes (and supports manual dispatch), recovering lost completion notifications and PRs with no App checks at all. Missing or retryable stale plans dispatch only @@ -247,10 +265,17 @@ An online/idle API observation is not a reservation. A runner can disappear between dispatch and start; queued self-hosted work remains pending. An operator can cancel the stale run and use `workflow_dispatch` with its PR number for a fresh protected hosted run. That dispatch always selects hosted execution. -Failed-only hosted retries are refused when they reuse an older successful -admission job; rerun all jobs for new hosted authorization. Membership revocation -or a stale tuple blocks privileged-event execution; a fresh plan can choose the -ordinary approved hosted path. Genuine test failures are never hidden by the +Every hosted matrix job checks run ID, attempt, deadline and the protected +admission output using main-owned code **before** candidate checkout, toolchain, +cache setup or tests. It receives no App credential. Individual/failed-only +reruns that reuse successful dependencies fail this local fence even when cached +`allowed=true` remains present; rerun all jobs for new plan/admission. Queue delay +beyond authorization expiry also rejects execution. Publication additionally +requires the fence step to succeed and the actual run attempt to match the plan. +Membership/source are checked at protected admission; the local fence does not +claim a new membership API check on the execution host. Revocation or a stale +tuple found at admission blocks execution; a fresh plan can choose the ordinary +approved hosted path. Genuine test failures are never hidden by the infrastructure fallback. ## Enable, verify and roll back @@ -262,7 +287,10 @@ infrastructure fallback. cache write denial, pollution isolation and capacity on disposable fixtures. Include an initially null/stale merge, zero App checks, stale completion, lost/forbidden recovery dispatch, recovery throttling and all three parallel - hosted results. Recovery must never execute a non-member under the trusted + hosted results. Include individual hosted reruns with reused successful + admission outputs and expired authorization, and a fork run with empty PR and + commit associations that completes all three ordinary checks. Recovery must + never execute a non-member under the trusted event or treat a marker/timeout as a pass. 3. Create GitHub Environment `astra-macos-control`, allowing only branch `main` via its custom deployment branch policy, with no allowed tags or wildcards. diff --git a/scripts/ci/macos_runner.py b/scripts/ci/macos_runner.py index 749f2610ae..72c79d8118 100755 --- a/scripts/ci/macos_runner.py +++ b/scripts/ci/macos_runner.py @@ -30,6 +30,9 @@ SOURCE_ATTEMPTS = 12 RECOVERY_COOLDOWN_SECONDS = 30 * 60 HOSTED_LANES = ("default-reader", "reflow", "coordination") +ORDINARY_RUN_TITLE = re.compile( + r"Test Suite PR (?P[1-9][0-9]*) base=(?P[0-9a-f]{40}) " + r"head=(?P[0-9a-f]{40}) merge=(?P[0-9a-f]{40})") class Unavailable(Exception): @@ -321,6 +324,23 @@ def admit(api, request, config): return plan +def guard_hosted(plan): + """Local execution fence for reused successful dependency outputs. + + This needs no App token or candidate code. A new attempt must rerun trusted + plan + admission; queue delay must not outlive the admitted authorization. + """ + require_unexpired(plan) + events = ("pull_request_target", "workflow_dispatch") if plan["kind"] == "pr" else ("push",) + if (plan.get("schema") != 1 or plan["route"] not in ("hosted", "self-hosted") + or os.environ.get("GITHUB_REPOSITORY") != REPO + or os.environ.get("GITHUB_EVENT_NAME") not in events + or os.environ.get("ASTRA_MACOS_HOSTED_ADMITTED") != "true" + or int(os.environ["GITHUB_RUN_ID"]) != plan["run_id"] + or int(os.environ["GITHUB_RUN_ATTEMPT"]) != plan["attempt"]): + raise Unavailable("hosted execution requires fresh plan and admission for this attempt") + + def verified_job(api, plan, name): run = validate_run(api, plan) jobs = api.pages(f"/repos/{REPO}/actions/runs/{run['id']}/jobs?filter=latest", "jobs") @@ -334,7 +354,10 @@ def verified_job(api, plan, name): expected_step = "Run all three macOS lanes" if name == "Migrated macOS lanes" else "Run macOS lane" if steps.get(expected_step) != "success": return "failure" - if name.startswith("Fallback macOS:") and "macos-15" not in job.get("labels", []): + if name.startswith("Fallback macOS:") and ( + "macos-15" not in job.get("labels", []) + or steps.get("Enforce current hosted authorization") != "success" + or run["run_attempt"] != plan["attempt"]): return "failure" if name == "Migrated macOS lanes" and job["runner_group_id"] != plan["group_id"]: return "failure" @@ -351,14 +374,42 @@ def can_fallback(api, plan): for s in primary[0]["steps"])) +def ordinary_source(api, run): + """Bind ordinary execution to its event tuple, even for empty fork links. + + GitHub records run-name's github-context expressions in run metadata, before + checkout. Recheck that tuple against the canonical PR and immutable merge. + Ordinary PR workflow definitions remain PR-modifiable, as do their tests; + this metadata does not grant member trust or privileged execution. + """ + if (run["event"] != "pull_request" or run["path"] != ".github/workflows/test.yml" + or run["repository"]["full_name"] != REPO): + raise Unavailable("not an ordinary Astra Test Suite run") + match = ORDINARY_RUN_TITLE.fullmatch(run.get("display_title", "")) + if not match: + raise Unavailable("ordinary run lacks source tuple; rerun approved Test Suite") + current = source(api, int(match["pr"])) + if (any(match[key] != current[key] for key in ("base", "head", "merge")) + or run["head_sha"] != current["head"] + or run["head_repository"]["id"] != current["head_repository_id"]): + raise Unavailable("ordinary run source differs from current PR/merge") + refs = run.get("pull_requests", []) + if refs and not any(p["number"] == current["pr"] and p["head"]["sha"] == current["head"] + and p["base"]["sha"] == current["base"] for p in refs): + raise Unavailable("ordinary run has a conflicting PR association") + return current + + def hosted_result(api, plan): runs = api.pages(f"/repos/{REPO}/actions/workflows/test.yml/runs?event=pull_request&head_sha={plan['head']}", "workflow_runs") for run in sorted(runs, key=lambda r: r["id"], reverse=True): - if run["head_repository"]["id"] != plan["head_repository_id"]: + if run["head_repository"]["id"] != plan["head_repository_id"] or run["head_sha"] != plan["head"]: continue - refs = run.get("pull_requests", []) - if not any(p["number"] == plan["pr"] and p["head"]["sha"] == plan["head"] - and p["base"]["sha"] == plan["base"] for p in refs): + try: + current = ordinary_source(api, run) + except Unavailable: + return "failure" if run["status"] == "completed" else None + if not same_source(current, plan): continue jobs = api.pages(f"/repos/{REPO}/actions/runs/{run['id']}/jobs?filter=latest", "jobs") wanted = {"Terminal: default-reader (macos-15)", "Terminal: reflow (macos-15)", @@ -586,11 +637,14 @@ def reconcile(api, app_id): def main(): parser = argparse.ArgumentParser() - parser.add_argument("command", choices=("plan", "hint", "finish", "refresh", "admit", "observe", "reconcile")) + parser.add_argument("command", choices=("plan", "hint", "finish", "refresh", "admit", "observe", "reconcile", "guard-hosted")) parser.add_argument("--plan") parser.add_argument("--config") args = parser.parse_args() try: + if args.command == "guard-hosted": + guard_hosted(json.loads(args.plan)) + return 0 if args.command == "admit": signal.alarm(90) # Hooks themselves have no GitHub timeout. config = json.loads(Path(args.config).read_text()) @@ -614,22 +668,19 @@ def main(): run = api.get(f"/repos/{REPO}/actions/runs/{int(event['workflow_run']['id'])}") if run["event"] != "pull_request" or run["path"] != ".github/workflows/test.yml": return 0 - for pr in api.get(f"/repos/{REPO}/commits/{run['head_sha']}/pulls"): - if pr["head"]["sha"] != run["head_sha"]: - continue - current = source(api, pr["number"]) - rows = checks(api, current["head"]) - plans = [] - for row in rows: - if row.get("app", {}).get("id") == app_id and row["name"] == CONTEXTS[0]: - try: - p = json.loads(row["output"]["text"]) - if p["route"] == "ordinary" and same_source(p, current): - plans.append((row["id"], p)) - except (KeyError, TypeError, ValueError): - pass - if plans: - finish(api, max(plans, key=lambda p: p[0])[1]) + current = ordinary_source(api, run) + rows = checks(api, current["head"]) + plans = [] + for row in rows: + if row.get("app", {}).get("id") == app_id and row["name"] == CONTEXTS[0]: + try: + p = json.loads(row["output"]["text"]) + if p["route"] == "ordinary" and same_source(p, current): + plans.append((row["id"], p)) + except (KeyError, TypeError, ValueError): + pass + if plans: + finish(api, max(plans, key=lambda p: p[0])[1]) else: plan = json.loads(args.plan) if args.command == "finish": diff --git a/scripts/ci/test_macos_runner.py b/scripts/ci/test_macos_runner.py index b1fd6b64e0..0d9e134faa 100755 --- a/scripts/ci/test_macos_runner.py +++ b/scripts/ci/test_macos_runner.py @@ -12,11 +12,20 @@ from unittest.mock import patch import macos_runner as m +from test_release_build_shells import workflow_run_script ROOT = Path(__file__).resolve().parents[2] B, H, M = "b" * 40, "a" * 40, "c" * 40 +def ordinary_run(): + return {"id": 200, "status": "completed", "event": "pull_request", + "path": ".github/workflows/test.yml", "repository": {"full_name": m.REPO}, + "head_sha": H, "head_repository": {"id": 2}, "run_attempt": 1, + "display_title": f"Test Suite PR 42 base={B} head={H} merge={M}", + "pull_requests": [{"number": 42, "head": {"sha": H}, "base": {"sha": B}}]} + + class FakeAPI(m.API): def __init__(self): self.data = {} @@ -246,7 +255,8 @@ def test_real_test_failure_cannot_be_masked_by_a_successful_fallback(self): "steps": [{"name": "Run all three macOS lanes", "conclusion": "failure"}]} fallbacks = [{**failed, "name": "Fallback macOS: " + lane, "conclusion": "success", "labels": ["macos-15"], - "steps": [{"name": "Run macOS lane", "conclusion": "success"}]} for lane in m.HOSTED_LANES] + "steps": [{"name": "Enforce current hosted authorization", "conclusion": "success"}, + {"name": "Run macOS lane", "conclusion": "success"}]} for lane in m.HOSTED_LANES] self.api.data[self.run_path + "/jobs?filter=latest"] = {"jobs": [failed, *fallbacks]} self.assertFalse(m.can_fallback(self.api, self.plan)) m.finish(self.api, self.plan) @@ -289,8 +299,7 @@ def test_missing_hint_keeps_hosted_execution_and_app_hint_is_not_a_result(self): def test_ordinary_hosted_results_require_matching_source_all_three_jobs_and_real_macos(self): runs_path = f"/repos/{m.REPO}/actions/workflows/test.yml/runs?event=pull_request&head_sha={H}" - self.api.data[runs_path] = {"workflow_runs": [{"id": 200, "status": "completed", "head_repository": {"id": 2}, - "pull_requests": [{"number": 42, "head": {"sha": H}, "base": {"sha": B}}]}]} + self.api.data[runs_path] = {"workflow_runs": [ordinary_run()]} names = ("Terminal: default-reader (macos-15)", "Terminal: reflow (macos-15)", "macOS workspace coordination contracts") jobs = [{"name": name, "status": "completed", "conclusion": "success", "labels": ["macos-15"]} for name in names] @@ -308,6 +317,20 @@ def test_ordinary_hosted_results_require_matching_source_all_three_jobs_and_real self.api.data[runs_path]["workflow_runs"][0]["pull_requests"][0]["base"]["sha"] = "d" * 40 self.assertIsNone(m.hosted_result(self.api, self.plan)) + def test_hosted_result_cannot_accept_a_rerun_with_successful_cached_steps(self): + plan = {**self.plan, "route": "hosted"} + job = {"name": "Fallback macOS: default-reader", "status": "completed", "conclusion": "success", + "labels": ["macos-15"], "steps": [ + {"name": "Enforce current hosted authorization", "conclusion": "success"}, + {"name": "Run macOS lane", "conclusion": "success"}]} + self.api.data[self.run_path + "/jobs?filter=latest"] = {"jobs": [job]} + self.assertEqual(m.verified_job(self.api, plan, job["name"]), "success") + self.api.data[self.run_path]["run_attempt"] = 2 + self.assertEqual(m.verified_job(self.api, plan, job["name"]), "failure") + self.api.data[self.run_path]["run_attempt"] = 1 + job["steps"].pop(0) + self.assertEqual(m.verified_job(self.api, plan, job["name"]), "failure") + def test_main_stays_hosted_without_explicit_isolation_acceptance(self): with tempfile.TemporaryDirectory() as directory, patch.dict(os.environ, { "GITHUB_REPOSITORY": m.REPO, "GITHUB_EVENT_NAME": "push", @@ -386,14 +409,79 @@ def prepare_ordinary_reconciliation(self): self.api.data[f"/repos/{m.REPO}/pulls?state=open&base=main"] = [ {"number": 42, "head": {"sha": H}}] self.api.data[f"/repos/{m.REPO}/actions/workflows/test.yml/runs?event=pull_request&head_sha={H}"] = { - "workflow_runs": [{"id": 200, "status": "completed", "head_repository": {"id": 2}, - "pull_requests": [{"number": 42, "head": {"sha": H}, "base": {"sha": B}}]}]} + "workflow_runs": [ordinary_run()]} self.jobs200 = f"/repos/{m.REPO}/actions/runs/200/jobs?filter=latest" self.api.data[self.jobs200] = {"jobs": [ {"name": name, "status": "completed", "conclusion": "success", "labels": ["macos-15"]} for name in ("Terminal: default-reader (macos-15)", "Terminal: reflow (macos-15)", "macOS workspace coordination contracts")]} + def test_empty_fork_associations_complete_ordinary_checks(self): + self.prepare_ordinary_reconciliation() + run = self.api.data[f"/repos/{m.REPO}/actions/workflows/test.yml/runs?event=pull_request&head_sha={H}"]["workflow_runs"][0] + run["pull_requests"] = [] + self.assertEqual(m.hosted_result(self.api, self.plan), "success") + self.assertTrue(m.finish(self.api, self.plan)) + self.assertTrue(all(c["conclusion"] == "success" for c in self.api.checks)) + + def test_empty_fork_associations_work_through_observer_and_scheduled_reconciliation(self): + for entrypoint in ("observe", "reconcile"): + with self.subTest(entrypoint=entrypoint): + self.api.checks.clear() + self.prepare_ordinary_reconciliation() + run = self.api.data[f"/repos/{m.REPO}/actions/workflows/test.yml/runs?event=pull_request&head_sha={H}"]["workflow_runs"][0] + run["pull_requests"] = [] + self.api.data[f"/repos/{m.REPO}/actions/runs/200"] = run + # This endpoint is also empty for the real reviewed fork run. + self.api.data[f"/repos/{m.REPO}/commits/{H}/pulls"] = [] + with tempfile.TemporaryDirectory() as directory: + event = Path(directory) / "event.json" + event.write_text(json.dumps({"workflow_run": {"id": 200}})) + with ( + patch.dict(os.environ, {"GH_TOKEN": "fixture", "ASTRA_MACOS_APP_ID": "5", + "GITHUB_EVENT_PATH": str(event)}), + patch.object(m, "API", return_value=self.api), + patch.object(sys, "argv", ["macos_runner.py", entrypoint]), + ): + self.assertEqual(m.main(), 0) + self.assertTrue(all(c["conclusion"] == "success" for c in self.api.checks)) + self.assertFalse(any(path.endswith(f"/commits/{H}/pulls") for path, _ in self.api.calls)) + + def test_ordinary_empty_links_retain_repository_event_workflow_and_merge_identity(self): + self.prepare_ordinary_reconciliation() + original = ordinary_run() + original["pull_requests"] = [] + mutations = ( + lambda r: r.update(event="push"), + lambda r: r.update(path=m.WORKFLOW), + lambda r: r["repository"].update(full_name="fixture/other"), + lambda r: r["head_repository"].update(id=99), + lambda r: r.update(head_sha="d" * 40), + lambda r: r.update(display_title=r["display_title"].replace(B, "d" * 40)), + lambda r: r.update(display_title=r["display_title"].replace(H, "d" * 40)), + lambda r: r.update(display_title=r["display_title"].replace(M, "d" * 40)), + lambda r: r.update(display_title=r["display_title"].replace("PR 42", "PR 43")), + lambda r: r.update(display_title="Historical title without source metadata"), + lambda r: r.update(pull_requests=[{"number": 99, "head": {"sha": H}, "base": {"sha": B}}])) + for mutate in mutations: + with self.subTest(mutate=mutate): + run = copy.deepcopy(original) + mutate(run) + with self.assertRaises(m.Unavailable): + m.ordinary_source(self.api, run) + # A plausible run title cannot replace validation of the immutable merge. + self.api.data[f"/repos/{m.REPO}/git/commits/{M}"]["parents"] = [{"sha": H}, {"sha": B}] + with self.assertRaises(m.Unavailable): + m.ordinary_source(self.api, original) + + def test_completed_ordinary_run_without_source_tuple_fails_instead_of_pending(self): + self.prepare_ordinary_reconciliation() + run = self.api.data[f"/repos/{m.REPO}/actions/workflows/test.yml/runs?event=pull_request&head_sha={H}"]["workflow_runs"][0] + run["pull_requests"] = [] + del run["display_title"] + self.assertTrue(m.finish(self.api, self.plan)) + self.assertTrue(all(c["conclusion"] == "failure" for c in self.api.checks)) + def test_reconciliation_recovers_a_lost_completion_notification(self): self.prepare_ordinary_reconciliation() m.reconcile(self.api, 5) @@ -557,7 +645,8 @@ def test_stale_old_publisher_cannot_close_new_attempt(self): def test_parallel_hosted_requires_every_real_lane_and_retains_failures(self): plan = {**self.plan, "route": "hosted"} jobs = [{"name": "Fallback macOS: " + lane, "status": "completed", "conclusion": "success", - "labels": ["macos-15"], "steps": [{"name": "Run macOS lane", "conclusion": "success"}]} + "labels": ["macos-15"], "steps": [{"name": "Enforce current hosted authorization", "conclusion": "success"}, + {"name": "Run macOS lane", "conclusion": "success"}]} for lane in m.HOSTED_LANES] self.api.data[self.run_path + "/jobs?filter=latest"] = {"jobs": jobs} m.save_checks(self.api, plan) @@ -629,6 +718,46 @@ def test_canonical_commands_and_all_failures_propagate(self): if not fail and lane == "reflow": self.assertEqual([c[:2] for c in commands], [["cargo", "build"], ["npm", "ci"], ["npm", "test"]]) + def test_hosted_execution_rejects_reused_outputs_before_any_candidate_command(self): + workflow = (ROOT / ".github/workflows/macos-trusted.yml").read_text() + hosted = workflow.split("\n hosted:\n", 1)[1].split("\n publish:\n", 1)[0] + guard_name = "Enforce current hosted authorization" + self.assertIn(f"- name: {guard_name}", hosted) + self.assertLess(hosted.index(guard_name), hosted.index("path: _mac_source")) + guard = workflow_run_script(".github/workflows/macos-trusted.yml", guard_name) + execution = hosted.split(" - name: Run macOS lane\n", 1)[1].split(" run: ", 1)[1].splitlines()[0] + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + control = root / "_mac_control/scripts/ci" + control.mkdir(parents=True) + for name in ("macos_runner.py", "ci_scope.py", "run-macos-lanes.sh"): + (control / name).write_text((ROOT / "scripts/ci" / name).read_text()) + source = root / "_mac_source" + source.mkdir() + subprocess.run(["git", "init", str(source)], check=True, capture_output=True) + cargo = root / "cargo" + cargo.write_text('#!/bin/sh\ntouch "$CALLS"\n') + cargo.chmod(0o755) + for name, attempt, expiry, run_id, allowed, succeeds in ( + ("cached-successful-admission", "2", int(time.time()) + 3600, "100", "true", False), + ("expired-before-start", "1", 1, "100", "true", False), + ("another-run", "1", int(time.time()) + 3600, "101", "true", False), + ("not-admitted", "1", int(time.time()) + 3600, "100", "false", False), + ("fresh-full-rerun", "2", int(time.time()) + 3600, "100", "true", True)): + with self.subTest(name=name): + plan = {"schema": 1, "kind": "pr", "route": "hosted", "head": H, "merge": M, + "run_id": 100, "attempt": 2 if succeeds else 1, "expires_at": expiry} + calls = root / "calls" + calls.unlink(missing_ok=True) + result = subprocess.run(["bash", "-euo", "pipefail", "-c", guard + '\ncd _mac_source\n' + execution], + cwd=root, env={**os.environ, "PLAN": json.dumps(plan), "PATH": f"{root}:{os.environ['PATH']}", + "CALLS": str(calls), "MACOS_LANE": "coordination", "GITHUB_REPOSITORY": m.REPO, + "GITHUB_EVENT_NAME": "pull_request_target", "GITHUB_RUN_ID": run_id, + "GITHUB_RUN_ATTEMPT": attempt, "ASTRA_MACOS_HOSTED_ADMITTED": allowed, + "GH_TOKEN": "", "ASTRA_MACOS_APP_ID": ""}, capture_output=True, text=True) + self.assertEqual(result.returncode == 0, succeeds, result.stderr) + self.assertEqual(calls.exists(), succeeds, "candidate Cargo ran before rejection") + def test_hint_permissions_are_scoped_and_reconciliation_stays_metadata_only(self): workflow = (ROOT / ".github/workflows/test.yml").read_text() top = workflow.split("\non:", 1)[0] From 92ad6e4035c62f18e4992501233e115b6ad293a9 Mon Sep 17 00:00:00 2001 From: lr90 Date: Sat, 24 Oct 2026 12:16:53 +0800 Subject: [PATCH 05/11] test(cli): make PTY draft setup independent of paste timing --- crates/astra-cli/src/tui/terminal_startup_pty_tests.rs | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/crates/astra-cli/src/tui/terminal_startup_pty_tests.rs b/crates/astra-cli/src/tui/terminal_startup_pty_tests.rs index 38edc0d99f..b03b207ba5 100644 --- a/crates/astra-cli/src/tui/terminal_startup_pty_tests.rs +++ b/crates/astra-cli/src/tui/terminal_startup_pty_tests.rs @@ -187,9 +187,10 @@ fn probe_child() { pane.set_slash_items(vec![crate::tui::slash_menu::SlashItem::simple( "/skill", "skills", )]); - for ch in draft_command.chars() { - pane.handle_key(KeyEvent::new(KeyCode::Char(ch), KeyModifiers::NONE)); - } + // Seed the command explicitly: scheduler-sized gaps in synthetic + // key loops can activate paste detection and consume Enter. This + // case tests draft custody across commands, not typing cadence. + pane.replace_composer_text(draft_command); pane.restore_into_composer("//returned"); let BottomPaneAction::SubmitInput(command) = pane.handle_key(KeyEvent::new(KeyCode::Enter, KeyModifiers::NONE)) From 80ff991e8374719b0fb00b1e5833ec588b9afc0d Mon Sep 17 00:00:00 2001 From: lr90 Date: Sat, 24 Oct 2026 15:26:05 +0800 Subject: [PATCH 06/11] fix(deps): remove vulnerable crypto-bigint paths from JWT and MySQL --- Cargo.lock | 233 +++++------------------------------------------------ Cargo.toml | 4 +- 2 files changed, 21 insertions(+), 216 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 8170f69da8..7d8e524733 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -914,6 +914,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b281d307588d634de920874890732659e2e7672f72b5e10e81badc1a8a83621e" dependencies = [ "aws-lc-sys", + "untrusted 0.7.1", "zeroize", ] @@ -985,12 +986,6 @@ dependencies = [ "tracing", ] -[[package]] -name = "base16ct" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c7f02d4ea65f2c1853089ffd8d2787bdbc63de2f0d29dedbcf8ccdfa0ccd4cf" - [[package]] name = "base16ct" version = "1.0.0" @@ -1662,21 +1657,9 @@ checksum = "460fbee9c2c2f33933d720630a6a0bac33ba7053db5344fac858d4b8952d77d5" [[package]] name = "crypto-bigint" -version = "0.5.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0dc92fb57ca44df6db8059111ab3af99a63d5d0f8375d9972e319a379c6bab76" -dependencies = [ - "generic-array", - "rand_core 0.6.4", - "subtle", - "zeroize", -] - -[[package]] -name = "crypto-bigint" -version = "0.7.5" +version = "0.7.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a52aa3fcda4e6302a9f48734f234d35d4721b96f8fe07d073f07ce9df4f0271" +checksum = "6bb08299a6046833a879ace115697f12fd2bd6c8d3e93c909ffc9e0738e7cae0" dependencies = [ "cpubits", "ctutils", @@ -1711,7 +1694,7 @@ version = "0.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3633a51a39c69ebbaa4feaa694bd83d241e4093901c84a0963b19d9bb3f0cf8f" dependencies = [ - "crypto-bigint 0.7.5", + "crypto-bigint", "rand_core 0.10.1", ] @@ -1750,40 +1733,13 @@ dependencies = [ [[package]] name = "ctutils" -version = "0.4.4" +version = "0.4.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "12218662ccac37d9714f2191252553d67585b97df8c839cbe49e0bde278dc865" +checksum = "38d28fb6aa7dd680b3ffd1a3e64e115c177bdba63c0ae7f9cd48129ff05c4fa0" dependencies = [ "cmov", ] -[[package]] -name = "curve25519-dalek" -version = "4.1.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "97fb8b7c4503de7d6ae7b42ab72a5a59857b4c937ec27a3d4539dba95b5ab2be" -dependencies = [ - "cfg-if", - "cpufeatures 0.2.17", - "curve25519-dalek-derive", - "digest 0.10.7", - "fiat-crypto", - "rustc_version", - "subtle", - "zeroize", -] - -[[package]] -name = "curve25519-dalek-derive" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f46882e17999c6cc590af592290432be3bce0428cb0d5f8b6715e4dc7b383eb3" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.117", -] - [[package]] name = "darling" version = "0.23.0" @@ -2071,44 +2027,6 @@ version = "1.0.20" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d0881ea181b1df73ff77ffaaf9c7544ecc11e82fba9b5f27b262a3c73a332555" -[[package]] -name = "ecdsa" -version = "0.16.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ee27f32b5c5292967d2d4a9d7f1e0b0aed2c15daded5a60300e4abb9d8020bca" -dependencies = [ - "der 0.7.10", - "digest 0.10.7", - "elliptic-curve", - "rfc6979", - "signature 2.2.0", - "spki 0.7.3", -] - -[[package]] -name = "ed25519" -version = "2.2.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "115531babc129696a58c64a4fef0a8bf9e9698629fb97e9e40767d235cfbcd53" -dependencies = [ - "pkcs8 0.10.2", - "signature 2.2.0", -] - -[[package]] -name = "ed25519-dalek" -version = "2.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "70e796c081cee67dc755e1a36a0a172b897fab85fc3f6bc48307991f64e4eca9" -dependencies = [ - "curve25519-dalek", - "ed25519", - "serde", - "sha2 0.10.9", - "subtle", - "zeroize", -] - [[package]] name = "ego-tree" version = "0.11.0" @@ -2124,27 +2042,6 @@ dependencies = [ "serde", ] -[[package]] -name = "elliptic-curve" -version = "0.13.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b5e6043086bf7973472e0c7dff2142ea0b680d30e18d9cc40f267efbf222bd47" -dependencies = [ - "base16ct 0.2.0", - "crypto-bigint 0.5.5", - "digest 0.10.7", - "ff", - "generic-array", - "group", - "hkdf 0.12.4", - "pem-rfc7468 0.7.0", - "pkcs8 0.10.2", - "rand_core 0.6.4", - "sec1", - "subtle", - "zeroize", -] - [[package]] name = "encode_unicode" version = "1.0.0" @@ -2279,22 +2176,6 @@ dependencies = [ "zeroize", ] -[[package]] -name = "ff" -version = "0.13.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c0b50bfb653653f9ca9095b427bed08ab8d75a137839d9ad64eb11810d5b6393" -dependencies = [ - "rand_core 0.6.4", - "subtle", -] - -[[package]] -name = "fiat-crypto" -version = "0.2.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "28dea519a9695b9977216879a3ebfddf92f1c08c05d984f8996aecd6ecdc811d" - [[package]] name = "filedescriptor" version = "0.8.3" @@ -2540,7 +2421,6 @@ checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" dependencies = [ "typenum", "version_check", - "zeroize", ] [[package]] @@ -3200,17 +3080,6 @@ dependencies = [ "regex-syntax", ] -[[package]] -name = "group" -version = "0.13.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f0f9ef7462f7c099f518d754361858f86d8a07af53ba9af0fe635bbccb151a63" -dependencies = [ - "ff", - "rand_core 0.6.4", - "subtle", -] - [[package]] name = "h2" version = "0.4.16" @@ -3394,15 +3263,6 @@ dependencies = [ "tracing", ] -[[package]] -name = "hkdf" -version = "0.12.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b5f8eb2ad728638ea2c7d47a21db23b7b58a72ed6a38256b8a1849f15fbbdf7" -dependencies = [ - "hmac 0.12.1", -] - [[package]] name = "hkdf" version = "0.13.0" @@ -4015,18 +3875,12 @@ version = "10.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "eba32bfb4ffdeaca3e34431072faf01745c9b26d25504aa7a6cf5684334fc4fc" dependencies = [ + "aws-lc-rs", "base64", - "ed25519-dalek", "getrandom 0.2.17", - "hmac 0.12.1", "js-sys", - "p256", - "p384", - "rand 0.8.6", - "rsa 0.9.10", "serde", "serde_json", - "sha2 0.10.9", "signature 2.2.0", "zeroize", ] @@ -4707,30 +4561,6 @@ dependencies = [ "num-traits", ] -[[package]] -name = "p256" -version = "0.13.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c9863ad85fa8f4460f9c48cb909d38a0d689dba1f6f6988a5e3e0d31071bcd4b" -dependencies = [ - "ecdsa", - "elliptic-curve", - "primeorder", - "sha2 0.10.9", -] - -[[package]] -name = "p384" -version = "0.13.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fe42f1670a52a47d448f14b6a5c61dd78fce51856e68edaa38f7ae3a46b8d6b6" -dependencies = [ - "ecdsa", - "elliptic-curve", - "primeorder", - "sha2 0.10.9", -] - [[package]] name = "palette" version = "0.7.6" @@ -5120,15 +4950,6 @@ dependencies = [ "syn 2.0.117", ] -[[package]] -name = "primeorder" -version = "0.13.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "353e1ca18966c16d9deb1c69278edbc5f194139612772bd9537af60ac231e1e6" -dependencies = [ - "elliptic-curve", -] - [[package]] name = "proc-macro-crate" version = "3.5.0" @@ -5712,16 +5533,6 @@ version = "0.7.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1e061d1b48cb8d38042de4ae0a7a6401009d6143dc80d2e2d6f31f0bdd6470c7" -[[package]] -name = "rfc6979" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f8dd2a808d456c4a54e300a23e9f5a67e122c3024119acbfd73e3bf664491cb2" -dependencies = [ - "hmac 0.12.1", - "subtle", -] - [[package]] name = "ring" version = "0.17.14" @@ -5732,7 +5543,7 @@ dependencies = [ "cfg-if", "getrandom 0.2.17", "libc", - "untrusted", + "untrusted 0.9.0", "windows-sys 0.52.0", ] @@ -5813,7 +5624,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "33405e321e16ef941ca66e254f6904c3aafd3c7ff001fb88bf11c73eb75c2b85" dependencies = [ "const-oid 0.10.2", - "crypto-bigint 0.7.5", + "crypto-bigint", "crypto-primes", "digest 0.11.3", "pkcs1 0.8.0-rc.5", @@ -5936,7 +5747,7 @@ dependencies = [ "aws-lc-rs", "ring", "rustls-pki-types", - "untrusted", + "untrusted 0.9.0", ] [[package]] @@ -6050,20 +5861,6 @@ dependencies = [ "tendril", ] -[[package]] -name = "sec1" -version = "0.7.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d3e97a565f76233a6003f9f5c54be1d9c5bdfa3eccfb189469f11ec4901c47dc" -dependencies = [ - "base16ct 0.2.0", - "der 0.7.10", - "generic-array", - "pkcs8 0.10.2", - "subtle", - "zeroize", -] - [[package]] name = "security-framework" version = "3.7.0" @@ -6217,7 +6014,7 @@ version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "66cf8fedced2fcf12406bcb34223dffb92eaf34908ede12fed414c82b7f00b3e" dependencies = [ - "base16ct 1.0.0", + "base16ct", "serde", ] @@ -6612,7 +6409,7 @@ dependencies = [ "futures-core", "futures-util", "hex", - "hkdf 0.13.0", + "hkdf", "hmac 0.13.0", "itoa", "log", @@ -7704,6 +7501,12 @@ version = "0.2.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "673aac59facbab8a9007c7f6108d11f63b603f7cabff99fabf650fea5c32b861" +[[package]] +name = "untrusted" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a156c684c91ea7d62626509bce3cb4e1d9ed5c4d978f7b4352658f96a4c26b4a" + [[package]] name = "untrusted" version = "0.9.0" diff --git a/Cargo.toml b/Cargo.toml index 987c8f9856..57ae577e9a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -73,7 +73,9 @@ fs2 = "0.4" tar = "0.4" futures-util = "0.3.31" hmac = "0.12.1" -jsonwebtoken = { version = "10.3.0", default-features = false, features = ["rust_crypto"] } +# Reuse AWS-LC: the RustCrypto backend pins crypto-bigint 0.5, affected by +# RUSTSEC-2026-0387 with no compatible patched release. +jsonwebtoken = { version = "10.3.0", default-features = false, features = ["aws_lc_rs"] } nix = { version = "0.31.2", features = ["fs", "signal", "term"] } regex = "1.12.2" reqwest = { version = "0.12.23", default-features = false, features = ["json", "rustls-tls", "rustls-tls-native-roots", "stream", "socks"] } From aedcee446b7610c0c4ee3b63efe38ceb6997e7a1 Mon Sep 17 00:00:00 2001 From: lr90 Date: Sat, 24 Oct 2026 16:16:53 +0800 Subject: [PATCH 07/11] ci: route all macOS test jobs directly to self-hosted runners --- .github/actionlint.yaml | 11 +- .github/actions/rust-setup/action.yml | 8 +- .github/workflows/macos-results.yml | 54 -- .github/workflows/macos-trusted.yml | 246 -------- .github/workflows/test.yml | 142 +++-- CONTRIBUTING.md | 14 +- docs/guides/macos-ci-runner.md | 333 ----------- docs/guides/repository-automation.md | 40 +- scripts/ci/macos-job-started.sh | 15 - scripts/ci/macos-runner-admit | 12 - scripts/ci/macos-runner-preflight.sh | 12 - scripts/ci/macos_runner.py | 701 ---------------------- scripts/ci/run-macos-lanes.sh | 45 -- scripts/ci/test_ci_build_coverage.py | 59 +- scripts/ci/test_ci_parallelism.py | 31 +- scripts/ci/test_macos_runner.py | 801 -------------------------- 16 files changed, 178 insertions(+), 2346 deletions(-) delete mode 100644 .github/workflows/macos-results.yml delete mode 100644 .github/workflows/macos-trusted.yml delete mode 100644 docs/guides/macos-ci-runner.md delete mode 100755 scripts/ci/macos-job-started.sh delete mode 100755 scripts/ci/macos-runner-admit delete mode 100755 scripts/ci/macos-runner-preflight.sh delete mode 100755 scripts/ci/macos_runner.py delete mode 100755 scripts/ci/run-macos-lanes.sh delete mode 100755 scripts/ci/test_macos_runner.py diff --git a/.github/actionlint.yaml b/.github/actionlint.yaml index 02d48fffcf..af57fe46ac 100644 --- a/.github/actionlint.yaml +++ b/.github/actionlint.yaml @@ -1,11 +1,2 @@ self-hosted-runner: - labels: [mac-arm64-mini, astra-macos-admitted] -paths: - .github/workflows/macos-trusted.yml: - ignore: - # GitHub supports cache-mode since 2026-09-10; actionlint 1.7.12 does not. - # Remove only this narrow compatibility filter when its parser catches up. - - '^unexpected key "cache-mode" for "workflow" section\.' - .github/workflows/macos-results.yml: - ignore: - - '^unexpected key "cache-mode" for "workflow" section\.' + labels: [mac-arm64-mini] diff --git a/.github/actions/rust-setup/action.yml b/.github/actions/rust-setup/action.yml index 51aee2cbb1..c0b961a0e7 100644 --- a/.github/actions/rust-setup/action.yml +++ b/.github/actions/rust-setup/action.yml @@ -27,9 +27,12 @@ runs: using: "composite" steps: # Avoid paying for cleanup when the filesystem containing the workspace has - # enough room. Keep the cleanup fallback for low or unreadable disk space. + # enough room. Only hosted images use the cleanup fallback; persistent + # self-hosted runners report low/unreadable disk without privileged cleanup. - name: Guard workspace disk space shell: bash + env: + ASTRA_CI_RUNNER_ENVIRONMENT: ${{ runner.environment }} run: | set -euxo pipefail disk_guard_workspace="${GITHUB_WORKSPACE:?GITHUB_WORKSPACE must be set}" @@ -51,6 +54,9 @@ runs: if [[ "${disk_guard_measurement}" == "valid" ]] && (( disk_guard_available_kib >= disk_guard_threshold_kib )); then echo "Disk guard decision: skip cleanup; workspace filesystem free=${disk_guard_available_kib} KiB; threshold=${disk_guard_threshold_kib} KiB (30 GiB)." + elif [[ "${ASTRA_CI_RUNNER_ENVIRONMENT:-}" == self-hosted ]]; then + echo "::error::Disk guard decision: fail self-hosted; measurement=${disk_guard_measurement}; workspace filesystem free=${disk_guard_available_kib:-unavailable} KiB; threshold=${disk_guard_threshold_kib} KiB (30 GiB). Reclaim runner disk before retrying." + exit 1 else echo "Disk guard decision: run cleanup; measurement=${disk_guard_measurement}; workspace filesystem free=${disk_guard_available_kib:-unavailable} KiB; threshold=${disk_guard_threshold_kib} KiB (30 GiB)." sudo rm -rf /usr/share/dotnet diff --git a/.github/workflows/macos-results.yml b/.github/workflows/macos-results.yml deleted file mode 100644 index 4e1316b265..0000000000 --- a/.github/workflows/macos-results.yml +++ /dev/null @@ -1,54 +0,0 @@ -name: Mac CI result observer -on: - workflow_run: - workflows: [Test Suite] - types: [completed] - schedule: - - cron: '*/15 * * * *' - workflow_dispatch: -permissions: - contents: read -cache-mode: none -concurrency: - group: mac-ci-result-controller - cancel-in-progress: false -jobs: - observe: - if: github.repository == 'matrixorigin/Astra' && vars.ASTRA_MACOS_MIGRATION_ENABLED == 'true' - runs-on: ubuntu-latest - environment: astra-macos-control - timeout-minutes: 5 - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 - with: - ref: main - persist-credentials: false - - uses: actions/create-github-app-token@64ab729a6294000f5aa4a4bc0b73d1f32bc45510 # v3 - id: app - with: - app-id: ${{ vars.ASTRA_MACOS_APP_ID }} - private-key: ${{ secrets.ASTRA_MACOS_APP_PRIVATE_KEY }} - owner: matrixorigin - repositories: Astra - permission-members: read - permission-organization-self-hosted-runners: read - permission-administration: read - # Only metadata control can dispatch the fixed main workflow. - permission-actions: write - permission-contents: read - permission-pull-requests: read - permission-checks: write - permission-environments: read - # No PR checkout, downloaded artifacts, code execution or cached scripts. - - name: Observe ordinary hosted results - if: github.event_name == 'workflow_run' - env: - GH_TOKEN: ${{ steps.app.outputs.token }} - ASTRA_MACOS_APP_ID: ${{ vars.ASTRA_MACOS_APP_ID }} - run: python3 scripts/ci/macos_runner.py observe - - name: Reconcile missing and expired results - if: github.event_name != 'workflow_run' - env: - GH_TOKEN: ${{ steps.app.outputs.token }} - ASTRA_MACOS_APP_ID: ${{ vars.ASTRA_MACOS_APP_ID }} - run: python3 scripts/ci/macos_runner.py reconcile diff --git a/.github/workflows/macos-trusted.yml b/.github/workflows/macos-trusted.yml deleted file mode 100644 index a619998fd8..0000000000 --- a/.github/workflows/macos-trusted.yml +++ /dev/null @@ -1,246 +0,0 @@ -name: Mac CI - -on: - pull_request_target: - branches: [main] - types: [opened, synchronize, reopened, ready_for_review] - push: - branches: [main] - workflow_dispatch: - inputs: - pr_number: - description: PR to re-authorize with hosted routing (non-members keep ordinary PR approval) - required: true - type: string - -permissions: - contents: read -cache-mode: read -concurrency: - group: mac-ci-${{ github.event.pull_request.number || inputs.pr_number || github.ref }} - cancel-in-progress: true - -env: - CARGO_INCREMENTAL: "0" - CARGO_TERM_COLOR: always - ASTRA_MACOS_APP_ID: ${{ vars.ASTRA_MACOS_APP_ID }} - -jobs: - plan: - if: >- - github.repository == 'matrixorigin/Astra' && - vars.ASTRA_MACOS_MIGRATION_ENABLED == 'true' && - (github.event_name != 'workflow_dispatch' || github.ref == 'refs/heads/main') - runs-on: ubuntu-latest - environment: astra-macos-control - timeout-minutes: 5 - outputs: - plan: ${{ steps.plan.outputs.plan }} - route: ${{ steps.plan.outputs.route }} - merge: ${{ steps.plan.outputs.merge }} - kind: ${{ steps.plan.outputs.kind }} - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 - with: - ref: main - persist-credentials: false - - uses: actions/create-github-app-token@64ab729a6294000f5aa4a4bc0b73d1f32bc45510 # v3 - id: app - with: - app-id: ${{ vars.ASTRA_MACOS_APP_ID }} - private-key: ${{ secrets.ASTRA_MACOS_APP_PRIVATE_KEY }} - owner: matrixorigin - repositories: Astra - permission-members: read - permission-organization-self-hosted-runners: read - permission-administration: read - permission-actions: read - permission-contents: read - permission-pull-requests: read - permission-checks: write - permission-environments: read - - id: plan - name: Verify source, membership and enforcement - env: - GH_TOKEN: ${{ steps.app.outputs.token }} - ASTRA_MACOS_GROUP_ID: ${{ vars.ASTRA_MACOS_GROUP_ID }} - ASTRA_MACOS_MAIN_GROUP_ID: ${{ vars.ASTRA_MACOS_MAIN_GROUP_ID }} - ASTRA_MACOS_MAIN_ISOLATED: ${{ vars.ASTRA_MACOS_MAIN_ISOLATED }} - run: python3 scripts/ci/macos_runner.py plan - - mini: - name: Migrated macOS lanes - needs: plan - if: needs.plan.outputs.route == 'self-hosted' - runs-on: - group: astra-macos-${{ needs.plan.outputs.kind }} - labels: [self-hosted, macOS, ARM64, mac-arm64-mini, astra-macos-admitted] - timeout-minutes: 90 - permissions: - contents: read - steps: - # The root-owned job-started hook runs before even this step, including - # individual/failed-only reruns. Its separate credential owner is mandatory. - - name: Verify admitted platform and capacity - timeout-minutes: 2 - shell: bash - run: /opt/astra-ci/bin/preflight - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 - with: - ref: main - path: _mac_control - persist-credentials: false - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 - with: - ref: ${{ needs.plan.outputs.merge }} - path: _mac_source - persist-credentials: false - - uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7 - with: - node-version-file: _mac_source/.nvmrc - - name: Run all three macOS lanes - shell: bash - working-directory: _mac_source - env: - CARGO_TARGET_DIR: ${{ runner.temp }}/astra-macos-${{ github.run_id }}-${{ github.run_attempt }}/target - npm_config_cache: ${{ runner.temp }}/astra-macos-${{ github.run_id }}-${{ github.run_attempt }}/npm - run: bash ../_mac_control/scripts/ci/run-macos-lanes.sh all - - name: Remove job artifacts - shell: bash - if: always() - run: rm -rf "${RUNNER_TEMP}/astra-macos-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" - - hosted-admission: - needs: [plan, mini] - if: >- - !cancelled() && needs.plan.result == 'success' && - (needs.plan.outputs.route == 'hosted' || - (needs.plan.outputs.route == 'self-hosted' && needs.mini.result == 'failure')) - runs-on: ubuntu-latest - environment: astra-macos-control - timeout-minutes: 5 - outputs: - allowed: ${{ steps.admission.outputs.allowed }} - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 - with: - ref: main - persist-credentials: false - - uses: actions/create-github-app-token@64ab729a6294000f5aa4a4bc0b73d1f32bc45510 # v3 - id: app - with: - app-id: ${{ vars.ASTRA_MACOS_APP_ID }} - private-key: ${{ secrets.ASTRA_MACOS_APP_PRIVATE_KEY }} - owner: matrixorigin - repositories: Astra - permission-members: read - permission-organization-self-hosted-runners: read - permission-administration: read - permission-actions: read - permission-contents: read - permission-pull-requests: read - permission-checks: write - permission-environments: read - - id: admission - name: Fresh authorization for hosted handoff - env: - GH_TOKEN: ${{ steps.app.outputs.token }} - PLAN: ${{ needs.plan.outputs.plan }} - run: python3 scripts/ci/macos_runner.py refresh --plan "$PLAN" - - hosted: - name: "Fallback macOS: ${{ matrix.lane }}" - needs: [plan, hosted-admission] - if: >- - !cancelled() && needs.hosted-admission.result == 'success' && - needs.hosted-admission.outputs.allowed == 'true' - runs-on: macos-15 - timeout-minutes: 45 - strategy: - fail-fast: false - matrix: - lane: [default-reader, reflow, coordination] - permissions: - contents: read - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 - with: - ref: main - path: _mac_control - persist-credentials: false - - name: Enforce current hosted authorization - shell: bash - env: - PLAN: ${{ needs.plan.outputs.plan }} - ASTRA_MACOS_HOSTED_ADMITTED: ${{ needs.hosted-admission.outputs.allowed }} - run: | - python3 _mac_control/scripts/ci/macos_runner.py guard-hosted --plan "$PLAN" - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 - with: - ref: ${{ needs.plan.outputs.merge }} - path: _mac_source - persist-credentials: false - - name: Select candidate Rust toolchain - working-directory: _mac_source - shell: bash - run: | - rustc --version - echo "RUSTUP_TOOLCHAIN=$(rustup show active-toolchain | cut -d ' ' -f 1)" >> "$GITHUB_ENV" - - uses: mozilla-actions/sccache-action@9e7fa8a12102821edf02ca5dbea1acd0f89a2696 # v0.0.10 - id: sccache - continue-on-error: true - - name: Configure read-only compiler cache - if: steps.sccache.outcome == 'success' - working-directory: _mac_source - shell: bash - run: bash ../_mac_control/scripts/ci/configure-sccache.sh - # Restore only. Candidate code cannot write the base branch cache. - - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2 - continue-on-error: true - with: - workspaces: _mac_source -> target - shared-key: ci-terminal-pty-main - save-if: "false" - - uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7 - if: matrix.lane == 'reflow' - with: - node-version-file: _mac_source/.nvmrc - - name: Run macOS lane - shell: bash - working-directory: _mac_source - env: - npm_config_cache: ${{ runner.temp }}/astra-macos-${{ github.run_id }}-${{ github.run_attempt }}/npm - MACOS_LANE: ${{ matrix.lane }} - run: bash ../_mac_control/scripts/ci/run-macos-lanes.sh "$MACOS_LANE" - - publish: - needs: [plan, mini, hosted-admission, hosted] - if: always() && needs.plan.result == 'success' - runs-on: ubuntu-latest - environment: astra-macos-control - timeout-minutes: 5 - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 - with: - ref: main - persist-credentials: false - - uses: actions/create-github-app-token@64ab729a6294000f5aa4a4bc0b73d1f32bc45510 # v3 - id: app - with: - app-id: ${{ vars.ASTRA_MACOS_APP_ID }} - private-key: ${{ secrets.ASTRA_MACOS_APP_PRIVATE_KEY }} - owner: matrixorigin - repositories: Astra - permission-members: read - permission-organization-self-hosted-runners: read - permission-administration: read - permission-actions: read - permission-contents: read - permission-pull-requests: read - permission-checks: write - permission-environments: read - - name: Publish verified App-owned results - env: - GH_TOKEN: ${{ steps.app.outputs.token }} - PLAN: ${{ needs.plan.outputs.plan }} - run: python3 scripts/ci/macos_runner.py finish --plan "$PLAN" diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 17239db857..7741c83add 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -1,11 +1,4 @@ name: Test Suite -# API pull_requests and commit->PR links can both be empty for fork runs. -# Record the event tuple before checkout; the observer validates it against GitHub. -run-name: >- - ${{ github.event_name == 'pull_request' && - format('Test Suite PR {0} base={1} head={2} merge={3}', github.event.number, - github.event.pull_request.base.sha, github.event.pull_request.head.sha, github.sha) || - format('Test Suite {0}', github.sha) }} permissions: contents: read @@ -20,6 +13,10 @@ concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} cancel-in-progress: true +defaults: + run: + shell: bash + env: CARGO_TERM_COLOR: always CARGO_INCREMENTAL: "0" @@ -27,8 +24,6 @@ env: jobs: scope: name: Determine test scope - permissions: - contents: read runs-on: ubuntu-latest timeout-minutes: 5 outputs: @@ -49,38 +44,6 @@ jobs: id: scope run: python3 scripts/ci/ci_scope.py - macos-routing: - name: Read macOS delegation - needs: scope - if: >- - !cancelled() && vars.ASTRA_MACOS_MIGRATION_ENABLED == 'true' && - (needs.scope.outputs.test_cli == 'true' || needs.scope.outputs.test_core == 'true') - runs-on: ubuntu-latest - timeout-minutes: 3 - permissions: - contents: read - checks: read - pull-requests: read - actions: read - outputs: - macos_delegated: ${{ steps.mac.outputs.delegated }} - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 - with: - persist-credentials: false - - name: Read trusted macOS execution hint - id: mac - if: >- - vars.ASTRA_MACOS_MIGRATION_ENABLED == 'true' && - (needs.scope.outputs.test_cli == 'true' || needs.scope.outputs.test_core == 'true') - timeout-minutes: 2 - continue-on-error: true - env: - GH_TOKEN: ${{ github.token }} - ASTRA_MACOS_APP_ID: ${{ vars.ASTRA_MACOS_APP_ID }} - # A missing/forged hint cannot satisfy the App-owned required results. - run: python3 scripts/ci/macos_runner.py hint - # Package groups remain parallel. CLI builds and executes its full inventory # once; stable protected labels require that same completed test result. # Short CLI partitions are not the workflow critical path, so separate @@ -232,7 +195,7 @@ jobs: name: "Test: vendored terminal (${{ matrix.os }})" needs: scope if: ${{ !cancelled() && (needs.scope.result != 'success' || needs.scope.outputs.test_cli == 'true' || needs.scope.outputs.test_core == 'true') }} - runs-on: ${{ matrix.os }} + runs-on: ${{ matrix.os == 'macos-15' && fromJSON('["self-hosted","macOS","ARM64","mac-arm64-mini"]') || matrix.os }} timeout-minutes: 15 strategy: fail-fast: false @@ -251,7 +214,7 @@ jobs: cargo test --locked --manifest-path vendor/crossterm/Cargo.toml --lib --features event-stream cargo test --locked --manifest-path vendor/crossterm/Cargo.toml --lib --features event-stream,use-dev-tty - # Preserve required check names while three builds run on separate runners. + # Preserve required check names while the reader/reflow builds stay isolated. # Both aggregate checks require the entire terminal matrix. terminal-pty: name: "Test: terminal PTY (${{ matrix.os }})" @@ -279,11 +242,9 @@ jobs: terminal-pty-shards: name: "Terminal: ${{ matrix.segment }} (${{ matrix.os }})" - needs: [scope, macos-routing] + needs: scope if: ${{ !cancelled() && (needs.scope.result != 'success' || needs.scope.outputs.test_cli == 'true' || needs.scope.outputs.test_core == 'true') }} - runs-on: ${{ matrix.os == 'macos-15' && matrix.segment != 'dev-tty-reader' && needs.macos-routing.outputs.macos_delegated == 'true' && 'ubuntu-latest' || matrix.os }} - env: - RUN_LANE: ${{ !(matrix.os == 'macos-15' && matrix.segment != 'dev-tty-reader' && needs.macos-routing.outputs.macos_delegated == 'true') }} + runs-on: ${{ matrix.os == 'macos-15' && fromJSON('["self-hosted","macOS","ARM64","mac-arm64-mini"]') || matrix.os }} timeout-minutes: 45 strategy: fail-fast: false @@ -291,34 +252,50 @@ jobs: os: [ubuntu-latest, macos-15] segment: [default-reader, dev-tty-reader, reflow] steps: - - name: Delegated to App-verified macOS execution - if: env.RUN_LANE != 'true' - run: | - echo 'Requested macOS lane delegated to protected Mac CI; this Ubuntu job only reports delegation.' - echo 'Execution results are the three App-owned Mac CI checks. The platform in this legacy check name is the requested platform.' >> "$GITHUB_STEP_SUMMARY" - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 - if: env.RUN_LANE == 'true' with: persist-credentials: false - uses: ./.github/actions/rust-setup - if: env.RUN_LANE == 'true' with: shared-key: ci-terminal-pty + # One writer per OS populates the shared dependency cache on pushes. save-cache: ${{ github.event_name == 'push' && matrix.segment == 'default-reader' }} install-nextest: false + # Separate Cargo invocations keep dev-tty feature unification out of + # the default reader; one self-hosted registration queues these jobs. + - name: Test default terminal reader through PTYs + if: matrix.segment == 'default-reader' + run: | + cargo test --locked -p astra-cli --lib tui::terminal_startup -- --test-threads=2 + - name: Initialize system Git for macOS CLI observation + if: matrix.os == 'macos-15' && matrix.segment == 'default-reader' + timeout-minutes: 1 + run: | + time /usr/bin/env -i PATH=/usr/bin:/bin HOME=/nonexistent LANG=C LC_ALL=C \ + GIT_CONFIG_NOSYSTEM=1 GIT_CONFIG_GLOBAL=/dev/null GIT_CONFIG_SYSTEM=/dev/null \ + GIT_OPTIONAL_LOCKS=0 GIT_TERMINAL_PROMPT=0 \ + /usr/bin/git -C "$GITHUB_WORKSPACE" -c core.fsmonitor=false \ + -c core.untrackedCache=false rev-parse --show-toplevel + - name: Test macOS CLI Bash execution + if: matrix.os == 'macos-15' && matrix.segment == 'default-reader' + # Reuse the default-feature CLI test executable already built above. + run: >- + cargo test --locked --manifest-path Cargo.toml -p astra-cli + --lib edge_tools::shell::tests::bash_echo_returns_output -- --exact + - name: Test dev-tty terminal reader through PTYs + if: matrix.segment == 'dev-tty-reader' + run: | + cargo test --locked -p astra-cli --lib --features crossterm/use-dev-tty tui::terminal_startup -- --test-threads=2 - uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7 - if: env.RUN_LANE == 'true' && matrix.segment == 'reflow' + if: matrix.segment == 'reflow' with: node-version-file: .nvmrc - - name: Test default terminal reader and macOS CLI Bash - if: env.RUN_LANE == 'true' && matrix.segment == 'default-reader' - run: bash scripts/ci/run-macos-lanes.sh default-reader - - name: Test dev-tty terminal reader through PTYs - if: matrix.segment == 'dev-tty-reader' - run: cargo test --locked -p astra-cli --lib --features crossterm/use-dev-tty tui::terminal_startup -- --test-threads=2 - name: Test inline terminal resize with xterm reflow - if: env.RUN_LANE == 'true' && matrix.segment == 'reflow' - run: bash scripts/ci/run-macos-lanes.sh reflow + if: matrix.segment == 'reflow' + run: | + cargo build --locked -p astra-cli --bin astra + npm ci --ignore-scripts --prefix scripts/tui-reflow + ASTRA_TEST_BINARY="$PWD/target/debug/astra" npm test --prefix scripts/tui-reflow macos-workspace-coordination: name: "Test: macOS workspace coordination" @@ -345,31 +322,44 @@ jobs: macos-workspace-coordination-tests: name: "macOS workspace coordination contracts" - needs: [scope, macos-routing] + needs: scope if: ${{ !cancelled() && (needs.scope.result != 'success' || needs.scope.outputs.test_core == 'true' || needs.scope.outputs.test_cli == 'true') }} - runs-on: ${{ needs.macos-routing.outputs.macos_delegated == 'true' && 'ubuntu-latest' || 'macos-15' }} - env: - RUN_LANE: ${{ needs.macos-routing.outputs.macos_delegated != 'true' }} + runs-on: [self-hosted, macOS, ARM64, mac-arm64-mini] timeout-minutes: 30 steps: - - name: Delegated to App-verified macOS execution - if: env.RUN_LANE != 'true' - run: | - echo 'Requested macOS lane delegated to protected Mac CI; this Ubuntu job only reports delegation.' - echo 'Execution results are the three App-owned Mac CI checks. The platform in this legacy check name is the requested platform.' >> "$GITHUB_STEP_SUMMARY" - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 - if: env.RUN_LANE == 'true' with: persist-credentials: false - uses: ./.github/actions/rust-setup - if: env.RUN_LANE == 'true' with: + # Reuse the CLI dependency cache; default-reader is its only writer. shared-key: ci-terminal-pty save-cache: "false" install-nextest: "false" + - name: Lint macOS workspace coordination + run: >- + cargo clippy --locked --manifest-path Cargo.toml -p astra-tools + --all-targets -- -D warnings + - name: Initialize system Git for bounded observation probes + timeout-minutes: 1 + # Checkout uses Homebrew Git, but the hardened observer selects + # /usr/bin/git with a cleared environment. Initialize that executable + # before its first use under the production two-second probe deadline. + # Keep the timing visible so runner startup failures are diagnosable. + run: | + time /usr/bin/env -i PATH=/usr/bin:/bin HOME=/nonexistent LANG=C LC_ALL=C \ + GIT_CONFIG_NOSYSTEM=1 GIT_CONFIG_GLOBAL=/dev/null GIT_CONFIG_SYSTEM=/dev/null \ + GIT_OPTIONAL_LOCKS=0 GIT_TERMINAL_PROMPT=0 \ + /usr/bin/git -C "$GITHUB_WORKSPACE" -c core.fsmonitor=false \ + -c core.untrackedCache=false rev-parse --show-toplevel - name: Test macOS workspace coordination invariants - if: env.RUN_LANE == 'true' - run: bash scripts/ci/run-macos-lanes.sh coordination + # Several invariants launch re-exec lock contenders while fingerprint + # probes deliberately have a short wall-clock deadline. Keep the module + # serial on shared CI hosts; the contender test supplies explicit + # cross-process concurrency without starving unrelated probes. + run: >- + cargo test --locked --manifest-path Cargo.toml -p astra-tools + workspace_observation::tests:: -- --test-threads=1 # Preserve the required core check while its package groups run separately. # The aggregate requires all selected online jobs, including integration. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index c3e8261768..ee9235851d 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -116,13 +116,13 @@ strict profile without retries, preserving the complete inventory's CI JUnit file and avoiding three separate Cargo test builds. On macOS, the default terminal-reader lane also runs the CLI Bash regression using its existing default-feature test executable. Tools lint and serial -workspace-coordination tests remain parallel; the protected coordination check -requires both paths to succeed. Feature-specific terminal tests remain separate. -The opt-in [self-hosted macOS migration](docs/guides/macos-ci-runner.md) runs -three lanes together for verified organization-member PRs. Its required results -come from a dedicated App; ordinary hosted delegation hints cannot satisfy them. -Non-members and unknown identities retain ordinary hosted PR approval. Main stays -hosted until an independently isolated main runner is accepted. +workspace-coordination tests retain independent jobs; the protected coordination +check requires both paths to succeed. Feature-specific terminal tests remain separate. +All macOS execution jobs in Test Suite use the organization's self-hosted ARM +runner labels for both pull requests and main/develop pushes, without author +classification. Existing fork approval and required-check names remain in place. +See [runner operations](docs/guides/repository-automation.md#self-hosted-macos-tests) +for registration, queueing and rollback. The lint gate retains restored artifacts; age-based cleanup is not a lint prerequisite. For fork pull requests, an update may require maintainer approval before the diff --git a/docs/guides/macos-ci-runner.md b/docs/guides/macos-ci-runner.md deleted file mode 100644 index 0e5210be59..0000000000 --- a/docs/guides/macos-ci-runner.md +++ /dev/null @@ -1,333 +0,0 @@ -# Self-hosted macOS CI - -This guide owns the rollout of the three Mac CI lanes tracked in -[issue #1106](https://github.com/matrixorigin/Astra/issues/1106). -The implementation is opt-in. A merged workflow is not evidence that the Mac -is configured, admitted, compatible, or faster than hosted runners. - -## Workload and routing - -The migrated unit runs default-reader PTYs and the CLI Bash regression, reflow, -and workspace coordination. On the mini it runs sequentially and reuses its -build directory. This avoids scheduling three simultaneous Rust builds on a -single small machine. The canonical commands live in -`scripts/ci/run-macos-lanes.sh` and are shared with ordinary hosted CI. -Protected hosted fallback runs three parallel jobs, with read-only Rust artifact -and compiler caches. It does not serialize three hosted Macs onto one job. -Build reuse on the mini and hosted cache hit rates still require measurement; -adding the durations of the previous independent jobs is not a latency forecast. - -| Event | Execution | -| --- | --- | -| Confirmed matrixorigin member PR, current verified merge, admitted idle Mac | Self-hosted ARM Mac | -| Confirmed member, Mac unavailable or admission/platform failure before tests | Protected hosted macOS fallback | -| Non-member, membership API failure, or unknown identity | Ordinary `pull_request` hosted workflow | -| Main push before isolated main account is accepted | Ordinary hosted workflow | -| Main push with accepted separate main account/group | Admitted main Mac runner; protected hosted fallback | -| Other branches/events | Existing hosted routing | - -macOS dev-tty and vendored terminal units remain hosted, as do Linux and release -builds. Path selection calls the existing `ci_scope.classify` on the verified -base-to-merge comparison. Unknown/truncated comparisons select the lanes. - -A PR author is identified by the actual GitHub PR record, including user ID. -Only `GET /orgs/matrixorigin/members/{author}` returning 204 qualifies. The -credential must see private membership. Fork ownership, actor, association and -public profiles are not substituted. Qualifying members vouch for their PR's -contents, including collaborators' changes; this does not prove every writer is -a member. - -## Control and required results - -`macos-trusted.yml` is base-owned `pull_request_target` control. Only confirmed -members execute PR code through that event. The controller independently reads -the current base B, head H and merge M and requires exactly two ordered parents -`M^1=B` and `M^2=H`. It checks out immutable M. Null/stale merges, force-pushes, -base advancement or changed repository identity cannot silently select another -revision. Planning retries a not-yet-ready merge up to 12 times (two seconds -between attempts), re-reading the full tuple and retaining the event's head. -Exhaustion creates three explicit failed, retryable App checks instead of -silently leaving the required contexts absent. A later head is never substituted -into an earlier event's plan. - -A dedicated GitHub App publishes three contexts on H: - -- `Mac CI: default reader` -- `Mac CI: reflow` -- `Mac CI: coordination` - -All three must be required with their source bound to that App, together with -strict up-to-date branch checks. Source binding prevents impersonation; the -up-to-date rule prevents a historical check on H from satisfying a different -base tuple after main advances. Control verifies both requirements before -dispatch. Its private key and installation token -remain on GitHub-hosted Ubuntu control jobs, never on PR execution machines. -Store the private key only in the `astra-macos-control` GitHub Environment, -restricted by a custom branch rule to branch `main` alone (no tag rule). -Do not duplicate it as a repository/organization secret: same-repository PRs -could otherwise request it. PR refs and feature-branch dispatches must be denied -by the environment policy. Control verifies that exact policy before dispatch. -The publisher reads actual GitHub run/job/step results; it does not accept PR -artifacts or shell-supplied success flags. A successful combined job satisfies -all three contexts; hosted fallback must prove all three parallel macOS jobs -and their execution steps succeeded. Any actual test failure fails the contexts -conservatively and cannot be masked by fallback. - -The ordinary `pull_request` workflow reads App-owned authorization as a -best-effort deduplication hint. Delegated copies use cheap Ubuntu notification -jobs. The hint cannot publish or replace the three required App checks. Missing -hints preserve hosted execution, so a control race can cause duplicate work, -not missing coverage. A forged hint leaves the member's trusted checks pending. -Only the separate `macos-routing` job receives the extra Actions/checks/PR read -permissions. Linux package groups and vendored tests depend directly on scope -and do not wait for this hint; the mixed Linux/macOS terminal matrix still waits -for routing. The hint waits up to a 90-second polling budget for delayed -control, bounded by a -two-minute step timeout; a known ordinary route or permission denial returns -immediately. Separate event queues can still outlast that budget. Such runs keep -hosted execution and can duplicate work; this is not claimed eliminated. The -activation pilot must demonstrate `delegated=true` with an actual fork PR's -`GITHUB_TOKEN`, not only a fixture or a personal token. - -Non-members retain ordinary fork approval. `macos-results.yml` observes the -completed Test Suite through `workflow_run`, reads GitHub metadata only, and -publishes the ordinary path's actual three hosted results. It never downloads -artifacts or executes PR code. Approval-pending, missing, stale, cancelled or -failed execution cannot become a successful skip. A completed ordinary run with -missing/skipped selected lanes fails explicitly, including scope disagreement, -instead of staying pending forever. This retains the pre-existing PR-modifiable -test-definition limitation for non-member hosted tests. - -Fork runs can have empty `pull_requests` arrays, and the commit-to-PR API can -also return no associations. Test Suite therefore sets `run-name` from GitHub's -PR number, event base/head and `github.sha` (the tested merge SHA), before any -checkout. Both observation and reconciliation read this tuple from authenticated -Actions metadata, fetch that PR directly, and verify repository/head identity, -current base and the immutable merge's exact ordered parents. Non-empty PR -associations must agree as well; an empty association alone never rejects a run. -No downloaded artifact, test stdout or current merge ref substitutes for the -recorded tested SHA. Completed historical runs without this tuple fail closed; -rerun the normal approved Test Suite after rollout. See GitHub's -[PR merge context](https://docs.github.com/en/actions/reference/workflows-and-actions/events-that-trigger-workflows#pull_request) -and [run-name expressions](https://docs.github.com/en/actions/reference/workflows-and-actions/workflow-syntax#run-name). - -This records the ordinary workflow's event context, not a signed attestation: -PR authors can modify that workflow and its tests, as before. It cannot grant -membership or authorize privileged execution. Member results still require the -protected path and fresh member/source checks. - -The same metadata-only observer reconciles every 15 minutes (and supports -manual dispatch), recovering lost completion notifications and PRs with no App -checks at all. Missing or retryable stale plans dispatch only -`macos-trusted.yml` at `main`, with a numeric PR number; that workflow repeats -all source and membership checks. Recovery dispatches intentionally use protected -hosted routing for members; non-members still use ordinary `pull_request` -approval and execution. The controller never executes PR code or auto-approves -forks. Its App token alone requests Actions **write** for dispatch; plan, -admission, publisher and runner-side credentials retain Actions read. See -[GitHub's dispatch permission](https://docs.github.com/en/rest/actions/workflows#create-a-workflow-dispatch-event). -The observer is serialized and writes a non-authorizing recovery marker before -dispatch. Same-source requests are throttled for 30 minutes, including lost -responses or failed dispatches; later real control supersedes the marker. - -When base/head/merge or author trust changes during execution, publication -immediately fails the old contexts with an explicit stale-source explanation. -Reconciliation requests fresh control once a verified merge is available; old -results cannot overwrite a newer run/attempt. Successful checks and actual test -failures are not automatically retried. A missing merge remains an explicit -retryable failure until it resolves. - -Authorization expires three hours after planning. At expiry, pending execution -contexts are failed with a recovery instruction; expiry never means tests -passed. Active API/Actions outages or delayed cron execution can delay recovery, -so this is not a platform availability guarantee. Expired authorization is -rejected by the runner hook and result publisher. After resolving fork approval -or runner unavailability, start fresh trusted control and rerun applicable tests -as required. The observer's five-minute job timeout also bounds an individual -scan; prolonged API failures may require manual recovery. - -Both privileged-event workflows explicitly restrict remote cache tokens with -`cache-mode: read` or `none`. Disabling a client upload is not the authorization -boundary. Actionlint 1.7.12 does not parse this GitHub-supported key: the config -filters only that exact unsupported-key diagnostic for these two files. Other -workflow errors remain checked. Remove that narrow filter when actionlint gains -support. See [GitHub cache permissions](https://docs.github.com/en/actions/reference/workflows-and-actions/workflow-syntax#cache-mode). - -## Mandatory runner-side admission - -Use a dedicated `astra-macos-pr` group, selecting only Astra, allowing its public -repository, restricted to exactly: - -```text -matrixorigin/Astra/.github/workflows/macos-trusted.yml@refs/heads/main -``` - -Remove every unrestricted registration/path to this same execution identity. -Labels and repository restrictions alone do not protect against PR YAML changing -`runs-on`. The group must enforce the workflow and protected ref. If the actual -GitHub plan cannot enforce that restriction, do not enable this design. - -Runner labels are `self-hosted`, `macOS`, `ARM64`, `mac-arm64-mini` and -`astra-macos-admitted`. Add the last label only after admission and acceptance -have actually passed. The runner's physical name/location is not part of the -public configuration. - -Install a root-owned job-started hook before accepting jobs. The PR runner must -be a non-admin account named `astra-ci-pr` without general sudo rights. A -separate non-login `astra-ci-admission` account owns a read-only API credential -with private Members read, organization runners/groups read, Actions read, -contents/pull-request/checks read. It must not publish checks or write contents. -No credential is readable by the runner account. - -Install these reviewed files outside all runner-writable paths: - -| Source | Root-owned destination | -| --- | --- | -| `scripts/ci/macos_runner.py`, `scripts/ci/ci_scope.py` | `/opt/astra-ci/lib/` | -| `scripts/ci/macos-runner-admit` | `/opt/astra-ci/bin/admit` | -| `scripts/ci/macos-job-started.sh` | `/opt/astra-ci/bin/job-started.sh` | -| `scripts/ci/macos-runner-preflight.sh` | `/opt/astra-ci/bin/preflight` | - -Use a root-managed Python 3.11+ installation at `/opt/astra-ci/python`, with no -runner-writable executable, library or parent directory. The small hook itself -uses system Python for JSON only; the policy helper uses isolated Python mode. -The fixed sudo target accepts no command arguments and selects a root-owned -policy by sudo's verified invoking user, not by JSON supplied by a PR. Configure -sudoers for exactly this command with no arguments and no environment override: - -```text -astra-ci-pr ALL=(astra-ci-admission) NOPASSWD: /opt/astra-ci/bin/admit "" -``` - -No general shell, Python, environment-preserving sudo, or arbitrary config-path -invocation is permitted. Protect every installed parent directory and config. -Set `ACTIONS_RUNNER_HOOK_JOB_STARTED=/opt/astra-ci/bin/job-started.sh` in the runner's -service environment, then restart that runner through its reviewed operations -process. Changing repository YAML does not install this hook. - -Example `/etc/astra-ci/pr.json` (root-owned, readable only by admission owner): - -```json -{ - "app_id": 12345, - "kind": "pr", - "group_id": 123, - "runner_name": "", - "token_file": "/etc/astra-ci/admission-token" -} -``` - -The helper has a 90-second deadline; the hook client also bounds the privileged -subprocess at 100 seconds. The `.sh` extension on the installed hook is required -by GitHub's hook launcher. - -The helper reads the publisher App's authorization from GitHub, validates the -actual workflow/repository/event/current attempt and active job on the expected -runner/group, rechecks the current tuple and private membership, and only then -admits. This runs for individual and failed-only reruns even when the previous -hosted router job was successful. Caller-controlled JSON cannot supply policy, -credentials, an API host or an arbitrary command. Hook rejection fails the job; -the workflow separately authorizes hosted handoff. - -## Host state and platform acceptance - -Keep main hosted by default. PR job artifacts/npm cache use a private per-run -scratch directory and are removed afterward. Persistent user state, including -HOME, Cargo/rustup, tool binaries, Git config/helpers, keychains, launchd agents, -background processes and writable shares, is still under the PR account's trust. -A clean checkout or scratch target does not erase those effects. - -Prefer disposable VMs per job. To enable main migration, use an independently -installed runner under a different non-admin OS account `astra-ci-main`, separate -home/toolchain/caches, and group `astra-macos-main` with the same workflow/ref -restriction. Provide `/etc/astra-ci/main.json` with kind `main`, the main runner -name/group, and add only the same fixed sudo command for that account. Two runner -registrations under the same OS account do not qualify: the fixed helper maps -the PR identity only to PR policy. Validate filesystem/process isolation and -harmless pollution sentinels for all state above before accepting main. Separate -accounts retain shared-kernel/host compromise risk; they are not disposable VMs. - -The installed preflight requires macOS 15+, native ARM64 and at least 30 GiB -free on the workspace filesystem. It records OS, CPU count, RAM and free disk -without host identity. A machine name containing `macos-13` is not proof of its -current OS. Upgrade/replace an older installation through its normal reviewed -process before assigning the admission label. Preinstall the repository's pinned -Rust toolchain, Node/action runtime dependencies and working system Git. The -runner must support the current `cache-mode` token enforcement. - -Before rollout measure cold/warm full-lane runs, queue delay, peak RSS/disk and -competing work. Start with one registration and serial combined execution; -do not multiply runner processes to hide capacity issues. Record acceptable -queue p50/p95 and job duration limits against observed PR arrival bursts. These -measurements have not been performed by the application PR. - -An online/idle API observation is not a reservation. A runner can disappear -between dispatch and start; queued self-hosted work remains pending. An operator -can cancel the stale run and use `workflow_dispatch` with its PR number for a -fresh protected hosted run. That dispatch always selects hosted execution. -Every hosted matrix job checks run ID, attempt, deadline and the protected -admission output using main-owned code **before** candidate checkout, toolchain, -cache setup or tests. It receives no App credential. Individual/failed-only -reruns that reuse successful dependencies fail this local fence even when cached -`allowed=true` remains present; rerun all jobs for new plan/admission. Queue delay -beyond authorization expiry also rejects execution. Publication additionally -requires the fence step to succeed and the actual run attempt to match the plan. -Membership/source are checked at protected admission; the local fence does not -claim a new membership API check on the execution host. Revocation or a stale -tuple found at admission blocks execution; a fresh plan can choose the ordinary -approved hosted path. Genuine test failures are never hidden by the -infrastructure fallback. - -## Enable, verify and roll back - -1. Merge the reviewed implementation while migration remains disabled. -2. Configure the dedicated publisher App, groups, isolated credential owner, - root-owned hook/helper and platform described above. Validate direct-label - bypass, fork/member/unknown routing, revoked-member reruns, tuple changes, - cache write denial, pollution isolation and capacity on disposable fixtures. - Include an initially null/stale merge, zero App checks, stale completion, - lost/forbidden recovery dispatch, recovery throttling and all three parallel - hosted results. Include individual hosted reruns with reused successful - admission outputs and expired authorization, and a fork run with empty PR and - commit associations that completes all three ordinary checks. Recovery must - never execute a non-member under the trusted - event or treat a marker/timeout as a pass. -3. Create GitHub Environment `astra-macos-control`, allowing only branch `main` - via its custom deployment branch policy, with no allowed tags or wildcards. - After verifying that PR refs and feature dispatches cannot access it, store - `ASTRA_MACOS_APP_PRIVATE_KEY` only as that environment's secret. Set repository - variable `ASTRA_MACOS_APP_ID` and `ASTRA_MACOS_GROUP_ID` to the App/PR group IDs. - The App installation needs repository Actions write (only the metadata - observer requests a write token), contents/PR read, administration read, - checks write and environments read; organization Members and self-hosted - runners/groups read. Other control jobs explicitly downscope Actions to read; - execution jobs receive no App token. -4. Make the three contexts required with source pinned to the App, and enforce - strict up-to-date branch checks. Preserve the existing protected checks. The migration control refuses dispatch until this - binding is visible through the rules/protection APIs. -5. Set `ASTRA_MACOS_MIGRATION_ENABLED=true` only after the preceding acceptance. - For main, separately set `ASTRA_MACOS_MAIN_GROUP_ID` and - `ASTRA_MACOS_MAIN_ISOLATED=true` after its isolation acceptance. -6. Verify actual job runner names/groups and reduced hosted Mac jobs. Record - hosted displacement, Mac execution and total elapsed time independently. - -Activation needs organization/repository administration and runner-host access. -The application PR does not modify live groups, secrets, branch protection, -remote clusters or runner services. Current caller API queries for runners and -groups returned 403; none of those settings is claimed verified or enabled. - -For rollback, pause merges and cancel in-flight Mac CI work/disable admitted -registrations through the reviewed operations process. Set -`ASTRA_MACOS_MIGRATION_ENABLED=false` first. Rerun ordinary Test Suite for -in-flight PRs and verify actual hosted macOS execution of the three lanes; an -older delegated echo is not proof. Only then remove the three App-required -contexts through reviewed protection configuration, and resume merges. Keep the -App contexts blocking during this transition. Disabling the flag does not revoke -already-issued runner authorization by itself. Never remove the App-required -contexts while delegated legacy checks alone could permit merging. - -Legacy job names retain the requested platform for protected-check compatibility. -A delegated job's summary explicitly identifies its Ubuntu notification role; -actual execution is attested by the three App-owned contexts and their run/job -metadata. Main push concurrency intentionally keeps only the newest generation, -as ordinary Test Suite already does; cancelled results do not become passes. diff --git a/docs/guides/repository-automation.md b/docs/guides/repository-automation.md index adb2f59313..51921a3319 100644 --- a/docs/guides/repository-automation.md +++ b/docs/guides/repository-automation.md @@ -120,10 +120,36 @@ before rerunning CI: This separation prevents permission failures from being treated as flaky CI and keeps external-fork behavior part of the repository's delivery contract. -## Self-hosted macOS migration - -The three-lane member/main migration, dedicated result source, runner-side -admission, isolation requirements and explicit activation/rollback steps are -documented in [Self-hosted macOS CI](macos-ci-runner.md). Migration is opt-in; -ordinary hosted execution remains the default until administration and host -acceptance are complete. +## Self-hosted macOS tests + +Test Suite sends all macOS execution to runners matching +`[self-hosted, macOS, ARM64, mac-arm64-mini]`: default reader/Bash, dev-tty reader, +reflow, workspace coordination and vendored terminal units. The same routing +applies to every PR author and to pushes on main/develop. Existing required-check +names retain their `macos-15` label for compatibility; inspect each job's actual +runner for execution provenance. Linux execution and release workflows retain +their existing platforms. + +This uses the ordinary `pull_request`/`push` workflow, GitHub's existing fork +approval and read-only workflow token. No App, organization membership query, +custom result contexts, routing flag, hook or hosted fallback is required by this +implementation. The selected runner must be available to Astra, including public +repository access when applicable, and have the exact labels above. Confirm its +OS/toolchain support for the pinned Actions/Rust/Node versions and at least +30 GiB workspace disk before assigning work. The shared Rust setup reports low +or unreadable disk as a failure on self-hosted runners, without running the +hosted-image sudo cleanup. All approved PR code executes on that +runner; operators must accept that persistent-host access and resource exposure. + +One registration executes one job at a time. The five macOS jobs queue on a +single Mac; adding registrations on the same machine changes resource contention +and does not create extra capacity. Reader feature builds remain separate jobs. +If the runner is offline, jobs stay queued rather than consuming a hosted Mac. +Measure queue delay and actual job duration before claiming an elapsed-time gain. + +If the earlier controller design was staged, remove its unused custom App +required contexts and retire its installed admission hook through the reviewed +operations process; this workflow no longer emits those contexts or App +admissions. Preserve the existing GitHub Actions required checks and fork +approval settings. Roll back execution by reverting the runner selectors in Test +Suite to the hosted matrix OS and `macos-15` coordination runner through a PR. diff --git a/scripts/ci/macos-job-started.sh b/scripts/ci/macos-job-started.sh deleted file mode 100755 index ca6dc8818a..0000000000 --- a/scripts/ci/macos-job-started.sh +++ /dev/null @@ -1,15 +0,0 @@ -#!/bin/bash -# Install root-owned outside every runner-writable path. No sourced PR code. -set -euo pipefail -/usr/bin/python3 -I -c ' -import json, os, subprocess -from pathlib import Path -event = json.loads(Path(os.environ["GITHUB_EVENT_PATH"]).read_text()) -head = event.get("pull_request", {}).get("head", {}).get("sha") or event.get("after") -if not head: - raise SystemExit("Use a fresh pull_request_target event for self-hosted admission") -request = {"run_id": int(os.environ["GITHUB_RUN_ID"]), - "attempt": int(os.environ["GITHUB_RUN_ATTEMPT"]), "head": head} -subprocess.run(["/usr/bin/sudo", "-n", "-u", "astra-ci-admission", "/opt/astra-ci/bin/admit"], - input=json.dumps(request), text=True, check=True, timeout=100) -' diff --git a/scripts/ci/macos-runner-admit b/scripts/ci/macos-runner-admit deleted file mode 100755 index df3eb60407..0000000000 --- a/scripts/ci/macos-runner-admit +++ /dev/null @@ -1,12 +0,0 @@ -#!/bin/bash -# Root-owned fixed sudo target; no command arguments or caller paths allowed. -set -euo pipefail -[ "$#" -eq 0 ] -case "${SUDO_USER:-}" in - astra-ci-pr) policy=/etc/astra-ci/pr.json ;; - astra-ci-main) policy=/etc/astra-ci/main.json ;; - *) echo 'Unrecognized runner identity' >&2; exit 1 ;; -esac -exec /usr/bin/env -i PATH=/usr/bin:/bin \ - /opt/astra-ci/python/bin/python3 -I /opt/astra-ci/lib/macos_runner.py \ - admit --config "$policy" diff --git a/scripts/ci/macos-runner-preflight.sh b/scripts/ci/macos-runner-preflight.sh deleted file mode 100755 index 3362832337..0000000000 --- a/scripts/ci/macos-runner-preflight.sh +++ /dev/null @@ -1,12 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail -[ "$(uname -s)" = Darwin ] -[ "$(uname -m)" = arm64 ] -version="$(sw_vers -productVersion)" -[ "${version%%.*}" -ge 15 ] -available="$(df -Pk "$GITHUB_WORKSPACE" | awk 'NR==2 {print $4}')" -[[ "$available" =~ ^[0-9]+$ ]] && [ "$available" -ge 31457280 ] -# Record capacity without host identity, network information or credentials. -printf 'macOS=%s arch=arm64 free_disk_KiB=%s\n' "$version" "$available" -sysctl -n hw.ncpu hw.memsize -rustc --version diff --git a/scripts/ci/macos_runner.py b/scripts/ci/macos_runner.py deleted file mode 100755 index 72c79d8118..0000000000 --- a/scripts/ci/macos_runner.py +++ /dev/null @@ -1,701 +0,0 @@ -#!/usr/bin/env python3 -"""Trusted macOS routing, admission and result publication (stdlib only). - -No PR artifacts or shell-provided success results are used as evidence. The -same policy is installed read-only outside the runner account for job hooks. -""" -from __future__ import annotations - -import argparse -import json -import os -from pathlib import Path -import re -import signal -import sys -import time -import urllib.error -import urllib.request - -sys.path.insert(0, str(Path(__file__).resolve().parent)) -from ci_scope import classify - -REPO = "matrixorigin/Astra" -WORKFLOW = ".github/workflows/macos-trusted.yml" -WORKFLOW_REF = f"{REPO}/{WORKFLOW}@refs/heads/main" -CONTEXTS = ("Mac CI: default reader", "Mac CI: reflow", "Mac CI: coordination") -SHA = re.compile(r"^[0-9a-f]{40}$") -AUTHORIZATION_SECONDS = 3 * 60 * 60 -HINT_WAIT_SECONDS = 90 -SOURCE_ATTEMPTS = 12 -RECOVERY_COOLDOWN_SECONDS = 30 * 60 -HOSTED_LANES = ("default-reader", "reflow", "coordination") -ORDINARY_RUN_TITLE = re.compile( - r"Test Suite PR (?P[1-9][0-9]*) base=(?P[0-9a-f]{40}) " - r"head=(?P[0-9a-f]{40}) merge=(?P[0-9a-f]{40})") - - -class Unavailable(Exception): - pass - - -class MergePending(Unavailable): - pass - - -class StaleSource(Unavailable): - pass - - -class PermissionDenied(Unavailable): - pass - - -class NoRedirect(urllib.request.HTTPRedirectHandler): - def redirect_request(self, *args, **kwargs): - return None - - -class API: - def __init__(self, token): - self.token = token - self.opener = urllib.request.build_opener(NoRedirect) - - def request(self, path, data=None, method=None): - if not path.startswith("/") or path.startswith("//"): - raise Unavailable("invalid API path") - req = urllib.request.Request("https://api.github.com" + path, - data=None if data is None else json.dumps(data).encode(), method=method, - headers={"Authorization": f"Bearer {self.token}", - "Accept": "application/vnd.github+json", - "X-GitHub-Api-Version": "2022-11-28", - "Content-Type": "application/json"}) - try: - with self.opener.open(req, timeout=20) as response: - raw = response.read() - return response.status, json.loads(raw) if raw else None - except urllib.error.HTTPError as error: - return error.code, None # Never print provider responses or credentials. - except (OSError, ValueError): - raise Unavailable("GitHub API unavailable") from None - - def get(self, path): - status, value = self.request(path) - if status in (401, 403): - raise PermissionDenied("API permission unavailable") - if status != 200: - raise Unavailable(f"GitHub API returned {status}") - return value - - def pages(self, path, key=None): - result = [] - for page in range(1, 101): - value = self.get(path + ("&" if "?" in path else "?") + f"per_page=100&page={page}") - rows = value[key] if key else value - result.extend(rows) - if len(rows) < 100: - return result - raise Unavailable("API pagination limit") - - -def member(api, login): - if not re.fullmatch(r"[A-Za-z0-9-]{1,39}", login): - return False - # A configured Members:read credential is required, including private members. - try: - return api.request(f"/orgs/matrixorigin/members/{login}")[0] == 204 - except Unavailable: - return False - - -def pr_identity(api, number, expected_head=None): - pr = api.get(f"/repos/{REPO}/pulls/{int(number)}") - if (pr["state"] != "open" or pr["base"]["repo"]["full_name"] != REPO - or pr["base"]["ref"] != "main"): - raise Unavailable("PR is not an open main pull request") - base = api.get(f"/repos/{REPO}/git/ref/heads/main")["object"]["sha"] - head = pr["head"]["sha"] - if not all(isinstance(s, str) and SHA.fullmatch(s) for s in (base, head)): - raise Unavailable("invalid base/head revision") - if expected_head is not None and head != expected_head: - raise StaleSource("trigger head has been superseded") - return {"kind": "pr", "pr": int(number), "base": base, "head": head, - "merge": pr.get("merge_commit_sha"), "head_repository_id": pr["head"]["repo"]["id"], - "author": pr["user"]["login"], "author_id": pr["user"]["id"]} - - -def source(api, number, *, attempts=1, expected_head=None): - # GitHub computes the test merge asynchronously. Re-read the entire tuple on - # each retry, never substitute a head from a newer synchronize event. - for attempt in range(attempts): - plan = pr_identity(api, number, expected_head) - merge = plan["merge"] - if isinstance(merge, str) and SHA.fullmatch(merge): - status, commit = api.request(f"/repos/{REPO}/git/commits/{merge}") - if status in (401, 403): - raise PermissionDenied("merge API permission unavailable") - if status == 200 and [p["sha"] for p in commit["parents"]] == [plan["base"], plan["head"]]: - return plan - if status not in (200, 404): - raise Unavailable("merge API unavailable") - if attempt + 1 < attempts: - time.sleep(2) - raise MergePending("merge revision not ready for current base/head") - - -def same_source(a, b): - return all(a.get(k) == b.get(k) for k in - ("kind", "pr", "base", "head", "merge", "head_repository_id", "author_id")) - - -def selected(api, plan): - try: - diff = api.get(f"/repos/{REPO}/compare/{plan['base']}...{plan['merge']}") - files = diff.get("files") - if files is None or len(files) >= 300: - return True # GitHub truncates compare's file inventory at 300. - paths = [name for f in files for name in (f["filename"], f.get("previous_filename")) if name] - scope = classify(paths) - return scope["test_cli"] or scope["test_core"] - except (Unavailable, KeyError, TypeError): - return True - - -def require_check_sources(api, app_id): - environment = api.get(f"/repos/{REPO}/environments/astra-macos-control") - policy = environment.get("deployment_branch_policy") or {} - if policy != {"protected_branches": False, "custom_branch_policies": True}: - raise Unavailable("control environment must select only the main branch") - branches = api.pages(f"/repos/{REPO}/environments/astra-macos-control/deployment-branch-policies", "branch_policies") - if len(branches) != 1 or branches[0].get("name") != "main" or branches[0].get("type") != "branch": - raise Unavailable("control environment allows an untrusted branch or tag") - required = {} - up_to_date = False - for rule in api.get(f"/repos/{REPO}/rules/branches/main"): - if rule["type"] == "required_status_checks": - up_to_date |= rule["parameters"].get("strict_required_status_checks_policy") is True - for check in rule["parameters"]["required_status_checks"]: - required[check["context"]] = check.get("integration_id") - status, protection = api.request(f"/repos/{REPO}/branches/main/protection") - if status == 200: - protected_checks = protection.get("required_status_checks") or {} - up_to_date |= protected_checks.get("strict") is True - for check in protected_checks.get("checks", []): - required[check["context"]] = check.get("app_id") - elif status != 404: - raise Unavailable("cannot verify required-check source binding") - if not up_to_date: - raise Unavailable("main must require checks on an up-to-date branch") - if any(required.get(name) != app_id for name in CONTEXTS): - raise Unavailable("three Mac CI checks must be required from the dedicated App") - - -def group_ready(api, group_id, kind): - group = api.get(f"/orgs/matrixorigin/actions/runner-groups/{int(group_id)}") - if (group["name"] != f"astra-macos-{kind}" or not group.get("restricted_to_workflows") - or group.get("selected_workflows") != [WORKFLOW_REF] - or group.get("visibility") != "selected" or not group.get("allows_public_repositories")): - raise Unavailable("runner group is not restricted to the protected workflow") - repos = api.pages(f"/orgs/matrixorigin/actions/runner-groups/{group_id}/repositories", "repositories") - if [r["full_name"] for r in repos] != [REPO]: - raise Unavailable("runner group must select only Astra") - runners = api.pages(f"/orgs/matrixorigin/actions/runner-groups/{group_id}/runners", "runners") - labels = {"self-hosted", "macOS", "ARM64", "mac-arm64-mini", "astra-macos-admitted"} - return any(r["status"] == "online" and not r["busy"] and - labels <= {label["name"] for label in r["labels"]} for r in runners) - - -def checks(api, head): - return api.pages(f"/repos/{REPO}/commits/{head}/check-runs", "check_runs") - - -def save_checks(api, plan, conclusion=None, summary="Awaiting verified execution", *, only_pending=False): - existing = {name: [] for name in CONTEXTS} - for check in checks(api, plan["head"]): - if check["name"] in existing and check.get("app", {}).get("id") == plan["app_id"]: - existing[check["name"]].append(check) - for name in CONTEXTS: - data = {"name": name, "head_sha": plan["head"], - "external_id": f"astra-macos-v1:{plan['run_id']}:{plan['head']}", - "status": "completed" if conclusion else "in_progress", - "output": {"title": summary, "summary": summary, "text": json.dumps(plan, sort_keys=True)}} - if conclusion: - data["conclusion"] = conclusion - previous = max(existing[name], key=lambda c: c["id"], default=None) - if previous: - if only_pending and previous["status"] == "completed": - continue - try: - old = json.loads(previous["output"]["text"]) - if (old["run_id"], old["attempt"]) > (plan["run_id"], plan["attempt"]): - continue # A late publisher cannot overwrite newer control. - reuse = old["run_id"] == plan["run_id"] and old["attempt"] == plan["attempt"] - except (ValueError, KeyError, TypeError): - reuse = False - else: - reuse = False - if reuse: - data.pop("head_sha") - status, _ = api.request(f"/repos/{REPO}/check-runs/{previous['id']}", data, "PATCH") - expected = 200 - else: - status, _ = api.request(f"/repos/{REPO}/check-runs", data) - expected = 201 - if status != expected: - raise Unavailable("could not publish dedicated App checks") - - -def authorization(api, head, run_id, app_id): - found = {} - for check in checks(api, head): - if check["name"] not in CONTEXTS or check.get("app", {}).get("id") != app_id: - continue - try: - plan = json.loads(check["output"]["text"]) - except (KeyError, TypeError, ValueError): - continue - if plan.get("schema") == 1 and plan.get("run_id") == run_id and plan.get("head") == head: - if check["name"] not in found or found[check["name"]][0] < check["id"]: - found[check["name"]] = (check["id"], plan) - if len(found) != len(CONTEXTS): - raise Unavailable("no complete App authorization") - plans = [p for _, p in found.values()] - if any(p != plans[0] for p in plans): - raise Unavailable("inconsistent App authorization") - return plans[0] - - -def validate_run(api, plan, attempt=None): - run = api.get(f"/repos/{REPO}/actions/runs/{int(plan['run_id'])}") - expected_events = ("pull_request_target", "workflow_dispatch") if plan["kind"] == "pr" else ("push",) - if (run["event"] not in expected_events or run["path"] != WORKFLOW - or run["repository"]["full_name"] != REPO - or (attempt is not None and run["run_attempt"] != attempt)): - raise Unavailable("unexpected workflow, repository, event or attempt") - return run - - -def expired(plan): - deadline = plan.get("expires_at") - return not isinstance(deadline, int) or time.time() >= deadline - - -def require_unexpired(plan): - if expired(plan): - raise Unavailable("authorization expired; start fresh trusted control") - - -def refresh(api, plan): - require_unexpired(plan) - if plan["kind"] == "pr": - identity = pr_identity(api, plan["pr"]) - if not same_source(identity, plan) or not member(api, identity["author"]): - raise StaleSource("source changed or membership is not confirmed") - try: - current = source(api, plan["pr"]) - except MergePending: - raise StaleSource("merge no longer matches current base/head") from None - if not same_source(current, plan): - raise StaleSource("source advanced during refresh") - else: - head = api.get(f"/repos/{REPO}/git/ref/heads/main")["object"]["sha"] - if plan["head"] != head: - raise StaleSource("main has advanced") - - -def admit(api, request, config): - # Invoked by a root-owned job-started hook under a distinct credential owner. - # No caller-supplied token, path, repository, API URL or command is accepted. - if set(request) != {"run_id", "attempt", "head"} or not SHA.fullmatch(request["head"]): - raise Unavailable("invalid admission request") - plan = authorization(api, request["head"], int(request["run_id"]), int(config["app_id"])) - if plan["route"] != "self-hosted" or plan["kind"] != config["kind"]: - raise Unavailable("authorization is for another execution boundary") - if int(config["group_id"]) != plan["group_id"]: - raise Unavailable("runner group differs from the authorized group") - group_ready(api, int(config["group_id"]), config["kind"]) - validate_run(api, plan, int(request["attempt"])) - refresh(api, plan) # Fresh private-membership check on every actual attempt. - jobs = api.pages(f"/repos/{REPO}/actions/runs/{plan['run_id']}/attempts/{request['attempt']}/jobs", "jobs") - if not any(j["name"] == "Migrated macOS lanes" and j["status"] == "in_progress" - and j["runner_name"] == config["runner_name"] - and j["runner_group_id"] == int(config["group_id"]) for j in jobs): - raise Unavailable("no matching active job on the admitted runner") - return plan - - -def guard_hosted(plan): - """Local execution fence for reused successful dependency outputs. - - This needs no App token or candidate code. A new attempt must rerun trusted - plan + admission; queue delay must not outlive the admitted authorization. - """ - require_unexpired(plan) - events = ("pull_request_target", "workflow_dispatch") if plan["kind"] == "pr" else ("push",) - if (plan.get("schema") != 1 or plan["route"] not in ("hosted", "self-hosted") - or os.environ.get("GITHUB_REPOSITORY") != REPO - or os.environ.get("GITHUB_EVENT_NAME") not in events - or os.environ.get("ASTRA_MACOS_HOSTED_ADMITTED") != "true" - or int(os.environ["GITHUB_RUN_ID"]) != plan["run_id"] - or int(os.environ["GITHUB_RUN_ATTEMPT"]) != plan["attempt"]): - raise Unavailable("hosted execution requires fresh plan and admission for this attempt") - - -def verified_job(api, plan, name): - run = validate_run(api, plan) - jobs = api.pages(f"/repos/{REPO}/actions/runs/{run['id']}/jobs?filter=latest", "jobs") - matching = [j for j in jobs if j["name"] == name] - if len(matching) != 1: - return "failure" if run.get("status") == "completed" else None - job = matching[0] - if job["conclusion"] != "success": - return job["conclusion"] if job["status"] == "completed" else None - steps = {s["name"]: s["conclusion"] for s in job["steps"]} - expected_step = "Run all three macOS lanes" if name == "Migrated macOS lanes" else "Run macOS lane" - if steps.get(expected_step) != "success": - return "failure" - if name.startswith("Fallback macOS:") and ( - "macos-15" not in job.get("labels", []) - or steps.get("Enforce current hosted authorization") != "success" - or run["run_attempt"] != plan["attempt"]): - return "failure" - if name == "Migrated macOS lanes" and job["runner_group_id"] != plan["group_id"]: - return "failure" - return "success" - - -def can_fallback(api, plan): - if plan["route"] == "hosted": - return True - jobs = api.pages(f"/repos/{REPO}/actions/runs/{plan['run_id']}/jobs?filter=latest", "jobs") - primary = [j for j in jobs if j["name"] == "Migrated macOS lanes"] - return (len(primary) == 1 and primary[0]["conclusion"] == "failure" and not any( - s["name"] == "Run all three macOS lanes" and s["conclusion"] not in (None, "skipped") - for s in primary[0]["steps"])) - - -def ordinary_source(api, run): - """Bind ordinary execution to its event tuple, even for empty fork links. - - GitHub records run-name's github-context expressions in run metadata, before - checkout. Recheck that tuple against the canonical PR and immutable merge. - Ordinary PR workflow definitions remain PR-modifiable, as do their tests; - this metadata does not grant member trust or privileged execution. - """ - if (run["event"] != "pull_request" or run["path"] != ".github/workflows/test.yml" - or run["repository"]["full_name"] != REPO): - raise Unavailable("not an ordinary Astra Test Suite run") - match = ORDINARY_RUN_TITLE.fullmatch(run.get("display_title", "")) - if not match: - raise Unavailable("ordinary run lacks source tuple; rerun approved Test Suite") - current = source(api, int(match["pr"])) - if (any(match[key] != current[key] for key in ("base", "head", "merge")) - or run["head_sha"] != current["head"] - or run["head_repository"]["id"] != current["head_repository_id"]): - raise Unavailable("ordinary run source differs from current PR/merge") - refs = run.get("pull_requests", []) - if refs and not any(p["number"] == current["pr"] and p["head"]["sha"] == current["head"] - and p["base"]["sha"] == current["base"] for p in refs): - raise Unavailable("ordinary run has a conflicting PR association") - return current - - -def hosted_result(api, plan): - runs = api.pages(f"/repos/{REPO}/actions/workflows/test.yml/runs?event=pull_request&head_sha={plan['head']}", "workflow_runs") - for run in sorted(runs, key=lambda r: r["id"], reverse=True): - if run["head_repository"]["id"] != plan["head_repository_id"] or run["head_sha"] != plan["head"]: - continue - try: - current = ordinary_source(api, run) - except Unavailable: - return "failure" if run["status"] == "completed" else None - if not same_source(current, plan): - continue - jobs = api.pages(f"/repos/{REPO}/actions/runs/{run['id']}/jobs?filter=latest", "jobs") - wanted = {"Terminal: default-reader (macos-15)", "Terminal: reflow (macos-15)", - "macOS workspace coordination contracts"} - actual = {j["name"]: j for j in jobs if j["name"] in wanted} - if set(actual) != wanted or any(j["status"] != "completed" for j in actual.values()): - # A completed run cannot later gain skipped/missing lane execution. - return "failure" if run["status"] == "completed" else None - return "success" if all(j["conclusion"] == "success" and "macos-15" in j["labels"] - for j in actual.values()) else "failure" - return None - - -def outputs(values): - with open(os.environ["GITHUB_OUTPUT"], "a") as out: - for key, value in values.items(): - out.write(f"{key}={json.dumps(value, separators=(',', ':')) if isinstance(value, dict) else value}\n") - - -def plan_command(api, event, app_id): - if os.environ["GITHUB_REPOSITORY"] != REPO: - raise Unavailable("wrong repository") - require_check_sources(api, app_id) - kind = "main" if os.environ["GITHUB_EVENT_NAME"] == "push" else "pr" - unresolved = False - if kind == "pr": - number = event.get("number") or event.get("inputs", {}).get("pr_number") - expected_head = event.get("pull_request", {}).get("head", {}).get("sha") - try: - plan = source(api, number, attempts=SOURCE_ATTEMPTS, expected_head=expected_head) - except MergePending: - plan = pr_identity(api, number, expected_head) - plan["merge"] = "" - unresolved = True - trusted = member(api, plan["author"]) - else: - head = event["after"] - if event["ref"] != "refs/heads/main" or not SHA.fullmatch(head): - raise Unavailable("unexpected push") - plan = {"kind": "main", "pr": 0, "base": event["before"], "head": head, "merge": head} - trusted = os.environ.get("ASTRA_MACOS_MAIN_ISOLATED") == "true" - plan.update(schema=1, run_id=int(os.environ["GITHUB_RUN_ID"]), app_id=app_id, - attempt=int(os.environ["GITHUB_RUN_ATTEMPT"]), route="ordinary", group_id=0, - expires_at=int(time.time()) + AUTHORIZATION_SECONDS) - if unresolved: - plan.update(route="unresolved", retryable=True) - save_checks(api, plan, "failure", "Merge revision not ready; trusted reconciliation will retry") - elif not selected(api, plan): - plan["route"] = "unaffected" - save_checks(api, plan, "success", "Canonical scope does not select these lanes") - elif trusted: - plan["route"] = "hosted" - group_id = os.environ.get("ASTRA_MACOS_GROUP_ID" if kind == "pr" else "ASTRA_MACOS_MAIN_GROUP_ID", "") - if os.environ["GITHUB_EVENT_NAME"] != "workflow_dispatch" and group_id.isdecimal(): - try: - if group_ready(api, int(group_id), kind): - plan.update(route="self-hosted", group_id=int(group_id)) - except Unavailable: - pass # A member still receives protected hosted execution. - save_checks(api, plan) - elif kind == "pr": - save_checks(api, plan) - else: - plan["route"] = "disabled" # Keep main hosted until isolation is accepted. - outputs({"plan": plan, "route": plan["route"], "merge": plan["merge"], "kind": kind}) - - -def hint(api, event, app_id): - if event.get("pull_request"): - if event["pull_request"].get("base", {}).get("ref") != "main": - return False - elif event.get("ref") != "refs/heads/main": - return False - head = event.get("pull_request", {}).get("head", {}).get("sha") or event.get("after") - number = event.get("number", 0) - if not isinstance(head, str) or not SHA.fullmatch(head): - return False - deadline = time.monotonic() + HINT_WAIT_SECONDS - for _ in range(HINT_WAIT_SECONDS // 2 + 1): - try: - rows = checks(api, head) - run_ids = {int(c["external_id"].split(":")[1]) for c in rows - if c.get("app", {}).get("id") == app_id - and c.get("external_id", "").startswith("astra-macos-v1:")} - for run_id in sorted(run_ids, reverse=True): - plan = authorization(api, head, run_id, app_id) - if plan["pr"] != number: - continue - require_unexpired(plan) - if plan["route"] in ("unresolved", "recovering"): - return False - validate_run(api, plan) - if number and not same_source(plan, source(api, number)): - continue - # Once trusted control has chosen ordinary hosted execution, - # there is no reason to keep delaying a non-member's scope job. - if plan["route"] in ("ordinary", "unaffected", "disabled", "unresolved", "recovering"): - return False - if plan["route"] in ("self-hosted", "hosted"): - return True - except PermissionDenied: - print("::notice::Mac CI hint lacks API access; retaining hosted execution.") - return False - except (Unavailable, ValueError, KeyError): - pass - remaining = deadline - time.monotonic() - if remaining <= 0: - break - time.sleep(min(2, remaining)) - print("::notice::Mac CI routing hint unavailable within budget; retaining hosted execution.") - return False - - -def finish(api, plan): - if plan["route"] in ("disabled", "unaffected", "unresolved", "recovering"): - return True - require_unexpired(plan) - try: - if plan["route"] == "ordinary": - if plan["kind"] != "pr" or not same_source(plan, pr_identity(api, plan["pr"])): - raise StaleSource("hosted result is stale") - try: - current = source(api, plan["pr"]) - except MergePending: - raise StaleSource("hosted merge is stale") from None - if not same_source(current, plan): - raise StaleSource("hosted source advanced during refresh") - if member(api, plan["author"]): - raise StaleSource("author now qualifies for protected execution") - result = hosted_result(api, plan) - else: - refresh(api, plan) - primary = verified_job(api, plan, "Migrated macOS lanes") if plan["route"] == "self-hosted" else None - if primary == "success" or not can_fallback(api, plan): - result = primary - else: - lanes = [verified_job(api, plan, "Fallback macOS: " + lane) for lane in HOSTED_LANES] - result = (None if any(r is None for r in lanes) else - "success" if all(r == "success" for r in lanes) else "failure") - except StaleSource: - # Failure, rather than neutral/skipped, keeps required checks fail-closed. - save_checks(api, {**plan, "retryable": True}, "failure", - "Stale macOS source or trust; trusted reconciliation will request fresh control") - return True - if result is not None: - observed = {**plan, "attempt": validate_run(api, plan)["run_attempt"]} - save_checks(api, observed, "success" if result == "success" else "failure", "Verified macOS execution: " + result) - return True - return False - - -def recover(api, pr, app_id, previous=None): - """Only the protected metadata controller may dispatch fixed main control. - - A non-authorizing marker bounds duplicate dispatches (including lost API - responses). A later real plan supersedes it by run id. No execution evidence - or member authorization is manufactured here. - """ - try: - current = source(api, pr["number"], expected_head=pr["head"]["sha"]) - except MergePending: - current = pr_identity(api, pr["number"], pr["head"]["sha"]) - marker = {**current, "schema": 1, "app_id": app_id, "run_id": 0, - "attempt": 0, "route": "unresolved", "group_id": 0, - "retryable": True, "expires_at": int(time.time()) + AUTHORIZATION_SECONDS} - save_checks(api, marker, "failure", "Merge revision not ready; trusted reconciliation will retry") - return - if (previous and same_source(previous, current) and - time.time() - previous.get("recovery_requested_at", 0) < RECOVERY_COOLDOWN_SECONDS): - return - marker = {**current, "schema": 1, "app_id": app_id, - "run_id": previous["run_id"] if previous else 0, - "attempt": previous["attempt"] if previous else 0, - "route": "recovering", "group_id": 0, "retryable": True, - "recovery_requested_at": int(time.time()), - "expires_at": int(time.time()) + AUTHORIZATION_SECONDS} - save_checks(api, marker, summary="Requesting fresh protected Mac CI control") - # A real plan may have appeared since the scan. Do not dispatch over it. - latest = {} - for row in checks(api, current["head"]): - if row["name"] in CONTEXTS and row.get("app", {}).get("id") == app_id: - if row["name"] not in latest or row["id"] > latest[row["name"]]["id"]: - latest[row["name"]] = row - if any(json.loads(row["output"]["text"]) != marker for row in latest.values()): - return - status, _ = api.request(f"/repos/{REPO}/actions/workflows/macos-trusted.yml/dispatches", - {"ref": "main", "inputs": {"pr_number": str(pr["number"])}}) - if status not in (200, 204): - raise Unavailable("could not request fresh trusted control") - - -def reconcile(api, app_id): - """Recover absent plans/stale sources; close expired or lost results.""" - require_check_sources(api, app_id) - prs = api.pages(f"/repos/{REPO}/pulls?state=open&base=main") - for pr in prs: - try: - rows = checks(api, pr["head"]["sha"]) - latest = {} - for row in rows: - if row["name"] in CONTEXTS and row.get("app", {}).get("id") == app_id: - if row["name"] not in latest or row["id"] > latest[row["name"]]["id"]: - latest[row["name"]] = row - if not latest: - recover(api, pr, app_id) - continue - plan = json.loads(max(latest.values(), key=lambda c: c["id"])["output"]["text"]) - if (plan["schema"] != 1 or plan["app_id"] != app_id or plan["kind"] != "pr" - or plan["head"] != pr["head"]["sha"] or plan["pr"] != pr["number"]): - continue - if plan.get("retryable"): - recover(api, pr, app_id, plan) - elif len(latest) == len(CONTEXTS) and all(c["status"] == "completed" for c in latest.values()): - continue - elif expired(plan): - save_checks(api, plan, "failure", - "Mac CI authorization expired; resolve approval/runner state and start fresh Mac CI control", - only_pending=True) - else: - finish(api, plan) - except (Unavailable, KeyError, TypeError, ValueError): - # Other PRs still get reconciliation during a transient per-PR failure. - print("::notice::A pending Mac CI result could not be reconciled; no success published.") - - -def main(): - parser = argparse.ArgumentParser() - parser.add_argument("command", choices=("plan", "hint", "finish", "refresh", "admit", "observe", "reconcile", "guard-hosted")) - parser.add_argument("--plan") - parser.add_argument("--config") - args = parser.parse_args() - try: - if args.command == "guard-hosted": - guard_hosted(json.loads(args.plan)) - return 0 - if args.command == "admit": - signal.alarm(90) # Hooks themselves have no GitHub timeout. - config = json.loads(Path(args.config).read_text()) - api = API(Path(config["token_file"]).read_text().strip()) - request = sys.stdin.read(4097) - if len(request) > 4096: - raise Unavailable("admission request too large") - admit(api, json.loads(request), config) - print("Runner admission accepted") - return 0 - api = API(os.environ["GH_TOKEN"]) - event = json.loads(Path(os.environ["GITHUB_EVENT_PATH"]).read_text()) - app_id = int(os.environ["ASTRA_MACOS_APP_ID"]) - if args.command == "plan": - plan_command(api, event, app_id) - elif args.command == "hint": - outputs({"delegated": str(hint(api, event, app_id)).lower()}) - elif args.command == "reconcile": - reconcile(api, app_id) - elif args.command == "observe": - run = api.get(f"/repos/{REPO}/actions/runs/{int(event['workflow_run']['id'])}") - if run["event"] != "pull_request" or run["path"] != ".github/workflows/test.yml": - return 0 - current = ordinary_source(api, run) - rows = checks(api, current["head"]) - plans = [] - for row in rows: - if row.get("app", {}).get("id") == app_id and row["name"] == CONTEXTS[0]: - try: - p = json.loads(row["output"]["text"]) - if p["route"] == "ordinary" and same_source(p, current): - plans.append((row["id"], p)) - except (KeyError, TypeError, ValueError): - pass - if plans: - finish(api, max(plans, key=lambda p: p[0])[1]) - else: - plan = json.loads(args.plan) - if args.command == "finish": - finish(api, plan) - else: - refresh(api, plan) - # Failed-only reruns cannot reuse a hosted authorization job. - if int(os.environ["GITHUB_RUN_ATTEMPT"]) != plan["attempt"]: - raise Unavailable("rerun all jobs for a fresh hosted admission") - outputs({"allowed": str(can_fallback(api, plan)).lower()}) - return 0 - except (Unavailable, KeyError, ValueError, TypeError, OSError): - print("::error::Mac CI admission/result unavailable; retain hosted checks or rerun trusted control. No success published.") - return 1 - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/scripts/ci/run-macos-lanes.sh b/scripts/ci/run-macos-lanes.sh deleted file mode 100755 index 03364fb0e8..0000000000 --- a/scripts/ci/run-macos-lanes.sh +++ /dev/null @@ -1,45 +0,0 @@ -#!/usr/bin/env bash -# Canonical commands shared by hosted jobs and the protected migration workflow. -set -euo pipefail -lane="${1:?lane required}" -lane_workspace="$PWD" -observe_git() { - python3 - "$lane_workspace" <<'PYTHON' -import os, subprocess, sys, time -started = time.monotonic() -subprocess.run(["/usr/bin/git", "-C", sys.argv[1], "-c", "core.fsmonitor=false", - "-c", "core.untrackedCache=false", "rev-parse", "--show-toplevel"], - env={"PATH": "/usr/bin:/bin", "HOME": "/nonexistent", "LANG": "C", "LC_ALL": "C", - "GIT_CONFIG_NOSYSTEM": "1", "GIT_CONFIG_GLOBAL": os.devnull, - "GIT_CONFIG_SYSTEM": os.devnull, "GIT_OPTIONAL_LOCKS": "0", - "GIT_TERMINAL_PROMPT": "0"}, check=True, timeout=60) -print(f"System Git observation initialized in {time.monotonic() - started:.3f}s") -PYTHON -} -case "$lane" in - default-reader) - cargo test --locked -p astra-cli --lib tui::terminal_startup -- --test-threads=2 - if [ "$(uname -s)" = Darwin ]; then - observe_git - cargo test --locked --manifest-path Cargo.toml -p astra-cli \ - --lib edge_tools::shell::tests::bash_echo_returns_output -- --exact - fi - ;; - reflow) - cargo build --locked -p astra-cli --bin astra - npm ci --ignore-scripts --prefix scripts/tui-reflow - ASTRA_TEST_BINARY="${CARGO_TARGET_DIR:-$PWD/target}/debug/astra" npm test --prefix scripts/tui-reflow - ;; - coordination) - cargo clippy --locked --manifest-path Cargo.toml -p astra-tools --all-targets -- -D warnings - observe_git - cargo test --locked --manifest-path Cargo.toml -p astra-tools \ - workspace_observation::tests:: -- --test-threads=1 - ;; - all) - bash "$0" default-reader - bash "$0" reflow - bash "$0" coordination - ;; - *) echo "Unknown macOS lane" >&2; exit 2 ;; -esac diff --git a/scripts/ci/test_ci_build_coverage.py b/scripts/ci/test_ci_build_coverage.py index f43d087270..06fb867399 100755 --- a/scripts/ci/test_ci_build_coverage.py +++ b/scripts/ci/test_ci_build_coverage.py @@ -2,6 +2,7 @@ """Guard workspace shard coverage and Docker's vendored path dependency inputs.""" from pathlib import Path +from itertools import product import os import re import shlex @@ -33,14 +34,14 @@ def test_rust_setup_disk_guard_uses_workspace_free_space_without_real_cleanup(se "docker image prune -af", ] - for mode, available_kib, should_skip in ( + for (mode, available_kib, should_skip), runner_environment in product(( ("above", threshold_kib + 1, True), ("equal", threshold_kib, True), ("below", threshold_kib - 1, False), ("invalid", "not-a-number", False), ("failure", None, False), - ): - with self.subTest(mode=mode), tempfile.TemporaryDirectory() as directory: + ), ("github-hosted", "self-hosted")): + with self.subTest(mode=mode, runner=runner_environment), tempfile.TemporaryDirectory() as directory: root = Path(directory) mock_bin = root / "mock-bin" mock_bin.mkdir() @@ -80,6 +81,7 @@ def test_rust_setup_disk_guard_uses_workspace_free_space_without_real_cleanup(se environment = { **os.environ, "GITHUB_WORKSPACE": str(workspace), + "ASTRA_CI_RUNNER_ENVIRONMENT": runner_environment, "MOCK_DF_MODE": mode, "MOCK_DF_AVAILABLE_KIB": str(available_kib or ""), "MOCK_SUDO_LOG": str(sudo_log), @@ -91,9 +93,11 @@ def test_rust_setup_disk_guard_uses_workspace_free_space_without_real_cleanup(se capture_output=True, text=True, ) - self.assertEqual(result.returncode, 0, result.stderr) + self_hosted_failure = runner_environment == "self-hosted" and not should_skip + self.assertEqual(result.returncode, 1 if self_hosted_failure else 0, result.stderr) output = result.stdout + result.stderr - decision = "skip cleanup" if should_skip else "run cleanup" + decision = ("skip cleanup" if should_skip else + "fail self-hosted" if self_hosted_failure else "run cleanup") self.assertIn(f"Disk guard decision: {decision}", output) self.assertIn( f"threshold={threshold_kib} KiB (30 GiB)", @@ -101,7 +105,7 @@ def test_rust_setup_disk_guard_uses_workspace_free_space_without_real_cleanup(se ) invocations = sudo_log.read_text().splitlines() if sudo_log.exists() else [] - self.assertEqual(invocations, [] if should_skip else cleanup) + self.assertEqual(invocations, [] if should_skip or self_hosted_failure else cleanup) def test_make_lint_preserves_restored_artifacts_and_checks_all_targets(self): result = subprocess.run( @@ -169,28 +173,39 @@ def test_vendored_terminal_units_are_locked_and_pty_tests_run_both_readers(self) pty = workflow.split("\n terminal-pty:\n", 1)[1].split("\n macos-workspace-coordination:", 1)[0] self.assertIn("os: [ubuntu-latest, macos-15]", pty) self.assertIn("segment: [default-reader, dev-tty-reader, reflow]", pty) - self.assertIn("--features crossterm/use-dev-tty tui::terminal_startup -- --test-threads=2", pty) - self.assertIn("bash scripts/ci/run-macos-lanes.sh default-reader", pty) + for segment, step, features in ( + ("default-reader", "Test default terminal reader through PTYs", ""), + ("dev-tty-reader", "Test dev-tty terminal reader through PTYs", "--features crossterm/use-dev-tty "), + ): + with self.subTest(segment=segment): + commands = workflow_run_script(".github/workflows/test.yml", step).strip() + self.assertEqual(commands, + f"cargo test --locked -p astra-cli --lib {features}tui::terminal_startup -- --test-threads=2") + block = pty.split(f" - name: {step}\n", 1)[1].split(" - ", 1)[0] + self.assertIn(f"if: matrix.segment == '{segment}'", block) def test_terminal_reflow_uses_real_binary_and_locked_emulator(self): - commands = (ROOT / "scripts/ci/run-macos-lanes.sh").read_text() + commands = workflow_run_script(".github/workflows/test.yml", "Test inline terminal resize with xterm reflow") self.assertIn("cargo build --locked -p astra-cli --bin astra", commands) self.assertIn("npm ci --ignore-scripts --prefix scripts/tui-reflow", commands) - self.assertIn('ASTRA_TEST_BINARY="${CARGO_TARGET_DIR:-$PWD/target}/debug/astra" npm test --prefix scripts/tui-reflow', commands) + self.assertIn('ASTRA_TEST_BINARY="$PWD/target/debug/astra" npm test --prefix scripts/tui-reflow', commands) def test_macos_cli_bash_reuses_default_reader_without_losing_coordination_contracts(self): - commands = (ROOT / "scripts/ci/run-macos-lanes.sh").read_text() - reader = commands.split(" default-reader)\n", 1)[1].split(" ;;", 1)[0] - self.assertLess(reader.index("tui::terminal_startup"), reader.index("observe_git")) - self.assertLess(reader.index("observe_git"), reader.index("bash_echo_returns_output")) - self.assertIn("--lib edge_tools::shell::tests::bash_echo_returns_output -- --exact", reader) - self.assertNotIn("--features", reader) - self.assertNotIn("cargo build", reader, "Reuse the default CLI library test executable") - contracts = commands.split(" coordination)\n", 1)[1].split(" ;;", 1)[0] - for protected in ("-p astra-tools", "--all-targets -- -D warnings", - "workspace_observation::tests:: -- --test-threads=1", "observe_git"): - self.assertIn(protected, contracts) - self.assertNotIn("-p astra-cli", contracts) + workflow = (ROOT / ".github/workflows/test.yml").read_text() + terminal = workflow.split("\n terminal-pty-shards:\n", 1)[1].split("\n macos-workspace-coordination:\n", 1)[0] + self.assertEqual(workflow.count("- name: Test macOS CLI Bash execution"), 1) + bash = terminal.split(" - name: Test macOS CLI Bash execution\n", 1)[1].split(" - ", 1)[0] + self.assertIn("if: matrix.os == 'macos-15' && matrix.segment == 'default-reader'", bash) + self.assertIn("--lib edge_tools::shell::tests::bash_echo_returns_output -- --exact", bash) + self.assertLess(terminal.index("Test default terminal reader through PTYs"), terminal.index("Initialize system Git for macOS CLI observation")) + self.assertLess(terminal.index("Initialize system Git for macOS CLI observation"), terminal.index("Test macOS CLI Bash execution")) + contracts = workflow.split("\n macos-workspace-coordination-tests:\n", 1)[1].split("\n test-online-core:", 1)[0] + self.assertIn('install-nextest: "false"', contracts) + self.assertIn("-p astra-tools", contracts) + self.assertIn("--all-targets -- -D warnings", contracts) + self.assertIn("workspace_observation::tests:: -- --test-threads=1", contracts) + self.assertIn("/usr/bin/git", contracts) + self.assertNotIn("-p astra-cli", contracts, "Do not compile the same CLI test executable twice on macOS") def test_ci_resolves_the_actual_builder_base_not_only_the_planner(self): dockerfile = (ROOT / "Dockerfile").read_text() diff --git a/scripts/ci/test_ci_parallelism.py b/scripts/ci/test_ci_parallelism.py index 2043f8bbd8..a9797a69c2 100755 --- a/scripts/ci/test_ci_parallelism.py +++ b/scripts/ci/test_ci_parallelism.py @@ -116,6 +116,30 @@ def test_status_labels_do_not_repeat_setup_builds_or_artifact_transfers(self): self.assertIn("cancel-in-progress: true", self.workflow) +class MacRunnerTests(unittest.TestCase): + def test_all_macos_execution_uses_the_same_self_hosted_labels_for_every_author(self): + workflow = (ROOT / ".github/workflows/test.yml").read_text() + labels = ["self-hosted", "macOS", "ARM64", "mac-arm64-mini"] + for job in ("vendored-terminal", "terminal-pty-shards"): + header = workflow.split(f"\n {job}:\n", 1)[1].split(" steps:\n", 1)[0] + selection = re.search(r"runs-on: \$\{\{ matrix\.os == 'macos-15' && fromJSON\('([^']+)'\) \|\| matrix\.os \}\}", header) + self.assertIsNotNone(selection, job) + self.assertEqual(json.loads(selection[1]), labels) + self.assertIn("os: [ubuntu-latest, macos-15]", header) + self.assertIn("needs: scope", header) + coordination = workflow.split("\n macos-workspace-coordination-tests:\n", 1)[1].split(" steps:\n", 1)[0] + self.assertIn("runs-on: [self-hosted, macOS, ARM64, mac-arm64-mini]", coordination) + for job in ("scope", "cli-test-build", "shard-b", "shard-c", "shard-d", "test-online"): + header = workflow.split(f"\n {job}:\n", 1)[1].split(" steps:\n", 1)[0] + self.assertIn("runs-on: ubuntu-latest", header) + self.assertIn("pull_request:", workflow) + self.assertIn("branches: [main, develop]", workflow) + self.assertNotIn("pull_request_target:", workflow) + self.assertNotIn("github.event.pull_request.user", workflow) + self.assertNotIn("ASTRA_MACOS_", workflow) + self.assertNotIn("macos-routing:", workflow) + + class ParallelGateTests(unittest.TestCase): def test_shard_gates_reject_failure_cancellation_and_unexpected_skip(self): for step, variables in ( @@ -143,12 +167,11 @@ def test_online_gate_requires_all_matrix_jobs_to_succeed(self): }, capture_output=True, text=True) self.assertEqual(result.returncode == 0, status == moi_status == "success") - def test_package_jobs_avoid_macos_routing_and_gates_depend_on_execution(self): + def test_parallel_jobs_depend_only_on_scope_and_gates_depend_on_shards(self): workflow = (ROOT / ".github/workflows/test.yml").read_text() for job in ("terminal-pty-shards", "test-online"): header = workflow.split(f"\n {job}:\n", 1)[1].split(" steps:\n", 1)[0] - expected = "[scope, macos-routing]" if job == "terminal-pty-shards" else "scope" - self.assertIn(f" needs: {expected}\n", header) + self.assertIn(" needs: scope\n", header) self.assertIn("!cancelled()", header) self.assertIn("fail-fast: false", header) terminal = workflow.split("\n terminal-pty:\n", 1)[1].split("\n terminal-pty-shards:", 1)[0] @@ -164,7 +187,7 @@ def test_package_jobs_avoid_macos_routing_and_gates_depend_on_execution(self): self.assertIn("needs.scope.result != 'success'", coordination) self.assertIn("if: ${{ !cancelled() }}", coordination) contracts = workflow.split("\n macos-workspace-coordination-tests:\n", 1)[1].split(" steps:\n", 1)[0] - self.assertIn("needs: [scope, macos-routing]", contracts) + self.assertIn("needs: scope", contracts) self.assertNotIn("terminal-pty-shards", contracts, "Tools contracts must not wait for the CLI build") online = workflow.split("\n test-online:\n", 1)[1] self.assertIn("lane: [core-runtime, core-turn-core, core-services, integration]", online) diff --git a/scripts/ci/test_macos_runner.py b/scripts/ci/test_macos_runner.py deleted file mode 100755 index 0d9e134faa..0000000000 --- a/scripts/ci/test_macos_runner.py +++ /dev/null @@ -1,801 +0,0 @@ -#!/usr/bin/env python3 -"""Exercise routing/admission against GitHub-shaped API data and real shell dispatch.""" -import copy -import json -import os -from pathlib import Path -import subprocess -import sys -import tempfile -import time -import unittest -from unittest.mock import patch - -import macos_runner as m -from test_release_build_shells import workflow_run_script - -ROOT = Path(__file__).resolve().parents[2] -B, H, M = "b" * 40, "a" * 40, "c" * 40 - - -def ordinary_run(): - return {"id": 200, "status": "completed", "event": "pull_request", - "path": ".github/workflows/test.yml", "repository": {"full_name": m.REPO}, - "head_sha": H, "head_repository": {"id": 2}, "run_attempt": 1, - "display_title": f"Test Suite PR 42 base={B} head={H} merge={M}", - "pull_requests": [{"number": 42, "head": {"sha": H}, "base": {"sha": B}}]} - - -class FakeAPI(m.API): - def __init__(self): - self.data = {} - self.status = {} - self.calls = [] - self.writes = [] - self.checks = [] - - def request(self, path, data=None, method=None): - self.calls.append((path, method)) - if data is not None: - self.writes.append(copy.deepcopy(data)) - if path.endswith("/dispatches"): - return self.status.get(path, 204), None - if method == "PATCH": - row = next(c for c in self.checks if c["id"] == int(path.rsplit("/", 1)[1])) - row.update(copy.deepcopy(data)) - return 200, row - row = {**copy.deepcopy(data), "id": len(self.checks) + 1, "app": {"id": 5}} - self.checks.append(row) - return 201, row - if path.endswith("/check-runs"): - return 200, {"check_runs": copy.deepcopy(self.checks)} - return self.status.get(path, 200 if path in self.data else 404), copy.deepcopy(self.data.get(path)) - - def pages(self, path, key=None): - value = self.get(path) - return value[key] if key else value - - -class PolicyTests(unittest.TestCase): - def setUp(self): - self.api = a = FakeAPI() - self.pr_path = f"/repos/{m.REPO}/pulls/42" - a.data[self.pr_path] = {"state": "open", "number": 42, - "base": {"repo": {"full_name": m.REPO}, "ref": "main"}, - "head": {"sha": H, "repo": {"id": 2}}, "merge_commit_sha": M, - "user": {"login": "fixture-member", "id": 7}} - a.data[f"/repos/{m.REPO}/git/ref/heads/main"] = {"object": {"sha": B}} - a.data[f"/repos/{m.REPO}/git/commits/{M}"] = {"parents": [{"sha": B}, {"sha": H}]} - a.data[f"/repos/{m.REPO}/compare/{B}...{M}"] = {"files": [{"filename": "crates/core/src/lib.rs"}]} - a.status["/orgs/matrixorigin/members/fixture-member"] = 204 - self.group_path = "/orgs/matrixorigin/actions/runner-groups/9" - a.data[self.group_path] = {"name": "astra-macos-pr", "restricted_to_workflows": True, - "selected_workflows": [m.WORKFLOW_REF], "visibility": "selected", "allows_public_repositories": True} - a.data[self.group_path + "/repositories"] = {"repositories": [{"full_name": m.REPO}]} - a.data[self.group_path + "/runners"] = {"runners": [{"name": "fixture-arm", "status": "online", "busy": False, - "labels": [{"name": n} for n in ("self-hosted", "macOS", "ARM64", "mac-arm64-mini", "astra-macos-admitted")]}]} - a.data[f"/repos/{m.REPO}/rules/branches/main"] = [{"type": "required_status_checks", "parameters": { - "strict_required_status_checks_policy": True, - "required_status_checks": [{"context": c, "integration_id": 5} for c in m.CONTEXTS]}}] - a.data[f"/repos/{m.REPO}/environments/astra-macos-control"] = { - "deployment_branch_policy": {"protected_branches": False, "custom_branch_policies": True}} - a.data[f"/repos/{m.REPO}/environments/astra-macos-control/deployment-branch-policies"] = { - "branch_policies": [{"name": "main", "type": "branch"}]} - self.run_path = f"/repos/{m.REPO}/actions/runs/100" - a.data[self.run_path] = {"id": 100, "repository": {"full_name": m.REPO}, - "event": "pull_request_target", "path": m.WORKFLOW, "run_attempt": 1} - self.active = {"name": "Migrated macOS lanes", "status": "in_progress", "conclusion": None, - "runner_name": "fixture-arm", "runner_group_id": 9, "steps": []} - a.data[self.run_path + "/attempts/1/jobs"] = {"jobs": [self.active]} - a.data[self.run_path + "/jobs?filter=latest"] = {"jobs": [self.active]} - self.plan = {**m.source(a, 42), "schema": 1, "app_id": 5, "run_id": 100, - "attempt": 1, "route": "self-hosted", "group_id": 9, - "expires_at": int(time.time()) + m.AUTHORIZATION_SECONDS} - self.config = {"app_id": 5, "kind": "pr", "group_id": 9, "runner_name": "fixture-arm"} - self.request = {"run_id": 100, "attempt": 1, "head": H} - - def authorize(self): - m.save_checks(self.api, self.plan) - - def test_merge_requires_current_base_head_and_exact_ordered_parents(self): - for parents in ([H, B], [B], [B, H, M], [B, "d" * 40]): - with self.subTest(parents=parents): - self.api.data[f"/repos/{m.REPO}/git/commits/{M}"]["parents"] = [{"sha": p} for p in parents] - with self.assertRaises(m.Unavailable): - m.source(self.api, 42) - self.api.data[self.pr_path]["merge_commit_sha"] = None - with self.assertRaises(m.Unavailable): - m.source(self.api, 42) - - def test_wrong_repository_branch_closed_pr_and_unknown_head_fail(self): - for mutate in (lambda p: p.update(state="closed"), - lambda p: p["base"].update(ref="develop"), - lambda p: p["base"]["repo"].update(full_name="fixture/other"), - lambda p: p["head"].update(sha="invalid")): - with self.subTest(mutate=mutate): - before = copy.deepcopy(self.api.data[self.pr_path]) - mutate(self.api.data[self.pr_path]) - with self.assertRaises(m.Unavailable): - m.source(self.api, 42) - self.api.data[self.pr_path] = before - - def test_membership_redirect_and_failures_never_authorize(self): - for status in (204, 302, 401, 403, 404, 429, 500): - with self.subTest(status=status): - self.api.status["/orgs/matrixorigin/members/fixture-member"] = status - self.assertEqual(m.member(self.api, "fixture-member"), status == 204) - self.assertFalse(m.member(self.api, "../fixture")) - self.assertIsNone(m.NoRedirect().redirect_request(None, None, 302, None, None, None)) - - def test_public_or_repo_only_runner_policy_cannot_authorize(self): - for key, value in (("name", "Default"), ("restricted_to_workflows", False), - ("selected_workflows", [m.WORKFLOW_REF, "fixture/other@main"]), - ("visibility", "all"), ("allows_public_repositories", False)): - with self.subTest(key=key): - before = copy.deepcopy(self.api.data[self.group_path]) - self.api.data[self.group_path][key] = value - with self.assertRaises(m.Unavailable): - m.group_ready(self.api, 9, "pr") - self.api.data[self.group_path] = before - self.api.data[self.group_path + "/repositories"]["repositories"].append({"full_name": "fixture/other"}) - with self.assertRaises(m.Unavailable): - m.group_ready(self.api, 9, "pr") - - def test_offline_busy_or_unadmitted_runner_uses_hosted_route(self): - runner = self.api.data[self.group_path + "/runners"]["runners"][0] - for change in ({"status": "offline"}, {"busy": True}, {"labels": []}): - with self.subTest(change=change): - before = copy.deepcopy(runner) - runner.update(change) - self.assertFalse(m.group_ready(self.api, 9, "pr")) - runner.update(before) - - def test_publisher_key_environment_cannot_allow_pr_refs_wildcards_or_tags(self): - path = f"/repos/{m.REPO}/environments/astra-macos-control/deployment-branch-policies" - for policies in ([], [{"name": "*", "type": "branch"}], [{"name": "main", "type": "tag"}], - [{"name": "main", "type": "branch"}, {"name": "develop", "type": "branch"}]): - with self.subTest(policies=policies): - self.api.data[path]["branch_policies"] = policies - with self.assertRaises(m.Unavailable): - m.require_check_sources(self.api, 5) - - def test_required_contexts_must_bind_to_the_publisher_app(self): - m.require_check_sources(self.api, 5) - for source in (None, 0, 6): - with self.subTest(source=source): - self.api.data[f"/repos/{m.REPO}/rules/branches/main"][0]["parameters"]["required_status_checks"][0]["integration_id"] = source - with self.assertRaises(m.Unavailable): - m.require_check_sources(self.api, 5) - - def test_up_to_date_branch_enforcement_is_required_for_tuple_bound_checks(self): - self.api.data[f"/repos/{m.REPO}/rules/branches/main"][0]["parameters"]["strict_required_status_checks_policy"] = False - with self.assertRaises(m.Unavailable): - m.require_check_sources(self.api, 5) - - def test_private_member_personal_fork_is_admitted_and_each_attempt_rechecks(self): - self.authorize() - self.assertEqual(m.admit(self.api, self.request, self.config), self.plan) - self.api.data[self.run_path]["run_attempt"] = 2 - self.api.data[self.run_path + "/attempts/2/jobs"] = {"jobs": [self.active]} - self.request["attempt"] = 2 - self.assertEqual(m.admit(self.api, self.request, self.config), self.plan) - self.api.status["/orgs/matrixorigin/members/fixture-member"] = 404 - with self.assertRaises(m.Unavailable): - m.admit(self.api, self.request, self.config) - self.assertGreaterEqual(sum(p.endswith("members/fixture-member") for p, _ in self.api.calls), 3) - - def test_forged_app_workflow_attempt_runner_and_event_are_rejected(self): - self.authorize() - cases = [lambda: self.api.checks[0]["app"].update(id=9), - lambda: self.api.data[self.run_path].update(path=".github/workflows/test.yml"), - lambda: self.api.data[self.run_path].update(event="pull_request"), - lambda: self.api.data[self.run_path].update(run_attempt=2), - lambda: self.active.update(runner_name="other"), - lambda: self.active.update(runner_group_id=1)] - for mutate in cases: - with self.subTest(mutate=mutate): - snapshot = copy.deepcopy((self.api.data, self.api.checks)) - mutate() - with self.assertRaises(m.Unavailable): - m.admit(self.api, self.request, self.config) - self.api.data, self.api.checks = snapshot - self.active = self.api.data[self.run_path + "/attempts/1/jobs"]["jobs"][0] - - def test_admission_cannot_cross_main_account_or_use_stale_source(self): - self.authorize() - with self.assertRaises(m.Unavailable): - m.admit(self.api, self.request, {**self.config, "kind": "main"}) - self.api.data[self.pr_path]["user"]["id"] = 8 - with self.assertRaises(m.Unavailable): - m.admit(self.api, self.request, self.config) - self.api.data[self.pr_path]["user"]["id"] = 7 - self.api.data[f"/repos/{m.REPO}/git/ref/heads/main"]["object"]["sha"] = "d" * 40 - with self.assertRaises(m.Unavailable): - m.admit(self.api, self.request, self.config) - - def test_scope_uses_canonical_classifier_and_truncation_is_full_scope(self): - path = f"/repos/{m.REPO}/compare/{B}...{M}" - self.api.data[path] = {"files": [{"filename": "docs/guide.md"}]} - self.assertFalse(m.selected(self.api, self.plan)) - self.api.data[path] = {"files": [{"filename": "docs/guide.md", "previous_filename": "crates/core/src/lib.rs"}]} - self.assertTrue(m.selected(self.api, self.plan)) - self.api.data[path] = {"files": [{"filename": "docs/a.md"}] * 300} - self.assertTrue(m.selected(self.api, self.plan)) - self.api.status[path] = 403 - self.assertTrue(m.selected(self.api, self.plan)) - - def test_ordinary_member_lookup_failure_and_resource_failure_choose_distinct_routes(self): - with tempfile.TemporaryDirectory() as directory, patch.dict(os.environ, { - "GITHUB_REPOSITORY": m.REPO, "GITHUB_EVENT_NAME": "pull_request_target", - "GITHUB_RUN_ID": "100", "GITHUB_RUN_ATTEMPT": "1", - "GITHUB_OUTPUT": str(Path(directory) / "outputs"), "ASTRA_MACOS_GROUP_ID": "9"}): - for status, busy, expected in ((204, False, "self-hosted"), (204, True, "hosted"), - (404, False, "ordinary"), (403, False, "ordinary"), (302, False, "ordinary")): - with self.subTest(status=status, busy=busy): - self.api.status["/orgs/matrixorigin/members/fixture-member"] = status - self.api.data[self.group_path + "/runners"]["runners"][0]["busy"] = busy - m.plan_command(self.api, {"number": 42}, 5) - plan = json.loads(self.api.checks[-1]["output"]["text"]) - self.assertEqual(plan["route"], expected) - - def test_recovery_dispatch_revalidates_members_and_uses_hosted_without_auto_approval(self): - with tempfile.TemporaryDirectory() as directory, patch.dict(os.environ, { - "GITHUB_REPOSITORY": m.REPO, "GITHUB_EVENT_NAME": "workflow_dispatch", - "GITHUB_RUN_ID": "100", "GITHUB_RUN_ATTEMPT": "1", - "GITHUB_OUTPUT": str(Path(directory) / "outputs"), "ASTRA_MACOS_GROUP_ID": "9"}): - for membership, route in ((204, "hosted"), (404, "ordinary"), (403, "ordinary")): - with self.subTest(membership=membership): - self.api.status["/orgs/matrixorigin/members/fixture-member"] = membership - m.plan_command(self.api, {"inputs": {"pr_number": "42"}}, 5) - self.assertEqual(json.loads(self.api.checks[-1]["output"]["text"])["route"], route) - - def test_real_test_failure_cannot_be_masked_by_a_successful_fallback(self): - self.authorize() - failed = {**self.active, "status": "completed", "conclusion": "failure", - "steps": [{"name": "Run all three macOS lanes", "conclusion": "failure"}]} - fallbacks = [{**failed, "name": "Fallback macOS: " + lane, "conclusion": "success", - "labels": ["macos-15"], - "steps": [{"name": "Enforce current hosted authorization", "conclusion": "success"}, - {"name": "Run macOS lane", "conclusion": "success"}]} for lane in m.HOSTED_LANES] - self.api.data[self.run_path + "/jobs?filter=latest"] = {"jobs": [failed, *fallbacks]} - self.assertFalse(m.can_fallback(self.api, self.plan)) - m.finish(self.api, self.plan) - self.assertTrue(all(c["conclusion"] == "failure" for c in self.api.checks)) - failed["steps"] = [] # Admission failed before executing PR tests. - self.assertTrue(m.can_fallback(self.api, self.plan)) - m.finish(self.api, self.plan) - self.assertTrue(all(c["conclusion"] == "success" for c in self.api.checks)) - - def test_success_requires_actual_test_step_and_expected_runner_group(self): - self.authorize() - self.active.update(status="completed", conclusion="success") - self.assertEqual(m.verified_job(self.api, self.plan, "Migrated macOS lanes"), "failure") - self.active["steps"] = [{"name": "Run all three macOS lanes", "conclusion": "success"}] - self.assertEqual(m.verified_job(self.api, self.plan, "Migrated macOS lanes"), "success") - self.active["runner_group_id"] = 1 - self.assertEqual(m.verified_job(self.api, self.plan, "Migrated macOS lanes"), "failure") - - def test_untrusted_ordinary_success_does_not_satisfy_member_results(self): - plan = {**self.plan, "route": "ordinary"} - self.authorize() - self.assertTrue(m.finish(self.api, plan)) - self.assertTrue(all(c.get("conclusion") == "failure" for c in self.api.checks)) - - def test_delayed_publication_cannot_overwrite_newer_attempt(self): - self.authorize() - newer = {**self.plan, "attempt": 2} - m.save_checks(self.api, newer) - count = len(self.api.writes) - m.save_checks(self.api, self.plan, "success") - self.assertEqual(len(self.api.writes), count) - self.assertFalse(any(c.get("conclusion") == "success" for c in self.api.checks)) - - def test_missing_hint_keeps_hosted_execution_and_app_hint_is_not_a_result(self): - with patch.object(m.time, "sleep"): - self.assertFalse(m.hint(self.api, {"number": 42, "pull_request": {"head": {"sha": H}, "base": {"ref": "main"}}}, 5)) - self.authorize() - self.assertTrue(m.hint(self.api, {"number": 42, "pull_request": {"head": {"sha": H}, "base": {"ref": "main"}}}, 5)) - self.assertTrue(all(c["status"] == "in_progress" for c in self.api.checks)) - - def test_ordinary_hosted_results_require_matching_source_all_three_jobs_and_real_macos(self): - runs_path = f"/repos/{m.REPO}/actions/workflows/test.yml/runs?event=pull_request&head_sha={H}" - self.api.data[runs_path] = {"workflow_runs": [ordinary_run()]} - names = ("Terminal: default-reader (macos-15)", "Terminal: reflow (macos-15)", - "macOS workspace coordination contracts") - jobs = [{"name": name, "status": "completed", "conclusion": "success", "labels": ["macos-15"]} for name in names] - self.api.data[f"/repos/{m.REPO}/actions/runs/200/jobs?filter=latest"] = {"jobs": jobs} - self.assertEqual(m.hosted_result(self.api, self.plan), "success") - for key, value in (("status", "in_progress"), ("conclusion", "cancelled"), ("labels", ["ubuntu-latest"])): - before = copy.deepcopy(jobs[0]) - jobs[0][key] = value - self.assertNotEqual(m.hosted_result(self.api, self.plan), "success") - jobs[0] = before - jobs.pop() - self.assertEqual(m.hosted_result(self.api, self.plan), "failure") - self.api.data[runs_path]["workflow_runs"][0]["status"] = "in_progress" - self.assertIsNone(m.hosted_result(self.api, self.plan)) - self.api.data[runs_path]["workflow_runs"][0]["pull_requests"][0]["base"]["sha"] = "d" * 40 - self.assertIsNone(m.hosted_result(self.api, self.plan)) - - def test_hosted_result_cannot_accept_a_rerun_with_successful_cached_steps(self): - plan = {**self.plan, "route": "hosted"} - job = {"name": "Fallback macOS: default-reader", "status": "completed", "conclusion": "success", - "labels": ["macos-15"], "steps": [ - {"name": "Enforce current hosted authorization", "conclusion": "success"}, - {"name": "Run macOS lane", "conclusion": "success"}]} - self.api.data[self.run_path + "/jobs?filter=latest"] = {"jobs": [job]} - self.assertEqual(m.verified_job(self.api, plan, job["name"]), "success") - self.api.data[self.run_path]["run_attempt"] = 2 - self.assertEqual(m.verified_job(self.api, plan, job["name"]), "failure") - self.api.data[self.run_path]["run_attempt"] = 1 - job["steps"].pop(0) - self.assertEqual(m.verified_job(self.api, plan, job["name"]), "failure") - - def test_main_stays_hosted_without_explicit_isolation_acceptance(self): - with tempfile.TemporaryDirectory() as directory, patch.dict(os.environ, { - "GITHUB_REPOSITORY": m.REPO, "GITHUB_EVENT_NAME": "push", - "GITHUB_RUN_ID": "100", "GITHUB_RUN_ATTEMPT": "1", - "GITHUB_OUTPUT": str(Path(directory) / "outputs"), "ASTRA_MACOS_MAIN_ISOLATED": "false"}): - m.plan_command(self.api, {"ref": "refs/heads/main", "before": B, "after": H}, 5) - out = dict(line.split("=", 1) for line in Path(os.environ["GITHUB_OUTPUT"]).read_text().splitlines()) - self.assertEqual(out["route"], "disabled") - self.assertEqual(self.api.writes, []) - - def test_hosted_failed_only_rerun_requires_new_admission_not_cached_router(self): - with tempfile.TemporaryDirectory() as directory: - event = Path(directory) / "event.json" - event.write_text("{}") - output = Path(directory) / "outputs" - with patch.dict(os.environ, {"GH_TOKEN": "fixture", "ASTRA_MACOS_APP_ID": "5", - "GITHUB_EVENT_PATH": str(event), "GITHUB_OUTPUT": str(output), "GITHUB_RUN_ATTEMPT": "2"}), \ - patch.object(m, "API", return_value=self.api), \ - patch.object(sys, "argv", ["macos_runner.py", "refresh", "--plan", json.dumps(self.plan)]): - self.assertEqual(m.main(), 1) - self.assertFalse(output.exists()) - - def test_api_pagination_collects_all_pages_and_errors_are_fail_closed(self): - api = m.API("fixture") - pages = [(200, {"jobs": [{"id": i} for i in range(100)]}), (200, {"jobs": [{"id": 100}]})] - with patch.object(api, "request", side_effect=pages) as request: - self.assertEqual(len(api.pages("/fixture/jobs", "jobs")), 101) - self.assertIn("page=2", request.call_args.args[0]) - with patch.object(api, "request", return_value=(403, None)): - with self.assertRaises(m.Unavailable): - api.pages("/fixture/jobs", "jobs") - with self.assertRaises(m.Unavailable): - api.request("https://fixture.invalid/token") - - def test_ordinary_hint_returns_without_sleep_and_permission_denial_is_immediate(self): - event = {"number": 42, "pull_request": {"head": {"sha": H}, "base": {"ref": "main"}}} - m.save_checks(self.api, {**self.plan, "route": "ordinary"}) - with patch.object(m.time, "sleep") as sleep: - self.assertFalse(m.hint(self.api, event, 5)) - sleep.assert_not_called() - self.api.checks.clear() - self.authorize() - self.api.status[self.run_path] = 403 - with patch.object(m.time, "sleep") as sleep: - self.assertFalse(m.hint(self.api, event, 5)) - sleep.assert_not_called() - - def test_hint_waits_for_delayed_plan_and_exhausted_budget_keeps_hosted(self): - event = {"number": 42, "pull_request": {"head": {"sha": H}, "base": {"ref": "main"}}} - self.authorize() - records = copy.deepcopy(self.api.checks) - self.api.checks.clear() - def publish_after_first_sleep(_seconds): - self.api.checks = records - with patch.object(m.time, "sleep", side_effect=publish_after_first_sleep) as sleep: - self.assertTrue(m.hint(self.api, event, 5)) - self.assertEqual(sleep.call_count, 1) - self.api.checks.clear() - with patch.object(m.time, "sleep") as sleep, patch.object(m.time, "monotonic", side_effect=[0, 91]): - self.assertFalse(m.hint(self.api, event, 5)) - sleep.assert_not_called() - - def test_expired_authorization_rejects_actual_rerun_and_publication(self): - self.plan["expires_at"] = 1 - self.authorize() - with self.assertRaises(m.Unavailable): - m.admit(self.api, self.request, self.config) - with self.assertRaises(m.Unavailable): - m.finish(self.api, self.plan) - self.assertTrue(all(c["status"] == "in_progress" for c in self.api.checks)) - - def prepare_ordinary_reconciliation(self): - self.api.status["/orgs/matrixorigin/members/fixture-member"] = 404 - self.plan["route"] = "ordinary" - self.authorize() - self.api.data[f"/repos/{m.REPO}/pulls?state=open&base=main"] = [ - {"number": 42, "head": {"sha": H}}] - self.api.data[f"/repos/{m.REPO}/actions/workflows/test.yml/runs?event=pull_request&head_sha={H}"] = { - "workflow_runs": [ordinary_run()]} - self.jobs200 = f"/repos/{m.REPO}/actions/runs/200/jobs?filter=latest" - self.api.data[self.jobs200] = {"jobs": [ - {"name": name, "status": "completed", "conclusion": "success", "labels": ["macos-15"]} - for name in ("Terminal: default-reader (macos-15)", "Terminal: reflow (macos-15)", - "macOS workspace coordination contracts")]} - - def test_empty_fork_associations_complete_ordinary_checks(self): - self.prepare_ordinary_reconciliation() - run = self.api.data[f"/repos/{m.REPO}/actions/workflows/test.yml/runs?event=pull_request&head_sha={H}"]["workflow_runs"][0] - run["pull_requests"] = [] - self.assertEqual(m.hosted_result(self.api, self.plan), "success") - self.assertTrue(m.finish(self.api, self.plan)) - self.assertTrue(all(c["conclusion"] == "success" for c in self.api.checks)) - - def test_empty_fork_associations_work_through_observer_and_scheduled_reconciliation(self): - for entrypoint in ("observe", "reconcile"): - with self.subTest(entrypoint=entrypoint): - self.api.checks.clear() - self.prepare_ordinary_reconciliation() - run = self.api.data[f"/repos/{m.REPO}/actions/workflows/test.yml/runs?event=pull_request&head_sha={H}"]["workflow_runs"][0] - run["pull_requests"] = [] - self.api.data[f"/repos/{m.REPO}/actions/runs/200"] = run - # This endpoint is also empty for the real reviewed fork run. - self.api.data[f"/repos/{m.REPO}/commits/{H}/pulls"] = [] - with tempfile.TemporaryDirectory() as directory: - event = Path(directory) / "event.json" - event.write_text(json.dumps({"workflow_run": {"id": 200}})) - with ( - patch.dict(os.environ, {"GH_TOKEN": "fixture", "ASTRA_MACOS_APP_ID": "5", - "GITHUB_EVENT_PATH": str(event)}), - patch.object(m, "API", return_value=self.api), - patch.object(sys, "argv", ["macos_runner.py", entrypoint]), - ): - self.assertEqual(m.main(), 0) - self.assertTrue(all(c["conclusion"] == "success" for c in self.api.checks)) - self.assertFalse(any(path.endswith(f"/commits/{H}/pulls") for path, _ in self.api.calls)) - - def test_ordinary_empty_links_retain_repository_event_workflow_and_merge_identity(self): - self.prepare_ordinary_reconciliation() - original = ordinary_run() - original["pull_requests"] = [] - mutations = ( - lambda r: r.update(event="push"), - lambda r: r.update(path=m.WORKFLOW), - lambda r: r["repository"].update(full_name="fixture/other"), - lambda r: r["head_repository"].update(id=99), - lambda r: r.update(head_sha="d" * 40), - lambda r: r.update(display_title=r["display_title"].replace(B, "d" * 40)), - lambda r: r.update(display_title=r["display_title"].replace(H, "d" * 40)), - lambda r: r.update(display_title=r["display_title"].replace(M, "d" * 40)), - lambda r: r.update(display_title=r["display_title"].replace("PR 42", "PR 43")), - lambda r: r.update(display_title="Historical title without source metadata"), - lambda r: r.update(pull_requests=[{"number": 99, "head": {"sha": H}, "base": {"sha": B}}])) - for mutate in mutations: - with self.subTest(mutate=mutate): - run = copy.deepcopy(original) - mutate(run) - with self.assertRaises(m.Unavailable): - m.ordinary_source(self.api, run) - # A plausible run title cannot replace validation of the immutable merge. - self.api.data[f"/repos/{m.REPO}/git/commits/{M}"]["parents"] = [{"sha": H}, {"sha": B}] - with self.assertRaises(m.Unavailable): - m.ordinary_source(self.api, original) - - def test_completed_ordinary_run_without_source_tuple_fails_instead_of_pending(self): - self.prepare_ordinary_reconciliation() - run = self.api.data[f"/repos/{m.REPO}/actions/workflows/test.yml/runs?event=pull_request&head_sha={H}"]["workflow_runs"][0] - run["pull_requests"] = [] - del run["display_title"] - self.assertTrue(m.finish(self.api, self.plan)) - self.assertTrue(all(c["conclusion"] == "failure" for c in self.api.checks)) - - def test_reconciliation_recovers_a_lost_completion_notification(self): - self.prepare_ordinary_reconciliation() - m.reconcile(self.api, 5) - self.assertTrue(all(c["status"] == "completed" and c["conclusion"] == "success" for c in self.api.checks)) - writes = len(self.api.writes) - m.reconcile(self.api, 5) - self.assertEqual(len(self.api.writes), writes) - - def test_completed_run_with_missing_or_skipped_lanes_fails_instead_of_pending(self): - for mode in ("missing", "skipped"): - with self.subTest(mode=mode): - self.api.checks.clear() - self.prepare_ordinary_reconciliation() - if mode == "missing": - self.api.data[self.jobs200]["jobs"].pop() - else: - self.api.data[self.jobs200]["jobs"][0]["conclusion"] = "skipped" - m.reconcile(self.api, 5) - self.assertTrue(all(c["status"] == "completed" and c["conclusion"] == "failure" for c in self.api.checks)) - - def test_approval_pending_expires_and_does_not_become_a_successful_skip(self): - self.prepare_ordinary_reconciliation() - self.api.data[f"/repos/{m.REPO}/actions/workflows/test.yml/runs?event=pull_request&head_sha={H}"]["workflow_runs"] = [] - m.reconcile(self.api, 5) - self.assertTrue(all(c["status"] == "in_progress" for c in self.api.checks)) - with patch.object(m.time, "time", return_value=self.plan["expires_at"] + 1): - m.reconcile(self.api, 5) - self.assertTrue(all(c["status"] == "completed" and c["conclusion"] == "failure" for c in self.api.checks)) - self.assertTrue(all("expired" in c["output"]["summary"] for c in self.api.checks)) - - def test_legacy_authorization_without_deadline_fails_closed_instead_of_staying_pending(self): - self.prepare_ordinary_reconciliation() - del self.plan["expires_at"] - m.save_checks(self.api, self.plan) - m.reconcile(self.api, 5) - self.assertTrue(all(c["conclusion"] == "failure" for c in self.api.checks)) - with self.assertRaises(m.Unavailable): - m.require_unexpired(self.plan) - - def test_one_unavailable_pr_does_not_starve_reconciliation_of_other_prs(self): - self.prepare_ordinary_reconciliation() - self.api.data[f"/repos/{m.REPO}/pulls?state=open&base=main"].insert(0, - {"number": 43, "head": {"sha": "d" * 40}}) - pages = self.api.pages - def flaky_pages(path, key=None): - if path == f"/repos/{m.REPO}/commits/{'d' * 40}/check-runs": - raise m.Unavailable("fixture unavailable") - return pages(path, key) - with patch.object(self.api, "pages", side_effect=flaky_pages): - m.reconcile(self.api, 5) - self.assertTrue(all(c["conclusion"] == "success" for c in self.api.checks)) - - def test_expiry_reconciliation_does_not_downgrade_already_successful_results(self): - self.prepare_ordinary_reconciliation() - m.save_checks(self.api, self.plan, "success") - with patch.object(m.time, "time", return_value=self.plan["expires_at"] + 1): - m.reconcile(self.api, 5) - m.save_checks(self.api, self.plan, "failure", only_pending=True) - self.assertTrue(all(c["conclusion"] == "success" for c in self.api.checks)) - - def test_merge_source_retries_null_and_old_parents_without_changing_trigger_head(self): - for mode in ("null", "old-parents"): - with self.subTest(mode=mode): - original = copy.deepcopy(self.api.data) - if mode == "null": - self.api.data[self.pr_path]["merge_commit_sha"] = None - else: - self.api.data[f"/repos/{m.REPO}/git/commits/{M}"]["parents"][0]["sha"] = "d" * 40 - with patch.object(m.time, "sleep", side_effect=lambda _: self.api.data.update(original)) as sleep: - self.assertEqual(m.source(self.api, 42, attempts=3, expected_head=H)["merge"], M) - self.assertEqual(sleep.call_count, 1) - self.api.data[self.pr_path]["head"]["sha"] = "e" * 40 - with patch.object(m.time, "sleep") as sleep, self.assertRaises(m.StaleSource): - m.source(self.api, 42, attempts=3, expected_head=H) - sleep.assert_not_called() - - def test_plan_exhaustion_publishes_retryable_failure_instead_of_missing_checks(self): - self.api.data[self.pr_path]["merge_commit_sha"] = None - with tempfile.TemporaryDirectory() as directory, patch.dict(os.environ, { - "GITHUB_REPOSITORY": m.REPO, "GITHUB_EVENT_NAME": "pull_request_target", - "GITHUB_RUN_ID": "100", "GITHUB_RUN_ATTEMPT": "1", - "GITHUB_OUTPUT": str(Path(directory) / "outputs")}), patch.object(m.time, "sleep"): - m.plan_command(self.api, {"number": 42, "pull_request": {"head": {"sha": H}}}, 5) - self.assertEqual(len(self.api.checks), 3) - self.assertTrue(all(c["conclusion"] == "failure" for c in self.api.checks)) - self.api.data[self.pr_path]["merge_commit_sha"] = M - self.api.data[f"/repos/{m.REPO}/pulls?state=open&base=main"] = [{"number": 42, "head": {"sha": H}}] - m.reconcile(self.api, 5) - self.assertEqual(self.api.writes[-1], {"ref": "main", "inputs": {"pr_number": "42"}}) - - def test_zero_checks_recovery_is_bounded_and_never_authorizes_execution(self): - self.api.data[f"/repos/{m.REPO}/pulls?state=open&base=main"] = [{"number": 42, "head": {"sha": H}}] - m.reconcile(self.api, 5) - self.assertEqual(sum(path.endswith("/dispatches") for path, _ in self.api.calls), 1) - marker = json.loads(self.api.checks[-1]["output"]["text"]) - self.assertEqual(marker["route"], "recovering") - with self.assertRaises(m.Unavailable): - m.admit(self.api, self.request, self.config) - m.reconcile(self.api, 5) - self.assertEqual(sum(path.endswith("/dispatches") for path, _ in self.api.calls), 1) - with patch.object(m.time, "time", return_value=marker["recovery_requested_at"] + m.RECOVERY_COOLDOWN_SECONDS): - m.reconcile(self.api, 5) - self.assertEqual(sum(path.endswith("/dispatches") for path, _ in self.api.calls), 2) - self.authorize() # Real plan replaces the non-authorizing marker. - self.assertEqual(m.authorization(self.api, H, 100, 5), self.plan) - - def test_recovery_does_not_dispatch_over_a_real_plan_or_retry_test_failures(self): - self.authorize() - m.recover(self.api, {"number": 42, "head": {"sha": H}}, 5) - self.assertFalse(any(path.endswith("/dispatches") for path, _ in self.api.calls)) - m.save_checks(self.api, self.plan, "failure", "Actual test failure") - self.api.data[f"/repos/{m.REPO}/pulls?state=open&base=main"] = [{"number": 42, "head": {"sha": H}}] - m.reconcile(self.api, 5) - self.assertFalse(any(path.endswith("/dispatches") for path, _ in self.api.calls)) - - def test_dispatch_permission_failure_is_throttled_and_recovers_after_cooldown(self): - pr = {"number": 42, "head": {"sha": H}} - path = f"/repos/{m.REPO}/actions/workflows/macos-trusted.yml/dispatches" - self.api.status[path] = 403 - with self.assertRaises(m.Unavailable): - m.recover(self.api, pr, 5) - marker = json.loads(self.api.checks[-1]["output"]["text"]) - m.recover(self.api, pr, 5, marker) - self.assertEqual(sum(p == path for p, _ in self.api.calls), 1) - self.api.status[path] = 204 - with patch.object(m.time, "time", return_value=marker["recovery_requested_at"] + m.RECOVERY_COOLDOWN_SECONDS): - m.recover(self.api, pr, 5, marker) - self.assertEqual(sum(p == path for p, _ in self.api.calls), 2) - - def test_refresh_rejects_a_source_changed_during_its_second_api_read(self): - with patch.object(m, "source", return_value={**self.plan, "head": "e" * 40}): - with self.assertRaises(m.StaleSource): - m.refresh(self.api, self.plan) - - def test_zero_checks_unresolved_merge_is_explicit_failure_and_does_not_dispatch(self): - self.api.data[self.pr_path]["merge_commit_sha"] = None - self.api.data[f"/repos/{m.REPO}/pulls?state=open&base=main"] = [{"number": 42, "head": {"sha": H}}] - m.reconcile(self.api, 5) - self.assertTrue(all(c["conclusion"] == "failure" for c in self.api.checks)) - self.assertFalse(any(path.endswith("/dispatches") for path, _ in self.api.calls)) - - def test_stale_source_closes_immediately_and_reconciliation_requests_fresh_control(self): - self.authorize() - newer_base = "d" * 40 - self.api.data[f"/repos/{m.REPO}/git/ref/heads/main"]["object"]["sha"] = newer_base - self.assertTrue(m.finish(self.api, self.plan)) - self.assertTrue(all(c["conclusion"] == "failure" and "Stale" in c["output"]["summary"] for c in self.api.checks)) - self.api.data[f"/repos/{m.REPO}/git/commits/{M}"]["parents"][0]["sha"] = newer_base - self.api.data[f"/repos/{m.REPO}/pulls?state=open&base=main"] = [{"number": 42, "head": {"sha": H}}] - m.reconcile(self.api, 5) - self.assertEqual(self.api.writes[-1], {"ref": "main", "inputs": {"pr_number": "42"}}) - - def test_stale_old_publisher_cannot_close_new_attempt(self): - self.authorize() - m.save_checks(self.api, {**self.plan, "attempt": 2}) - self.api.data[f"/repos/{m.REPO}/git/ref/heads/main"]["object"]["sha"] = "d" * 40 - writes = len(self.api.writes) - self.assertTrue(m.finish(self.api, self.plan)) - self.assertEqual(len(self.api.writes), writes) - - def test_parallel_hosted_requires_every_real_lane_and_retains_failures(self): - plan = {**self.plan, "route": "hosted"} - jobs = [{"name": "Fallback macOS: " + lane, "status": "completed", "conclusion": "success", - "labels": ["macos-15"], "steps": [{"name": "Enforce current hosted authorization", "conclusion": "success"}, - {"name": "Run macOS lane", "conclusion": "success"}]} - for lane in m.HOSTED_LANES] - self.api.data[self.run_path + "/jobs?filter=latest"] = {"jobs": jobs} - m.save_checks(self.api, plan) - for mutation in ("missing", "failed", "echo", "ubuntu", "success"): - with self.subTest(mutation=mutation): - before = copy.deepcopy(jobs) - if mutation == "missing": - jobs.pop() - elif mutation == "failed": - jobs[0]["conclusion"] = "failure" - elif mutation == "echo": - jobs[0]["steps"] = [] - elif mutation == "ubuntu": - jobs[0]["labels"] = ["ubuntu-latest"] - result = m.finish(self.api, plan) - if mutation == "missing": - self.assertFalse(result) - self.api.data[self.run_path]["status"] = "completed" - self.assertTrue(m.finish(self.api, plan)) - self.assertTrue(all(c["conclusion"] == "failure" for c in self.api.checks)) - self.api.data[self.run_path].pop("status") - else: - self.assertTrue(result) - self.assertTrue(all(c["conclusion"] == ("success" if mutation == "success" else "failure") for c in self.api.checks)) - jobs[:] = before - - - - -class LaneShellTests(unittest.TestCase): - def test_canonical_commands_and_all_failures_propagate(self): - with tempfile.TemporaryDirectory() as directory: - root = Path(directory) - for name in ("cargo", "npm"): - path = root / name - path.write_text(f"#!{sys.executable}\n" + ''' -import json, os, sys -with open(os.environ["CALLS"], "a") as out: out.write(json.dumps([os.path.basename(sys.argv[0]), *sys.argv[1:]]) + "\\n") -failed = os.environ.get("FAIL") and os.environ["FAIL"] in " ".join(sys.argv[1:]) -if not failed and os.path.basename(sys.argv[0]) == "cargo" and sys.argv[1] == "build": - from pathlib import Path - binary = Path(os.environ["CARGO_TARGET_DIR"]) / "debug/astra" - binary.parent.mkdir(parents=True, exist_ok=True) - binary.write_text("fixture binary") -if not failed and os.path.basename(sys.argv[0]) == "npm" and sys.argv[1] == "test": - failed = not os.path.isfile(os.environ["ASTRA_TEST_BINARY"]) -sys.exit(17 if failed else 0) -''') - path.chmod(0o755) - # Keep the real system Git observation, with a harmless synthetic repo. - subprocess.run(["git", "init", str(root)], check=True, capture_output=True) - for lane, failures in (("default-reader", ("", "tui::terminal_startup")), - ("reflow", ("", "build", "ci", "test")), - ("coordination", ("", "clippy", "workspace_observation::tests::"))): - for fail in failures: - with self.subTest(lane=lane, fail=fail): - calls = root / "calls" - calls.unlink(missing_ok=True) - result = subprocess.run(["bash", str(ROOT / "scripts/ci/run-macos-lanes.sh"), lane], - cwd=root, env={**os.environ, "PATH": f"{root}:{os.environ['PATH']}", - "CALLS": str(calls), "FAIL": fail, "GITHUB_WORKSPACE": str(root), - "CARGO_TARGET_DIR": str(root / "scratch-target")}, - capture_output=True, text=True) - self.assertEqual(result.returncode, 17 if fail else 0, result.stderr) - commands = [json.loads(line) for line in calls.read_text().splitlines()] - if not fail and lane == "coordination": - self.assertEqual(len(commands), 2) - self.assertIn("--test-threads=1", commands[-1]) - if not fail and lane == "reflow": - self.assertEqual([c[:2] for c in commands], [["cargo", "build"], ["npm", "ci"], ["npm", "test"]]) - - def test_hosted_execution_rejects_reused_outputs_before_any_candidate_command(self): - workflow = (ROOT / ".github/workflows/macos-trusted.yml").read_text() - hosted = workflow.split("\n hosted:\n", 1)[1].split("\n publish:\n", 1)[0] - guard_name = "Enforce current hosted authorization" - self.assertIn(f"- name: {guard_name}", hosted) - self.assertLess(hosted.index(guard_name), hosted.index("path: _mac_source")) - guard = workflow_run_script(".github/workflows/macos-trusted.yml", guard_name) - execution = hosted.split(" - name: Run macOS lane\n", 1)[1].split(" run: ", 1)[1].splitlines()[0] - with tempfile.TemporaryDirectory() as directory: - root = Path(directory) - control = root / "_mac_control/scripts/ci" - control.mkdir(parents=True) - for name in ("macos_runner.py", "ci_scope.py", "run-macos-lanes.sh"): - (control / name).write_text((ROOT / "scripts/ci" / name).read_text()) - source = root / "_mac_source" - source.mkdir() - subprocess.run(["git", "init", str(source)], check=True, capture_output=True) - cargo = root / "cargo" - cargo.write_text('#!/bin/sh\ntouch "$CALLS"\n') - cargo.chmod(0o755) - for name, attempt, expiry, run_id, allowed, succeeds in ( - ("cached-successful-admission", "2", int(time.time()) + 3600, "100", "true", False), - ("expired-before-start", "1", 1, "100", "true", False), - ("another-run", "1", int(time.time()) + 3600, "101", "true", False), - ("not-admitted", "1", int(time.time()) + 3600, "100", "false", False), - ("fresh-full-rerun", "2", int(time.time()) + 3600, "100", "true", True)): - with self.subTest(name=name): - plan = {"schema": 1, "kind": "pr", "route": "hosted", "head": H, "merge": M, - "run_id": 100, "attempt": 2 if succeeds else 1, "expires_at": expiry} - calls = root / "calls" - calls.unlink(missing_ok=True) - result = subprocess.run(["bash", "-euo", "pipefail", "-c", guard + '\ncd _mac_source\n' + execution], - cwd=root, env={**os.environ, "PLAN": json.dumps(plan), "PATH": f"{root}:{os.environ['PATH']}", - "CALLS": str(calls), "MACOS_LANE": "coordination", "GITHUB_REPOSITORY": m.REPO, - "GITHUB_EVENT_NAME": "pull_request_target", "GITHUB_RUN_ID": run_id, - "GITHUB_RUN_ATTEMPT": attempt, "ASTRA_MACOS_HOSTED_ADMITTED": allowed, - "GH_TOKEN": "", "ASTRA_MACOS_APP_ID": ""}, capture_output=True, text=True) - self.assertEqual(result.returncode == 0, succeeds, result.stderr) - self.assertEqual(calls.exists(), succeeds, "candidate Cargo ran before rejection") - - def test_hint_permissions_are_scoped_and_reconciliation_stays_metadata_only(self): - workflow = (ROOT / ".github/workflows/test.yml").read_text() - top = workflow.split("\non:", 1)[0] - self.assertNotIn("actions: read", top) - self.assertNotIn("checks: read", top) - scope = workflow.split("\n macos-routing:\n", 1)[1].split("\n cli-test-build:\n", 1)[0] - for permission in ("actions: read", "checks: read", "pull-requests: read", "contents: read"): - self.assertIn(permission, scope) - observer = (ROOT / ".github/workflows/macos-results.yml").read_text() - self.assertIn("cron: '*/15 * * * *'", observer) - self.assertIn("macos_runner.py reconcile", observer) - self.assertIn("permission-actions: write", observer) - self.assertIn("cancel-in-progress: false", observer) - classifier = workflow.split("\n scope:\n", 1)[1].split("\n macos-routing:\n", 1)[0] - self.assertNotIn("macos_runner.py hint", classifier) - self.assertNotIn("actions: read", classifier) - self.assertNotIn("_mac_source", observer) - - def test_credential_owner_and_cache_boundary_are_outside_pr_execution(self): - workflow = (ROOT / ".github/workflows/macos-trusted.yml").read_text() - for job, end in (("mini", "hosted-admission"), ("hosted", "publish")): - execution = workflow.split(f"\n {job}:\n", 1)[1].split(f"\n {end}:\n", 1)[0] - self.assertNotIn("secrets.", execution) - self.assertNotIn("create-github-app-token", execution) - self.assertNotIn("GH_TOKEN", execution) - self.assertNotIn("environment:", execution) - self.assertIn("cache-mode: read", workflow) - self.assertNotIn("cache-mode: write", workflow) - self.assertNotIn("workflow_call:", workflow) - observer = (ROOT / ".github/workflows/macos-results.yml").read_text() - self.assertNotIn("download-artifact", observer) - self.assertNotIn("_mac_source", observer) - self.assertIn("cache-mode: none", observer) - hook = (ROOT / "scripts/ci/macos-runner-admit").read_text() - self.assertIn('[ "$#" -eq 0 ]', hook) - self.assertIn("SUDO_USER", hook) - self.assertNotIn("eval", hook) - - -if __name__ == "__main__": - unittest.main() From ca3fb5bd4c3ae01f92b1f5f1b5ddcb80c22feb2c Mon Sep 17 00:00:00 2001 From: lr90 Date: Sat, 24 Oct 2026 16:25:36 +0800 Subject: [PATCH 08/11] fix(ci): bootstrap repository Rust toolchain on self-hosted runners --- .github/actions/rust-setup/action.yml | 4 +- .github/actions/rust-toolchain/action.yml | 20 +++++++++ .github/workflows/test.yml | 2 +- docs/guides/repository-automation.md | 5 ++- scripts/ci/test_ci_build_coverage.py | 54 +++++++++++++++++++++++ 5 files changed, 79 insertions(+), 6 deletions(-) create mode 100644 .github/actions/rust-toolchain/action.yml diff --git a/.github/actions/rust-setup/action.yml b/.github/actions/rust-setup/action.yml index c0b961a0e7..f26f92269a 100644 --- a/.github/actions/rust-setup/action.yml +++ b/.github/actions/rust-setup/action.yml @@ -69,9 +69,7 @@ runs: df -h "${disk_guard_workspace}" - name: Install pinned Rust toolchain - # The repository toolchain also owns test-created Cargo fixtures. - shell: bash - run: rustc --version + uses: ./.github/actions/rust-toolchain - uses: taiki-e/install-action@d438492cf8a250514fa2d34b30bc3c0dc37c65ff # v2 if: ${{ inputs.install-nextest == 'true' }} diff --git a/.github/actions/rust-toolchain/action.yml b/.github/actions/rust-toolchain/action.yml new file mode 100644 index 0000000000..aff6b0838d --- /dev/null +++ b/.github/actions/rust-toolchain/action.yml @@ -0,0 +1,20 @@ +name: Install repository Rust toolchain +description: Bootstrap rustup and install the toolchain from rust-toolchain.toml +runs: + using: composite + steps: + - name: Install pinned Rust toolchain + shell: bash + run: | + set -euo pipefail + ci_cargo_bin="${CARGO_HOME:-$HOME/.cargo}/bin" + export PATH="$ci_cargo_bin:$PATH" + if ! command -v rustup >/dev/null 2>&1; then + curl --proto '=https' --tlsv1.2 --retry 3 --location --silent --show-error --fail \ + https://sh.rustup.rs | sh -s -- --default-toolchain none --no-modify-path -y + fi + echo "$ci_cargo_bin" >> "$GITHUB_PATH" + # Older rustup installs implicitly on show; newer versions require the + # explicit install. Both read the repository file, including components. + rustup show active-toolchain || rustup toolchain install + rustc --version diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 7741c83add..db1adc8141 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -206,7 +206,7 @@ jobs: with: persist-credentials: false - name: Install pinned Rust toolchain - run: rustc --version + uses: ./.github/actions/rust-toolchain # The patched crate is deliberately outside the application workspace. # These are parser/unit tests, not the real-reader PTY regression suite. - name: Test vendored terminal units diff --git a/docs/guides/repository-automation.md b/docs/guides/repository-automation.md index 51921a3319..949651cf41 100644 --- a/docs/guides/repository-automation.md +++ b/docs/guides/repository-automation.md @@ -136,8 +136,9 @@ custom result contexts, routing flag, hook or hosted fallback is required by thi implementation. The selected runner must be available to Astra, including public repository access when applicable, and have the exact labels above. Confirm its OS/toolchain support for the pinned Actions/Rust/Node versions and at least -30 GiB workspace disk before assigning work. The shared Rust setup reports low -or unreadable disk as a failure on self-hosted runners, without running the +30 GiB workspace disk before assigning work. CI bootstraps rustup when missing, +adds Cargo binaries to PATH and installs the repository-pinned Rust toolchain. +The shared Rust setup reports low or unreadable disk as a failure on self-hosted runners, without running the hosted-image sudo cleanup. All approved PR code executes on that runner; operators must accept that persistent-host access and resource exposure. diff --git a/scripts/ci/test_ci_build_coverage.py b/scripts/ci/test_ci_build_coverage.py index 06fb867399..6fa1518582 100755 --- a/scripts/ci/test_ci_build_coverage.py +++ b/scripts/ci/test_ci_build_coverage.py @@ -20,6 +20,60 @@ class BuildCoverageTests(unittest.TestCase): + def test_rust_toolchain_bootstraps_missing_rust_and_reuses_cargo_bin(self): + action = (ROOT / ".github/actions/rust-toolchain/action.yml").read_text() + script = textwrap.dedent(action.split(" run: |\n", 1)[1]) + for installed, install_failure in ((False, False), (True, False), (False, True)): + with self.subTest(installed=installed, failure=install_failure), tempfile.TemporaryDirectory() as directory: + root = Path(directory) + mock_bin = root / "mock-bin" + cargo_home = root / "cargo home" + mock_bin.mkdir() + (cargo_home / "bin").mkdir(parents=True) + calls = root / "calls" + github_path = root / "github-path" + installer = root / "installer" + installer.write_text("""#!/bin/sh +set -eu +printf 'installer %s\\n' "$*" >> "$MOCK_CALLS" +cp "$MOCK_BIN/rustup-source" "$CARGO_HOME/bin/rustup" +cp "$MOCK_BIN/rustc-source" "$CARGO_HOME/bin/rustc" +""") + programs = { + "curl": 'printf "curl\\n" >> "$MOCK_CALLS"; if [ "$MOCK_FAILURE" = 1 ]; then exit 22; fi; cat "$MOCK_INSTALLER"', + "rustup-source": 'printf "rustup %s\\n" "$*" >> "$MOCK_CALLS"; if [ "$1" = show ]; then test -f "$CARGO_HOME/ready"; else touch "$CARGO_HOME/ready"; fi', + "rustc-source": 'test -f "$CARGO_HOME/ready"; printf "rustc pinned-fixture\\n"', + } + for name, body in programs.items(): + path = mock_bin / name + path.write_text("#!/bin/sh\nset -eu\n" + body + "\n") + path.chmod(0o755) + if installed: + for name in ("rustup", "rustc"): + target = cargo_home / "bin" / name + target.write_bytes((mock_bin / (name + "-source")).read_bytes()) + target.chmod(0o755) + (cargo_home / "ready").touch() + result = subprocess.run( + ["/bin/bash", "-e", "-c", script], cwd=ROOT, capture_output=True, text=True, + env={**os.environ, "PATH": str(mock_bin) + ":/usr/bin:/bin", + "CARGO_HOME": str(cargo_home), "GITHUB_PATH": str(github_path), + "MOCK_BIN": str(mock_bin), "MOCK_CALLS": str(calls), + "MOCK_INSTALLER": str(installer), "MOCK_FAILURE": str(int(install_failure))}, + ) + invocations = calls.read_text().splitlines() + if install_failure: + self.assertNotEqual(result.returncode, 0) + self.assertNotIn("rustc pinned-fixture", result.stdout) + continue + self.assertEqual(result.returncode, 0, result.stderr) + self.assertIn("rustc pinned-fixture", result.stdout) + self.assertEqual(github_path.read_text().strip(), str(cargo_home / "bin")) + self.assertEqual("curl" in invocations, not installed) + self.assertEqual("rustup toolchain install" in invocations, not installed) + for path in (".github/actions/rust-setup/action.yml", ".github/workflows/test.yml"): + self.assertIn("uses: ./.github/actions/rust-toolchain", (ROOT / path).read_text()) + def test_rust_setup_disk_guard_uses_workspace_free_space_without_real_cleanup(self): action = (ROOT / ".github/actions/rust-setup/action.yml").read_text() section = action.split("\n - name: Guard workspace disk space\n", 1)[1] From b7efd37aefb9d1773be469224b5d93ddeb2af153 Mon Sep 17 00:00:00 2001 From: lr90 Date: Sat, 24 Oct 2026 16:30:04 +0800 Subject: [PATCH 09/11] ci: select self-hosted macOS runners by contributor author ID --- .github/workflows/test.yml | 9 +++--- CONTRIBUTING.md | 5 ++-- docs/guides/repository-automation.md | 24 +++++++++------ scripts/ci/ci_scope.py | 44 +++++++++++++++++++++++++++- scripts/ci/test_ci_parallelism.py | 13 ++++---- scripts/ci/test_ci_scope.py | 39 +++++++++++++++++++++++- 6 files changed, 109 insertions(+), 25 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index db1adc8141..a119b55c19 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -27,6 +27,7 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 5 outputs: + macos_runner: ${{ steps.scope.outputs.macos_runner }} test_cli: ${{ steps.scope.outputs.test_cli }} test_runtime: ${{ steps.scope.outputs.test_runtime }} test_services: ${{ steps.scope.outputs.test_services }} @@ -195,7 +196,7 @@ jobs: name: "Test: vendored terminal (${{ matrix.os }})" needs: scope if: ${{ !cancelled() && (needs.scope.result != 'success' || needs.scope.outputs.test_cli == 'true' || needs.scope.outputs.test_core == 'true') }} - runs-on: ${{ matrix.os == 'macos-15' && fromJSON('["self-hosted","macOS","ARM64","mac-arm64-mini"]') || matrix.os }} + runs-on: ${{ matrix.os == 'macos-15' && fromJSON(needs.scope.outputs.macos_runner || '["macos-15"]') || matrix.os }} timeout-minutes: 15 strategy: fail-fast: false @@ -244,7 +245,7 @@ jobs: name: "Terminal: ${{ matrix.segment }} (${{ matrix.os }})" needs: scope if: ${{ !cancelled() && (needs.scope.result != 'success' || needs.scope.outputs.test_cli == 'true' || needs.scope.outputs.test_core == 'true') }} - runs-on: ${{ matrix.os == 'macos-15' && fromJSON('["self-hosted","macOS","ARM64","mac-arm64-mini"]') || matrix.os }} + runs-on: ${{ matrix.os == 'macos-15' && fromJSON(needs.scope.outputs.macos_runner || '["macos-15"]') || matrix.os }} timeout-minutes: 45 strategy: fail-fast: false @@ -262,7 +263,7 @@ jobs: save-cache: ${{ github.event_name == 'push' && matrix.segment == 'default-reader' }} install-nextest: false # Separate Cargo invocations keep dev-tty feature unification out of - # the default reader; one self-hosted registration queues these jobs. + # the default reader; one self-hosted registration queues allowlisted jobs. - name: Test default terminal reader through PTYs if: matrix.segment == 'default-reader' run: | @@ -324,7 +325,7 @@ jobs: name: "macOS workspace coordination contracts" needs: scope if: ${{ !cancelled() && (needs.scope.result != 'success' || needs.scope.outputs.test_core == 'true' || needs.scope.outputs.test_cli == 'true') }} - runs-on: [self-hosted, macOS, ARM64, mac-arm64-mini] + runs-on: ${{ fromJSON(needs.scope.outputs.macos_runner || '["macos-15"]') }} timeout-minutes: 30 steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index ee9235851d..3628499f3d 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -119,8 +119,9 @@ using its existing default-feature test executable. Tools lint and serial workspace-coordination tests retain independent jobs; the protected coordination check requires both paths to succeed. Feature-specific terminal tests remain separate. All macOS execution jobs in Test Suite use the organization's self-hosted ARM -runner labels for both pull requests and main/develop pushes, without author -classification. Existing fork approval and required-check names remain in place. +runner for allowlisted PR author IDs and main/develop pushes. Other or unknown +PR authors use hosted macOS. The allowlist lives in `scripts/ci/ci_scope.py`; +existing fork approval and required-check names remain in place. See [runner operations](docs/guides/repository-automation.md#self-hosted-macos-tests) for registration, queueing and rollback. The lint gate retains restored artifacts; age-based cleanup is not a lint prerequisite. diff --git a/docs/guides/repository-automation.md b/docs/guides/repository-automation.md index 949651cf41..f9f3ec860c 100644 --- a/docs/guides/repository-automation.md +++ b/docs/guides/repository-automation.md @@ -122,30 +122,36 @@ and keeps external-fork behavior part of the repository's delivery contract. ## Self-hosted macOS tests -Test Suite sends all macOS execution to runners matching -`[self-hosted, macOS, ARM64, mac-arm64-mini]`: default reader/Bash, dev-tty reader, -reflow, workspace coordination and vendored terminal units. The same routing -applies to every PR author and to pushes on main/develop. Existing required-check +Test Suite routes default reader/Bash, dev-tty reader, reflow, workspace +coordination and vendored terminal units by PR author ID. IDs in +`MACOS_RUNNER_AUTHORS` in `scripts/ci/ci_scope.py` use runners matching +`[self-hosted, macOS, ARM64, mac-arm64-mini]`; other or unknown authors use hosted +`macos-15`. The list includes Dependabot. Numeric IDs remain stable across login +renames; login strings in the list are review annotations. The author is +`pull_request.user.id`, not the sender or re-run actor. Pushes on main/develop +also use the self-hosted labels. Existing required-check names retain their `macos-15` label for compatibility; inspect each job's actual runner for execution provenance. Linux execution and release workflows retain their existing platforms. This uses the ordinary `pull_request`/`push` workflow, GitHub's existing fork approval and read-only workflow token. No App, organization membership query, -custom result contexts, routing flag, hook or hosted fallback is required by this -implementation. The selected runner must be available to Astra, including public +custom result contexts, routing flag or hook is required by this implementation. +If scope output is missing, Mac jobs use hosted `macos-15` and still run tests. The selected runner must be available to Astra, including public repository access when applicable, and have the exact labels above. Confirm its OS/toolchain support for the pinned Actions/Rust/Node versions and at least 30 GiB workspace disk before assigning work. CI bootstraps rustup when missing, adds Cargo binaries to PATH and installs the repository-pinned Rust toolchain. The shared Rust setup reports low or unreadable disk as a failure on self-hosted runners, without running the -hosted-image sudo cleanup. All approved PR code executes on that -runner; operators must accept that persistent-host access and resource exposure. +hosted-image sudo cleanup. Approved PR code executes on that +runner when the author is allowlisted; operators accept that persistent-host +access and resource exposure. One registration executes one job at a time. The five macOS jobs queue on a single Mac; adding registrations on the same machine changes resource contention and does not create extra capacity. Reader feature builds remain separate jobs. -If the runner is offline, jobs stay queued rather than consuming a hosted Mac. +For allowlisted PRs and pushes, a busy/offline runner leaves jobs queued; there +is no automatic availability-based hosted fallback. Measure queue delay and actual job duration before claiming an elapsed-time gain. If the earlier controller design was staged, remove its unused custom App diff --git a/scripts/ci/ci_scope.py b/scripts/ci/ci_scope.py index 9676920a4c..e9869141ff 100755 --- a/scripts/ci/ci_scope.py +++ b/scripts/ci/ci_scope.py @@ -3,7 +3,9 @@ The classifier is deliberately conservative: an unknown diff or a change to CI itself enables every scope. Documentation and repository-governance changes are -handled by the always-on lightweight repository checks. +handled by the always-on lightweight repository checks. The same event also +selects macOS runners by a static PR-author ID allowlist, with hosted fallback +for unknown authors or unavailable metadata. """ from __future__ import annotations @@ -16,6 +18,35 @@ import sys +# GitHub account IDs survive login renames. Login values are review annotations. +MACOS_RUNNER_AUTHORS = { + 39627130: "XuPeng-SH", + 52204121: "loveRhythm1990", + 26336316: "gouhongshen", + 49699333: "dependabot[bot]", + 2245572: "zuyu", + 4965857: "dengn", + 85048713: "MatrixAdventurer", + 176287284: "endlesschasey-ai", + 49832303: "aptend", + 169294: "fengttt", + 28526327: "plane-hjh", +} +MACOS_SELF_HOSTED_LABELS = ["self-hosted", "macOS", "ARM64", "mac-arm64-mini"] + + +def select_macos_runner(event_name: str, event: dict, ref: str) -> list[str]: + if event_name == "push" and ref in {"refs/heads/main", "refs/heads/develop"}: + return MACOS_SELF_HOSTED_LABELS.copy() + if event_name == "pull_request" and isinstance(event, dict): + pull_request = event.get("pull_request") + author = pull_request.get("user") if isinstance(pull_request, dict) else None + author_id = author.get("id") if isinstance(author, dict) else None + if type(author_id) is int and author_id in MACOS_RUNNER_AUTHORS: + return MACOS_SELF_HOSTED_LABELS.copy() + return ["macos-15"] + + SCOPES = ( "rust", "sdk", @@ -255,11 +286,22 @@ def emit(scopes: dict[str, bool], paths: list[str], *, fallback: str | None = No print(f"{scope}={'true' if scopes[scope] else 'false'}") print("changed_paths=" + (", ".join(sorted(paths)) or "")) + try: + event = json.loads(Path(os.environ["GITHUB_EVENT_PATH"]).read_text(encoding="utf-8")) + labels = select_macos_runner( + os.environ.get("GITHUB_EVENT_NAME", ""), event, os.environ.get("GITHUB_REF", "") + ) + except (KeyError, OSError, ValueError, TypeError): + labels = ["macos-15"] + runner_json = json.dumps(labels, separators=(",", ":")) + print("macos_runner=" + runner_json) + github_output = os.environ.get("GITHUB_OUTPUT") if github_output: with Path(github_output).open("a", encoding="utf-8") as output: for scope in SCOPES: output.write(f"{scope}={'true' if scopes[scope] else 'false'}\n") + output.write("macos_runner=" + runner_json + "\n") def main() -> int: diff --git a/scripts/ci/test_ci_parallelism.py b/scripts/ci/test_ci_parallelism.py index a9797a69c2..6620b27deb 100755 --- a/scripts/ci/test_ci_parallelism.py +++ b/scripts/ci/test_ci_parallelism.py @@ -117,25 +117,22 @@ def test_status_labels_do_not_repeat_setup_builds_or_artifact_transfers(self): class MacRunnerTests(unittest.TestCase): - def test_all_macos_execution_uses_the_same_self_hosted_labels_for_every_author(self): + def test_macos_execution_uses_scope_routing_and_keeps_a_hosted_fallback(self): workflow = (ROOT / ".github/workflows/test.yml").read_text() - labels = ["self-hosted", "macOS", "ARM64", "mac-arm64-mini"] + selector = "fromJSON(needs.scope.outputs.macos_runner || '[\"macos-15\"]')" for job in ("vendored-terminal", "terminal-pty-shards"): header = workflow.split(f"\n {job}:\n", 1)[1].split(" steps:\n", 1)[0] - selection = re.search(r"runs-on: \$\{\{ matrix\.os == 'macos-15' && fromJSON\('([^']+)'\) \|\| matrix\.os \}\}", header) - self.assertIsNotNone(selection, job) - self.assertEqual(json.loads(selection[1]), labels) + self.assertIn("runs-on: ${{ matrix.os == 'macos-15' && " + selector + " || matrix.os }}", header) self.assertIn("os: [ubuntu-latest, macos-15]", header) self.assertIn("needs: scope", header) coordination = workflow.split("\n macos-workspace-coordination-tests:\n", 1)[1].split(" steps:\n", 1)[0] - self.assertIn("runs-on: [self-hosted, macOS, ARM64, mac-arm64-mini]", coordination) + self.assertIn("runs-on: ${{ " + selector + " }}", coordination) + self.assertIn("macos_runner: ${{ steps.scope.outputs.macos_runner }}", workflow) for job in ("scope", "cli-test-build", "shard-b", "shard-c", "shard-d", "test-online"): header = workflow.split(f"\n {job}:\n", 1)[1].split(" steps:\n", 1)[0] self.assertIn("runs-on: ubuntu-latest", header) self.assertIn("pull_request:", workflow) - self.assertIn("branches: [main, develop]", workflow) self.assertNotIn("pull_request_target:", workflow) - self.assertNotIn("github.event.pull_request.user", workflow) self.assertNotIn("ASTRA_MACOS_", workflow) self.assertNotIn("macos-routing:", workflow) diff --git a/scripts/ci/test_ci_scope.py b/scripts/ci/test_ci_scope.py index 0976a5cfbc..ae246b5147 100755 --- a/scripts/ci/test_ci_scope.py +++ b/scripts/ci/test_ci_scope.py @@ -11,10 +11,47 @@ from io import StringIO from unittest.mock import patch -from ci_scope import classify, main +from ci_scope import classify, main, select_macos_runner class CiScopeTests(unittest.TestCase): + def test_macos_allowlist_uses_author_id_including_bots_and_login_renames(self): + for author_id in (39627130, 52204121, 49699333): + with self.subTest(author_id=author_id): + event = {"pull_request": {"user": {"id": author_id, "login": "renamed-login"}}, + "sender": {"id": 999999999}} + self.assertEqual(select_macos_runner("pull_request", event, "refs/pull/1/merge"), + ["self-hosted", "macOS", "ARM64", "mac-arm64-mini"]) + + def test_macos_unknown_author_stays_hosted_even_when_actor_or_login_matches(self): + for user in ({}, {"login": "XuPeng-SH"}, {"id": 999999999, "login": "XuPeng-SH"}, + {"id": "39627130"}, {"id": True}, None): + with self.subTest(user=user): + event = {"pull_request": {"user": user}, "sender": {"id": 39627130}} + self.assertEqual(select_macos_runner("pull_request", event, "refs/pull/1/merge"), + ["macos-15"]) + for event in ({}, {"pull_request": None}, []): + self.assertEqual(select_macos_runner("pull_request", event, ""), ["macos-15"]) + + def test_macos_only_main_and_develop_pushes_use_mini_without_a_pr_author(self): + for ref in ("refs/heads/main", "refs/heads/develop"): + self.assertEqual(select_macos_runner("push", {}, ref), + ["self-hosted", "macOS", "ARM64", "mac-arm64-mini"]) + self.assertEqual(select_macos_runner("push", {}, "refs/heads/feature"), ["macos-15"]) + self.assertEqual(select_macos_runner("workflow_dispatch", {}, "refs/heads/main"), ["macos-15"]) + + def test_macos_missing_event_routes_to_hosted_without_omitting_scope_checks(self): + with tempfile.TemporaryDirectory() as directory: + output = Path(directory) / "output" + with patch.dict(os.environ, {"GITHUB_EVENT_PATH": str(Path(directory) / "missing"), + "GITHUB_OUTPUT": str(output)}, clear=True), \ + patch.object(sys, "argv", ["ci_scope.py", "crates/core/src/lib.rs"]), \ + redirect_stdout(StringIO()), redirect_stderr(StringIO()): + self.assertEqual(main(), 0) + values = dict(line.split("=", 1) for line in output.read_text().splitlines()) + self.assertEqual(json.loads(values["macos_runner"]), ["macos-15"]) + self.assertEqual(values["test_core"], "true") + def assert_rename_selects_expected_scopes( self, source: str, From 8a40aafa9c54f43518171709f50171df6527f042 Mon Sep 17 00:00:00 2001 From: lr90 Date: Sat, 24 Oct 2026 17:05:59 +0800 Subject: [PATCH 10/11] ci: bound mini queue time and restore parallel hosted fallback --- .github/actionlint.yaml | 5 + .github/workflows/macos-mini.yml | 63 ++++ .github/workflows/macos-scheduler.yml | 30 ++ .github/workflows/test.yml | 209 +++++++++----- CONTRIBUTING.md | 17 +- docs/guides/repository-automation.md | 101 ++++--- scripts/ci/cancel_stale_pr_runs.py | 4 +- scripts/ci/macos_dispatch.py | 293 +++++++++++++++++++ scripts/ci/run-macos-lanes.sh | 54 ++++ scripts/ci/test_ci_build_coverage.py | 51 ++-- scripts/ci/test_ci_parallelism.py | 46 +-- scripts/ci/test_macos_dispatch.py | 401 ++++++++++++++++++++++++++ 12 files changed, 1102 insertions(+), 172 deletions(-) create mode 100644 .github/workflows/macos-mini.yml create mode 100644 .github/workflows/macos-scheduler.yml create mode 100755 scripts/ci/macos_dispatch.py create mode 100755 scripts/ci/run-macos-lanes.sh create mode 100755 scripts/ci/test_macos_dispatch.py diff --git a/.github/actionlint.yaml b/.github/actionlint.yaml index af57fe46ac..10c7250fcb 100644 --- a/.github/actionlint.yaml +++ b/.github/actionlint.yaml @@ -1,2 +1,7 @@ self-hosted-runner: labels: [mac-arm64-mini] +# GitHub supports cache-mode; actionlint 1.7.12 predates the syntax. +paths: + .github/workflows/macos-mini.yml: + ignore: + - 'unexpected key "cache-mode" for "workflow" section' diff --git a/.github/workflows/macos-mini.yml b/.github/workflows/macos-mini.yml new file mode 100644 index 0000000000..0cfe1364e5 --- /dev/null +++ b/.github/workflows/macos-mini.yml @@ -0,0 +1,63 @@ +name: macOS mini worker +run-name: Mac mini for Test Suite ${{ inputs.source_run_id }}/${{ inputs.source_attempt }} ${{ inputs.source_sha }} +on: + workflow_dispatch: + inputs: + source_run_id: + required: true + type: string + source_attempt: + required: true + type: string + source_sha: + required: true + type: string +permissions: + contents: read + actions: read + pull-requests: read +cache-mode: read +concurrency: + group: macos-mini-${{ inputs.source_run_id }}-${{ inputs.source_attempt }} + cancel-in-progress: false +defaults: + run: + shell: bash +env: + CARGO_INCREMENTAL: "0" + CARGO_TERM_COLOR: always +jobs: + execute: + runs-on: [self-hosted, macOS, ARM64, mac-arm64-mini] + timeout-minutes: 65 + steps: + - name: Checkout control policy + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + with: + ref: ${{ github.sha }} + path: .ci-control + persist-credentials: false + - name: Validate mini dispatch + env: + GH_TOKEN: ${{ github.token }} + SOURCE_RUN_ID: ${{ inputs.source_run_id }} + SOURCE_ATTEMPT: ${{ inputs.source_attempt }} + SOURCE_SHA: ${{ inputs.source_sha }} + run: python3 .ci-control/scripts/ci/macos_dispatch.py admit --run-id "$SOURCE_RUN_ID" --attempt "$SOURCE_ATTEMPT" --sha "$SOURCE_SHA" + - name: Checkout candidate + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + with: + ref: ${{ inputs.source_sha }} + persist-credentials: false + - name: Prepare Rust + uses: ./.github/actions/rust-setup + with: + shared-key: ci-terminal-pty + save-cache: "false" + install-nextest: "false" + - name: Prepare Node + uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7 + with: + node-version-file: .nvmrc + - name: Run all macOS lanes + run: bash scripts/ci/run-macos-lanes.sh all diff --git a/.github/workflows/macos-scheduler.yml b/.github/workflows/macos-scheduler.yml new file mode 100644 index 0000000000..a39cc2cea7 --- /dev/null +++ b/.github/workflows/macos-scheduler.yml @@ -0,0 +1,30 @@ +name: macOS mini scheduler +on: + workflow_run: + workflows: [Test Suite] + types: [in_progress, completed] +permissions: + contents: read + actions: write + pull-requests: read +concurrency: + group: macos-scheduler-${{ github.event.workflow_run.id }}-${{ github.event.workflow_run.run_attempt }} + cancel-in-progress: false +jobs: + schedule: + if: github.event.workflow_run.run_attempt == 1 + runs-on: ubuntu-latest + timeout-minutes: 70 + steps: + # Only the default-branch controller is loaded; it never executes PR code. + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + with: + ref: ${{ github.sha }} + persist-credentials: false + - name: Schedule and bound the mini attempt + env: + GH_TOKEN: ${{ github.token }} + DISPATCH_MODE: ${{ github.event.action == 'completed' && 'cleanup' || 'schedule' }} + SOURCE_RUN_ID: ${{ github.event.workflow_run.id }} + SOURCE_ATTEMPT: ${{ github.event.workflow_run.run_attempt }} + run: python3 scripts/ci/macos_dispatch.py "$DISPATCH_MODE" --run-id "$SOURCE_RUN_ID" --attempt "$SOURCE_ATTEMPT" diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index a119b55c19..74f2a5dd83 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -1,4 +1,6 @@ name: Test Suite +run-name: >- + Test Suite pr=${{ github.event.pull_request.number || 0 }} base=${{ github.event.pull_request.base.sha || github.event.before }} head=${{ github.event.pull_request.head.sha || github.sha }} merge=${{ github.sha }} permissions: contents: read @@ -45,6 +47,31 @@ jobs: id: scope run: python3 scripts/ci/ci_scope.py + macos-plan: + name: Resolve macOS execution + needs: scope + if: >- + !cancelled() && github.run_attempt == 1 && + (needs.scope.outputs.test_cli == 'true' || needs.scope.outputs.test_core == 'true') && + contains(fromJSON(needs.scope.outputs.macos_runner || '[]'), 'self-hosted') + runs-on: ubuntu-latest + timeout-minutes: 67 + permissions: + contents: read + actions: read + pull-requests: read + outputs: + result: ${{ steps.mini.outputs.result }} + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + with: + persist-credentials: false + - name: Wait for a bounded mini attempt + id: mini + env: + GH_TOKEN: ${{ github.token }} + run: python3 scripts/ci/macos_dispatch.py wait --run-id "$GITHUB_RUN_ID" --attempt "$GITHUB_RUN_ATTEMPT" + # Package groups remain parallel. CLI builds and executes its full inventory # once; stable protected labels require that same completed test result. # Short CLI partitions are not the workflow critical path, so separate @@ -196,12 +223,12 @@ jobs: name: "Test: vendored terminal (${{ matrix.os }})" needs: scope if: ${{ !cancelled() && (needs.scope.result != 'success' || needs.scope.outputs.test_cli == 'true' || needs.scope.outputs.test_core == 'true') }} - runs-on: ${{ matrix.os == 'macos-15' && fromJSON(needs.scope.outputs.macos_runner || '["macos-15"]') || matrix.os }} + runs-on: ubuntu-latest timeout-minutes: 15 strategy: fail-fast: false matrix: - os: [ubuntu-latest, macos-15] + os: [ubuntu-latest] steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 with: @@ -211,15 +238,13 @@ jobs: # The patched crate is deliberately outside the application workspace. # These are parser/unit tests, not the real-reader PTY regression suite. - name: Test vendored terminal units - run: | - cargo test --locked --manifest-path vendor/crossterm/Cargo.toml --lib --features event-stream - cargo test --locked --manifest-path vendor/crossterm/Cargo.toml --lib --features event-stream,use-dev-tty + run: bash scripts/ci/run-macos-lanes.sh vendored # Preserve required check names while the reader/reflow builds stay isolated. # Both aggregate checks require the entire terminal matrix. terminal-pty: name: "Test: terminal PTY (${{ matrix.os }})" - needs: [scope, terminal-pty-shards] + needs: [scope, terminal-pty-shards, terminal-pty-shards-macos] if: ${{ !cancelled() }} runs-on: ubuntu-latest timeout-minutes: 5 @@ -232,25 +257,28 @@ jobs: shell: bash env: SHARDS_RESULT: ${{ needs.terminal-pty-shards.result }} + MACOS_RESULT: ${{ needs.terminal-pty-shards-macos.result }} SHARDS_REQUIRED: ${{ needs.scope.result != 'success' || needs.scope.outputs.test_cli == 'true' || needs.scope.outputs.test_core == 'true' }} run: | set -euo pipefail - if [ "$SHARDS_REQUIRED" = true ]; then - test "$SHARDS_RESULT" = success - else - case "$SHARDS_RESULT" in success|skipped) ;; *) exit 1 ;; esac - fi + for result in "$SHARDS_RESULT" "$MACOS_RESULT"; do + if [ "$SHARDS_REQUIRED" = true ]; then + test "$result" = success + else + case "$result" in success|skipped) ;; *) exit 1 ;; esac + fi + done terminal-pty-shards: name: "Terminal: ${{ matrix.segment }} (${{ matrix.os }})" needs: scope if: ${{ !cancelled() && (needs.scope.result != 'success' || needs.scope.outputs.test_cli == 'true' || needs.scope.outputs.test_core == 'true') }} - runs-on: ${{ matrix.os == 'macos-15' && fromJSON(needs.scope.outputs.macos_runner || '["macos-15"]') || matrix.os }} + runs-on: ubuntu-latest timeout-minutes: 45 strategy: fail-fast: false matrix: - os: [ubuntu-latest, macos-15] + os: [ubuntu-latest] segment: [default-reader, dev-tty-reader, reflow] steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 @@ -262,45 +290,18 @@ jobs: # One writer per OS populates the shared dependency cache on pushes. save-cache: ${{ github.event_name == 'push' && matrix.segment == 'default-reader' }} install-nextest: false - # Separate Cargo invocations keep dev-tty feature unification out of - # the default reader; one self-hosted registration queues allowlisted jobs. - - name: Test default terminal reader through PTYs - if: matrix.segment == 'default-reader' - run: | - cargo test --locked -p astra-cli --lib tui::terminal_startup -- --test-threads=2 - - name: Initialize system Git for macOS CLI observation - if: matrix.os == 'macos-15' && matrix.segment == 'default-reader' - timeout-minutes: 1 - run: | - time /usr/bin/env -i PATH=/usr/bin:/bin HOME=/nonexistent LANG=C LC_ALL=C \ - GIT_CONFIG_NOSYSTEM=1 GIT_CONFIG_GLOBAL=/dev/null GIT_CONFIG_SYSTEM=/dev/null \ - GIT_OPTIONAL_LOCKS=0 GIT_TERMINAL_PROMPT=0 \ - /usr/bin/git -C "$GITHUB_WORKSPACE" -c core.fsmonitor=false \ - -c core.untrackedCache=false rev-parse --show-toplevel - - name: Test macOS CLI Bash execution - if: matrix.os == 'macos-15' && matrix.segment == 'default-reader' - # Reuse the default-feature CLI test executable already built above. - run: >- - cargo test --locked --manifest-path Cargo.toml -p astra-cli - --lib edge_tools::shell::tests::bash_echo_returns_output -- --exact - - name: Test dev-tty terminal reader through PTYs - if: matrix.segment == 'dev-tty-reader' - run: | - cargo test --locked -p astra-cli --lib --features crossterm/use-dev-tty tui::terminal_startup -- --test-threads=2 - uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7 if: matrix.segment == 'reflow' with: node-version-file: .nvmrc - - name: Test inline terminal resize with xterm reflow - if: matrix.segment == 'reflow' - run: | - cargo build --locked -p astra-cli --bin astra - npm ci --ignore-scripts --prefix scripts/tui-reflow - ASTRA_TEST_BINARY="$PWD/target/debug/astra" npm test --prefix scripts/tui-reflow + - name: Test terminal lane + env: + MACOS_LANE: ${{ matrix.segment }} + run: bash scripts/ci/run-macos-lanes.sh "$MACOS_LANE" macos-workspace-coordination: name: "Test: macOS workspace coordination" - needs: [scope, macos-workspace-coordination-tests, terminal-pty-shards] + needs: [scope, macos-workspace-coordination-tests, terminal-pty-shards, terminal-pty-shards-macos] if: ${{ !cancelled() }} runs-on: ubuntu-latest timeout-minutes: 5 @@ -310,10 +311,11 @@ jobs: env: COORDINATION_RESULT: ${{ needs.macos-workspace-coordination-tests.result }} TERMINAL_RESULT: ${{ needs.terminal-pty-shards.result }} + MACOS_TERMINAL_RESULT: ${{ needs.terminal-pty-shards-macos.result }} SHARDS_REQUIRED: ${{ needs.scope.result != 'success' || needs.scope.outputs.test_core == 'true' || needs.scope.outputs.test_cli == 'true' }} run: | set -euo pipefail - for result in "$COORDINATION_RESULT" "$TERMINAL_RESULT"; do + for result in "$COORDINATION_RESULT" "$TERMINAL_RESULT" "$MACOS_TERMINAL_RESULT"; do if [ "$SHARDS_REQUIRED" = true ]; then test "$result" = success else @@ -321,46 +323,105 @@ jobs: fi done + vendored-terminal-macos: + name: "Test: vendored terminal (macos-15)" + needs: [scope, macos-plan] + if: ${{ !cancelled() && (needs.scope.result != 'success' || needs.scope.outputs.test_cli == 'true' || needs.scope.outputs.test_core == 'true') }} + runs-on: ${{ (github.run_attempt > 1 || !contains(fromJSON('["success","failure"]'), needs.macos-plan.outputs.result)) && 'macos-15' || 'ubuntu-latest' }} + timeout-minutes: 15 + env: + RUN_LANE: ${{ github.run_attempt > 1 || !contains(fromJSON('["success","failure"]'), needs.macos-plan.outputs.result) }} + MINI_RESULT: ${{ needs.macos-plan.outputs.result }} + steps: + - name: Require actual mini result + if: env.RUN_LANE != 'true' + run: | + set -euo pipefail + test "$MINI_RESULT" = success + echo 'All five macOS lanes passed on the mini; see Resolve macOS execution for the worker run.' >> "$GITHUB_STEP_SUMMARY" + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + if: env.RUN_LANE == 'true' + with: + persist-credentials: false + - uses: ./.github/actions/rust-toolchain + if: env.RUN_LANE == 'true' + - name: Run hosted macOS lane + if: env.RUN_LANE == 'true' + env: + MACOS_LANE: vendored + run: bash scripts/ci/run-macos-lanes.sh "$MACOS_LANE" + + terminal-pty-shards-macos: + name: "Terminal: ${{ matrix.segment }} (macos-15)" + needs: [scope, macos-plan] + if: ${{ !cancelled() && (needs.scope.result != 'success' || needs.scope.outputs.test_cli == 'true' || needs.scope.outputs.test_core == 'true') }} + runs-on: ${{ (github.run_attempt > 1 || !contains(fromJSON('["success","failure"]'), needs.macos-plan.outputs.result)) && 'macos-15' || 'ubuntu-latest' }} + timeout-minutes: 45 + env: + RUN_LANE: ${{ github.run_attempt > 1 || !contains(fromJSON('["success","failure"]'), needs.macos-plan.outputs.result) }} + MINI_RESULT: ${{ needs.macos-plan.outputs.result }} + strategy: + fail-fast: false + matrix: + segment: [default-reader, dev-tty-reader, reflow] + steps: + - name: Require actual mini result + if: env.RUN_LANE != 'true' + run: | + set -euo pipefail + test "$MINI_RESULT" = success + echo 'All five macOS lanes passed on the mini; see Resolve macOS execution for the worker run.' >> "$GITHUB_STEP_SUMMARY" + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + if: env.RUN_LANE == 'true' + with: + persist-credentials: false + - uses: ./.github/actions/rust-setup + if: env.RUN_LANE == 'true' + with: + shared-key: ci-terminal-pty + save-cache: ${{ github.event_name == 'push' && matrix.segment == 'default-reader' }} + install-nextest: "false" + - uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7 + if: env.RUN_LANE == 'true' && matrix.segment == 'reflow' + with: + node-version-file: .nvmrc + - name: Run hosted macOS lane + if: env.RUN_LANE == 'true' + env: + MACOS_LANE: ${{ matrix.segment }} + run: bash scripts/ci/run-macos-lanes.sh "$MACOS_LANE" + macos-workspace-coordination-tests: name: "macOS workspace coordination contracts" - needs: scope - if: ${{ !cancelled() && (needs.scope.result != 'success' || needs.scope.outputs.test_core == 'true' || needs.scope.outputs.test_cli == 'true') }} - runs-on: ${{ fromJSON(needs.scope.outputs.macos_runner || '["macos-15"]') }} + needs: [scope, macos-plan] + if: ${{ !cancelled() && (needs.scope.result != 'success' || needs.scope.outputs.test_cli == 'true' || needs.scope.outputs.test_core == 'true') }} + runs-on: ${{ (github.run_attempt > 1 || !contains(fromJSON('["success","failure"]'), needs.macos-plan.outputs.result)) && 'macos-15' || 'ubuntu-latest' }} timeout-minutes: 30 + env: + RUN_LANE: ${{ github.run_attempt > 1 || !contains(fromJSON('["success","failure"]'), needs.macos-plan.outputs.result) }} + MINI_RESULT: ${{ needs.macos-plan.outputs.result }} steps: + - name: Require actual mini result + if: env.RUN_LANE != 'true' + run: | + set -euo pipefail + test "$MINI_RESULT" = success + echo 'All five macOS lanes passed on the mini; see Resolve macOS execution for the worker run.' >> "$GITHUB_STEP_SUMMARY" - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + if: env.RUN_LANE == 'true' with: persist-credentials: false - uses: ./.github/actions/rust-setup + if: env.RUN_LANE == 'true' with: - # Reuse the CLI dependency cache; default-reader is its only writer. shared-key: ci-terminal-pty save-cache: "false" install-nextest: "false" - - name: Lint macOS workspace coordination - run: >- - cargo clippy --locked --manifest-path Cargo.toml -p astra-tools - --all-targets -- -D warnings - - name: Initialize system Git for bounded observation probes - timeout-minutes: 1 - # Checkout uses Homebrew Git, but the hardened observer selects - # /usr/bin/git with a cleared environment. Initialize that executable - # before its first use under the production two-second probe deadline. - # Keep the timing visible so runner startup failures are diagnosable. - run: | - time /usr/bin/env -i PATH=/usr/bin:/bin HOME=/nonexistent LANG=C LC_ALL=C \ - GIT_CONFIG_NOSYSTEM=1 GIT_CONFIG_GLOBAL=/dev/null GIT_CONFIG_SYSTEM=/dev/null \ - GIT_OPTIONAL_LOCKS=0 GIT_TERMINAL_PROMPT=0 \ - /usr/bin/git -C "$GITHUB_WORKSPACE" -c core.fsmonitor=false \ - -c core.untrackedCache=false rev-parse --show-toplevel - - name: Test macOS workspace coordination invariants - # Several invariants launch re-exec lock contenders while fingerprint - # probes deliberately have a short wall-clock deadline. Keep the module - # serial on shared CI hosts; the contender test supplies explicit - # cross-process concurrency without starving unrelated probes. - run: >- - cargo test --locked --manifest-path Cargo.toml -p astra-tools - workspace_observation::tests:: -- --test-threads=1 + - name: Run hosted macOS lane + if: env.RUN_LANE == 'true' + env: + MACOS_LANE: coordination + run: bash scripts/ci/run-macos-lanes.sh "$MACOS_LANE" # Preserve the required core check while its package groups run separately. # The aggregate requires all selected online jobs, including integration. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 3628499f3d..ae5097b028 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -114,16 +114,15 @@ The three ignored production-supervisor smokes use one additional nextest invocation, with exact names and separate serial test processes. They use the strict profile without retries, preserving the complete inventory's CI JUnit file and avoiding three separate Cargo test builds. -On macOS, the default terminal-reader lane also runs the CLI Bash regression -using its existing default-feature test executable. Tools lint and serial -workspace-coordination tests retain independent jobs; the protected coordination -check requires both paths to succeed. Feature-specific terminal tests remain separate. -All macOS execution jobs in Test Suite use the organization's self-hosted ARM -runner for allowlisted PR author IDs and main/develop pushes. Other or unknown -PR authors use hosted macOS. The allowlist lives in `scripts/ci/ci_scope.py`; -existing fork approval and required-check names remain in place. +On macOS, the default terminal-reader lane runs the CLI Bash regression using +its existing test executable. A static contributor ID allowlist and main/develop +pushes can attempt one self-hosted ARM bundle containing all five Mac lanes. +A bounded queue/setup failure falls back to hosted jobs in parallel; real test +failures remain failures. Other authors and reruns use hosted macOS. Linux +execution starts independently. The ordinary workflow retains required checks +and fork approval; no App or custom check source is needed. See [runner operations](docs/guides/repository-automation.md#self-hosted-macos-tests) -for registration, queueing and rollback. +for dispatch bootstrap, budgets, queueing and rollback. The lint gate retains restored artifacts; age-based cleanup is not a lint prerequisite. For fork pull requests, an update may require maintainer approval before the diff --git a/docs/guides/repository-automation.md b/docs/guides/repository-automation.md index f9f3ec860c..a1358dcdbf 100644 --- a/docs/guides/repository-automation.md +++ b/docs/guides/repository-automation.md @@ -122,41 +122,66 @@ and keeps external-fork behavior part of the repository's delivery contract. ## Self-hosted macOS tests -Test Suite routes default reader/Bash, dev-tty reader, reflow, workspace -coordination and vendored terminal units by PR author ID. IDs in -`MACOS_RUNNER_AUTHORS` in `scripts/ci/ci_scope.py` use runners matching -`[self-hosted, macOS, ARM64, mac-arm64-mini]`; other or unknown authors use hosted -`macos-15`. The list includes Dependabot. Numeric IDs remain stable across login -renames; login strings in the list are review annotations. The author is -`pull_request.user.id`, not the sender or re-run actor. Pushes on main/develop -also use the self-hosted labels. Existing required-check -names retain their `macos-15` label for compatibility; inspect each job's actual -runner for execution provenance. Linux execution and release workflows retain -their existing platforms. - -This uses the ordinary `pull_request`/`push` workflow, GitHub's existing fork -approval and read-only workflow token. No App, organization membership query, -custom result contexts, routing flag or hook is required by this implementation. -If scope output is missing, Mac jobs use hosted `macos-15` and still run tests. The selected runner must be available to Astra, including public -repository access when applicable, and have the exact labels above. Confirm its -OS/toolchain support for the pinned Actions/Rust/Node versions and at least -30 GiB workspace disk before assigning work. CI bootstraps rustup when missing, -adds Cargo binaries to PATH and installs the repository-pinned Rust toolchain. -The shared Rust setup reports low or unreadable disk as a failure on self-hosted runners, without running the -hosted-image sudo cleanup. Approved PR code executes on that -runner when the author is allowlisted; operators accept that persistent-host -access and resource exposure. - -One registration executes one job at a time. The five macOS jobs queue on a -single Mac; adding registrations on the same machine changes resource contention -and does not create extra capacity. Reader feature builds remain separate jobs. -For allowlisted PRs and pushes, a busy/offline runner leaves jobs queued; there -is no automatic availability-based hosted fallback. -Measure queue delay and actual job duration before claiming an elapsed-time gain. - -If the earlier controller design was staged, remove its unused custom App -required contexts and retire its installed admission hook through the reviewed -operations process; this workflow no longer emits those contexts or App -admissions. Preserve the existing GitHub Actions required checks and fork -approval settings. Roll back execution by reverting the runner selectors in Test -Suite to the hosted matrix OS and `macos-15` coordination runner through a PR. +The static author-ID allowlist in `scripts/ci/ci_scope.py` makes the eleven +selected contributors (including Dependabot) eligible for a mini attempt. +Main/develop pushes are eligible too. Other or unknown PR authors run on hosted +macOS immediately; GitHub's existing fork approval remains in effect. Every Mac +job re-executed on attempts greater than one runs hosted, even when its planner's +old output is reused. GitHub may retain successful Mac jobs on a failed-only +rerun; those checks still refer to the original tested merge. Required-check names retain +`macos-15` for compatibility; job summaries link the actual worker execution. + +The ordinary Test Suite owns all required checks. Linux jobs depend only on +scope and start without waiting for the mini. `Resolve macOS execution` has +Actions/Pull requests read access and waits for a worker bound to its exact run +ID, attempt and recorded merge SHA. Missing controller, API errors or missing +routing output select real hosted tests, never a successful skip. + +`macos-scheduler.yml` runs on the default branch on Test Suite's `workflow_run` +start/completion events. It uses the built-in token's Actions write permission +to dispatch/cancel `macos-mini.yml`; it never checks out PR code. No App, +organization membership query, additional secret, custom required context or +admission hook is needed. These workflows must exist on main before dispatch +works. Before that, Test Suite falls back to hosted execution. + +A mini attempt gets a 180-second queue/discovery budget, with bounded API +requests, then falls back to the original hosted jobs in parallel. The +reader/controller enforce a 60-minute execution budget even if the runner +disappears; the worker has a separate 65-minute hard backstop. Thus single-machine +unavailability does not hold a +required check for GitHub's 24-hour queue limit. API/control outages can prevent +immediate child cancellation; the completion observer retries cleanup, and a +late worker rejects an ended parent gate before candidate checkout. + +The worker performs one candidate checkout, toolchain setup and cache restore, +then runs default reader/Bash, reflow, coordination, vendored units and dev-tty +through the same `scripts/ci/run-macos-lanes.sh` used by hosted/Linux jobs. Cargo +invocations remain separate and dev-tty is last. A real compile/test failure is +reported as failure, without hosted retry. Failed runs with incomplete test +metadata also stay failed rather than hiding a possible test failure. Checkout, +low-disk/toolchain setup, +agent cancellation/loss or the execution budget select hosted fallback. The +worker gets a read-only token and cache-mode read; PR work cannot save default +branch Actions/sccache cache entries. + +The runner must match `[self-hosted, macOS, ARM64, mac-arm64-mini]`, be available +to Astra (including public repository access), and support the pinned Actions, +Rust and Node versions. Rustup is bootstrapped if absent and Cargo bin is added +to PATH. At least 30 GiB workspace disk is required. On self-hosted runners the +disk guard reports low/unreadable disk without privileged hosted-image cleanup; +the hosted fallback proceeds while operators restore mini capacity. + +One registration runs one bundle at a time. Other eligible runs use hosted Macs +when their queue budget expires. Measure mini execution time, fallback rate, +queue delay, API usage and hosted minutes; running attempts poll once per minute +and filter worker history by source creation time. Bundling reduces repeated +preparation, but +neither creates physical capacity nor guarantees the previous hosted elapsed +time. The initial 60-minute budget needs qualification against actual load. + +If the old App/controller design was staged, retire its unused App required +contexts and admission hook, and permit the new `macos-mini.yml@main` worker in +the runner group. Preserve existing GitHub Actions checks and fork approval. +Rollback through a PR by routing all authors to hosted macOS; existing reruns +already take that recovery path. No remote runner/group or protection settings +are changed by the implementation PR. diff --git a/scripts/ci/cancel_stale_pr_runs.py b/scripts/ci/cancel_stale_pr_runs.py index c605a11932..8f9b8982a9 100755 --- a/scripts/ci/cancel_stale_pr_runs.py +++ b/scripts/ci/cancel_stale_pr_runs.py @@ -70,7 +70,7 @@ def __init__(self, token: str, api_url: str = "https://api.github.com") -> None: self._api_origin = (parsed_api_url.scheme.lower(), parsed_api_url.netloc.lower()) self._opener = build_opener(RejectRedirects()) - def _request(self, method: str, url: str) -> tuple[bytes, Any]: + def _request(self, method: str, url: str, *, data: dict[str, Any] | None = None) -> tuple[bytes, Any]: target = url if urlsplit(url).scheme else urljoin(self._api_url, url.lstrip("/")) parsed_target = urlsplit(target) target_origin = (parsed_target.scheme.lower(), parsed_target.netloc.lower()) @@ -79,8 +79,10 @@ def _request(self, method: str, url: str) -> tuple[bytes, Any]: request = Request( target, method=method, + data=json.dumps(data).encode() if data is not None else None, headers={ "Accept": "application/vnd.github+json", + "Content-Type": "application/json", "Authorization": f"Bearer {self._token}", "User-Agent": "astra-stale-pr-run-controller", "X-GitHub-Api-Version": "2022-11-28", diff --git a/scripts/ci/macos_dispatch.py b/scripts/ci/macos_dispatch.py new file mode 100755 index 0000000000..27a56c5fb0 --- /dev/null +++ b/scripts/ci/macos_dispatch.py @@ -0,0 +1,293 @@ +#!/usr/bin/env python3 +"""Bounded mini attempt; ordinary Test Suite owns hosted fallback and checks.""" + +from __future__ import annotations + +import argparse +from dataclasses import dataclass +import json +import os +from pathlib import Path +import re +import time +from urllib.parse import urlencode + +from cancel_stale_pr_runs import GitHubApi, MAX_PAGES +from ci_scope import MACOS_SELF_HOSTED_LABELS, select_macos_runner + + +WORKFLOW = "macos-mini.yml" +GATE = "Resolve macOS execution" +WAIT_STEP = "Wait for a bounded mini attempt" +TEST_STEP = "Run all macOS lanes" +PREFLIGHT_STEPS = {"Set up job", "Checkout control policy", "Validate mini dispatch", + "Checkout candidate", "Prepare Rust", "Prepare Node"} +QUEUE_SECONDS = 180 +EXECUTION_SECONDS = 3600 +POLL_SECONDS = 20 +RUNNING_POLL_SECONDS = 60 +TITLE = re.compile(r"Test Suite pr=(\d+) base=([0-9a-f]{40}) head=([0-9a-f]{40}) merge=([0-9a-f]{40})") + + +class Unavailable(RuntimeError): + pass + + +class Api(GitHubApi): + def get(self, path): + return self._get_json(path) + + def post(self, path, data): + self._request("POST", path, data=data) + + def collection(self, path, key): + result, visited = [], set() + while path: + if path in visited or len(visited) >= MAX_PAGES: + raise Unavailable("pagination bound exceeded") + visited.add(path) + body, headers = self._request("GET", path) + page = json.loads(body) + if not isinstance(page.get(key), list): + raise Unavailable("missing API collection") + result.extend(page[key]) + path = self._next_link(headers) + return result + + +@dataclass(frozen=True) +class Source: + run_id: int + attempt: int + sha: str + created_at: str | None = None + + @property + def title(self): + return f"Mac mini for Test Suite {self.run_id}/{self.attempt} {self.sha}" + + +def source(api, repo, run_id, attempt): + run = api.get(f"repos/{repo}/actions/runs/{run_id}") + match = TITLE.fullmatch(run.get("display_title", "")) + if (run.get("id") != run_id or run.get("name") != "Test Suite" or run.get("path") != ".github/workflows/test.yml" + or run.get("run_attempt") != attempt or attempt != 1 + or run.get("status") == "completed" or not match): + raise Unavailable("source run is not a current mini candidate") + pr_number, base, head, merge = match.groups() + if run.get("head_sha") != head: + raise Unavailable("source head mismatch") + if run.get("event") == "pull_request" and int(pr_number) > 0: + # Fork run.pull_requests can be empty. The event title records the PR + # and immutable merge tuple, rather than relying on that association. + pr = api.get(f"repos/{repo}/pulls/{pr_number}") + if (pr["head"]["sha"] != head or pr["state"] != "open" + or pr["head"]["repo"]["id"] != run["head_repository"]["id"]): + raise Unavailable("PR source changed") + parents = api.get(f"repos/{repo}/commits/{merge}")["parents"] + if [parent["sha"] for parent in parents] != [base, head]: + raise Unavailable("unverified merge tuple") + labels = select_macos_runner("pull_request", {"pull_request": {"user": pr["user"]}}, "") + elif run.get("event") == "push" and pr_number == "0" and merge == head: + if run["head_repository"]["id"] != run["repository"]["id"]: + raise Unavailable("push repository mismatch") + labels = select_macos_runner("push", {}, "refs/heads/" + run["head_branch"]) + else: + raise Unavailable("unsupported source event") + if labels != MACOS_SELF_HOSTED_LABELS: + raise Unavailable("source does not prefer mini") + return Source(run_id, attempt, merge, run.get("created_at")) + + +def jobs(api, repo, run_id, attempt): + return api.collection( + f"repos/{repo}/actions/runs/{run_id}/attempts/{attempt}/jobs?per_page=100", "jobs") + + +def gate_active(api, repo, src): + run = api.get(f"repos/{repo}/actions/runs/{src.run_id}") + if run.get("run_attempt") != src.attempt or run.get("status") == "completed": + return False + return any(job.get("name") == GATE and job.get("status") == "in_progress" + and any(step.get("name") == WAIT_STEP and step.get("status") == "in_progress" + for step in job.get("steps", [])) + for job in jobs(api, repo, src.run_id, src.attempt)) + + +def workers(api, repo, src): + filters = {"event": "workflow_dispatch", "branch": "main", "per_page": 100} + if src.created_at: + filters["created"] = ">=" + src.created_at + query = urlencode(filters) + runs = api.collection(f"repos/{repo}/actions/workflows/{WORKFLOW}/runs?{query}", "workflow_runs") + return [run for run in runs if run.get("display_title") == src.title + and run.get("path") == f".github/workflows/{WORKFLOW}" + and run.get("event") == "workflow_dispatch"] + + +def worker_state(api, repo, run): + if run.get("run_attempt") != 1: + raise Unavailable("worker reruns are not reusable") + execution = jobs(api, repo, run["id"], 1) + if len(execution) > 1: + raise Unavailable("unexpected mini job inventory") + job = execution[0] if execution else {} + assigned = bool(job.get("runner_id")) and job.get("status") == "in_progress" + tests = [step for step in job.get("steps", []) if step.get("name") == TEST_STEP] + # A known test failure wins even while post-job cleanup is still running. + if any(step.get("conclusion") == "failure" for step in tests): + return "failure", True + if run.get("status") != "completed": + return None, assigned + if len(tests) == 1: + if run.get("conclusion") == "success" and tests[0].get("conclusion") == "success": + return "success", True + preflight_failed = any(step.get("name") in PREFLIGHT_STEPS and step.get("conclusion") == "failure" + for step in job.get("steps", [])) + if (run.get("conclusion") == "failure" and not preflight_failed + and not (len(tests) == 1 and tests[0].get("conclusion") == "success")): + # Incomplete/eventually consistent job metadata must not turn a failed + # test run into a successful hosted retry. Only proven setup failures + # or an explicit infrastructure terminal state get that fallback. + return "failure", assigned + # Checkout, admission, low disk, toolchain, agent loss or an execution + # deadline are infrastructure outcomes. Never infer success from a skip. + print("::notice::Mini ended without a completed passing test step; using hosted macOS") + return "hosted", assigned + + +def wait_for_mini(api, repo, src, *, clock=time.monotonic, sleep=time.sleep): + queued_at, assigned_at = clock(), None + announced = None + while True: + run = api.get(f"repos/{repo}/actions/runs/{src.run_id}") + if run.get("run_attempt") != src.attempt or run.get("status") == "completed": + raise Unavailable("source attempt ended") + candidates = workers(api, repo, src) + if len(candidates) > 1: + raise Unavailable("duplicate mini dispatch") + if candidates: + if candidates[0]["id"] != announced: + announced = candidates[0]["id"] + print(f"Mini worker run: {announced}") + if os.environ.get("GITHUB_STEP_SUMMARY"): + with Path(os.environ["GITHUB_STEP_SUMMARY"]).open("a") as summary: + summary.write(f"Mini worker: https://github.com/{repo}/actions/runs/{announced}\n") + outcome, assigned = worker_state(api, repo, candidates[0]) + if outcome: + return outcome + if assigned and assigned_at is None: + assigned_at = clock() + if assigned_at is None and clock() - queued_at >= QUEUE_SECONDS: + print("::notice::Mini queue/discovery budget expired; using hosted macOS") + return "hosted" + if assigned_at is not None and clock() - assigned_at >= EXECUTION_SECONDS: + print("::notice::Mini execution budget expired; using hosted macOS") + return "hosted" + sleep(RUNNING_POLL_SECONDS if assigned_at is not None else POLL_SECONDS) + + +def cancel_workers(api, repo, src): + failures = 0 + for run in workers(api, repo, src): + if run.get("status") != "completed": + try: + api.cancel_run(repo, run["id"]) + except Exception: + # A transient failure for one duplicate must not prevent + # cleanup of the other workers belonging to this source. + failures += 1 + if failures: + raise Unavailable(f"cancellation failed for {failures} worker(s)") + + +def schedule(api, repo, src, *, clock=time.monotonic, sleep=time.sleep): + queued_at, assigned_at = clock(), None + try: + # No worker is dispatched unless the approved ordinary workflow has + # actually started its gate. A delayed scheduler cannot resurrect it. + while not gate_active(api, repo, src): + if any(job.get("name") == GATE and (job.get("status") == "completed" + or any(step.get("name") == WAIT_STEP and step.get("status") == "completed" + for step in job.get("steps", []))) + for job in jobs(api, repo, src.run_id, src.attempt)): + return + run = api.get(f"repos/{repo}/actions/runs/{src.run_id}") + if run.get("status") == "completed" or run.get("run_attempt") != src.attempt: + return + if clock() - queued_at >= QUEUE_SECONDS: + return + sleep(POLL_SECONDS) + queued_at = clock() + if not workers(api, repo, src): + api.post(f"repos/{repo}/actions/workflows/{WORKFLOW}/dispatches", { + "ref": "main", "inputs": {"source_run_id": str(src.run_id), + "source_attempt": str(src.attempt), "source_sha": src.sha}}) + while gate_active(api, repo, src): + candidates = workers(api, repo, src) + if len(candidates) > 1: + raise Unavailable("duplicate mini dispatch") + if candidates: + outcome, assigned = worker_state(api, repo, candidates[0]) + if outcome: + return + if assigned and assigned_at is None: + assigned_at = clock() + if assigned_at is None and clock() - queued_at >= QUEUE_SECONDS: + return + if assigned_at is not None and clock() - assigned_at >= EXECUTION_SECONDS: + return + sleep(RUNNING_POLL_SECONDS if assigned_at is not None else POLL_SECONDS) + finally: + # Also runs when the parent is cancelled/superseded or has already + # selected hosted fallback. Only this exact source/attempt is touched. + cancel_workers(api, repo, src) + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("mode", choices=("schedule", "wait", "admit", "cleanup")) + parser.add_argument("--run-id", type=int, required=True) + parser.add_argument("--attempt", type=int, required=True) + parser.add_argument("--sha") + args = parser.parse_args() + repo = os.environ["GITHUB_REPOSITORY"] + api = Api(os.environ.get("GH_TOKEN", ""), os.environ.get("GITHUB_API_URL", "https://api.github.com")) + outcome = "hosted" + try: + if args.mode == "cleanup": + run = api.get(f"repos/{repo}/actions/runs/{args.run_id}") + match = TITLE.fullmatch(run.get("display_title", "")) + if run.get("path") == ".github/workflows/test.yml" and match: + # Completion also repairs an orphan left by a scheduler/API + # failure; the PR may already be closed or on a newer attempt. + cancel_workers(api, repo, Source(args.run_id, args.attempt, match[4], run.get("created_at"))) + return 0 + src = source(api, repo, args.run_id, args.attempt) + if args.mode == "schedule": + schedule(api, repo, src) + return 0 + if args.mode == "admit": + if os.environ.get("GITHUB_RUN_ATTEMPT") != "1": + raise Unavailable("worker reruns must use ordinary hosted recovery") + if src.sha != args.sha or not gate_active(api, repo, src): + raise Unavailable("dispatch expired before execution") + return 0 + outcome = wait_for_mini(api, repo, src) + except Exception as error: + # Do not print API bodies, credential-bearing URLs or private config. + reason = str(error) if isinstance(error, Unavailable) else type(error).__name__ + if isinstance(getattr(error, "code", None), int): + reason += f" HTTP {error.code}" + print(f"::notice::Mini unavailable ({reason}); using hosted macOS") + if args.mode == "admit": + return 1 + if args.mode == "wait": + with Path(os.environ["GITHUB_OUTPUT"]).open("a") as output: + output.write(f"result={outcome}\n") + print(f"macOS execution result: {outcome}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/ci/run-macos-lanes.sh b/scripts/ci/run-macos-lanes.sh new file mode 100755 index 0000000000..84d1da2dee --- /dev/null +++ b/scripts/ci/run-macos-lanes.sh @@ -0,0 +1,54 @@ +#!/usr/bin/env bash +# Canonical test commands shared by the mini bundle and hosted fallback. +set -euo pipefail +lane="${1:?lane required}" +lane_workspace="$PWD" +observe_git() { + python3 - "$lane_workspace" <<'PYTHON' +import os, subprocess, sys, time +started = time.monotonic() +subprocess.run(["/usr/bin/git", "-C", sys.argv[1], "-c", "core.fsmonitor=false", + "-c", "core.untrackedCache=false", "rev-parse", "--show-toplevel"], + env={"PATH": "/usr/bin:/bin", "HOME": "/nonexistent", "LANG": "C", "LC_ALL": "C", + "GIT_CONFIG_NOSYSTEM": "1", "GIT_CONFIG_GLOBAL": os.devnull, + "GIT_CONFIG_SYSTEM": os.devnull, "GIT_OPTIONAL_LOCKS": "0", + "GIT_TERMINAL_PROMPT": "0"}, check=True, timeout=60) +print(f"System Git observation initialized in {time.monotonic() - started:.3f}s") +PYTHON +} +case "$lane" in + default-reader) + cargo test --locked -p astra-cli --lib tui::terminal_startup -- --test-threads=2 + if [ "$(uname -s)" = Darwin ]; then + observe_git + cargo test --locked --manifest-path Cargo.toml -p astra-cli \ + --lib edge_tools::shell::tests::bash_echo_returns_output -- --exact + fi + ;; + reflow) + cargo build --locked -p astra-cli --bin astra + npm ci --ignore-scripts --prefix scripts/tui-reflow + ASTRA_TEST_BINARY="${CARGO_TARGET_DIR:-$PWD/target}/debug/astra" npm test --prefix scripts/tui-reflow + ;; + coordination) + cargo clippy --locked --manifest-path Cargo.toml -p astra-tools --all-targets -- -D warnings + observe_git + cargo test --locked --manifest-path Cargo.toml -p astra-tools \ + workspace_observation::tests:: -- --test-threads=1 + ;; + dev-tty-reader) + cargo test --locked -p astra-cli --lib --features crossterm/use-dev-tty tui::terminal_startup -- --test-threads=2 + ;; + vendored) + cargo test --locked --manifest-path vendor/crossterm/Cargo.toml --lib --features event-stream + cargo test --locked --manifest-path vendor/crossterm/Cargo.toml --lib --features event-stream,use-dev-tty + ;; + all) + bash "$0" default-reader + bash "$0" reflow + bash "$0" coordination + bash "$0" vendored + bash "$0" dev-tty-reader + ;; + *) echo "Unknown macOS lane" >&2; exit 2 ;; +esac diff --git a/scripts/ci/test_ci_build_coverage.py b/scripts/ci/test_ci_build_coverage.py index 6fa1518582..6165c39f03 100755 --- a/scripts/ci/test_ci_build_coverage.py +++ b/scripts/ci/test_ci_build_coverage.py @@ -217,49 +217,36 @@ def test_complete_cli_run_prebuilds_required_standalone_mcp_fixture(self): def test_vendored_terminal_units_are_locked_and_pty_tests_run_both_readers(self): workflow = (ROOT / ".github/workflows/test.yml").read_text() - section = workflow.split("\n vendored-terminal:\n", 1)[1].split("\n terminal-pty:", 1)[0] - self.assertIn("os: [ubuntu-latest, macos-15]", section) + lanes = (ROOT / "scripts/ci/run-macos-lanes.sh").read_text() + self.assertIn("run-macos-lanes.sh vendored", workflow) for features in ("event-stream", "event-stream,use-dev-tty"): - self.assertIn(f"cargo test --locked --manifest-path vendor/crossterm/Cargo.toml --lib --features {features}\n", section) + self.assertIn(f"cargo test --locked --manifest-path vendor/crossterm/Cargo.toml --lib --features {features}\n", lanes) lock = tomllib.loads((ROOT / "vendor/crossterm/Cargo.lock").read_text()) self.assertTrue(any(p["name"] == "crossterm" for p in lock["package"])) self.assertNotIn("/Cargo.lock", (ROOT / "vendor/crossterm/.gitignore").read_text().splitlines()) - pty = workflow.split("\n terminal-pty:\n", 1)[1].split("\n macos-workspace-coordination:", 1)[0] - self.assertIn("os: [ubuntu-latest, macos-15]", pty) - self.assertIn("segment: [default-reader, dev-tty-reader, reflow]", pty) - for segment, step, features in ( - ("default-reader", "Test default terminal reader through PTYs", ""), - ("dev-tty-reader", "Test dev-tty terminal reader through PTYs", "--features crossterm/use-dev-tty "), - ): - with self.subTest(segment=segment): - commands = workflow_run_script(".github/workflows/test.yml", step).strip() - self.assertEqual(commands, - f"cargo test --locked -p astra-cli --lib {features}tui::terminal_startup -- --test-threads=2") - block = pty.split(f" - name: {step}\n", 1)[1].split(" - ", 1)[0] - self.assertIn(f"if: matrix.segment == '{segment}'", block) + self.assertEqual(workflow.count("segment: [default-reader, dev-tty-reader, reflow]"), 2) + for features in ("", "--features crossterm/use-dev-tty "): + self.assertIn(f"cargo test --locked -p astra-cli --lib {features}tui::terminal_startup -- --test-threads=2", lanes) def test_terminal_reflow_uses_real_binary_and_locked_emulator(self): - commands = workflow_run_script(".github/workflows/test.yml", "Test inline terminal resize with xterm reflow") + commands = (ROOT / "scripts/ci/run-macos-lanes.sh").read_text() self.assertIn("cargo build --locked -p astra-cli --bin astra", commands) self.assertIn("npm ci --ignore-scripts --prefix scripts/tui-reflow", commands) - self.assertIn('ASTRA_TEST_BINARY="$PWD/target/debug/astra" npm test --prefix scripts/tui-reflow', commands) + self.assertIn('ASTRA_TEST_BINARY="${CARGO_TARGET_DIR:-$PWD/target}/debug/astra" npm test', commands) def test_macos_cli_bash_reuses_default_reader_without_losing_coordination_contracts(self): - workflow = (ROOT / ".github/workflows/test.yml").read_text() - terminal = workflow.split("\n terminal-pty-shards:\n", 1)[1].split("\n macos-workspace-coordination:\n", 1)[0] - self.assertEqual(workflow.count("- name: Test macOS CLI Bash execution"), 1) - bash = terminal.split(" - name: Test macOS CLI Bash execution\n", 1)[1].split(" - ", 1)[0] - self.assertIn("if: matrix.os == 'macos-15' && matrix.segment == 'default-reader'", bash) - self.assertIn("--lib edge_tools::shell::tests::bash_echo_returns_output -- --exact", bash) - self.assertLess(terminal.index("Test default terminal reader through PTYs"), terminal.index("Initialize system Git for macOS CLI observation")) - self.assertLess(terminal.index("Initialize system Git for macOS CLI observation"), terminal.index("Test macOS CLI Bash execution")) - contracts = workflow.split("\n macos-workspace-coordination-tests:\n", 1)[1].split("\n test-online-core:", 1)[0] - self.assertIn('install-nextest: "false"', contracts) - self.assertIn("-p astra-tools", contracts) - self.assertIn("--all-targets -- -D warnings", contracts) + commands = (ROOT / "scripts/ci/run-macos-lanes.sh").read_text() + reader = commands.split(" default-reader)\n", 1)[1].split(" ;;", 1)[0] + self.assertLess(reader.index("tui::terminal_startup"), reader.index("observe_git")) + self.assertLess(reader.index("observe_git"), reader.index("bash_echo_returns_output")) + self.assertEqual(reader.count("cargo test"), 2) + contracts = commands.split(" coordination)\n", 1)[1].split(" ;;", 1)[0] + self.assertIn("cargo clippy --locked --manifest-path Cargo.toml -p astra-tools --all-targets -- -D warnings", contracts) self.assertIn("workspace_observation::tests:: -- --test-threads=1", contracts) - self.assertIn("/usr/bin/git", contracts) - self.assertNotIn("-p astra-cli", contracts, "Do not compile the same CLI test executable twice on macOS") + self.assertIn("observe_git", contracts) + self.assertNotIn("-p astra-cli", contracts) + self.assertIn('"/usr/bin/git"', commands) + self.assertIn("timeout=60", commands) def test_ci_resolves_the_actual_builder_base_not_only_the_planner(self): dockerfile = (ROOT / "Dockerfile").read_text() diff --git a/scripts/ci/test_ci_parallelism.py b/scripts/ci/test_ci_parallelism.py index 6620b27deb..3b5a530d4c 100755 --- a/scripts/ci/test_ci_parallelism.py +++ b/scripts/ci/test_ci_parallelism.py @@ -117,31 +117,41 @@ def test_status_labels_do_not_repeat_setup_builds_or_artifact_transfers(self): class MacRunnerTests(unittest.TestCase): - def test_macos_execution_uses_scope_routing_and_keeps_a_hosted_fallback(self): + def test_macos_fallback_is_parallel_and_does_not_block_linux(self): workflow = (ROOT / ".github/workflows/test.yml").read_text() - selector = "fromJSON(needs.scope.outputs.macos_runner || '[\"macos-15\"]')" - for job in ("vendored-terminal", "terminal-pty-shards"): + for job in ("vendored-terminal", "terminal-pty-shards", "test-online"): header = workflow.split(f"\n {job}:\n", 1)[1].split(" steps:\n", 1)[0] - self.assertIn("runs-on: ${{ matrix.os == 'macos-15' && " + selector + " || matrix.os }}", header) - self.assertIn("os: [ubuntu-latest, macos-15]", header) self.assertIn("needs: scope", header) - coordination = workflow.split("\n macos-workspace-coordination-tests:\n", 1)[1].split(" steps:\n", 1)[0] - self.assertIn("runs-on: ${{ " + selector + " }}", coordination) - self.assertIn("macos_runner: ${{ steps.scope.outputs.macos_runner }}", workflow) - for job in ("scope", "cli-test-build", "shard-b", "shard-c", "shard-d", "test-online"): - header = workflow.split(f"\n {job}:\n", 1)[1].split(" steps:\n", 1)[0] self.assertIn("runs-on: ubuntu-latest", header) - self.assertIn("pull_request:", workflow) + self.assertNotIn("macos-plan", header) + for job in ("vendored-terminal-macos", "terminal-pty-shards-macos", "macos-workspace-coordination-tests"): + section = workflow.split(f"\n {job}:\n", 1)[1].split(" steps:\n", 1)[0] + self.assertIn("needs: [scope, macos-plan]", section) + self.assertIn("github.run_attempt > 1", section, "Failed-only reruns must ignore reused mini outputs") + self.assertIn("'macos-15' || 'ubuntu-latest'", section) self.assertNotIn("pull_request_target:", workflow) - self.assertNotIn("ASTRA_MACOS_", workflow) - self.assertNotIn("macos-routing:", workflow) + self.assertNotIn("self-hosted", workflow.split(" vendored-terminal:\n", 1)[1]) + + def test_remote_result_gate_never_accepts_failure_or_missing_output(self): + workflow = (ROOT / ".github/workflows/test.yml").read_text() + for job in ("vendored-terminal-macos", "terminal-pty-shards-macos", "macos-workspace-coordination-tests"): + section = workflow.split(f"\n {job}:\n", 1)[1] + block = section.split(" run: |\n", 1)[1].split(" - ", 1)[0] + script = "\n".join(line[10:] for line in block.splitlines()) + for outcome in ("success", "failure", "hosted", "", "skipped"): + with self.subTest(job=job, outcome=outcome), tempfile.TemporaryDirectory() as directory: + result = subprocess.run(["bash", "-c", script], env={ + **os.environ, "MINI_RESULT": outcome, + "GITHUB_STEP_SUMMARY": str(Path(directory) / "summary"), + }, capture_output=True, text=True) + self.assertEqual(result.returncode == 0, outcome == "success") class ParallelGateTests(unittest.TestCase): def test_shard_gates_reject_failure_cancellation_and_unexpected_skip(self): for step, variables in ( - ("Require terminal PTY shards", ("SHARDS_RESULT",)), - ("Require macOS coordination and CLI execution", ("COORDINATION_RESULT", "TERMINAL_RESULT")), + ("Require terminal PTY shards", ("SHARDS_RESULT", "MACOS_RESULT")), + ("Require macOS coordination and CLI execution", ("COORDINATION_RESULT", "TERMINAL_RESULT", "MACOS_TERMINAL_RESULT")), ): script = workflow_run_script(".github/workflows/test.yml", step) for required in (True, False): @@ -172,7 +182,7 @@ def test_parallel_jobs_depend_only_on_scope_and_gates_depend_on_shards(self): self.assertIn("!cancelled()", header) self.assertIn("fail-fast: false", header) terminal = workflow.split("\n terminal-pty:\n", 1)[1].split("\n terminal-pty-shards:", 1)[0] - self.assertIn("needs: [scope, terminal-pty-shards]", terminal) + self.assertIn("needs: [scope, terminal-pty-shards, terminal-pty-shards-macos]", terminal) self.assertIn("needs.scope.result != 'success'", terminal) self.assertIn("needs.scope.outputs.test_cli == 'true'", terminal) self.assertIn("needs.scope.outputs.test_core == 'true'", terminal) @@ -180,11 +190,11 @@ def test_parallel_jobs_depend_only_on_scope_and_gates_depend_on_shards(self): self.assertIn("if: ${{ !cancelled() }}", terminal) coordination = workflow.split("\n macos-workspace-coordination:\n", 1)[1].split("\n macos-workspace-coordination-tests:", 1)[0] self.assertIn('name: "Test: macOS workspace coordination"', coordination) - self.assertIn("needs: [scope, macos-workspace-coordination-tests, terminal-pty-shards]", coordination) + self.assertIn("needs: [scope, macos-workspace-coordination-tests, terminal-pty-shards, terminal-pty-shards-macos]", coordination) self.assertIn("needs.scope.result != 'success'", coordination) self.assertIn("if: ${{ !cancelled() }}", coordination) contracts = workflow.split("\n macos-workspace-coordination-tests:\n", 1)[1].split(" steps:\n", 1)[0] - self.assertIn("needs: scope", contracts) + self.assertIn("needs: [scope, macos-plan]", contracts) self.assertNotIn("terminal-pty-shards", contracts, "Tools contracts must not wait for the CLI build") online = workflow.split("\n test-online:\n", 1)[1] self.assertIn("lane: [core-runtime, core-turn-core, core-services, integration]", online) diff --git a/scripts/ci/test_macos_dispatch.py b/scripts/ci/test_macos_dispatch.py new file mode 100755 index 0000000000..1da53ff177 --- /dev/null +++ b/scripts/ci/test_macos_dispatch.py @@ -0,0 +1,401 @@ +#!/usr/bin/env python3 +"""Exercise dispatch/queue/fallback against the production entrypoints.""" + +from copy import deepcopy +import json +import os +from pathlib import Path +import subprocess +import sys +import tempfile +import unittest +from unittest.mock import patch + +import macos_dispatch as dispatch + + +ROOT = Path(__file__).resolve().parents[2] +REPO = "matrixorigin/Astra" +BASE, HEAD, MERGE = "a" * 40, "b" * 40, "c" * 40 + + +class Clock: + def __init__(self): + self.now = 0 + + def __call__(self): + return self.now + + def sleep(self, seconds): + self.now += seconds + + +class ApiFixture: + def __init__(self): + self.clock = Clock() + self.run = {"id": 1, "run_attempt": 1, "name": "Test Suite", + "path": ".github/workflows/test.yml", "event": "pull_request", + "display_title": f"Test Suite pr=1126 base={BASE} head={HEAD} merge={MERGE}", + "head_sha": HEAD, "status": "in_progress", "pull_requests": [], + "head_repository": {"id": 42}, "repository": {"id": 7}} + self.pr = {"state": "open", "user": {"id": 52204121}, + "head": {"sha": HEAD, "repo": {"id": 42}}} + self.parents = [BASE, HEAD] + self.src = dispatch.Source(1, 1, MERGE) + self.enabled = True + self.assign_at, self.finish_at = 0, 30 + self.run_conclusion, self.test_conclusion = "success", "success" + self.gate_end_at = None + self.wait_end_at = None + self.gate_start_at = 0 + self.gate_conclusion = "success" + self.source_end_at = None + self.other_runs = [] + self.posts, self.cancelled = [], [] + self.calls = [] + + def get(self, path): + self.calls.append(path) + if path.endswith("/actions/runs/1"): + run = deepcopy(self.run) + if self.source_end_at is not None and self.clock() >= self.source_end_at: + run["status"] = "completed" + return run + if path.endswith("/pulls/1126"): + return deepcopy(self.pr) + if path.endswith("/commits/" + MERGE): + return {"parents": [{"sha": sha} for sha in self.parents]} + raise AssertionError(path) + + def worker(self): + status = "completed" if self.clock() >= self.finish_at else "in_progress" + return {"id": 10, "run_attempt": 1, "display_title": self.src.title, + "path": ".github/workflows/macos-mini.yml", "event": "workflow_dispatch", + "status": status, "conclusion": self.run_conclusion if status == "completed" else None} + + def collection(self, path, key): + self.calls.append(path) + if "/workflows/macos-mini.yml/runs?" in path: + return ([self.worker()] if self.enabled else []) + deepcopy(self.other_runs) + if "/runs/1/attempts/1/jobs?" in path: + completed = self.gate_end_at is not None and self.clock() >= self.gate_end_at + status = "in_progress" if self.clock() >= self.gate_start_at else "queued" + wait_status = "completed" if completed or (self.wait_end_at is not None and self.clock() >= self.wait_end_at) else status + return [{"name": dispatch.GATE, "status": "completed" if completed else status, + "conclusion": self.gate_conclusion if completed else None, + "steps": [{"name": dispatch.WAIT_STEP, "status": wait_status}]}] + if "/runs/10/attempts/1/jobs?" in path: + complete = self.clock() >= self.finish_at + assigned = self.clock() >= self.assign_at + steps = [{"name": dispatch.TEST_STEP, "conclusion": self.test_conclusion}] if complete else [] + if complete and self.test_conclusion is None: + steps.append({"name": "Prepare Rust", "conclusion": "failure"}) + return [{"name": "execute", "runner_id": 99 if assigned else 0, + "status": "completed" if complete else "in_progress" if assigned else "queued", + "steps": steps}] + raise AssertionError(path) + + def post(self, path, data): + self.posts.append((path, data)) + self.enabled = True + + def cancel_run(self, repo, run_id): + self.cancelled.append(run_id) + return True + + +class SourceTests(unittest.TestCase): + def test_dispatch_serializes_json_through_the_shared_origin_checked_client(self): + api = dispatch.Api("fixture") + data = {"ref": "main", "inputs": {"source_run_id": "1"}} + with patch.object(api, "_opener") as opener: + response = opener.open.return_value.__enter__.return_value + response.read.return_value = b"" + response.headers = {} + api.post(f"repos/{REPO}/actions/workflows/macos-mini.yml/dispatches", data) + request = opener.open.call_args.args[0] + self.assertEqual(request.get_method(), "POST") + self.assertEqual(json.loads(request.data), data) + self.assertEqual(request.get_header("Content-type"), "application/json") + self.assertEqual(opener.open.call_args.kwargs["timeout"], 10) + + def test_empty_fork_pr_association_uses_verified_merge_tuple(self): + api = ApiFixture() + self.assertEqual(dispatch.source(api, REPO, 1, 1), api.src) + self.assertFalse(any("members" in call for call in api.calls)) + + def test_wrong_parent_order_head_repository_author_and_attempt_are_rejected(self): + mutations = ( + lambda a: setattr(a, "parents", [HEAD, BASE]), + lambda a: a.pr["head"].update(sha="d" * 40), + lambda a: a.pr["head"]["repo"].update(id=43), + lambda a: a.pr["user"].update(id=999999999), + lambda a: a.run.update(run_attempt=2), + ) + for mutate in mutations: + with self.subTest(mutation=mutate): + api = ApiFixture() + mutate(api) + with self.assertRaises(dispatch.Unavailable): + dispatch.source(api, REPO, 1, 1) + + def test_main_develop_pushes_need_no_pr_identity(self): + for branch in ("main", "develop", "feature"): + api = ApiFixture() + api.run.update(event="push", head_branch=branch, head_repository={"id": 7}, + display_title=f"Test Suite pr=0 base={BASE} head={HEAD} merge={HEAD}") + if branch == "feature": + with self.assertRaises(dispatch.Unavailable): + dispatch.source(api, REPO, 1, 1) + else: + self.assertEqual(dispatch.source(api, REPO, 1, 1).sha, HEAD) + + +class WaitTests(unittest.TestCase): + def wait(self, api): + return dispatch.wait_for_mini(api, REPO, api.src, clock=api.clock, sleep=api.clock.sleep) + + def test_offline_queued_worker_has_bounded_wait(self): + api = ApiFixture() + api.assign_at = api.finish_at = 100000 + self.assertEqual(self.wait(api), "hosted") + self.assertEqual(api.clock(), dispatch.QUEUE_SECONDS) + + def test_missing_dispatch_has_the_same_bounded_wait(self): + api = ApiFixture() + api.enabled = False + self.assertEqual(self.wait(api), "hosted") + self.assertEqual(api.clock(), dispatch.QUEUE_SECONDS) + + def test_assigned_worker_may_complete_after_queue_deadline(self): + api = ApiFixture() + api.assign_at, api.finish_at = 170, 200 + self.assertEqual(self.wait(api), "success") + + def test_execution_deadline_and_agent_loss_use_hosted(self): + api = ApiFixture() + api.finish_at = 100000 + self.assertEqual(self.wait(api), "hosted") + self.assertEqual(api.clock(), dispatch.EXECUTION_SECONDS) + + def test_one_hour_attempt_stays_below_half_the_default_rest_budget(self): + api = ApiFixture() + api.finish_at = 100000 + self.assertEqual(self.wait(api), "hosted") + read_calls = len(api.calls) + api = ApiFixture() + api.finish_at = 100000 + dispatch.schedule(api, REPO, api.src, clock=api.clock, sleep=api.clock.sleep) + self.assertLess(read_calls + len(api.calls), 500) + + def test_actual_test_failure_never_becomes_hosted_retry(self): + api = ApiFixture() + api.run_conclusion = api.test_conclusion = "failure" + self.assertEqual(self.wait(api), "failure") + + def test_failed_run_with_incomplete_test_metadata_is_not_retried(self): + api = ApiFixture() + api.finish_at = 0 + api.run_conclusion, api.test_conclusion = "failure", "pending" + self.assertEqual(self.wait(api), "failure") + + def test_known_test_failure_wins_before_post_job_cleanup_finishes(self): + api = ApiFixture() + api.finish_at = 0 + api.test_conclusion = "failure" + run = api.worker() + run.update(status="in_progress", conclusion=None) + self.assertEqual(dispatch.worker_state(api, REPO, run)[0], "failure") + + def test_worker_lookup_filters_history_to_the_source_creation_window(self): + api = ApiFixture() + src = dispatch.Source(1, 1, MERGE, "2026-10-24T08:00:00Z") + dispatch.workers(api, REPO, src) + from urllib.parse import parse_qs, urlsplit + self.assertEqual(parse_qs(urlsplit(api.calls[-1]).query)["created"], [">=2026-10-24T08:00:00Z"]) + + def test_low_disk_checkout_toolchain_skips_and_cancellation_are_not_success(self): + for conclusion, test in (("failure", None), ("success", "skipped"), + ("cancelled", "cancelled"), ("timed_out", "cancelled")): + with self.subTest(conclusion=conclusion, test=test): + api = ApiFixture() + api.run_conclusion, api.test_conclusion = conclusion, test + self.assertEqual(self.wait(api), "hosted") + + def test_same_sha_different_source_run_and_attempt_cannot_supply_results(self): + for title in (dispatch.Source(2, 1, MERGE).title, dispatch.Source(1, 2, MERGE).title): + api = ApiFixture() + api.enabled = False + other = api.worker() + other.update(id=20, status="completed", conclusion="success", display_title=title) + api.other_runs = [other] + self.assertEqual(self.wait(api), "hosted") + self.assertEqual(api.clock(), dispatch.QUEUE_SECONDS) + + def test_parent_cancellation_does_not_reuse_old_mini_result(self): + api = ApiFixture() + api.source_end_at = 0 + with self.assertRaises(dispatch.Unavailable): + self.wait(api) + + def test_worker_individual_rerun_is_unavailable(self): + api = ApiFixture() + worker = api.worker() + worker["run_attempt"] = 2 + with self.assertRaises(dispatch.Unavailable): + dispatch.worker_state(api, REPO, worker) + + +class SchedulerTests(unittest.TestCase): + def run_scheduler(self, api): + dispatch.schedule(api, REPO, api.src, clock=api.clock, sleep=api.clock.sleep) + + def test_offline_worker_is_cancelled_without_touching_other_pr(self): + api = ApiFixture() + api.enabled = False + api.assign_at = api.finish_at = 100000 + other = api.worker() + other.update(id=20, display_title=dispatch.Source(2, 1, MERGE).title) + api.other_runs = [other] + self.run_scheduler(api) + self.assertEqual(api.cancelled, [10]) + self.assertEqual(api.clock(), dispatch.QUEUE_SECONDS) + self.assertEqual(api.posts[0][1], {"ref": "main", "inputs": { + "source_run_id": "1", "source_attempt": "1", "source_sha": MERGE}}) + + def test_completed_gate_cannot_dispatch_a_late_worker(self): + api = ApiFixture() + api.enabled = False + api.gate_end_at = 0 + self.run_scheduler(api) + self.assertEqual(api.posts, []) + + def test_finished_wait_step_blocks_dispatch_during_post_job_cleanup(self): + api = ApiFixture() + api.enabled = False + api.wait_end_at = 0 + self.run_scheduler(api) + self.assertEqual(api.posts, []) + + def test_gate_start_delay_does_not_consume_the_worker_queue_budget(self): + api = ApiFixture() + api.enabled = False + api.gate_start_at, api.assign_at, api.finish_at = 100, 270, 300 + self.run_scheduler(api) + self.assertEqual(api.cancelled, []) + self.assertGreaterEqual(api.clock(), 300) + self.assertLessEqual(api.clock(), 300 + dispatch.RUNNING_POLL_SECONDS) + + def test_cancel_failure_does_not_prevent_other_owned_workers_from_cancelling(self): + api = ApiFixture() + duplicate = api.worker() + duplicate["id"] = 11 + api.other_runs = [duplicate] + original = api.cancel_run + + def cancel(repo, run_id): + original(repo, run_id) + if run_id == 10: + raise RuntimeError("temporary API failure") + + api.cancel_run = cancel + with self.assertRaises(dispatch.Unavailable): + dispatch.cancel_workers(api, REPO, api.src) + self.assertEqual(api.cancelled, [10, 11]) + + def test_source_cancel_and_gate_fallback_cancel_only_the_owned_worker(self): + for field in ("source_end_at", "gate_end_at"): + api = ApiFixture() + api.finish_at = 100000 + setattr(api, field, 20) + self.run_scheduler(api) + self.assertEqual(api.cancelled, [10]) + self.assertLessEqual(api.clock(), 20 + dispatch.RUNNING_POLL_SECONDS) + + def test_success_or_real_test_failure_does_not_cancel_completed_worker(self): + for result in ("success", "failure"): + api = ApiFixture() + api.run_conclusion = api.test_conclusion = result + self.run_scheduler(api) + self.assertEqual(api.cancelled, []) + + +class EntrypointTests(unittest.TestCase): + def invoke(self, api, mode, *, worker_attempt="1"): + with tempfile.TemporaryDirectory() as directory: + output = Path(directory) / "output" + args = ["macos_dispatch.py", mode, "--run-id", "1", "--attempt", "1", "--sha", MERGE] + with patch.object(dispatch, "Api", return_value=api), patch.object(sys, "argv", args), \ + patch.dict(os.environ, {"GITHUB_REPOSITORY": REPO, "GH_TOKEN": "fixture", + "GITHUB_OUTPUT": str(output), "GITHUB_RUN_ATTEMPT": worker_attempt}, clear=True): + status = dispatch.main() + return status, output.read_text() if output.exists() else "" + + def test_unavailable_api_and_missing_controller_fail_open_to_actual_hosted_tests(self): + api = ApiFixture() + with patch.object(api, "get", side_effect=RuntimeError("unavailable")): + self.assertEqual(self.invoke(api, "wait"), (0, "result=hosted\n")) + + def test_failure_is_published_as_failure_not_fallback(self): + api = ApiFixture() + api.finish_at = 0 + api.run_conclusion = api.test_conclusion = "failure" + self.assertEqual(self.invoke(api, "wait"), (0, "result=failure\n")) + + def test_completion_cancels_orphan_even_after_pr_close_or_attempt_change(self): + api = ApiFixture() + api.run.update(status="completed", run_attempt=2) + api.pr["state"] = "closed" + self.assertEqual(self.invoke(api, "cleanup")[0], 0) + self.assertEqual(api.cancelled, [10]) + + def test_late_or_reused_worker_rejected_before_candidate_checkout(self): + for completed, attempt in ((True, "1"), (False, "2")): + api = ApiFixture() + if completed: + api.gate_end_at = 0 + self.assertEqual(self.invoke(api, "admit", worker_attempt=attempt)[0], 1) + workflow = (ROOT / ".github/workflows/macos-mini.yml").read_text() + self.assertLess(workflow.index("macos_dispatch.py admit"), workflow.index("ref: ${{ inputs.source_sha }}")) + self.assertIn("cache-mode: read", workflow) + self.assertNotIn("actions: write", workflow) + + def test_worker_cannot_start_during_parent_post_job_cleanup(self): + api = ApiFixture() + api.wait_end_at = 0 + self.assertEqual(self.invoke(api, "admit")[0], 1) + + +class LaneTests(unittest.TestCase): + def test_bundle_executes_all_real_lane_commands_and_propagates_failure(self): + for fail in (False, True): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + tools = root / "bin" + tools.mkdir() + log = root / "commands" + for command in ("cargo", "npm"): + program = tools / command + program.write_text('#!/bin/bash\nprintf "%s %s\\n" "${0##*/}" "$*" >> "$COMMANDS"\n' + 'if [[ "${FAIL_CARGO:-}" == 1 && "${0##*/}" == cargo ]]; then exit 17; fi\n') + program.chmod(0o755) + # Keep the native Git probe real, but use a disposable repository. + subprocess.run(["git", "init", "-q", str(root)], check=True) + result = subprocess.run(["bash", str(ROOT / "scripts/ci/run-macos-lanes.sh"), "all"], + cwd=root, env={**os.environ, "PATH": str(tools) + ":" + os.environ["PATH"], + "COMMANDS": str(log), "FAIL_CARGO": "1" if fail else "0"}, capture_output=True, text=True) + calls = log.read_text().splitlines() + if fail: + self.assertEqual(result.returncode, 17) + self.assertEqual(len(calls), 1) + else: + self.assertEqual(result.returncode, 0, result.stderr) + self.assertTrue(any("--features crossterm/use-dev-tty tui::terminal_startup" in c for c in calls)) + self.assertEqual(sum("vendor/crossterm" in c for c in calls), 2) + self.assertTrue(any("workspace_observation::tests:: -- --test-threads=1" in c for c in calls)) + self.assertTrue(any("npm test --prefix scripts/tui-reflow" in c for c in calls)) + self.assertIn("crossterm/use-dev-tty", calls[-1]) + + +if __name__ == "__main__": + unittest.main() From 58b1c7fab476c098a9f31e1083672ae64a3d7f03 Mon Sep 17 00:00:00 2001 From: lr90 Date: Sat, 24 Oct 2026 17:18:36 +0800 Subject: [PATCH 11/11] fix(ci): isolate push runners and preserve compiler and run identity --- .github/actions/rust-toolchain/action.yml | 7 +++++ .github/workflows/macos-scheduler.yml | 2 +- CONTRIBUTING.md | 5 +-- docs/guides/repository-automation.md | 10 +++--- scripts/ci/ci_scope.py | 7 ++--- scripts/ci/macos_dispatch.py | 10 +++--- scripts/ci/test_ci_build_coverage.py | 38 +++++++++++++++++++---- scripts/ci/test_ci_scope.py | 19 ++++++------ scripts/ci/test_macos_dispatch.py | 21 ++++++++----- 9 files changed, 79 insertions(+), 40 deletions(-) diff --git a/.github/actions/rust-toolchain/action.yml b/.github/actions/rust-toolchain/action.yml index aff6b0838d..44bc3411c1 100644 --- a/.github/actions/rust-toolchain/action.yml +++ b/.github/actions/rust-toolchain/action.yml @@ -14,7 +14,14 @@ runs: https://sh.rustup.rs | sh -s -- --default-toolchain none --no-modify-path -y fi echo "$ci_cargo_bin" >> "$GITHUB_PATH" + # Resolve the repository pin rather than a runner service override. + unset RUSTUP_TOOLCHAIN # Older rustup installs implicitly on show; newer versions require the # explicit install. Both read the repository file, including components. rustup show active-toolchain || rustup toolchain install + ci_rust_toolchain="$(rustup show active-toolchain)" + ci_rust_toolchain="${ci_rust_toolchain%% *}" + test -n "$ci_rust_toolchain" + export RUSTUP_TOOLCHAIN="$ci_rust_toolchain" + echo "RUSTUP_TOOLCHAIN=$RUSTUP_TOOLCHAIN" >> "$GITHUB_ENV" rustc --version diff --git a/.github/workflows/macos-scheduler.yml b/.github/workflows/macos-scheduler.yml index a39cc2cea7..c913a62356 100644 --- a/.github/workflows/macos-scheduler.yml +++ b/.github/workflows/macos-scheduler.yml @@ -12,7 +12,7 @@ concurrency: cancel-in-progress: false jobs: schedule: - if: github.event.workflow_run.run_attempt == 1 + if: github.event.workflow_run.run_attempt == 1 && github.event.workflow_run.event == 'pull_request' runs-on: ubuntu-latest timeout-minutes: 70 steps: diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index ae5097b028..34a4176222 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -115,8 +115,9 @@ invocation, with exact names and separate serial test processes. They use the strict profile without retries, preserving the complete inventory's CI JUnit file and avoiding three separate Cargo test builds. On macOS, the default terminal-reader lane runs the CLI Bash regression using -its existing test executable. A static contributor ID allowlist and main/develop -pushes can attempt one self-hosted ARM bundle containing all five Mac lanes. +its existing test executable. PR authors in a static contributor ID allowlist +can attempt one self-hosted ARM bundle containing all five Mac lanes. Trusted +main/develop pushes and release builds use hosted runners. A bounded queue/setup failure falls back to hosted jobs in parallel; real test failures remain failures. Other authors and reruns use hosted macOS. Linux execution starts independently. The ordinary workflow retains required checks diff --git a/docs/guides/repository-automation.md b/docs/guides/repository-automation.md index a1358dcdbf..c27e83c7bf 100644 --- a/docs/guides/repository-automation.md +++ b/docs/guides/repository-automation.md @@ -124,8 +124,9 @@ and keeps external-fork behavior part of the repository's delivery contract. The static author-ID allowlist in `scripts/ci/ci_scope.py` makes the eleven selected contributors (including Dependabot) eligible for a mini attempt. -Main/develop pushes are eligible too. Other or unknown PR authors run on hosted -macOS immediately; GitHub's existing fork approval remains in effect. Every Mac +Trusted main/develop pushes and release builds remain on hosted runners, so +unmerged PR state in the mini's HOME/Cargo/rustup directories is not reused by +trusted builds. Other or unknown PR authors run on hosted macOS immediately; GitHub's existing fork approval remains in effect. Every Mac job re-executed on attempts greater than one runs hosted, even when its planner's old output is reused. GitHub may retain successful Mac jobs on a failed-only rerun; those checks still refer to the original tested merge. Required-check names retain @@ -166,8 +167,9 @@ branch Actions/sccache cache entries. The runner must match `[self-hosted, macOS, ARM64, mac-arm64-mini]`, be available to Astra (including public repository access), and support the pinned Actions, -Rust and Node versions. Rustup is bootstrapped if absent and Cargo bin is added -to PATH. At least 30 GiB workspace disk is required. On self-hosted runners the +Rust and Node versions. Rustup is bootstrapped if absent, Cargo bin is added to PATH, and the selected +repository toolchain is exported through GITHUB_ENV for later sccache compiler +probes outside the checkout. At least 30 GiB workspace disk is required. On self-hosted runners the disk guard reports low/unreadable disk without privileged hosted-image cleanup; the hosted fallback proceeds while operators restore mini capacity. diff --git a/scripts/ci/ci_scope.py b/scripts/ci/ci_scope.py index e9869141ff..e0f06f8146 100755 --- a/scripts/ci/ci_scope.py +++ b/scripts/ci/ci_scope.py @@ -35,9 +35,8 @@ MACOS_SELF_HOSTED_LABELS = ["self-hosted", "macOS", "ARM64", "mac-arm64-mini"] -def select_macos_runner(event_name: str, event: dict, ref: str) -> list[str]: - if event_name == "push" and ref in {"refs/heads/main", "refs/heads/develop"}: - return MACOS_SELF_HOSTED_LABELS.copy() +def select_macos_runner(event_name: str, event: dict) -> list[str]: + # Trusted pushes must not consume persistent state left by unmerged PRs. if event_name == "pull_request" and isinstance(event, dict): pull_request = event.get("pull_request") author = pull_request.get("user") if isinstance(pull_request, dict) else None @@ -289,7 +288,7 @@ def emit(scopes: dict[str, bool], paths: list[str], *, fallback: str | None = No try: event = json.loads(Path(os.environ["GITHUB_EVENT_PATH"]).read_text(encoding="utf-8")) labels = select_macos_runner( - os.environ.get("GITHUB_EVENT_NAME", ""), event, os.environ.get("GITHUB_REF", "") + os.environ.get("GITHUB_EVENT_NAME", ""), event ) except (KeyError, OSError, ValueError, TypeError): labels = ["macos-15"] diff --git a/scripts/ci/macos_dispatch.py b/scripts/ci/macos_dispatch.py index 27a56c5fb0..e6bc24f0eb 100755 --- a/scripts/ci/macos_dispatch.py +++ b/scripts/ci/macos_dispatch.py @@ -70,7 +70,9 @@ def title(self): def source(api, repo, run_id, attempt): run = api.get(f"repos/{repo}/actions/runs/{run_id}") match = TITLE.fullmatch(run.get("display_title", "")) - if (run.get("id") != run_id or run.get("name") != "Test Suite" or run.get("path") != ".github/workflows/test.yml" + # GitHub may populate name with the dynamic run-name as well. The workflow + # path is the identity; the title carries the immutable event tuple. + if (run.get("id") != run_id or run.get("path") != ".github/workflows/test.yml" or run.get("run_attempt") != attempt or attempt != 1 or run.get("status") == "completed" or not match): raise Unavailable("source run is not a current mini candidate") @@ -87,11 +89,7 @@ def source(api, repo, run_id, attempt): parents = api.get(f"repos/{repo}/commits/{merge}")["parents"] if [parent["sha"] for parent in parents] != [base, head]: raise Unavailable("unverified merge tuple") - labels = select_macos_runner("pull_request", {"pull_request": {"user": pr["user"]}}, "") - elif run.get("event") == "push" and pr_number == "0" and merge == head: - if run["head_repository"]["id"] != run["repository"]["id"]: - raise Unavailable("push repository mismatch") - labels = select_macos_runner("push", {}, "refs/heads/" + run["head_branch"]) + labels = select_macos_runner("pull_request", {"pull_request": {"user": pr["user"]}}) else: raise Unavailable("unsupported source event") if labels != MACOS_SELF_HOSTED_LABELS: diff --git a/scripts/ci/test_ci_build_coverage.py b/scripts/ci/test_ci_build_coverage.py index 6165c39f03..27535f4313 100755 --- a/scripts/ci/test_ci_build_coverage.py +++ b/scripts/ci/test_ci_build_coverage.py @@ -32,6 +32,9 @@ def test_rust_toolchain_bootstraps_missing_rust_and_reuses_cargo_bin(self): (cargo_home / "bin").mkdir(parents=True) calls = root / "calls" github_path = root / "github-path" + github_env = root / "github-env" + outside = root / "outside-workspace" + outside.mkdir() installer = root / "installer" installer.write_text("""#!/bin/sh set -eu @@ -41,8 +44,9 @@ def test_rust_toolchain_bootstraps_missing_rust_and_reuses_cargo_bin(self): """) programs = { "curl": 'printf "curl\\n" >> "$MOCK_CALLS"; if [ "$MOCK_FAILURE" = 1 ]; then exit 22; fi; cat "$MOCK_INSTALLER"', - "rustup-source": 'printf "rustup %s\\n" "$*" >> "$MOCK_CALLS"; if [ "$1" = show ]; then test -f "$CARGO_HOME/ready"; else touch "$CARGO_HOME/ready"; fi', - "rustc-source": 'test -f "$CARGO_HOME/ready"; printf "rustc pinned-fixture\\n"', + "rustup-source": 'printf "rustup %s\\n" "$*" >> "$MOCK_CALLS"; if [ "$1" = show ]; then test -f "$CARGO_HOME/ready"; printf "1.97.0-fixture (overridden by repository)\\n"; else touch "$CARGO_HOME/ready"; fi', + "rustc-source": 'test -f "$CARGO_HOME/ready"; if [ "${RUSTUP_TOOLCHAIN:-}" != 1.97.0-fixture ] && [ "$PWD" != "$MOCK_WORKSPACE" ]; then exit 42; fi; printf "rustc pinned-fixture\\n"', + "sccache": 'compiler="$1"; shift; cd "$MOCK_OUTSIDE"; "$compiler" "$@"', } for name, body in programs.items(): path = mock_bin / name @@ -54,12 +58,15 @@ def test_rust_toolchain_bootstraps_missing_rust_and_reuses_cargo_bin(self): target.write_bytes((mock_bin / (name + "-source")).read_bytes()) target.chmod(0o755) (cargo_home / "ready").touch() - result = subprocess.run( - ["/bin/bash", "-e", "-c", script], cwd=ROOT, capture_output=True, text=True, - env={**os.environ, "PATH": str(mock_bin) + ":/usr/bin:/bin", + environment = {**os.environ, "PATH": str(mock_bin) + ":/usr/bin:/bin", "CARGO_HOME": str(cargo_home), "GITHUB_PATH": str(github_path), + "GITHUB_ENV": str(github_env), "MOCK_WORKSPACE": str(ROOT), + "MOCK_OUTSIDE": str(outside), "SCCACHE_PATH": str(mock_bin / "sccache"), "MOCK_BIN": str(mock_bin), "MOCK_CALLS": str(calls), - "MOCK_INSTALLER": str(installer), "MOCK_FAILURE": str(int(install_failure))}, + "MOCK_INSTALLER": str(installer), "MOCK_FAILURE": str(int(install_failure))} + result = subprocess.run( + ["/bin/bash", "-e", "-c", script], cwd=ROOT, capture_output=True, text=True, + env=environment, ) invocations = calls.read_text().splitlines() if install_failure: @@ -71,6 +78,25 @@ def test_rust_toolchain_bootstraps_missing_rust_and_reuses_cargo_bin(self): self.assertEqual(github_path.read_text().strip(), str(cargo_home / "bin")) self.assertEqual("curl" in invocations, not installed) self.assertEqual("rustup toolchain install" in invocations, not installed) + selected = dict(line.split("=", 1) for line in github_env.read_text().splitlines()) + self.assertEqual(selected["RUSTUP_TOOLCHAIN"], "1.97.0-fixture") + next_environment = {**environment, **selected, + "PATH": str(cargo_home / "bin") + ":" + environment["PATH"]} + probe = subprocess.run(["bash", str(ROOT / "scripts/ci/configure-sccache.sh")], + cwd=ROOT, env=next_environment, capture_output=True, text=True) + self.assertEqual(probe.returncode, 0, probe.stderr) + self.assertIn("RUSTC_WRAPPER=" + str(mock_bin / "sccache"), github_env.read_text()) + # Without the exported selection, the same proxy fails only + # when sccache invokes it outside the repository. The compiler + # itself still works in ROOT, reproducing the old fail-open path. + negative_env = root / "negative-env" + next_environment.pop("RUSTUP_TOOLCHAIN") + next_environment["GITHUB_ENV"] = str(negative_env) + probe = subprocess.run(["bash", str(ROOT / "scripts/ci/configure-sccache.sh")], + cwd=ROOT, env=next_environment, capture_output=True, text=True) + self.assertEqual(probe.returncode, 0, probe.stderr) + self.assertIn("Rust cache disabled", probe.stderr) + self.assertTrue(negative_env.read_text().endswith("RUSTC_WRAPPER=\n")) for path in (".github/actions/rust-setup/action.yml", ".github/workflows/test.yml"): self.assertIn("uses: ./.github/actions/rust-toolchain", (ROOT / path).read_text()) diff --git a/scripts/ci/test_ci_scope.py b/scripts/ci/test_ci_scope.py index ae246b5147..5d1d3c8d20 100755 --- a/scripts/ci/test_ci_scope.py +++ b/scripts/ci/test_ci_scope.py @@ -20,7 +20,7 @@ def test_macos_allowlist_uses_author_id_including_bots_and_login_renames(self): with self.subTest(author_id=author_id): event = {"pull_request": {"user": {"id": author_id, "login": "renamed-login"}}, "sender": {"id": 999999999}} - self.assertEqual(select_macos_runner("pull_request", event, "refs/pull/1/merge"), + self.assertEqual(select_macos_runner("pull_request", event), ["self-hosted", "macOS", "ARM64", "mac-arm64-mini"]) def test_macos_unknown_author_stays_hosted_even_when_actor_or_login_matches(self): @@ -28,17 +28,16 @@ def test_macos_unknown_author_stays_hosted_even_when_actor_or_login_matches(self {"id": "39627130"}, {"id": True}, None): with self.subTest(user=user): event = {"pull_request": {"user": user}, "sender": {"id": 39627130}} - self.assertEqual(select_macos_runner("pull_request", event, "refs/pull/1/merge"), + self.assertEqual(select_macos_runner("pull_request", event), ["macos-15"]) for event in ({}, {"pull_request": None}, []): - self.assertEqual(select_macos_runner("pull_request", event, ""), ["macos-15"]) - - def test_macos_only_main_and_develop_pushes_use_mini_without_a_pr_author(self): - for ref in ("refs/heads/main", "refs/heads/develop"): - self.assertEqual(select_macos_runner("push", {}, ref), - ["self-hosted", "macOS", "ARM64", "mac-arm64-mini"]) - self.assertEqual(select_macos_runner("push", {}, "refs/heads/feature"), ["macos-15"]) - self.assertEqual(select_macos_runner("workflow_dispatch", {}, "refs/heads/main"), ["macos-15"]) + self.assertEqual(select_macos_runner("pull_request", event), ["macos-15"]) + + def test_macos_trusted_pushes_and_manual_events_remain_hosted(self): + for event_name in ("push", "workflow_dispatch", "pull_request_target"): + self.assertEqual(select_macos_runner(event_name, + {"ref": "refs/heads/main", "sender": {"id": 39627130}}), ["macos-15"]) + self.assertEqual(select_macos_runner("push", {"ref": "refs/heads/develop"}), ["macos-15"]) def test_macos_missing_event_routes_to_hosted_without_omitting_scope_checks(self): with tempfile.TemporaryDirectory() as directory: diff --git a/scripts/ci/test_macos_dispatch.py b/scripts/ci/test_macos_dispatch.py index 1da53ff177..7c32378421 100755 --- a/scripts/ci/test_macos_dispatch.py +++ b/scripts/ci/test_macos_dispatch.py @@ -38,6 +38,7 @@ def __init__(self): "display_title": f"Test Suite pr=1126 base={BASE} head={HEAD} merge={MERGE}", "head_sha": HEAD, "status": "in_progress", "pull_requests": [], "head_repository": {"id": 42}, "repository": {"id": 7}} + self.run["name"] = self.run["display_title"] self.pr = {"state": "open", "user": {"id": 52204121}, "head": {"sha": HEAD, "repo": {"id": 42}}} self.parents = [BASE, HEAD] @@ -124,6 +125,15 @@ def test_empty_fork_pr_association_uses_verified_merge_tuple(self): self.assertEqual(dispatch.source(api, REPO, 1, 1), api.src) self.assertFalse(any("members" in call for call in api.calls)) + def test_static_and_dynamic_run_names_use_the_workflow_path_as_identity(self): + for name in ("Test Suite", ApiFixture().run["display_title"]): + api = ApiFixture() + api.run["name"] = name + self.assertEqual(dispatch.source(api, REPO, 1, 1), api.src) + api.run["path"] = ".github/workflows/other.yml" + with self.assertRaises(dispatch.Unavailable): + dispatch.source(api, REPO, 1, 1) + def test_wrong_parent_order_head_repository_author_and_attempt_are_rejected(self): mutations = ( lambda a: setattr(a, "parents", [HEAD, BASE]), @@ -139,16 +149,13 @@ def test_wrong_parent_order_head_repository_author_and_attempt_are_rejected(self with self.assertRaises(dispatch.Unavailable): dispatch.source(api, REPO, 1, 1) - def test_main_develop_pushes_need_no_pr_identity(self): - for branch in ("main", "develop", "feature"): + def test_main_develop_pushes_cannot_enter_the_pr_runner(self): + for branch in ("main", "develop"): api = ApiFixture() api.run.update(event="push", head_branch=branch, head_repository={"id": 7}, display_title=f"Test Suite pr=0 base={BASE} head={HEAD} merge={HEAD}") - if branch == "feature": - with self.assertRaises(dispatch.Unavailable): - dispatch.source(api, REPO, 1, 1) - else: - self.assertEqual(dispatch.source(api, REPO, 1, 1).sha, HEAD) + with self.assertRaises(dispatch.Unavailable): + dispatch.source(api, REPO, 1, 1) class WaitTests(unittest.TestCase):