diff --git a/.github/actionlint.yaml b/.github/actionlint.yaml new file mode 100644 index 0000000000..10c7250fcb --- /dev/null +++ b/.github/actionlint.yaml @@ -0,0 +1,7 @@ +self-hosted-runner: + labels: [mac-arm64-mini] +# GitHub supports cache-mode; actionlint 1.7.12 predates the syntax. +paths: + .github/workflows/macos-mini.yml: + ignore: + - 'unexpected key "cache-mode" for "workflow" section' diff --git a/.github/actions/rust-setup/action.yml b/.github/actions/rust-setup/action.yml index 51aee2cbb1..f26f92269a 100644 --- a/.github/actions/rust-setup/action.yml +++ b/.github/actions/rust-setup/action.yml @@ -27,9 +27,12 @@ runs: using: "composite" steps: # Avoid paying for cleanup when the filesystem containing the workspace has - # enough room. Keep the cleanup fallback for low or unreadable disk space. + # enough room. Only hosted images use the cleanup fallback; persistent + # self-hosted runners report low/unreadable disk without privileged cleanup. - name: Guard workspace disk space shell: bash + env: + ASTRA_CI_RUNNER_ENVIRONMENT: ${{ runner.environment }} run: | set -euxo pipefail disk_guard_workspace="${GITHUB_WORKSPACE:?GITHUB_WORKSPACE must be set}" @@ -51,6 +54,9 @@ runs: if [[ "${disk_guard_measurement}" == "valid" ]] && (( disk_guard_available_kib >= disk_guard_threshold_kib )); then echo "Disk guard decision: skip cleanup; workspace filesystem free=${disk_guard_available_kib} KiB; threshold=${disk_guard_threshold_kib} KiB (30 GiB)." + elif [[ "${ASTRA_CI_RUNNER_ENVIRONMENT:-}" == self-hosted ]]; then + echo "::error::Disk guard decision: fail self-hosted; measurement=${disk_guard_measurement}; workspace filesystem free=${disk_guard_available_kib:-unavailable} KiB; threshold=${disk_guard_threshold_kib} KiB (30 GiB). Reclaim runner disk before retrying." + exit 1 else echo "Disk guard decision: run cleanup; measurement=${disk_guard_measurement}; workspace filesystem free=${disk_guard_available_kib:-unavailable} KiB; threshold=${disk_guard_threshold_kib} KiB (30 GiB)." sudo rm -rf /usr/share/dotnet @@ -63,9 +69,7 @@ runs: df -h "${disk_guard_workspace}" - name: Install pinned Rust toolchain - # The repository toolchain also owns test-created Cargo fixtures. - shell: bash - run: rustc --version + uses: ./.github/actions/rust-toolchain - uses: taiki-e/install-action@d438492cf8a250514fa2d34b30bc3c0dc37c65ff # v2 if: ${{ inputs.install-nextest == 'true' }} diff --git a/.github/actions/rust-toolchain/action.yml b/.github/actions/rust-toolchain/action.yml new file mode 100644 index 0000000000..44bc3411c1 --- /dev/null +++ b/.github/actions/rust-toolchain/action.yml @@ -0,0 +1,27 @@ +name: Install repository Rust toolchain +description: Bootstrap rustup and install the toolchain from rust-toolchain.toml +runs: + using: composite + steps: + - name: Install pinned Rust toolchain + shell: bash + run: | + set -euo pipefail + ci_cargo_bin="${CARGO_HOME:-$HOME/.cargo}/bin" + export PATH="$ci_cargo_bin:$PATH" + if ! command -v rustup >/dev/null 2>&1; then + curl --proto '=https' --tlsv1.2 --retry 3 --location --silent --show-error --fail \ + https://sh.rustup.rs | sh -s -- --default-toolchain none --no-modify-path -y + fi + echo "$ci_cargo_bin" >> "$GITHUB_PATH" + # Resolve the repository pin rather than a runner service override. + unset RUSTUP_TOOLCHAIN + # Older rustup installs implicitly on show; newer versions require the + # explicit install. Both read the repository file, including components. + rustup show active-toolchain || rustup toolchain install + ci_rust_toolchain="$(rustup show active-toolchain)" + ci_rust_toolchain="${ci_rust_toolchain%% *}" + test -n "$ci_rust_toolchain" + export RUSTUP_TOOLCHAIN="$ci_rust_toolchain" + echo "RUSTUP_TOOLCHAIN=$RUSTUP_TOOLCHAIN" >> "$GITHUB_ENV" + rustc --version diff --git a/.github/workflows/macos-mini.yml b/.github/workflows/macos-mini.yml new file mode 100644 index 0000000000..0cfe1364e5 --- /dev/null +++ b/.github/workflows/macos-mini.yml @@ -0,0 +1,63 @@ +name: macOS mini worker +run-name: Mac mini for Test Suite ${{ inputs.source_run_id }}/${{ inputs.source_attempt }} ${{ inputs.source_sha }} +on: + workflow_dispatch: + inputs: + source_run_id: + required: true + type: string + source_attempt: + required: true + type: string + source_sha: + required: true + type: string +permissions: + contents: read + actions: read + pull-requests: read +cache-mode: read +concurrency: + group: macos-mini-${{ inputs.source_run_id }}-${{ inputs.source_attempt }} + cancel-in-progress: false +defaults: + run: + shell: bash +env: + CARGO_INCREMENTAL: "0" + CARGO_TERM_COLOR: always +jobs: + execute: + runs-on: [self-hosted, macOS, ARM64, mac-arm64-mini] + timeout-minutes: 65 + steps: + - name: Checkout control policy + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + with: + ref: ${{ github.sha }} + path: .ci-control + persist-credentials: false + - name: Validate mini dispatch + env: + GH_TOKEN: ${{ github.token }} + SOURCE_RUN_ID: ${{ inputs.source_run_id }} + SOURCE_ATTEMPT: ${{ inputs.source_attempt }} + SOURCE_SHA: ${{ inputs.source_sha }} + run: python3 .ci-control/scripts/ci/macos_dispatch.py admit --run-id "$SOURCE_RUN_ID" --attempt "$SOURCE_ATTEMPT" --sha "$SOURCE_SHA" + - name: Checkout candidate + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + with: + ref: ${{ inputs.source_sha }} + persist-credentials: false + - name: Prepare Rust + uses: ./.github/actions/rust-setup + with: + shared-key: ci-terminal-pty + save-cache: "false" + install-nextest: "false" + - name: Prepare Node + uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7 + with: + node-version-file: .nvmrc + - name: Run all macOS lanes + run: bash scripts/ci/run-macos-lanes.sh all diff --git a/.github/workflows/macos-scheduler.yml b/.github/workflows/macos-scheduler.yml new file mode 100644 index 0000000000..c913a62356 --- /dev/null +++ b/.github/workflows/macos-scheduler.yml @@ -0,0 +1,30 @@ +name: macOS mini scheduler +on: + workflow_run: + workflows: [Test Suite] + types: [in_progress, completed] +permissions: + contents: read + actions: write + pull-requests: read +concurrency: + group: macos-scheduler-${{ github.event.workflow_run.id }}-${{ github.event.workflow_run.run_attempt }} + cancel-in-progress: false +jobs: + schedule: + if: github.event.workflow_run.run_attempt == 1 && github.event.workflow_run.event == 'pull_request' + runs-on: ubuntu-latest + timeout-minutes: 70 + steps: + # Only the default-branch controller is loaded; it never executes PR code. + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + with: + ref: ${{ github.sha }} + persist-credentials: false + - name: Schedule and bound the mini attempt + env: + GH_TOKEN: ${{ github.token }} + DISPATCH_MODE: ${{ github.event.action == 'completed' && 'cleanup' || 'schedule' }} + SOURCE_RUN_ID: ${{ github.event.workflow_run.id }} + SOURCE_ATTEMPT: ${{ github.event.workflow_run.run_attempt }} + run: python3 scripts/ci/macos_dispatch.py "$DISPATCH_MODE" --run-id "$SOURCE_RUN_ID" --attempt "$SOURCE_ATTEMPT" diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 88250cc5a0..74f2a5dd83 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -1,4 +1,6 @@ name: Test Suite +run-name: >- + Test Suite pr=${{ github.event.pull_request.number || 0 }} base=${{ github.event.pull_request.base.sha || github.event.before }} head=${{ github.event.pull_request.head.sha || github.sha }} merge=${{ github.sha }} permissions: contents: read @@ -13,6 +15,10 @@ concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} cancel-in-progress: true +defaults: + run: + shell: bash + env: CARGO_TERM_COLOR: always CARGO_INCREMENTAL: "0" @@ -23,6 +29,7 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 5 outputs: + macos_runner: ${{ steps.scope.outputs.macos_runner }} test_cli: ${{ steps.scope.outputs.test_cli }} test_runtime: ${{ steps.scope.outputs.test_runtime }} test_services: ${{ steps.scope.outputs.test_services }} @@ -40,6 +47,31 @@ jobs: id: scope run: python3 scripts/ci/ci_scope.py + macos-plan: + name: Resolve macOS execution + needs: scope + if: >- + !cancelled() && github.run_attempt == 1 && + (needs.scope.outputs.test_cli == 'true' || needs.scope.outputs.test_core == 'true') && + contains(fromJSON(needs.scope.outputs.macos_runner || '[]'), 'self-hosted') + runs-on: ubuntu-latest + timeout-minutes: 67 + permissions: + contents: read + actions: read + pull-requests: read + outputs: + result: ${{ steps.mini.outputs.result }} + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + with: + persist-credentials: false + - name: Wait for a bounded mini attempt + id: mini + env: + GH_TOKEN: ${{ github.token }} + run: python3 scripts/ci/macos_dispatch.py wait --run-id "$GITHUB_RUN_ID" --attempt "$GITHUB_RUN_ATTEMPT" + # Package groups remain parallel. CLI builds and executes its full inventory # once; stable protected labels require that same completed test result. # Short CLI partitions are not the workflow critical path, so separate @@ -191,30 +223,28 @@ jobs: name: "Test: vendored terminal (${{ matrix.os }})" needs: scope if: ${{ !cancelled() && (needs.scope.result != 'success' || needs.scope.outputs.test_cli == 'true' || needs.scope.outputs.test_core == 'true') }} - runs-on: ${{ matrix.os }} + runs-on: ubuntu-latest timeout-minutes: 15 strategy: fail-fast: false matrix: - os: [ubuntu-latest, macos-15] + os: [ubuntu-latest] steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 with: persist-credentials: false - name: Install pinned Rust toolchain - run: rustc --version + uses: ./.github/actions/rust-toolchain # The patched crate is deliberately outside the application workspace. # These are parser/unit tests, not the real-reader PTY regression suite. - name: Test vendored terminal units - run: | - cargo test --locked --manifest-path vendor/crossterm/Cargo.toml --lib --features event-stream - cargo test --locked --manifest-path vendor/crossterm/Cargo.toml --lib --features event-stream,use-dev-tty + run: bash scripts/ci/run-macos-lanes.sh vendored - # Preserve required check names while three builds run on separate runners. + # Preserve required check names while the reader/reflow builds stay isolated. # Both aggregate checks require the entire terminal matrix. terminal-pty: name: "Test: terminal PTY (${{ matrix.os }})" - needs: [scope, terminal-pty-shards] + needs: [scope, terminal-pty-shards, terminal-pty-shards-macos] if: ${{ !cancelled() }} runs-on: ubuntu-latest timeout-minutes: 5 @@ -227,25 +257,28 @@ jobs: shell: bash env: SHARDS_RESULT: ${{ needs.terminal-pty-shards.result }} + MACOS_RESULT: ${{ needs.terminal-pty-shards-macos.result }} SHARDS_REQUIRED: ${{ needs.scope.result != 'success' || needs.scope.outputs.test_cli == 'true' || needs.scope.outputs.test_core == 'true' }} run: | set -euo pipefail - if [ "$SHARDS_REQUIRED" = true ]; then - test "$SHARDS_RESULT" = success - else - case "$SHARDS_RESULT" in success|skipped) ;; *) exit 1 ;; esac - fi + for result in "$SHARDS_RESULT" "$MACOS_RESULT"; do + if [ "$SHARDS_REQUIRED" = true ]; then + test "$result" = success + else + case "$result" in success|skipped) ;; *) exit 1 ;; esac + fi + done terminal-pty-shards: name: "Terminal: ${{ matrix.segment }} (${{ matrix.os }})" needs: scope if: ${{ !cancelled() && (needs.scope.result != 'success' || needs.scope.outputs.test_cli == 'true' || needs.scope.outputs.test_core == 'true') }} - runs-on: ${{ matrix.os }} + runs-on: ubuntu-latest timeout-minutes: 45 strategy: fail-fast: false matrix: - os: [ubuntu-latest, macos-15] + os: [ubuntu-latest] segment: [default-reader, dev-tty-reader, reflow] steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 @@ -257,45 +290,18 @@ jobs: # One writer per OS populates the shared dependency cache on pushes. save-cache: ${{ github.event_name == 'push' && matrix.segment == 'default-reader' }} install-nextest: false - # Separate runners keep dev-tty feature unification out of the default - # reader and avoid serializing two lib builds and the reflow binary build. - - name: Test default terminal reader through PTYs - if: matrix.segment == 'default-reader' - run: | - cargo test --locked -p astra-cli --lib tui::terminal_startup -- --test-threads=2 - - name: Initialize system Git for macOS CLI observation - if: matrix.os == 'macos-15' && matrix.segment == 'default-reader' - timeout-minutes: 1 - run: | - time /usr/bin/env -i PATH=/usr/bin:/bin HOME=/nonexistent LANG=C LC_ALL=C \ - GIT_CONFIG_NOSYSTEM=1 GIT_CONFIG_GLOBAL=/dev/null GIT_CONFIG_SYSTEM=/dev/null \ - GIT_OPTIONAL_LOCKS=0 GIT_TERMINAL_PROMPT=0 \ - /usr/bin/git -C "$GITHUB_WORKSPACE" -c core.fsmonitor=false \ - -c core.untrackedCache=false rev-parse --show-toplevel - - name: Test macOS CLI Bash execution - if: matrix.os == 'macos-15' && matrix.segment == 'default-reader' - # Reuse the default-feature CLI test executable already built above. - run: >- - cargo test --locked --manifest-path Cargo.toml -p astra-cli - --lib edge_tools::shell::tests::bash_echo_returns_output -- --exact - - name: Test dev-tty terminal reader through PTYs - if: matrix.segment == 'dev-tty-reader' - run: | - cargo test --locked -p astra-cli --lib --features crossterm/use-dev-tty tui::terminal_startup -- --test-threads=2 - uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7 if: matrix.segment == 'reflow' with: node-version-file: .nvmrc - - name: Test inline terminal resize with xterm reflow - if: matrix.segment == 'reflow' - run: | - cargo build --locked -p astra-cli --bin astra - npm ci --ignore-scripts --prefix scripts/tui-reflow - ASTRA_TEST_BINARY="$PWD/target/debug/astra" npm test --prefix scripts/tui-reflow + - name: Test terminal lane + env: + MACOS_LANE: ${{ matrix.segment }} + run: bash scripts/ci/run-macos-lanes.sh "$MACOS_LANE" macos-workspace-coordination: name: "Test: macOS workspace coordination" - needs: [scope, macos-workspace-coordination-tests, terminal-pty-shards] + needs: [scope, macos-workspace-coordination-tests, terminal-pty-shards, terminal-pty-shards-macos] if: ${{ !cancelled() }} runs-on: ubuntu-latest timeout-minutes: 5 @@ -305,10 +311,11 @@ jobs: env: COORDINATION_RESULT: ${{ needs.macos-workspace-coordination-tests.result }} TERMINAL_RESULT: ${{ needs.terminal-pty-shards.result }} + MACOS_TERMINAL_RESULT: ${{ needs.terminal-pty-shards-macos.result }} SHARDS_REQUIRED: ${{ needs.scope.result != 'success' || needs.scope.outputs.test_core == 'true' || needs.scope.outputs.test_cli == 'true' }} run: | set -euo pipefail - for result in "$COORDINATION_RESULT" "$TERMINAL_RESULT"; do + for result in "$COORDINATION_RESULT" "$TERMINAL_RESULT" "$MACOS_TERMINAL_RESULT"; do if [ "$SHARDS_REQUIRED" = true ]; then test "$result" = success else @@ -316,46 +323,105 @@ jobs: fi done + vendored-terminal-macos: + name: "Test: vendored terminal (macos-15)" + needs: [scope, macos-plan] + if: ${{ !cancelled() && (needs.scope.result != 'success' || needs.scope.outputs.test_cli == 'true' || needs.scope.outputs.test_core == 'true') }} + runs-on: ${{ (github.run_attempt > 1 || !contains(fromJSON('["success","failure"]'), needs.macos-plan.outputs.result)) && 'macos-15' || 'ubuntu-latest' }} + timeout-minutes: 15 + env: + RUN_LANE: ${{ github.run_attempt > 1 || !contains(fromJSON('["success","failure"]'), needs.macos-plan.outputs.result) }} + MINI_RESULT: ${{ needs.macos-plan.outputs.result }} + steps: + - name: Require actual mini result + if: env.RUN_LANE != 'true' + run: | + set -euo pipefail + test "$MINI_RESULT" = success + echo 'All five macOS lanes passed on the mini; see Resolve macOS execution for the worker run.' >> "$GITHUB_STEP_SUMMARY" + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + if: env.RUN_LANE == 'true' + with: + persist-credentials: false + - uses: ./.github/actions/rust-toolchain + if: env.RUN_LANE == 'true' + - name: Run hosted macOS lane + if: env.RUN_LANE == 'true' + env: + MACOS_LANE: vendored + run: bash scripts/ci/run-macos-lanes.sh "$MACOS_LANE" + + terminal-pty-shards-macos: + name: "Terminal: ${{ matrix.segment }} (macos-15)" + needs: [scope, macos-plan] + if: ${{ !cancelled() && (needs.scope.result != 'success' || needs.scope.outputs.test_cli == 'true' || needs.scope.outputs.test_core == 'true') }} + runs-on: ${{ (github.run_attempt > 1 || !contains(fromJSON('["success","failure"]'), needs.macos-plan.outputs.result)) && 'macos-15' || 'ubuntu-latest' }} + timeout-minutes: 45 + env: + RUN_LANE: ${{ github.run_attempt > 1 || !contains(fromJSON('["success","failure"]'), needs.macos-plan.outputs.result) }} + MINI_RESULT: ${{ needs.macos-plan.outputs.result }} + strategy: + fail-fast: false + matrix: + segment: [default-reader, dev-tty-reader, reflow] + steps: + - name: Require actual mini result + if: env.RUN_LANE != 'true' + run: | + set -euo pipefail + test "$MINI_RESULT" = success + echo 'All five macOS lanes passed on the mini; see Resolve macOS execution for the worker run.' >> "$GITHUB_STEP_SUMMARY" + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + if: env.RUN_LANE == 'true' + with: + persist-credentials: false + - uses: ./.github/actions/rust-setup + if: env.RUN_LANE == 'true' + with: + shared-key: ci-terminal-pty + save-cache: ${{ github.event_name == 'push' && matrix.segment == 'default-reader' }} + install-nextest: "false" + - uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7 + if: env.RUN_LANE == 'true' && matrix.segment == 'reflow' + with: + node-version-file: .nvmrc + - name: Run hosted macOS lane + if: env.RUN_LANE == 'true' + env: + MACOS_LANE: ${{ matrix.segment }} + run: bash scripts/ci/run-macos-lanes.sh "$MACOS_LANE" + macos-workspace-coordination-tests: name: "macOS workspace coordination contracts" - needs: scope - if: ${{ !cancelled() && (needs.scope.result != 'success' || needs.scope.outputs.test_core == 'true' || needs.scope.outputs.test_cli == 'true') }} - runs-on: macos-15 + needs: [scope, macos-plan] + if: ${{ !cancelled() && (needs.scope.result != 'success' || needs.scope.outputs.test_cli == 'true' || needs.scope.outputs.test_core == 'true') }} + runs-on: ${{ (github.run_attempt > 1 || !contains(fromJSON('["success","failure"]'), needs.macos-plan.outputs.result)) && 'macos-15' || 'ubuntu-latest' }} timeout-minutes: 30 + env: + RUN_LANE: ${{ github.run_attempt > 1 || !contains(fromJSON('["success","failure"]'), needs.macos-plan.outputs.result) }} + MINI_RESULT: ${{ needs.macos-plan.outputs.result }} steps: + - name: Require actual mini result + if: env.RUN_LANE != 'true' + run: | + set -euo pipefail + test "$MINI_RESULT" = success + echo 'All five macOS lanes passed on the mini; see Resolve macOS execution for the worker run.' >> "$GITHUB_STEP_SUMMARY" - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + if: env.RUN_LANE == 'true' with: persist-credentials: false - uses: ./.github/actions/rust-setup + if: env.RUN_LANE == 'true' with: - # Reuse the CLI dependency cache; default-reader is its only writer. shared-key: ci-terminal-pty save-cache: "false" install-nextest: "false" - - name: Lint macOS workspace coordination - run: >- - cargo clippy --locked --manifest-path Cargo.toml -p astra-tools - --all-targets -- -D warnings - - name: Initialize system Git for bounded observation probes - timeout-minutes: 1 - # Checkout uses Homebrew Git, but the hardened observer selects - # /usr/bin/git with a cleared environment. Initialize that executable - # before its first use under the production two-second probe deadline. - # Keep the timing visible so runner startup failures are diagnosable. - run: | - time /usr/bin/env -i PATH=/usr/bin:/bin HOME=/nonexistent LANG=C LC_ALL=C \ - GIT_CONFIG_NOSYSTEM=1 GIT_CONFIG_GLOBAL=/dev/null GIT_CONFIG_SYSTEM=/dev/null \ - GIT_OPTIONAL_LOCKS=0 GIT_TERMINAL_PROMPT=0 \ - /usr/bin/git -C "$GITHUB_WORKSPACE" -c core.fsmonitor=false \ - -c core.untrackedCache=false rev-parse --show-toplevel - - name: Test macOS workspace coordination invariants - # Several invariants launch re-exec lock contenders while fingerprint - # probes deliberately have a short wall-clock deadline. Keep the module - # serial on shared CI hosts; the contender test supplies explicit - # cross-process concurrency without starving unrelated probes. - run: >- - cargo test --locked --manifest-path Cargo.toml -p astra-tools - workspace_observation::tests:: -- --test-threads=1 + - name: Run hosted macOS lane + if: env.RUN_LANE == 'true' + env: + MACOS_LANE: coordination + run: bash scripts/ci/run-macos-lanes.sh "$MACOS_LANE" # Preserve the required core check while its package groups run separately. # The aggregate requires all selected online jobs, including integration. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 3b884b8b31..34a4176222 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -114,10 +114,16 @@ The three ignored production-supervisor smokes use one additional nextest invocation, with exact names and separate serial test processes. They use the strict profile without retries, preserving the complete inventory's CI JUnit file and avoiding three separate Cargo test builds. -On macOS, the default terminal-reader lane also runs the CLI Bash regression -using its existing default-feature test executable. Tools lint and serial -workspace-coordination tests remain parallel; the protected coordination check -requires both paths to succeed. Feature-specific terminal tests remain separate. +On macOS, the default terminal-reader lane runs the CLI Bash regression using +its existing test executable. PR authors in a static contributor ID allowlist +can attempt one self-hosted ARM bundle containing all five Mac lanes. Trusted +main/develop pushes and release builds use hosted runners. +A bounded queue/setup failure falls back to hosted jobs in parallel; real test +failures remain failures. Other authors and reruns use hosted macOS. Linux +execution starts independently. The ordinary workflow retains required checks +and fork approval; no App or custom check source is needed. +See [runner operations](docs/guides/repository-automation.md#self-hosted-macos-tests) +for dispatch bootstrap, budgets, queueing and rollback. The lint gate retains restored artifacts; age-based cleanup is not a lint prerequisite. For fork pull requests, an update may require maintainer approval before the diff --git a/Cargo.lock b/Cargo.lock index 8170f69da8..7d8e524733 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -914,6 +914,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b281d307588d634de920874890732659e2e7672f72b5e10e81badc1a8a83621e" dependencies = [ "aws-lc-sys", + "untrusted 0.7.1", "zeroize", ] @@ -985,12 +986,6 @@ dependencies = [ "tracing", ] -[[package]] -name = "base16ct" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c7f02d4ea65f2c1853089ffd8d2787bdbc63de2f0d29dedbcf8ccdfa0ccd4cf" - [[package]] name = "base16ct" version = "1.0.0" @@ -1662,21 +1657,9 @@ checksum = "460fbee9c2c2f33933d720630a6a0bac33ba7053db5344fac858d4b8952d77d5" [[package]] name = "crypto-bigint" -version = "0.5.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0dc92fb57ca44df6db8059111ab3af99a63d5d0f8375d9972e319a379c6bab76" -dependencies = [ - "generic-array", - "rand_core 0.6.4", - "subtle", - "zeroize", -] - -[[package]] -name = "crypto-bigint" -version = "0.7.5" +version = "0.7.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a52aa3fcda4e6302a9f48734f234d35d4721b96f8fe07d073f07ce9df4f0271" +checksum = "6bb08299a6046833a879ace115697f12fd2bd6c8d3e93c909ffc9e0738e7cae0" dependencies = [ "cpubits", "ctutils", @@ -1711,7 +1694,7 @@ version = "0.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3633a51a39c69ebbaa4feaa694bd83d241e4093901c84a0963b19d9bb3f0cf8f" dependencies = [ - "crypto-bigint 0.7.5", + "crypto-bigint", "rand_core 0.10.1", ] @@ -1750,40 +1733,13 @@ dependencies = [ [[package]] name = "ctutils" -version = "0.4.4" +version = "0.4.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "12218662ccac37d9714f2191252553d67585b97df8c839cbe49e0bde278dc865" +checksum = "38d28fb6aa7dd680b3ffd1a3e64e115c177bdba63c0ae7f9cd48129ff05c4fa0" dependencies = [ "cmov", ] -[[package]] -name = "curve25519-dalek" -version = "4.1.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "97fb8b7c4503de7d6ae7b42ab72a5a59857b4c937ec27a3d4539dba95b5ab2be" -dependencies = [ - "cfg-if", - "cpufeatures 0.2.17", - "curve25519-dalek-derive", - "digest 0.10.7", - "fiat-crypto", - "rustc_version", - "subtle", - "zeroize", -] - -[[package]] -name = "curve25519-dalek-derive" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f46882e17999c6cc590af592290432be3bce0428cb0d5f8b6715e4dc7b383eb3" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.117", -] - [[package]] name = "darling" version = "0.23.0" @@ -2071,44 +2027,6 @@ version = "1.0.20" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d0881ea181b1df73ff77ffaaf9c7544ecc11e82fba9b5f27b262a3c73a332555" -[[package]] -name = "ecdsa" -version = "0.16.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ee27f32b5c5292967d2d4a9d7f1e0b0aed2c15daded5a60300e4abb9d8020bca" -dependencies = [ - "der 0.7.10", - "digest 0.10.7", - "elliptic-curve", - "rfc6979", - "signature 2.2.0", - "spki 0.7.3", -] - -[[package]] -name = "ed25519" -version = "2.2.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "115531babc129696a58c64a4fef0a8bf9e9698629fb97e9e40767d235cfbcd53" -dependencies = [ - "pkcs8 0.10.2", - "signature 2.2.0", -] - -[[package]] -name = "ed25519-dalek" -version = "2.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "70e796c081cee67dc755e1a36a0a172b897fab85fc3f6bc48307991f64e4eca9" -dependencies = [ - "curve25519-dalek", - "ed25519", - "serde", - "sha2 0.10.9", - "subtle", - "zeroize", -] - [[package]] name = "ego-tree" version = "0.11.0" @@ -2124,27 +2042,6 @@ dependencies = [ "serde", ] -[[package]] -name = "elliptic-curve" -version = "0.13.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b5e6043086bf7973472e0c7dff2142ea0b680d30e18d9cc40f267efbf222bd47" -dependencies = [ - "base16ct 0.2.0", - "crypto-bigint 0.5.5", - "digest 0.10.7", - "ff", - "generic-array", - "group", - "hkdf 0.12.4", - "pem-rfc7468 0.7.0", - "pkcs8 0.10.2", - "rand_core 0.6.4", - "sec1", - "subtle", - "zeroize", -] - [[package]] name = "encode_unicode" version = "1.0.0" @@ -2279,22 +2176,6 @@ dependencies = [ "zeroize", ] -[[package]] -name = "ff" -version = "0.13.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c0b50bfb653653f9ca9095b427bed08ab8d75a137839d9ad64eb11810d5b6393" -dependencies = [ - "rand_core 0.6.4", - "subtle", -] - -[[package]] -name = "fiat-crypto" -version = "0.2.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "28dea519a9695b9977216879a3ebfddf92f1c08c05d984f8996aecd6ecdc811d" - [[package]] name = "filedescriptor" version = "0.8.3" @@ -2540,7 +2421,6 @@ checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" dependencies = [ "typenum", "version_check", - "zeroize", ] [[package]] @@ -3200,17 +3080,6 @@ dependencies = [ "regex-syntax", ] -[[package]] -name = "group" -version = "0.13.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f0f9ef7462f7c099f518d754361858f86d8a07af53ba9af0fe635bbccb151a63" -dependencies = [ - "ff", - "rand_core 0.6.4", - "subtle", -] - [[package]] name = "h2" version = "0.4.16" @@ -3394,15 +3263,6 @@ dependencies = [ "tracing", ] -[[package]] -name = "hkdf" -version = "0.12.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b5f8eb2ad728638ea2c7d47a21db23b7b58a72ed6a38256b8a1849f15fbbdf7" -dependencies = [ - "hmac 0.12.1", -] - [[package]] name = "hkdf" version = "0.13.0" @@ -4015,18 +3875,12 @@ version = "10.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "eba32bfb4ffdeaca3e34431072faf01745c9b26d25504aa7a6cf5684334fc4fc" dependencies = [ + "aws-lc-rs", "base64", - "ed25519-dalek", "getrandom 0.2.17", - "hmac 0.12.1", "js-sys", - "p256", - "p384", - "rand 0.8.6", - "rsa 0.9.10", "serde", "serde_json", - "sha2 0.10.9", "signature 2.2.0", "zeroize", ] @@ -4707,30 +4561,6 @@ dependencies = [ "num-traits", ] -[[package]] -name = "p256" -version = "0.13.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c9863ad85fa8f4460f9c48cb909d38a0d689dba1f6f6988a5e3e0d31071bcd4b" -dependencies = [ - "ecdsa", - "elliptic-curve", - "primeorder", - "sha2 0.10.9", -] - -[[package]] -name = "p384" -version = "0.13.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fe42f1670a52a47d448f14b6a5c61dd78fce51856e68edaa38f7ae3a46b8d6b6" -dependencies = [ - "ecdsa", - "elliptic-curve", - "primeorder", - "sha2 0.10.9", -] - [[package]] name = "palette" version = "0.7.6" @@ -5120,15 +4950,6 @@ dependencies = [ "syn 2.0.117", ] -[[package]] -name = "primeorder" -version = "0.13.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "353e1ca18966c16d9deb1c69278edbc5f194139612772bd9537af60ac231e1e6" -dependencies = [ - "elliptic-curve", -] - [[package]] name = "proc-macro-crate" version = "3.5.0" @@ -5712,16 +5533,6 @@ version = "0.7.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1e061d1b48cb8d38042de4ae0a7a6401009d6143dc80d2e2d6f31f0bdd6470c7" -[[package]] -name = "rfc6979" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f8dd2a808d456c4a54e300a23e9f5a67e122c3024119acbfd73e3bf664491cb2" -dependencies = [ - "hmac 0.12.1", - "subtle", -] - [[package]] name = "ring" version = "0.17.14" @@ -5732,7 +5543,7 @@ dependencies = [ "cfg-if", "getrandom 0.2.17", "libc", - "untrusted", + "untrusted 0.9.0", "windows-sys 0.52.0", ] @@ -5813,7 +5624,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "33405e321e16ef941ca66e254f6904c3aafd3c7ff001fb88bf11c73eb75c2b85" dependencies = [ "const-oid 0.10.2", - "crypto-bigint 0.7.5", + "crypto-bigint", "crypto-primes", "digest 0.11.3", "pkcs1 0.8.0-rc.5", @@ -5936,7 +5747,7 @@ dependencies = [ "aws-lc-rs", "ring", "rustls-pki-types", - "untrusted", + "untrusted 0.9.0", ] [[package]] @@ -6050,20 +5861,6 @@ dependencies = [ "tendril", ] -[[package]] -name = "sec1" -version = "0.7.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d3e97a565f76233a6003f9f5c54be1d9c5bdfa3eccfb189469f11ec4901c47dc" -dependencies = [ - "base16ct 0.2.0", - "der 0.7.10", - "generic-array", - "pkcs8 0.10.2", - "subtle", - "zeroize", -] - [[package]] name = "security-framework" version = "3.7.0" @@ -6217,7 +6014,7 @@ version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "66cf8fedced2fcf12406bcb34223dffb92eaf34908ede12fed414c82b7f00b3e" dependencies = [ - "base16ct 1.0.0", + "base16ct", "serde", ] @@ -6612,7 +6409,7 @@ dependencies = [ "futures-core", "futures-util", "hex", - "hkdf 0.13.0", + "hkdf", "hmac 0.13.0", "itoa", "log", @@ -7704,6 +7501,12 @@ version = "0.2.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "673aac59facbab8a9007c7f6108d11f63b603f7cabff99fabf650fea5c32b861" +[[package]] +name = "untrusted" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a156c684c91ea7d62626509bce3cb4e1d9ed5c4d978f7b4352658f96a4c26b4a" + [[package]] name = "untrusted" version = "0.9.0" diff --git a/Cargo.toml b/Cargo.toml index 987c8f9856..57ae577e9a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -73,7 +73,9 @@ fs2 = "0.4" tar = "0.4" futures-util = "0.3.31" hmac = "0.12.1" -jsonwebtoken = { version = "10.3.0", default-features = false, features = ["rust_crypto"] } +# Reuse AWS-LC: the RustCrypto backend pins crypto-bigint 0.5, affected by +# RUSTSEC-2026-0387 with no compatible patched release. +jsonwebtoken = { version = "10.3.0", default-features = false, features = ["aws_lc_rs"] } nix = { version = "0.31.2", features = ["fs", "signal", "term"] } regex = "1.12.2" reqwest = { version = "0.12.23", default-features = false, features = ["json", "rustls-tls", "rustls-tls-native-roots", "stream", "socks"] } diff --git a/crates/astra-cli/src/tui/terminal_startup_pty_tests.rs b/crates/astra-cli/src/tui/terminal_startup_pty_tests.rs index 38edc0d99f..b03b207ba5 100644 --- a/crates/astra-cli/src/tui/terminal_startup_pty_tests.rs +++ b/crates/astra-cli/src/tui/terminal_startup_pty_tests.rs @@ -187,9 +187,10 @@ fn probe_child() { pane.set_slash_items(vec![crate::tui::slash_menu::SlashItem::simple( "/skill", "skills", )]); - for ch in draft_command.chars() { - pane.handle_key(KeyEvent::new(KeyCode::Char(ch), KeyModifiers::NONE)); - } + // Seed the command explicitly: scheduler-sized gaps in synthetic + // key loops can activate paste detection and consume Enter. This + // case tests draft custody across commands, not typing cadence. + pane.replace_composer_text(draft_command); pane.restore_into_composer("//returned"); let BottomPaneAction::SubmitInput(command) = pane.handle_key(KeyEvent::new(KeyCode::Enter, KeyModifiers::NONE)) diff --git a/docs/guides/repository-automation.md b/docs/guides/repository-automation.md index 46fa750ed6..c27e83c7bf 100644 --- a/docs/guides/repository-automation.md +++ b/docs/guides/repository-automation.md @@ -119,3 +119,71 @@ before rerunning CI: This separation prevents permission failures from being treated as flaky CI and keeps external-fork behavior part of the repository's delivery contract. + +## Self-hosted macOS tests + +The static author-ID allowlist in `scripts/ci/ci_scope.py` makes the eleven +selected contributors (including Dependabot) eligible for a mini attempt. +Trusted main/develop pushes and release builds remain on hosted runners, so +unmerged PR state in the mini's HOME/Cargo/rustup directories is not reused by +trusted builds. Other or unknown PR authors run on hosted macOS immediately; GitHub's existing fork approval remains in effect. Every Mac +job re-executed on attempts greater than one runs hosted, even when its planner's +old output is reused. GitHub may retain successful Mac jobs on a failed-only +rerun; those checks still refer to the original tested merge. Required-check names retain +`macos-15` for compatibility; job summaries link the actual worker execution. + +The ordinary Test Suite owns all required checks. Linux jobs depend only on +scope and start without waiting for the mini. `Resolve macOS execution` has +Actions/Pull requests read access and waits for a worker bound to its exact run +ID, attempt and recorded merge SHA. Missing controller, API errors or missing +routing output select real hosted tests, never a successful skip. + +`macos-scheduler.yml` runs on the default branch on Test Suite's `workflow_run` +start/completion events. It uses the built-in token's Actions write permission +to dispatch/cancel `macos-mini.yml`; it never checks out PR code. No App, +organization membership query, additional secret, custom required context or +admission hook is needed. These workflows must exist on main before dispatch +works. Before that, Test Suite falls back to hosted execution. + +A mini attempt gets a 180-second queue/discovery budget, with bounded API +requests, then falls back to the original hosted jobs in parallel. The +reader/controller enforce a 60-minute execution budget even if the runner +disappears; the worker has a separate 65-minute hard backstop. Thus single-machine +unavailability does not hold a +required check for GitHub's 24-hour queue limit. API/control outages can prevent +immediate child cancellation; the completion observer retries cleanup, and a +late worker rejects an ended parent gate before candidate checkout. + +The worker performs one candidate checkout, toolchain setup and cache restore, +then runs default reader/Bash, reflow, coordination, vendored units and dev-tty +through the same `scripts/ci/run-macos-lanes.sh` used by hosted/Linux jobs. Cargo +invocations remain separate and dev-tty is last. A real compile/test failure is +reported as failure, without hosted retry. Failed runs with incomplete test +metadata also stay failed rather than hiding a possible test failure. Checkout, +low-disk/toolchain setup, +agent cancellation/loss or the execution budget select hosted fallback. The +worker gets a read-only token and cache-mode read; PR work cannot save default +branch Actions/sccache cache entries. + +The runner must match `[self-hosted, macOS, ARM64, mac-arm64-mini]`, be available +to Astra (including public repository access), and support the pinned Actions, +Rust and Node versions. Rustup is bootstrapped if absent, Cargo bin is added to PATH, and the selected +repository toolchain is exported through GITHUB_ENV for later sccache compiler +probes outside the checkout. At least 30 GiB workspace disk is required. On self-hosted runners the +disk guard reports low/unreadable disk without privileged hosted-image cleanup; +the hosted fallback proceeds while operators restore mini capacity. + +One registration runs one bundle at a time. Other eligible runs use hosted Macs +when their queue budget expires. Measure mini execution time, fallback rate, +queue delay, API usage and hosted minutes; running attempts poll once per minute +and filter worker history by source creation time. Bundling reduces repeated +preparation, but +neither creates physical capacity nor guarantees the previous hosted elapsed +time. The initial 60-minute budget needs qualification against actual load. + +If the old App/controller design was staged, retire its unused App required +contexts and admission hook, and permit the new `macos-mini.yml@main` worker in +the runner group. Preserve existing GitHub Actions checks and fork approval. +Rollback through a PR by routing all authors to hosted macOS; existing reruns +already take that recovery path. No remote runner/group or protection settings +are changed by the implementation PR. diff --git a/scripts/ci/cancel_stale_pr_runs.py b/scripts/ci/cancel_stale_pr_runs.py index c605a11932..8f9b8982a9 100755 --- a/scripts/ci/cancel_stale_pr_runs.py +++ b/scripts/ci/cancel_stale_pr_runs.py @@ -70,7 +70,7 @@ def __init__(self, token: str, api_url: str = "https://api.github.com") -> None: self._api_origin = (parsed_api_url.scheme.lower(), parsed_api_url.netloc.lower()) self._opener = build_opener(RejectRedirects()) - def _request(self, method: str, url: str) -> tuple[bytes, Any]: + def _request(self, method: str, url: str, *, data: dict[str, Any] | None = None) -> tuple[bytes, Any]: target = url if urlsplit(url).scheme else urljoin(self._api_url, url.lstrip("/")) parsed_target = urlsplit(target) target_origin = (parsed_target.scheme.lower(), parsed_target.netloc.lower()) @@ -79,8 +79,10 @@ def _request(self, method: str, url: str) -> tuple[bytes, Any]: request = Request( target, method=method, + data=json.dumps(data).encode() if data is not None else None, headers={ "Accept": "application/vnd.github+json", + "Content-Type": "application/json", "Authorization": f"Bearer {self._token}", "User-Agent": "astra-stale-pr-run-controller", "X-GitHub-Api-Version": "2022-11-28", diff --git a/scripts/ci/ci_scope.py b/scripts/ci/ci_scope.py index 9676920a4c..e0f06f8146 100755 --- a/scripts/ci/ci_scope.py +++ b/scripts/ci/ci_scope.py @@ -3,7 +3,9 @@ The classifier is deliberately conservative: an unknown diff or a change to CI itself enables every scope. Documentation and repository-governance changes are -handled by the always-on lightweight repository checks. +handled by the always-on lightweight repository checks. The same event also +selects macOS runners by a static PR-author ID allowlist, with hosted fallback +for unknown authors or unavailable metadata. """ from __future__ import annotations @@ -16,6 +18,34 @@ import sys +# GitHub account IDs survive login renames. Login values are review annotations. +MACOS_RUNNER_AUTHORS = { + 39627130: "XuPeng-SH", + 52204121: "loveRhythm1990", + 26336316: "gouhongshen", + 49699333: "dependabot[bot]", + 2245572: "zuyu", + 4965857: "dengn", + 85048713: "MatrixAdventurer", + 176287284: "endlesschasey-ai", + 49832303: "aptend", + 169294: "fengttt", + 28526327: "plane-hjh", +} +MACOS_SELF_HOSTED_LABELS = ["self-hosted", "macOS", "ARM64", "mac-arm64-mini"] + + +def select_macos_runner(event_name: str, event: dict) -> list[str]: + # Trusted pushes must not consume persistent state left by unmerged PRs. + if event_name == "pull_request" and isinstance(event, dict): + pull_request = event.get("pull_request") + author = pull_request.get("user") if isinstance(pull_request, dict) else None + author_id = author.get("id") if isinstance(author, dict) else None + if type(author_id) is int and author_id in MACOS_RUNNER_AUTHORS: + return MACOS_SELF_HOSTED_LABELS.copy() + return ["macos-15"] + + SCOPES = ( "rust", "sdk", @@ -255,11 +285,22 @@ def emit(scopes: dict[str, bool], paths: list[str], *, fallback: str | None = No print(f"{scope}={'true' if scopes[scope] else 'false'}") print("changed_paths=" + (", ".join(sorted(paths)) or "")) + try: + event = json.loads(Path(os.environ["GITHUB_EVENT_PATH"]).read_text(encoding="utf-8")) + labels = select_macos_runner( + os.environ.get("GITHUB_EVENT_NAME", ""), event + ) + except (KeyError, OSError, ValueError, TypeError): + labels = ["macos-15"] + runner_json = json.dumps(labels, separators=(",", ":")) + print("macos_runner=" + runner_json) + github_output = os.environ.get("GITHUB_OUTPUT") if github_output: with Path(github_output).open("a", encoding="utf-8") as output: for scope in SCOPES: output.write(f"{scope}={'true' if scopes[scope] else 'false'}\n") + output.write("macos_runner=" + runner_json + "\n") def main() -> int: diff --git a/scripts/ci/macos_dispatch.py b/scripts/ci/macos_dispatch.py new file mode 100755 index 0000000000..e6bc24f0eb --- /dev/null +++ b/scripts/ci/macos_dispatch.py @@ -0,0 +1,291 @@ +#!/usr/bin/env python3 +"""Bounded mini attempt; ordinary Test Suite owns hosted fallback and checks.""" + +from __future__ import annotations + +import argparse +from dataclasses import dataclass +import json +import os +from pathlib import Path +import re +import time +from urllib.parse import urlencode + +from cancel_stale_pr_runs import GitHubApi, MAX_PAGES +from ci_scope import MACOS_SELF_HOSTED_LABELS, select_macos_runner + + +WORKFLOW = "macos-mini.yml" +GATE = "Resolve macOS execution" +WAIT_STEP = "Wait for a bounded mini attempt" +TEST_STEP = "Run all macOS lanes" +PREFLIGHT_STEPS = {"Set up job", "Checkout control policy", "Validate mini dispatch", + "Checkout candidate", "Prepare Rust", "Prepare Node"} +QUEUE_SECONDS = 180 +EXECUTION_SECONDS = 3600 +POLL_SECONDS = 20 +RUNNING_POLL_SECONDS = 60 +TITLE = re.compile(r"Test Suite pr=(\d+) base=([0-9a-f]{40}) head=([0-9a-f]{40}) merge=([0-9a-f]{40})") + + +class Unavailable(RuntimeError): + pass + + +class Api(GitHubApi): + def get(self, path): + return self._get_json(path) + + def post(self, path, data): + self._request("POST", path, data=data) + + def collection(self, path, key): + result, visited = [], set() + while path: + if path in visited or len(visited) >= MAX_PAGES: + raise Unavailable("pagination bound exceeded") + visited.add(path) + body, headers = self._request("GET", path) + page = json.loads(body) + if not isinstance(page.get(key), list): + raise Unavailable("missing API collection") + result.extend(page[key]) + path = self._next_link(headers) + return result + + +@dataclass(frozen=True) +class Source: + run_id: int + attempt: int + sha: str + created_at: str | None = None + + @property + def title(self): + return f"Mac mini for Test Suite {self.run_id}/{self.attempt} {self.sha}" + + +def source(api, repo, run_id, attempt): + run = api.get(f"repos/{repo}/actions/runs/{run_id}") + match = TITLE.fullmatch(run.get("display_title", "")) + # GitHub may populate name with the dynamic run-name as well. The workflow + # path is the identity; the title carries the immutable event tuple. + if (run.get("id") != run_id or run.get("path") != ".github/workflows/test.yml" + or run.get("run_attempt") != attempt or attempt != 1 + or run.get("status") == "completed" or not match): + raise Unavailable("source run is not a current mini candidate") + pr_number, base, head, merge = match.groups() + if run.get("head_sha") != head: + raise Unavailable("source head mismatch") + if run.get("event") == "pull_request" and int(pr_number) > 0: + # Fork run.pull_requests can be empty. The event title records the PR + # and immutable merge tuple, rather than relying on that association. + pr = api.get(f"repos/{repo}/pulls/{pr_number}") + if (pr["head"]["sha"] != head or pr["state"] != "open" + or pr["head"]["repo"]["id"] != run["head_repository"]["id"]): + raise Unavailable("PR source changed") + parents = api.get(f"repos/{repo}/commits/{merge}")["parents"] + if [parent["sha"] for parent in parents] != [base, head]: + raise Unavailable("unverified merge tuple") + labels = select_macos_runner("pull_request", {"pull_request": {"user": pr["user"]}}) + else: + raise Unavailable("unsupported source event") + if labels != MACOS_SELF_HOSTED_LABELS: + raise Unavailable("source does not prefer mini") + return Source(run_id, attempt, merge, run.get("created_at")) + + +def jobs(api, repo, run_id, attempt): + return api.collection( + f"repos/{repo}/actions/runs/{run_id}/attempts/{attempt}/jobs?per_page=100", "jobs") + + +def gate_active(api, repo, src): + run = api.get(f"repos/{repo}/actions/runs/{src.run_id}") + if run.get("run_attempt") != src.attempt or run.get("status") == "completed": + return False + return any(job.get("name") == GATE and job.get("status") == "in_progress" + and any(step.get("name") == WAIT_STEP and step.get("status") == "in_progress" + for step in job.get("steps", [])) + for job in jobs(api, repo, src.run_id, src.attempt)) + + +def workers(api, repo, src): + filters = {"event": "workflow_dispatch", "branch": "main", "per_page": 100} + if src.created_at: + filters["created"] = ">=" + src.created_at + query = urlencode(filters) + runs = api.collection(f"repos/{repo}/actions/workflows/{WORKFLOW}/runs?{query}", "workflow_runs") + return [run for run in runs if run.get("display_title") == src.title + and run.get("path") == f".github/workflows/{WORKFLOW}" + and run.get("event") == "workflow_dispatch"] + + +def worker_state(api, repo, run): + if run.get("run_attempt") != 1: + raise Unavailable("worker reruns are not reusable") + execution = jobs(api, repo, run["id"], 1) + if len(execution) > 1: + raise Unavailable("unexpected mini job inventory") + job = execution[0] if execution else {} + assigned = bool(job.get("runner_id")) and job.get("status") == "in_progress" + tests = [step for step in job.get("steps", []) if step.get("name") == TEST_STEP] + # A known test failure wins even while post-job cleanup is still running. + if any(step.get("conclusion") == "failure" for step in tests): + return "failure", True + if run.get("status") != "completed": + return None, assigned + if len(tests) == 1: + if run.get("conclusion") == "success" and tests[0].get("conclusion") == "success": + return "success", True + preflight_failed = any(step.get("name") in PREFLIGHT_STEPS and step.get("conclusion") == "failure" + for step in job.get("steps", [])) + if (run.get("conclusion") == "failure" and not preflight_failed + and not (len(tests) == 1 and tests[0].get("conclusion") == "success")): + # Incomplete/eventually consistent job metadata must not turn a failed + # test run into a successful hosted retry. Only proven setup failures + # or an explicit infrastructure terminal state get that fallback. + return "failure", assigned + # Checkout, admission, low disk, toolchain, agent loss or an execution + # deadline are infrastructure outcomes. Never infer success from a skip. + print("::notice::Mini ended without a completed passing test step; using hosted macOS") + return "hosted", assigned + + +def wait_for_mini(api, repo, src, *, clock=time.monotonic, sleep=time.sleep): + queued_at, assigned_at = clock(), None + announced = None + while True: + run = api.get(f"repos/{repo}/actions/runs/{src.run_id}") + if run.get("run_attempt") != src.attempt or run.get("status") == "completed": + raise Unavailable("source attempt ended") + candidates = workers(api, repo, src) + if len(candidates) > 1: + raise Unavailable("duplicate mini dispatch") + if candidates: + if candidates[0]["id"] != announced: + announced = candidates[0]["id"] + print(f"Mini worker run: {announced}") + if os.environ.get("GITHUB_STEP_SUMMARY"): + with Path(os.environ["GITHUB_STEP_SUMMARY"]).open("a") as summary: + summary.write(f"Mini worker: https://github.com/{repo}/actions/runs/{announced}\n") + outcome, assigned = worker_state(api, repo, candidates[0]) + if outcome: + return outcome + if assigned and assigned_at is None: + assigned_at = clock() + if assigned_at is None and clock() - queued_at >= QUEUE_SECONDS: + print("::notice::Mini queue/discovery budget expired; using hosted macOS") + return "hosted" + if assigned_at is not None and clock() - assigned_at >= EXECUTION_SECONDS: + print("::notice::Mini execution budget expired; using hosted macOS") + return "hosted" + sleep(RUNNING_POLL_SECONDS if assigned_at is not None else POLL_SECONDS) + + +def cancel_workers(api, repo, src): + failures = 0 + for run in workers(api, repo, src): + if run.get("status") != "completed": + try: + api.cancel_run(repo, run["id"]) + except Exception: + # A transient failure for one duplicate must not prevent + # cleanup of the other workers belonging to this source. + failures += 1 + if failures: + raise Unavailable(f"cancellation failed for {failures} worker(s)") + + +def schedule(api, repo, src, *, clock=time.monotonic, sleep=time.sleep): + queued_at, assigned_at = clock(), None + try: + # No worker is dispatched unless the approved ordinary workflow has + # actually started its gate. A delayed scheduler cannot resurrect it. + while not gate_active(api, repo, src): + if any(job.get("name") == GATE and (job.get("status") == "completed" + or any(step.get("name") == WAIT_STEP and step.get("status") == "completed" + for step in job.get("steps", []))) + for job in jobs(api, repo, src.run_id, src.attempt)): + return + run = api.get(f"repos/{repo}/actions/runs/{src.run_id}") + if run.get("status") == "completed" or run.get("run_attempt") != src.attempt: + return + if clock() - queued_at >= QUEUE_SECONDS: + return + sleep(POLL_SECONDS) + queued_at = clock() + if not workers(api, repo, src): + api.post(f"repos/{repo}/actions/workflows/{WORKFLOW}/dispatches", { + "ref": "main", "inputs": {"source_run_id": str(src.run_id), + "source_attempt": str(src.attempt), "source_sha": src.sha}}) + while gate_active(api, repo, src): + candidates = workers(api, repo, src) + if len(candidates) > 1: + raise Unavailable("duplicate mini dispatch") + if candidates: + outcome, assigned = worker_state(api, repo, candidates[0]) + if outcome: + return + if assigned and assigned_at is None: + assigned_at = clock() + if assigned_at is None and clock() - queued_at >= QUEUE_SECONDS: + return + if assigned_at is not None and clock() - assigned_at >= EXECUTION_SECONDS: + return + sleep(RUNNING_POLL_SECONDS if assigned_at is not None else POLL_SECONDS) + finally: + # Also runs when the parent is cancelled/superseded or has already + # selected hosted fallback. Only this exact source/attempt is touched. + cancel_workers(api, repo, src) + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("mode", choices=("schedule", "wait", "admit", "cleanup")) + parser.add_argument("--run-id", type=int, required=True) + parser.add_argument("--attempt", type=int, required=True) + parser.add_argument("--sha") + args = parser.parse_args() + repo = os.environ["GITHUB_REPOSITORY"] + api = Api(os.environ.get("GH_TOKEN", ""), os.environ.get("GITHUB_API_URL", "https://api.github.com")) + outcome = "hosted" + try: + if args.mode == "cleanup": + run = api.get(f"repos/{repo}/actions/runs/{args.run_id}") + match = TITLE.fullmatch(run.get("display_title", "")) + if run.get("path") == ".github/workflows/test.yml" and match: + # Completion also repairs an orphan left by a scheduler/API + # failure; the PR may already be closed or on a newer attempt. + cancel_workers(api, repo, Source(args.run_id, args.attempt, match[4], run.get("created_at"))) + return 0 + src = source(api, repo, args.run_id, args.attempt) + if args.mode == "schedule": + schedule(api, repo, src) + return 0 + if args.mode == "admit": + if os.environ.get("GITHUB_RUN_ATTEMPT") != "1": + raise Unavailable("worker reruns must use ordinary hosted recovery") + if src.sha != args.sha or not gate_active(api, repo, src): + raise Unavailable("dispatch expired before execution") + return 0 + outcome = wait_for_mini(api, repo, src) + except Exception as error: + # Do not print API bodies, credential-bearing URLs or private config. + reason = str(error) if isinstance(error, Unavailable) else type(error).__name__ + if isinstance(getattr(error, "code", None), int): + reason += f" HTTP {error.code}" + print(f"::notice::Mini unavailable ({reason}); using hosted macOS") + if args.mode == "admit": + return 1 + if args.mode == "wait": + with Path(os.environ["GITHUB_OUTPUT"]).open("a") as output: + output.write(f"result={outcome}\n") + print(f"macOS execution result: {outcome}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/ci/run-macos-lanes.sh b/scripts/ci/run-macos-lanes.sh new file mode 100755 index 0000000000..84d1da2dee --- /dev/null +++ b/scripts/ci/run-macos-lanes.sh @@ -0,0 +1,54 @@ +#!/usr/bin/env bash +# Canonical test commands shared by the mini bundle and hosted fallback. +set -euo pipefail +lane="${1:?lane required}" +lane_workspace="$PWD" +observe_git() { + python3 - "$lane_workspace" <<'PYTHON' +import os, subprocess, sys, time +started = time.monotonic() +subprocess.run(["/usr/bin/git", "-C", sys.argv[1], "-c", "core.fsmonitor=false", + "-c", "core.untrackedCache=false", "rev-parse", "--show-toplevel"], + env={"PATH": "/usr/bin:/bin", "HOME": "/nonexistent", "LANG": "C", "LC_ALL": "C", + "GIT_CONFIG_NOSYSTEM": "1", "GIT_CONFIG_GLOBAL": os.devnull, + "GIT_CONFIG_SYSTEM": os.devnull, "GIT_OPTIONAL_LOCKS": "0", + "GIT_TERMINAL_PROMPT": "0"}, check=True, timeout=60) +print(f"System Git observation initialized in {time.monotonic() - started:.3f}s") +PYTHON +} +case "$lane" in + default-reader) + cargo test --locked -p astra-cli --lib tui::terminal_startup -- --test-threads=2 + if [ "$(uname -s)" = Darwin ]; then + observe_git + cargo test --locked --manifest-path Cargo.toml -p astra-cli \ + --lib edge_tools::shell::tests::bash_echo_returns_output -- --exact + fi + ;; + reflow) + cargo build --locked -p astra-cli --bin astra + npm ci --ignore-scripts --prefix scripts/tui-reflow + ASTRA_TEST_BINARY="${CARGO_TARGET_DIR:-$PWD/target}/debug/astra" npm test --prefix scripts/tui-reflow + ;; + coordination) + cargo clippy --locked --manifest-path Cargo.toml -p astra-tools --all-targets -- -D warnings + observe_git + cargo test --locked --manifest-path Cargo.toml -p astra-tools \ + workspace_observation::tests:: -- --test-threads=1 + ;; + dev-tty-reader) + cargo test --locked -p astra-cli --lib --features crossterm/use-dev-tty tui::terminal_startup -- --test-threads=2 + ;; + vendored) + cargo test --locked --manifest-path vendor/crossterm/Cargo.toml --lib --features event-stream + cargo test --locked --manifest-path vendor/crossterm/Cargo.toml --lib --features event-stream,use-dev-tty + ;; + all) + bash "$0" default-reader + bash "$0" reflow + bash "$0" coordination + bash "$0" vendored + bash "$0" dev-tty-reader + ;; + *) echo "Unknown macOS lane" >&2; exit 2 ;; +esac diff --git a/scripts/ci/test_ci_build_coverage.py b/scripts/ci/test_ci_build_coverage.py index 7affe0952d..27535f4313 100755 --- a/scripts/ci/test_ci_build_coverage.py +++ b/scripts/ci/test_ci_build_coverage.py @@ -2,6 +2,7 @@ """Guard workspace shard coverage and Docker's vendored path dependency inputs.""" from pathlib import Path +from itertools import product import os import re import shlex @@ -19,6 +20,86 @@ class BuildCoverageTests(unittest.TestCase): + def test_rust_toolchain_bootstraps_missing_rust_and_reuses_cargo_bin(self): + action = (ROOT / ".github/actions/rust-toolchain/action.yml").read_text() + script = textwrap.dedent(action.split(" run: |\n", 1)[1]) + for installed, install_failure in ((False, False), (True, False), (False, True)): + with self.subTest(installed=installed, failure=install_failure), tempfile.TemporaryDirectory() as directory: + root = Path(directory) + mock_bin = root / "mock-bin" + cargo_home = root / "cargo home" + mock_bin.mkdir() + (cargo_home / "bin").mkdir(parents=True) + calls = root / "calls" + github_path = root / "github-path" + github_env = root / "github-env" + outside = root / "outside-workspace" + outside.mkdir() + installer = root / "installer" + installer.write_text("""#!/bin/sh +set -eu +printf 'installer %s\\n' "$*" >> "$MOCK_CALLS" +cp "$MOCK_BIN/rustup-source" "$CARGO_HOME/bin/rustup" +cp "$MOCK_BIN/rustc-source" "$CARGO_HOME/bin/rustc" +""") + programs = { + "curl": 'printf "curl\\n" >> "$MOCK_CALLS"; if [ "$MOCK_FAILURE" = 1 ]; then exit 22; fi; cat "$MOCK_INSTALLER"', + "rustup-source": 'printf "rustup %s\\n" "$*" >> "$MOCK_CALLS"; if [ "$1" = show ]; then test -f "$CARGO_HOME/ready"; printf "1.97.0-fixture (overridden by repository)\\n"; else touch "$CARGO_HOME/ready"; fi', + "rustc-source": 'test -f "$CARGO_HOME/ready"; if [ "${RUSTUP_TOOLCHAIN:-}" != 1.97.0-fixture ] && [ "$PWD" != "$MOCK_WORKSPACE" ]; then exit 42; fi; printf "rustc pinned-fixture\\n"', + "sccache": 'compiler="$1"; shift; cd "$MOCK_OUTSIDE"; "$compiler" "$@"', + } + for name, body in programs.items(): + path = mock_bin / name + path.write_text("#!/bin/sh\nset -eu\n" + body + "\n") + path.chmod(0o755) + if installed: + for name in ("rustup", "rustc"): + target = cargo_home / "bin" / name + target.write_bytes((mock_bin / (name + "-source")).read_bytes()) + target.chmod(0o755) + (cargo_home / "ready").touch() + environment = {**os.environ, "PATH": str(mock_bin) + ":/usr/bin:/bin", + "CARGO_HOME": str(cargo_home), "GITHUB_PATH": str(github_path), + "GITHUB_ENV": str(github_env), "MOCK_WORKSPACE": str(ROOT), + "MOCK_OUTSIDE": str(outside), "SCCACHE_PATH": str(mock_bin / "sccache"), + "MOCK_BIN": str(mock_bin), "MOCK_CALLS": str(calls), + "MOCK_INSTALLER": str(installer), "MOCK_FAILURE": str(int(install_failure))} + result = subprocess.run( + ["/bin/bash", "-e", "-c", script], cwd=ROOT, capture_output=True, text=True, + env=environment, + ) + invocations = calls.read_text().splitlines() + if install_failure: + self.assertNotEqual(result.returncode, 0) + self.assertNotIn("rustc pinned-fixture", result.stdout) + continue + self.assertEqual(result.returncode, 0, result.stderr) + self.assertIn("rustc pinned-fixture", result.stdout) + self.assertEqual(github_path.read_text().strip(), str(cargo_home / "bin")) + self.assertEqual("curl" in invocations, not installed) + self.assertEqual("rustup toolchain install" in invocations, not installed) + selected = dict(line.split("=", 1) for line in github_env.read_text().splitlines()) + self.assertEqual(selected["RUSTUP_TOOLCHAIN"], "1.97.0-fixture") + next_environment = {**environment, **selected, + "PATH": str(cargo_home / "bin") + ":" + environment["PATH"]} + probe = subprocess.run(["bash", str(ROOT / "scripts/ci/configure-sccache.sh")], + cwd=ROOT, env=next_environment, capture_output=True, text=True) + self.assertEqual(probe.returncode, 0, probe.stderr) + self.assertIn("RUSTC_WRAPPER=" + str(mock_bin / "sccache"), github_env.read_text()) + # Without the exported selection, the same proxy fails only + # when sccache invokes it outside the repository. The compiler + # itself still works in ROOT, reproducing the old fail-open path. + negative_env = root / "negative-env" + next_environment.pop("RUSTUP_TOOLCHAIN") + next_environment["GITHUB_ENV"] = str(negative_env) + probe = subprocess.run(["bash", str(ROOT / "scripts/ci/configure-sccache.sh")], + cwd=ROOT, env=next_environment, capture_output=True, text=True) + self.assertEqual(probe.returncode, 0, probe.stderr) + self.assertIn("Rust cache disabled", probe.stderr) + self.assertTrue(negative_env.read_text().endswith("RUSTC_WRAPPER=\n")) + for path in (".github/actions/rust-setup/action.yml", ".github/workflows/test.yml"): + self.assertIn("uses: ./.github/actions/rust-toolchain", (ROOT / path).read_text()) + def test_rust_setup_disk_guard_uses_workspace_free_space_without_real_cleanup(self): action = (ROOT / ".github/actions/rust-setup/action.yml").read_text() section = action.split("\n - name: Guard workspace disk space\n", 1)[1] @@ -33,14 +114,14 @@ def test_rust_setup_disk_guard_uses_workspace_free_space_without_real_cleanup(se "docker image prune -af", ] - for mode, available_kib, should_skip in ( + for (mode, available_kib, should_skip), runner_environment in product(( ("above", threshold_kib + 1, True), ("equal", threshold_kib, True), ("below", threshold_kib - 1, False), ("invalid", "not-a-number", False), ("failure", None, False), - ): - with self.subTest(mode=mode), tempfile.TemporaryDirectory() as directory: + ), ("github-hosted", "self-hosted")): + with self.subTest(mode=mode, runner=runner_environment), tempfile.TemporaryDirectory() as directory: root = Path(directory) mock_bin = root / "mock-bin" mock_bin.mkdir() @@ -80,6 +161,7 @@ def test_rust_setup_disk_guard_uses_workspace_free_space_without_real_cleanup(se environment = { **os.environ, "GITHUB_WORKSPACE": str(workspace), + "ASTRA_CI_RUNNER_ENVIRONMENT": runner_environment, "MOCK_DF_MODE": mode, "MOCK_DF_AVAILABLE_KIB": str(available_kib or ""), "MOCK_SUDO_LOG": str(sudo_log), @@ -91,9 +173,11 @@ def test_rust_setup_disk_guard_uses_workspace_free_space_without_real_cleanup(se capture_output=True, text=True, ) - self.assertEqual(result.returncode, 0, result.stderr) + self_hosted_failure = runner_environment == "self-hosted" and not should_skip + self.assertEqual(result.returncode, 1 if self_hosted_failure else 0, result.stderr) output = result.stdout + result.stderr - decision = "skip cleanup" if should_skip else "run cleanup" + decision = ("skip cleanup" if should_skip else + "fail self-hosted" if self_hosted_failure else "run cleanup") self.assertIn(f"Disk guard decision: {decision}", output) self.assertIn( f"threshold={threshold_kib} KiB (30 GiB)", @@ -101,7 +185,7 @@ def test_rust_setup_disk_guard_uses_workspace_free_space_without_real_cleanup(se ) invocations = sudo_log.read_text().splitlines() if sudo_log.exists() else [] - self.assertEqual(invocations, [] if should_skip else cleanup) + self.assertEqual(invocations, [] if should_skip or self_hosted_failure else cleanup) def test_make_lint_preserves_restored_artifacts_and_checks_all_targets(self): result = subprocess.run( @@ -159,49 +243,36 @@ def test_complete_cli_run_prebuilds_required_standalone_mcp_fixture(self): def test_vendored_terminal_units_are_locked_and_pty_tests_run_both_readers(self): workflow = (ROOT / ".github/workflows/test.yml").read_text() - section = workflow.split("\n vendored-terminal:\n", 1)[1].split("\n terminal-pty:", 1)[0] - self.assertIn("os: [ubuntu-latest, macos-15]", section) + lanes = (ROOT / "scripts/ci/run-macos-lanes.sh").read_text() + self.assertIn("run-macos-lanes.sh vendored", workflow) for features in ("event-stream", "event-stream,use-dev-tty"): - self.assertIn(f"cargo test --locked --manifest-path vendor/crossterm/Cargo.toml --lib --features {features}\n", section) + self.assertIn(f"cargo test --locked --manifest-path vendor/crossterm/Cargo.toml --lib --features {features}\n", lanes) lock = tomllib.loads((ROOT / "vendor/crossterm/Cargo.lock").read_text()) self.assertTrue(any(p["name"] == "crossterm" for p in lock["package"])) self.assertNotIn("/Cargo.lock", (ROOT / "vendor/crossterm/.gitignore").read_text().splitlines()) - pty = workflow.split("\n terminal-pty:\n", 1)[1].split("\n macos-workspace-coordination:", 1)[0] - self.assertIn("os: [ubuntu-latest, macos-15]", pty) - self.assertIn("segment: [default-reader, dev-tty-reader, reflow]", pty) - for segment, step, features in ( - ("default-reader", "Test default terminal reader through PTYs", ""), - ("dev-tty-reader", "Test dev-tty terminal reader through PTYs", "--features crossterm/use-dev-tty "), - ): - with self.subTest(segment=segment): - commands = workflow_run_script(".github/workflows/test.yml", step).strip() - self.assertEqual(commands, - f"cargo test --locked -p astra-cli --lib {features}tui::terminal_startup -- --test-threads=2") - block = pty.split(f" - name: {step}\n", 1)[1].split(" - ", 1)[0] - self.assertIn(f"if: matrix.segment == '{segment}'", block) + self.assertEqual(workflow.count("segment: [default-reader, dev-tty-reader, reflow]"), 2) + for features in ("", "--features crossterm/use-dev-tty "): + self.assertIn(f"cargo test --locked -p astra-cli --lib {features}tui::terminal_startup -- --test-threads=2", lanes) def test_terminal_reflow_uses_real_binary_and_locked_emulator(self): - commands = workflow_run_script(".github/workflows/test.yml", "Test inline terminal resize with xterm reflow") + commands = (ROOT / "scripts/ci/run-macos-lanes.sh").read_text() self.assertIn("cargo build --locked -p astra-cli --bin astra", commands) self.assertIn("npm ci --ignore-scripts --prefix scripts/tui-reflow", commands) - self.assertIn('ASTRA_TEST_BINARY="$PWD/target/debug/astra" npm test --prefix scripts/tui-reflow', commands) + self.assertIn('ASTRA_TEST_BINARY="${CARGO_TARGET_DIR:-$PWD/target}/debug/astra" npm test', commands) def test_macos_cli_bash_reuses_default_reader_without_losing_coordination_contracts(self): - workflow = (ROOT / ".github/workflows/test.yml").read_text() - terminal = workflow.split("\n terminal-pty-shards:\n", 1)[1].split("\n macos-workspace-coordination:\n", 1)[0] - self.assertEqual(workflow.count("- name: Test macOS CLI Bash execution"), 1) - bash = terminal.split(" - name: Test macOS CLI Bash execution\n", 1)[1].split(" - ", 1)[0] - self.assertIn("if: matrix.os == 'macos-15' && matrix.segment == 'default-reader'", bash) - self.assertIn("--lib edge_tools::shell::tests::bash_echo_returns_output -- --exact", bash) - self.assertLess(terminal.index("Test default terminal reader through PTYs"), terminal.index("Initialize system Git for macOS CLI observation")) - self.assertLess(terminal.index("Initialize system Git for macOS CLI observation"), terminal.index("Test macOS CLI Bash execution")) - contracts = workflow.split("\n macos-workspace-coordination-tests:\n", 1)[1].split("\n test-online-core:", 1)[0] - self.assertIn('install-nextest: "false"', contracts) - self.assertIn("-p astra-tools", contracts) - self.assertIn("--all-targets -- -D warnings", contracts) + commands = (ROOT / "scripts/ci/run-macos-lanes.sh").read_text() + reader = commands.split(" default-reader)\n", 1)[1].split(" ;;", 1)[0] + self.assertLess(reader.index("tui::terminal_startup"), reader.index("observe_git")) + self.assertLess(reader.index("observe_git"), reader.index("bash_echo_returns_output")) + self.assertEqual(reader.count("cargo test"), 2) + contracts = commands.split(" coordination)\n", 1)[1].split(" ;;", 1)[0] + self.assertIn("cargo clippy --locked --manifest-path Cargo.toml -p astra-tools --all-targets -- -D warnings", contracts) self.assertIn("workspace_observation::tests:: -- --test-threads=1", contracts) - self.assertIn("/usr/bin/git", contracts) - self.assertNotIn("-p astra-cli", contracts, "Do not compile the same CLI test executable twice on macOS") + self.assertIn("observe_git", contracts) + self.assertNotIn("-p astra-cli", contracts) + self.assertIn('"/usr/bin/git"', commands) + self.assertIn("timeout=60", commands) def test_ci_resolves_the_actual_builder_base_not_only_the_planner(self): dockerfile = (ROOT / "Dockerfile").read_text() diff --git a/scripts/ci/test_ci_parallelism.py b/scripts/ci/test_ci_parallelism.py index ac5ae5961c..3b5a530d4c 100755 --- a/scripts/ci/test_ci_parallelism.py +++ b/scripts/ci/test_ci_parallelism.py @@ -116,11 +116,42 @@ def test_status_labels_do_not_repeat_setup_builds_or_artifact_transfers(self): self.assertIn("cancel-in-progress: true", self.workflow) +class MacRunnerTests(unittest.TestCase): + def test_macos_fallback_is_parallel_and_does_not_block_linux(self): + workflow = (ROOT / ".github/workflows/test.yml").read_text() + for job in ("vendored-terminal", "terminal-pty-shards", "test-online"): + header = workflow.split(f"\n {job}:\n", 1)[1].split(" steps:\n", 1)[0] + self.assertIn("needs: scope", header) + self.assertIn("runs-on: ubuntu-latest", header) + self.assertNotIn("macos-plan", header) + for job in ("vendored-terminal-macos", "terminal-pty-shards-macos", "macos-workspace-coordination-tests"): + section = workflow.split(f"\n {job}:\n", 1)[1].split(" steps:\n", 1)[0] + self.assertIn("needs: [scope, macos-plan]", section) + self.assertIn("github.run_attempt > 1", section, "Failed-only reruns must ignore reused mini outputs") + self.assertIn("'macos-15' || 'ubuntu-latest'", section) + self.assertNotIn("pull_request_target:", workflow) + self.assertNotIn("self-hosted", workflow.split(" vendored-terminal:\n", 1)[1]) + + def test_remote_result_gate_never_accepts_failure_or_missing_output(self): + workflow = (ROOT / ".github/workflows/test.yml").read_text() + for job in ("vendored-terminal-macos", "terminal-pty-shards-macos", "macos-workspace-coordination-tests"): + section = workflow.split(f"\n {job}:\n", 1)[1] + block = section.split(" run: |\n", 1)[1].split(" - ", 1)[0] + script = "\n".join(line[10:] for line in block.splitlines()) + for outcome in ("success", "failure", "hosted", "", "skipped"): + with self.subTest(job=job, outcome=outcome), tempfile.TemporaryDirectory() as directory: + result = subprocess.run(["bash", "-c", script], env={ + **os.environ, "MINI_RESULT": outcome, + "GITHUB_STEP_SUMMARY": str(Path(directory) / "summary"), + }, capture_output=True, text=True) + self.assertEqual(result.returncode == 0, outcome == "success") + + class ParallelGateTests(unittest.TestCase): def test_shard_gates_reject_failure_cancellation_and_unexpected_skip(self): for step, variables in ( - ("Require terminal PTY shards", ("SHARDS_RESULT",)), - ("Require macOS coordination and CLI execution", ("COORDINATION_RESULT", "TERMINAL_RESULT")), + ("Require terminal PTY shards", ("SHARDS_RESULT", "MACOS_RESULT")), + ("Require macOS coordination and CLI execution", ("COORDINATION_RESULT", "TERMINAL_RESULT", "MACOS_TERMINAL_RESULT")), ): script = workflow_run_script(".github/workflows/test.yml", step) for required in (True, False): @@ -151,7 +182,7 @@ def test_parallel_jobs_depend_only_on_scope_and_gates_depend_on_shards(self): self.assertIn("!cancelled()", header) self.assertIn("fail-fast: false", header) terminal = workflow.split("\n terminal-pty:\n", 1)[1].split("\n terminal-pty-shards:", 1)[0] - self.assertIn("needs: [scope, terminal-pty-shards]", terminal) + self.assertIn("needs: [scope, terminal-pty-shards, terminal-pty-shards-macos]", terminal) self.assertIn("needs.scope.result != 'success'", terminal) self.assertIn("needs.scope.outputs.test_cli == 'true'", terminal) self.assertIn("needs.scope.outputs.test_core == 'true'", terminal) @@ -159,11 +190,11 @@ def test_parallel_jobs_depend_only_on_scope_and_gates_depend_on_shards(self): self.assertIn("if: ${{ !cancelled() }}", terminal) coordination = workflow.split("\n macos-workspace-coordination:\n", 1)[1].split("\n macos-workspace-coordination-tests:", 1)[0] self.assertIn('name: "Test: macOS workspace coordination"', coordination) - self.assertIn("needs: [scope, macos-workspace-coordination-tests, terminal-pty-shards]", coordination) + self.assertIn("needs: [scope, macos-workspace-coordination-tests, terminal-pty-shards, terminal-pty-shards-macos]", coordination) self.assertIn("needs.scope.result != 'success'", coordination) self.assertIn("if: ${{ !cancelled() }}", coordination) contracts = workflow.split("\n macos-workspace-coordination-tests:\n", 1)[1].split(" steps:\n", 1)[0] - self.assertIn("needs: scope", contracts) + self.assertIn("needs: [scope, macos-plan]", contracts) self.assertNotIn("terminal-pty-shards", contracts, "Tools contracts must not wait for the CLI build") online = workflow.split("\n test-online:\n", 1)[1] self.assertIn("lane: [core-runtime, core-turn-core, core-services, integration]", online) diff --git a/scripts/ci/test_ci_scope.py b/scripts/ci/test_ci_scope.py index 0976a5cfbc..5d1d3c8d20 100755 --- a/scripts/ci/test_ci_scope.py +++ b/scripts/ci/test_ci_scope.py @@ -11,10 +11,46 @@ from io import StringIO from unittest.mock import patch -from ci_scope import classify, main +from ci_scope import classify, main, select_macos_runner class CiScopeTests(unittest.TestCase): + def test_macos_allowlist_uses_author_id_including_bots_and_login_renames(self): + for author_id in (39627130, 52204121, 49699333): + with self.subTest(author_id=author_id): + event = {"pull_request": {"user": {"id": author_id, "login": "renamed-login"}}, + "sender": {"id": 999999999}} + self.assertEqual(select_macos_runner("pull_request", event), + ["self-hosted", "macOS", "ARM64", "mac-arm64-mini"]) + + def test_macos_unknown_author_stays_hosted_even_when_actor_or_login_matches(self): + for user in ({}, {"login": "XuPeng-SH"}, {"id": 999999999, "login": "XuPeng-SH"}, + {"id": "39627130"}, {"id": True}, None): + with self.subTest(user=user): + event = {"pull_request": {"user": user}, "sender": {"id": 39627130}} + self.assertEqual(select_macos_runner("pull_request", event), + ["macos-15"]) + for event in ({}, {"pull_request": None}, []): + self.assertEqual(select_macos_runner("pull_request", event), ["macos-15"]) + + def test_macos_trusted_pushes_and_manual_events_remain_hosted(self): + for event_name in ("push", "workflow_dispatch", "pull_request_target"): + self.assertEqual(select_macos_runner(event_name, + {"ref": "refs/heads/main", "sender": {"id": 39627130}}), ["macos-15"]) + self.assertEqual(select_macos_runner("push", {"ref": "refs/heads/develop"}), ["macos-15"]) + + def test_macos_missing_event_routes_to_hosted_without_omitting_scope_checks(self): + with tempfile.TemporaryDirectory() as directory: + output = Path(directory) / "output" + with patch.dict(os.environ, {"GITHUB_EVENT_PATH": str(Path(directory) / "missing"), + "GITHUB_OUTPUT": str(output)}, clear=True), \ + patch.object(sys, "argv", ["ci_scope.py", "crates/core/src/lib.rs"]), \ + redirect_stdout(StringIO()), redirect_stderr(StringIO()): + self.assertEqual(main(), 0) + values = dict(line.split("=", 1) for line in output.read_text().splitlines()) + self.assertEqual(json.loads(values["macos_runner"]), ["macos-15"]) + self.assertEqual(values["test_core"], "true") + def assert_rename_selects_expected_scopes( self, source: str, diff --git a/scripts/ci/test_macos_dispatch.py b/scripts/ci/test_macos_dispatch.py new file mode 100755 index 0000000000..7c32378421 --- /dev/null +++ b/scripts/ci/test_macos_dispatch.py @@ -0,0 +1,408 @@ +#!/usr/bin/env python3 +"""Exercise dispatch/queue/fallback against the production entrypoints.""" + +from copy import deepcopy +import json +import os +from pathlib import Path +import subprocess +import sys +import tempfile +import unittest +from unittest.mock import patch + +import macos_dispatch as dispatch + + +ROOT = Path(__file__).resolve().parents[2] +REPO = "matrixorigin/Astra" +BASE, HEAD, MERGE = "a" * 40, "b" * 40, "c" * 40 + + +class Clock: + def __init__(self): + self.now = 0 + + def __call__(self): + return self.now + + def sleep(self, seconds): + self.now += seconds + + +class ApiFixture: + def __init__(self): + self.clock = Clock() + self.run = {"id": 1, "run_attempt": 1, "name": "Test Suite", + "path": ".github/workflows/test.yml", "event": "pull_request", + "display_title": f"Test Suite pr=1126 base={BASE} head={HEAD} merge={MERGE}", + "head_sha": HEAD, "status": "in_progress", "pull_requests": [], + "head_repository": {"id": 42}, "repository": {"id": 7}} + self.run["name"] = self.run["display_title"] + self.pr = {"state": "open", "user": {"id": 52204121}, + "head": {"sha": HEAD, "repo": {"id": 42}}} + self.parents = [BASE, HEAD] + self.src = dispatch.Source(1, 1, MERGE) + self.enabled = True + self.assign_at, self.finish_at = 0, 30 + self.run_conclusion, self.test_conclusion = "success", "success" + self.gate_end_at = None + self.wait_end_at = None + self.gate_start_at = 0 + self.gate_conclusion = "success" + self.source_end_at = None + self.other_runs = [] + self.posts, self.cancelled = [], [] + self.calls = [] + + def get(self, path): + self.calls.append(path) + if path.endswith("/actions/runs/1"): + run = deepcopy(self.run) + if self.source_end_at is not None and self.clock() >= self.source_end_at: + run["status"] = "completed" + return run + if path.endswith("/pulls/1126"): + return deepcopy(self.pr) + if path.endswith("/commits/" + MERGE): + return {"parents": [{"sha": sha} for sha in self.parents]} + raise AssertionError(path) + + def worker(self): + status = "completed" if self.clock() >= self.finish_at else "in_progress" + return {"id": 10, "run_attempt": 1, "display_title": self.src.title, + "path": ".github/workflows/macos-mini.yml", "event": "workflow_dispatch", + "status": status, "conclusion": self.run_conclusion if status == "completed" else None} + + def collection(self, path, key): + self.calls.append(path) + if "/workflows/macos-mini.yml/runs?" in path: + return ([self.worker()] if self.enabled else []) + deepcopy(self.other_runs) + if "/runs/1/attempts/1/jobs?" in path: + completed = self.gate_end_at is not None and self.clock() >= self.gate_end_at + status = "in_progress" if self.clock() >= self.gate_start_at else "queued" + wait_status = "completed" if completed or (self.wait_end_at is not None and self.clock() >= self.wait_end_at) else status + return [{"name": dispatch.GATE, "status": "completed" if completed else status, + "conclusion": self.gate_conclusion if completed else None, + "steps": [{"name": dispatch.WAIT_STEP, "status": wait_status}]}] + if "/runs/10/attempts/1/jobs?" in path: + complete = self.clock() >= self.finish_at + assigned = self.clock() >= self.assign_at + steps = [{"name": dispatch.TEST_STEP, "conclusion": self.test_conclusion}] if complete else [] + if complete and self.test_conclusion is None: + steps.append({"name": "Prepare Rust", "conclusion": "failure"}) + return [{"name": "execute", "runner_id": 99 if assigned else 0, + "status": "completed" if complete else "in_progress" if assigned else "queued", + "steps": steps}] + raise AssertionError(path) + + def post(self, path, data): + self.posts.append((path, data)) + self.enabled = True + + def cancel_run(self, repo, run_id): + self.cancelled.append(run_id) + return True + + +class SourceTests(unittest.TestCase): + def test_dispatch_serializes_json_through_the_shared_origin_checked_client(self): + api = dispatch.Api("fixture") + data = {"ref": "main", "inputs": {"source_run_id": "1"}} + with patch.object(api, "_opener") as opener: + response = opener.open.return_value.__enter__.return_value + response.read.return_value = b"" + response.headers = {} + api.post(f"repos/{REPO}/actions/workflows/macos-mini.yml/dispatches", data) + request = opener.open.call_args.args[0] + self.assertEqual(request.get_method(), "POST") + self.assertEqual(json.loads(request.data), data) + self.assertEqual(request.get_header("Content-type"), "application/json") + self.assertEqual(opener.open.call_args.kwargs["timeout"], 10) + + def test_empty_fork_pr_association_uses_verified_merge_tuple(self): + api = ApiFixture() + self.assertEqual(dispatch.source(api, REPO, 1, 1), api.src) + self.assertFalse(any("members" in call for call in api.calls)) + + def test_static_and_dynamic_run_names_use_the_workflow_path_as_identity(self): + for name in ("Test Suite", ApiFixture().run["display_title"]): + api = ApiFixture() + api.run["name"] = name + self.assertEqual(dispatch.source(api, REPO, 1, 1), api.src) + api.run["path"] = ".github/workflows/other.yml" + with self.assertRaises(dispatch.Unavailable): + dispatch.source(api, REPO, 1, 1) + + def test_wrong_parent_order_head_repository_author_and_attempt_are_rejected(self): + mutations = ( + lambda a: setattr(a, "parents", [HEAD, BASE]), + lambda a: a.pr["head"].update(sha="d" * 40), + lambda a: a.pr["head"]["repo"].update(id=43), + lambda a: a.pr["user"].update(id=999999999), + lambda a: a.run.update(run_attempt=2), + ) + for mutate in mutations: + with self.subTest(mutation=mutate): + api = ApiFixture() + mutate(api) + with self.assertRaises(dispatch.Unavailable): + dispatch.source(api, REPO, 1, 1) + + def test_main_develop_pushes_cannot_enter_the_pr_runner(self): + for branch in ("main", "develop"): + api = ApiFixture() + api.run.update(event="push", head_branch=branch, head_repository={"id": 7}, + display_title=f"Test Suite pr=0 base={BASE} head={HEAD} merge={HEAD}") + with self.assertRaises(dispatch.Unavailable): + dispatch.source(api, REPO, 1, 1) + + +class WaitTests(unittest.TestCase): + def wait(self, api): + return dispatch.wait_for_mini(api, REPO, api.src, clock=api.clock, sleep=api.clock.sleep) + + def test_offline_queued_worker_has_bounded_wait(self): + api = ApiFixture() + api.assign_at = api.finish_at = 100000 + self.assertEqual(self.wait(api), "hosted") + self.assertEqual(api.clock(), dispatch.QUEUE_SECONDS) + + def test_missing_dispatch_has_the_same_bounded_wait(self): + api = ApiFixture() + api.enabled = False + self.assertEqual(self.wait(api), "hosted") + self.assertEqual(api.clock(), dispatch.QUEUE_SECONDS) + + def test_assigned_worker_may_complete_after_queue_deadline(self): + api = ApiFixture() + api.assign_at, api.finish_at = 170, 200 + self.assertEqual(self.wait(api), "success") + + def test_execution_deadline_and_agent_loss_use_hosted(self): + api = ApiFixture() + api.finish_at = 100000 + self.assertEqual(self.wait(api), "hosted") + self.assertEqual(api.clock(), dispatch.EXECUTION_SECONDS) + + def test_one_hour_attempt_stays_below_half_the_default_rest_budget(self): + api = ApiFixture() + api.finish_at = 100000 + self.assertEqual(self.wait(api), "hosted") + read_calls = len(api.calls) + api = ApiFixture() + api.finish_at = 100000 + dispatch.schedule(api, REPO, api.src, clock=api.clock, sleep=api.clock.sleep) + self.assertLess(read_calls + len(api.calls), 500) + + def test_actual_test_failure_never_becomes_hosted_retry(self): + api = ApiFixture() + api.run_conclusion = api.test_conclusion = "failure" + self.assertEqual(self.wait(api), "failure") + + def test_failed_run_with_incomplete_test_metadata_is_not_retried(self): + api = ApiFixture() + api.finish_at = 0 + api.run_conclusion, api.test_conclusion = "failure", "pending" + self.assertEqual(self.wait(api), "failure") + + def test_known_test_failure_wins_before_post_job_cleanup_finishes(self): + api = ApiFixture() + api.finish_at = 0 + api.test_conclusion = "failure" + run = api.worker() + run.update(status="in_progress", conclusion=None) + self.assertEqual(dispatch.worker_state(api, REPO, run)[0], "failure") + + def test_worker_lookup_filters_history_to_the_source_creation_window(self): + api = ApiFixture() + src = dispatch.Source(1, 1, MERGE, "2026-10-24T08:00:00Z") + dispatch.workers(api, REPO, src) + from urllib.parse import parse_qs, urlsplit + self.assertEqual(parse_qs(urlsplit(api.calls[-1]).query)["created"], [">=2026-10-24T08:00:00Z"]) + + def test_low_disk_checkout_toolchain_skips_and_cancellation_are_not_success(self): + for conclusion, test in (("failure", None), ("success", "skipped"), + ("cancelled", "cancelled"), ("timed_out", "cancelled")): + with self.subTest(conclusion=conclusion, test=test): + api = ApiFixture() + api.run_conclusion, api.test_conclusion = conclusion, test + self.assertEqual(self.wait(api), "hosted") + + def test_same_sha_different_source_run_and_attempt_cannot_supply_results(self): + for title in (dispatch.Source(2, 1, MERGE).title, dispatch.Source(1, 2, MERGE).title): + api = ApiFixture() + api.enabled = False + other = api.worker() + other.update(id=20, status="completed", conclusion="success", display_title=title) + api.other_runs = [other] + self.assertEqual(self.wait(api), "hosted") + self.assertEqual(api.clock(), dispatch.QUEUE_SECONDS) + + def test_parent_cancellation_does_not_reuse_old_mini_result(self): + api = ApiFixture() + api.source_end_at = 0 + with self.assertRaises(dispatch.Unavailable): + self.wait(api) + + def test_worker_individual_rerun_is_unavailable(self): + api = ApiFixture() + worker = api.worker() + worker["run_attempt"] = 2 + with self.assertRaises(dispatch.Unavailable): + dispatch.worker_state(api, REPO, worker) + + +class SchedulerTests(unittest.TestCase): + def run_scheduler(self, api): + dispatch.schedule(api, REPO, api.src, clock=api.clock, sleep=api.clock.sleep) + + def test_offline_worker_is_cancelled_without_touching_other_pr(self): + api = ApiFixture() + api.enabled = False + api.assign_at = api.finish_at = 100000 + other = api.worker() + other.update(id=20, display_title=dispatch.Source(2, 1, MERGE).title) + api.other_runs = [other] + self.run_scheduler(api) + self.assertEqual(api.cancelled, [10]) + self.assertEqual(api.clock(), dispatch.QUEUE_SECONDS) + self.assertEqual(api.posts[0][1], {"ref": "main", "inputs": { + "source_run_id": "1", "source_attempt": "1", "source_sha": MERGE}}) + + def test_completed_gate_cannot_dispatch_a_late_worker(self): + api = ApiFixture() + api.enabled = False + api.gate_end_at = 0 + self.run_scheduler(api) + self.assertEqual(api.posts, []) + + def test_finished_wait_step_blocks_dispatch_during_post_job_cleanup(self): + api = ApiFixture() + api.enabled = False + api.wait_end_at = 0 + self.run_scheduler(api) + self.assertEqual(api.posts, []) + + def test_gate_start_delay_does_not_consume_the_worker_queue_budget(self): + api = ApiFixture() + api.enabled = False + api.gate_start_at, api.assign_at, api.finish_at = 100, 270, 300 + self.run_scheduler(api) + self.assertEqual(api.cancelled, []) + self.assertGreaterEqual(api.clock(), 300) + self.assertLessEqual(api.clock(), 300 + dispatch.RUNNING_POLL_SECONDS) + + def test_cancel_failure_does_not_prevent_other_owned_workers_from_cancelling(self): + api = ApiFixture() + duplicate = api.worker() + duplicate["id"] = 11 + api.other_runs = [duplicate] + original = api.cancel_run + + def cancel(repo, run_id): + original(repo, run_id) + if run_id == 10: + raise RuntimeError("temporary API failure") + + api.cancel_run = cancel + with self.assertRaises(dispatch.Unavailable): + dispatch.cancel_workers(api, REPO, api.src) + self.assertEqual(api.cancelled, [10, 11]) + + def test_source_cancel_and_gate_fallback_cancel_only_the_owned_worker(self): + for field in ("source_end_at", "gate_end_at"): + api = ApiFixture() + api.finish_at = 100000 + setattr(api, field, 20) + self.run_scheduler(api) + self.assertEqual(api.cancelled, [10]) + self.assertLessEqual(api.clock(), 20 + dispatch.RUNNING_POLL_SECONDS) + + def test_success_or_real_test_failure_does_not_cancel_completed_worker(self): + for result in ("success", "failure"): + api = ApiFixture() + api.run_conclusion = api.test_conclusion = result + self.run_scheduler(api) + self.assertEqual(api.cancelled, []) + + +class EntrypointTests(unittest.TestCase): + def invoke(self, api, mode, *, worker_attempt="1"): + with tempfile.TemporaryDirectory() as directory: + output = Path(directory) / "output" + args = ["macos_dispatch.py", mode, "--run-id", "1", "--attempt", "1", "--sha", MERGE] + with patch.object(dispatch, "Api", return_value=api), patch.object(sys, "argv", args), \ + patch.dict(os.environ, {"GITHUB_REPOSITORY": REPO, "GH_TOKEN": "fixture", + "GITHUB_OUTPUT": str(output), "GITHUB_RUN_ATTEMPT": worker_attempt}, clear=True): + status = dispatch.main() + return status, output.read_text() if output.exists() else "" + + def test_unavailable_api_and_missing_controller_fail_open_to_actual_hosted_tests(self): + api = ApiFixture() + with patch.object(api, "get", side_effect=RuntimeError("unavailable")): + self.assertEqual(self.invoke(api, "wait"), (0, "result=hosted\n")) + + def test_failure_is_published_as_failure_not_fallback(self): + api = ApiFixture() + api.finish_at = 0 + api.run_conclusion = api.test_conclusion = "failure" + self.assertEqual(self.invoke(api, "wait"), (0, "result=failure\n")) + + def test_completion_cancels_orphan_even_after_pr_close_or_attempt_change(self): + api = ApiFixture() + api.run.update(status="completed", run_attempt=2) + api.pr["state"] = "closed" + self.assertEqual(self.invoke(api, "cleanup")[0], 0) + self.assertEqual(api.cancelled, [10]) + + def test_late_or_reused_worker_rejected_before_candidate_checkout(self): + for completed, attempt in ((True, "1"), (False, "2")): + api = ApiFixture() + if completed: + api.gate_end_at = 0 + self.assertEqual(self.invoke(api, "admit", worker_attempt=attempt)[0], 1) + workflow = (ROOT / ".github/workflows/macos-mini.yml").read_text() + self.assertLess(workflow.index("macos_dispatch.py admit"), workflow.index("ref: ${{ inputs.source_sha }}")) + self.assertIn("cache-mode: read", workflow) + self.assertNotIn("actions: write", workflow) + + def test_worker_cannot_start_during_parent_post_job_cleanup(self): + api = ApiFixture() + api.wait_end_at = 0 + self.assertEqual(self.invoke(api, "admit")[0], 1) + + +class LaneTests(unittest.TestCase): + def test_bundle_executes_all_real_lane_commands_and_propagates_failure(self): + for fail in (False, True): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + tools = root / "bin" + tools.mkdir() + log = root / "commands" + for command in ("cargo", "npm"): + program = tools / command + program.write_text('#!/bin/bash\nprintf "%s %s\\n" "${0##*/}" "$*" >> "$COMMANDS"\n' + 'if [[ "${FAIL_CARGO:-}" == 1 && "${0##*/}" == cargo ]]; then exit 17; fi\n') + program.chmod(0o755) + # Keep the native Git probe real, but use a disposable repository. + subprocess.run(["git", "init", "-q", str(root)], check=True) + result = subprocess.run(["bash", str(ROOT / "scripts/ci/run-macos-lanes.sh"), "all"], + cwd=root, env={**os.environ, "PATH": str(tools) + ":" + os.environ["PATH"], + "COMMANDS": str(log), "FAIL_CARGO": "1" if fail else "0"}, capture_output=True, text=True) + calls = log.read_text().splitlines() + if fail: + self.assertEqual(result.returncode, 17) + self.assertEqual(len(calls), 1) + else: + self.assertEqual(result.returncode, 0, result.stderr) + self.assertTrue(any("--features crossterm/use-dev-tty tui::terminal_startup" in c for c in calls)) + self.assertEqual(sum("vendor/crossterm" in c for c in calls), 2) + self.assertTrue(any("workspace_observation::tests:: -- --test-threads=1" in c for c in calls)) + self.assertTrue(any("npm test --prefix scripts/tui-reflow" in c for c in calls)) + self.assertIn("crossterm/use-dev-tty", calls[-1]) + + +if __name__ == "__main__": + unittest.main()