From f26e4e1e40767f22cf18bdbe49987414ddfd1607 Mon Sep 17 00:00:00 2001 From: Magnus Gille Date: Wed, 14 Oct 2026 22:28:39 +0000 Subject: [PATCH 1/7] ci: share setup, cargo gates and CLI smoke tests across the OS lanes (#358) - .github/actions/setup: the toolchain preamble (Python, Rust, MSRV toolchain, Node, rust-cache, model cache, npm ci) that six lanes repeated. - scripts/ci/cargo-gates.mjs: the one list of cargo gates, run by every lane with --os (and --profile msrv); per-OS exclusions carry their reason in the list. Workspace clippy and test replace the bare app/CLI invocations, and the cargo check steps clippy --all-targets already covers are gone. - scripts/ci/smoke-cli.mjs: one cross-platform smoke script replaces the four inline bash and PowerShell blocks; Windows now verifies the JSON, JSONL and error streams like macOS. - scripts/ci/assert-lanes.mjs: the check-* aggregators run one tested script. - windows-package.yml: x64 no longer repeats the workspace tests and clippy on pull requests (test-windows owns them); ARM64, releases and manual runs keep them. - The ci tests read the workflows as parsed YAML (scripts/lib/ci-workflow.mjs) instead of matching text; the parity test expands the gate list and the setup action. Co-Authored-By: Claude Sonnet 5.5 --- .github/actions/setup/action.yml | 96 +++ .github/dependabot.yml | 5 +- .github/workflows/ci.yml | 887 +++----------------- .github/workflows/windows-package.yml | 15 +- CONTRIBUTING.md | 17 +- docs/ARCHITECTURE.md | 53 +- docs/adr/0001-cli-first.md | 2 +- docs/recipes/add-cli-subcommand.md | 12 +- docs/windows-release.md | 4 +- eslint-suppressions.json | 5 - scripts/ci/assert-lanes.mjs | 52 ++ scripts/ci/cargo-gates.mjs | 257 ++++++ scripts/ci/smoke-cli.mjs | 244 ++++++ scripts/dictation_eval/test_evaluate_cli.py | 15 - scripts/lib/ci-workflow.mjs | 128 +++ scripts/node-test-groups.mjs | 7 +- scripts/test-ci-cargo-gates.mjs | 229 +++++ scripts/test-ci-cli-feature-coverage.mjs | 122 ++- scripts/test-ci-docs-workflow.mjs | 143 ++-- scripts/test-ci-parallel-gates.mjs | 561 +++++-------- scripts/test-ci-smoke-cli.mjs | 244 ++++++ scripts/test-ci-unrun-suites.mjs | 215 +++-- scripts/test-clippy-policy.mjs | 36 +- scripts/test-dependency-policy.mjs | 89 +- scripts/test-frontend-test-groups.mjs | 27 +- scripts/test-pianissimo-requirements.mjs | 22 +- scripts/test-pipeline-cache.mjs | 17 +- scripts/test-release-promotion.mjs | 11 +- scripts/test-rust-toolchain-pin.mjs | 38 +- scripts/test-verify-ci-parity.mjs | 129 +-- scripts/test-windows-package-workflow.mjs | 14 +- scripts/test-workflow-hardening.mjs | 84 +- scripts/verify.mjs | 46 +- 33 files changed, 2254 insertions(+), 1572 deletions(-) create mode 100644 .github/actions/setup/action.yml create mode 100644 scripts/ci/assert-lanes.mjs create mode 100644 scripts/ci/cargo-gates.mjs create mode 100644 scripts/ci/smoke-cli.mjs create mode 100644 scripts/lib/ci-workflow.mjs create mode 100644 scripts/test-ci-cargo-gates.mjs create mode 100644 scripts/test-ci-smoke-cli.mjs diff --git a/.github/actions/setup/action.yml b/.github/actions/setup/action.yml new file mode 100644 index 00000000..a7c25ec5 --- /dev/null +++ b/.github/actions/setup/action.yml @@ -0,0 +1,96 @@ +name: Set up the CI toolchain +description: >- + The preamble of every ci.yml lane (#358): Python, the Rust toolchain, Node.js, the Rust + dependency cache, the downloaded-model cache of the native build lanes and `npm ci`. The job + checks out the repository first, so this action always comes from the revision under test. + The toolchain and Node versions are pinned here once; scripts/test-rust-toolchain-pin.mjs + holds them to rust-toolchain.toml and .nvmrc. + +inputs: + python: + description: Install Python 3.12 (the dictation and diarization tests, the JSON stream verifier). + default: "true" + rust: + description: >- + "true" installs the pinned Rust toolchain. "msrv" installs the one `rust-version` declares + (minimal profile) and exports RUSTUP_TOOLCHAIN for every later step (#353). "false" installs none. + default: "true" + rust-components: + description: Extra toolchain components, comma separated (clippy, rustfmt). + default: "" + cache-key: + description: Swatinem/rust-cache shared-key. Lanes that compile alike share one; rust-cache keeps OS and target entries apart. + required: true + cache-workspaces: + description: Cargo workspaces to cache, one per line. The standalone ORT engine host is its own workspace (#357). + default: src-tauri + models: + description: Cache the models the CLI smoke tests download (macOS and Windows build lanes). + default: "false" + linux-deps: + description: Install the Tauri system libraries and xdotool with apt (Linux lanes that compile the app crate). + default: "false" + npm-ci: + description: Run `npm ci`. test-windows turns it off to run its build identity gates first, then installs. + default: "true" + +runs: + using: composite + steps: + - if: ${{ inputs.linux-deps == 'true' }} + name: Install Tauri system dependencies + shell: bash + run: | + sudo apt-get update + sudo apt-get install -y libwebkit2gtk-4.1-dev libayatana-appindicator3-dev librsvg2-dev libasound2-dev build-essential file xdotool + + - if: ${{ inputs.python == 'true' }} + name: Select evaluation Python runtime + uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6 + with: + python-version: "3.12" + + - if: ${{ inputs.rust == 'true' }} + name: Install Rust + uses: dtolnay/rust-toolchain@e2a55d2ffb04f378e9626c28d38b36d230d1e12f # v1; toolchain must match rust-toolchain.toml + with: + toolchain: 1.99.0 + components: ${{ inputs.rust-components }} + + - if: ${{ inputs.rust == 'msrv' }} + name: Install the declared MSRV toolchain + shell: bash + run: | + msrv=$(sed -n 's/^rust-version = "\(.*\)"$/\1/p' src-tauri/Cargo.toml) + ort_msrv=$(sed -n 's/^rust-version = "\(.*\)"$/\1/p' src-tauri/engine-host/ort/Cargo.toml) + test -n "$msrv" + test "$msrv" = "$ort_msrv" + rustup toolchain install "$msrv" --profile minimal + echo "RUSTUP_TOOLCHAIN=$msrv" >> "$GITHUB_ENV" + echo "Checking against Rust $msrv" + + - name: Install Node.js + uses: actions/setup-node@249970729cb0ef3589644e2896645e5dc5ba9c38 # v6.5.0 + with: + node-version: 22 + cache: npm + + - name: Cache Rust dependencies + uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 + with: + workspaces: ${{ inputs.cache-workspaces }} + shared-key: ${{ inputs.cache-key }} + + - if: ${{ inputs.models == 'true' }} + name: Cache downloaded models + uses: actions/cache@caa296126883cff596d87d8935842f9db880ef25 # v5.1.0 + with: + path: ${{ runner.os == 'Windows' && '~\AppData\Roaming\Sagascript\Models' || '~/Library/Application Support/Sagascript/Models' }} + key: ${{ runner.os == 'Windows' && 'windows' || 'macos' }}-models-${{ hashFiles('src-tauri/crates/sagascript-core/src/settings/manager.rs', 'src-tauri/crates/sagascript-core/src/transcription/model.rs', 'src-tauri/crates/sagascript-core/src/diarization/model.rs') }} + restore-keys: | + ${{ runner.os == 'Windows' && 'windows' || 'macos' }}-models- + + - if: ${{ inputs.npm-ci == 'true' }} + name: Install npm dependencies + shell: bash + run: npm ci diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 512521d8..dcde65ec 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -33,8 +33,11 @@ updates: patterns: ["*"] update-types: ["minor", "patch"] + # The local composite actions hold SHA pins too (#358): list each one, or its pins go stale. - package-ecosystem: github-actions - directory: "/" + directories: + - "/" + - "/.github/actions/setup" schedule: interval: monthly open-pull-requests-limit: 5 diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7a058b1a..c03208d7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,40 +1,40 @@ name: CI +# Shared parts live once, outside this file (#358): .github/actions/setup (toolchain preamble), +# scripts/ci/cargo-gates.mjs (THE list of cargo gates, run by every lane; per-OS exclusions are +# documented there), scripts/ci/smoke-cli.mjs (CLI smoke tests), scripts/ci/assert-lanes.mjs (the +# check-* aggregators). Branch protection requires check-macos and check-windows by name: keep both +# and keep them `needs:` every lane of their OS. +# +# Windows tests have one owner per architecture: test-windows owns the x64 tests of every pull +# request (the required lane). windows-package.yml does not repeat `cargo test --workspace` and +# `cargo clippy --workspace` for x64 on pull requests; it keeps them for the native ARM64 runner +# (the only place tests execute on ARM64), for releases and for manual runs. + on: push: branches: [main] pull_request: branches: [main] -# Least privilege (#354): no CI job writes to the repository, so the token is -# read-only. Every job also sets timeout-minutes (60 for native build/test -# lanes, 20 for the rest; at least twice the slowest observed run) instead of -# the 360-minute default. +# Least privilege (#354): read-only token; every job sets timeout-minutes. permissions: contents: read -# Cancel superseded PR runs (#178): pushes to a PR share one group, so a fix -# cancels the stale run instead of burning runner-minutes compiling it twice. -# Pushes (e.g. to `main`) get a unique group per run, so a newer push can -# neither cancel an in-progress run nor replace a pending one — with the -# default single-slot queue, same-group pending runs would otherwise be -# dropped. Never touches release.yml (tag builds must never cancel). +# Cancel superseded PR runs (#178); pushes (e.g. to `main`) get a unique group per run. concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.run_id }} cancel-in-progress: true -# Keep source/line backtraces for CI failures without full variable/type debug -# information in every dependency and PDB. Release profiles are unchanged. -# rust-cache includes CARGO_* environment variables in its compatibility key. +# Line-table backtraces without full debug information (rust-cache keys on CARGO_* variables). env: CARGO_PROFILE_DEV_DEBUG: line-tables-only CARGO_PROFILE_TEST_DEBUG: line-tables-only jobs: - # This job is deliberately dependency-free and checks out the PR base so a - # PR cannot change the allowlist used to decide whether native CI is needed. - # Pushes use --full because the workflow only runs on main and no comparison - # range is needed to keep that required path fail-closed. + # Checks out the PR base so a PR cannot change the allowlist that decides whether native CI is + # needed. Inline on purpose: the base may predate any script file (adoption), and a missing file + # would fail the required checks instead of falling back to full CI. Pushes use --full. scope: runs-on: ubuntu-24.04 timeout-minutes: 20 @@ -48,7 +48,6 @@ jobs: with: fetch-depth: 0 ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.base.sha || github.sha }} - - name: Classify changed paths id: classify env: @@ -66,9 +65,7 @@ jobs: } if [ "$EVENT_NAME" = pull_request ]; then - # The policy is read from the base checkout. During adoption the - # base may predate this script; a fork head may also be absent from - # the base repository. Both cases safely run the full CI lanes. + # An absent policy (adoption) or head commit (fork) safely runs the full CI lanes. if [ -f scripts/ci-docs-scope.mjs ] \ && [[ "$PR_BASE_SHA" =~ ^[0-9a-f]{40}$ ]] \ && [[ "$PR_HEAD_SHA" =~ ^[0-9a-f]{40}$ ]] \ @@ -87,77 +84,31 @@ jobs: if: ${{ needs.scope.result == 'success' && needs.scope.outputs.docs_only != 'true' }} runs-on: macos-14 timeout-minutes: 60 - steps: - - name: Checkout - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0 - - - name: Select evaluation Python runtime - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6 - with: - python-version: "3.12" - - - name: Test offline dictation evaluation tooling - run: python -m unittest discover -s scripts/dictation_eval -p 'test_*.py' - - - name: Test diarization reference contracts - run: python -m unittest discover -s scripts/diarization_sv -p 'test_reference_dataset.py' - - - name: Install Rust - uses: dtolnay/rust-toolchain@e2a55d2ffb04f378e9626c28d38b36d230d1e12f # v1; toolchain must match rust-toolchain.toml + - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0 + - name: Set up toolchain + uses: ./.github/actions/setup with: - toolchain: 1.99.0 - components: clippy - - - name: Install Node.js - uses: actions/setup-node@249970729cb0ef3589644e2896645e5dc5ba9c38 # v6.5.0 - with: - node-version: 22 - cache: npm - - - name: Cache Rust dependencies - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 - with: - workspaces: src-tauri - shared-key: sagascript-ci-checks - - - name: Install npm dependencies - run: npm ci - - # npm advisories (#355): fail on a high or critical finding anywhere in - # the root package-lock.json, dev dependencies included, since they - # build the shipped frontend bundle. site/ and tooling/code-health/ have - # their own lockfiles; neither ships in the app and neither is audited - # here. - - name: Audit npm dependencies - run: npm audit --audit-level=high - - # Prettier over src/, scripts/ and tests/, ESLint, and the Vitest unit - # and component tests (src/**/*.test.ts, #361). verify runs them as - # `format`, `lint` and `unit`. - - name: Check frontend formatting - run: npm run format:check - - - name: Lint frontend and tooling - run: npm run lint - - - name: Unit-test frontend modules - run: npm run test:unit - - - name: Test frontend lifecycle helpers - run: | + rust-components: clippy + cache-key: sagascript-ci-checks + - name: Test offline dictation evaluation tooling and diarization reference contracts + run: | + python -m unittest discover -s scripts/dictation_eval -p 'test_*.py' + python -m unittest discover -s scripts/diarization_sv -p 'test_reference_dataset.py' + - name: Frontend checks (npm audit, format, lint, unit tests, Node tests, types) + run: | + npm audit --audit-level=high + npm run format:check + npm run lint + npm run test:unit npm run test:frontend - node scripts/test-generate-updater-manifest.mjs - node --test scripts/diarization_sv/test_reference_review.mjs - - - name: Install browser regression runtime - run: npx playwright install chromium - + npx svelte-check --tsconfig ./tsconfig.json - name: Test transcription result tabs in browser env: QA_OUTPUT_DIR: ${{ runner.temp }}/transcription-browser - run: npm run test:browser - + run: | + npx playwright install chromium + npm run test:browser - name: Upload browser regression diagnostics if: failure() uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 @@ -165,70 +116,20 @@ jobs: name: transcription-browser-diagnostics path: ${{ runner.temp }}/transcription-browser if-no-files-found: ignore - - name: Build frontend run: npm run build - - - name: Cargo check - working-directory: src-tauri - run: cargo check - - - name: Cargo test - working-directory: src-tauri - run: cargo test - - - name: Cargo clippy - working-directory: src-tauri - run: cargo clippy --all-targets -- -D warnings - - # Separate invocations preserve the core no-features build; combining - # package flags would unify features with the CLI defaults. - - name: Cargo test (sagascript-core, no features) - working-directory: src-tauri - run: cargo test -p sagascript-core - - - name: Cargo test (sagascript-cli) - working-directory: src-tauri - run: cargo test -p sagascript-cli - - - name: Cargo test (sagascript-cli, no features) - working-directory: src-tauri - run: cargo test -p sagascript-cli --no-default-features - - - name: Cargo clippy (sagascript-core, no features) - working-directory: src-tauri - run: cargo clippy -p sagascript-core --all-targets -- -D warnings - - - name: Cargo clippy (sagascript-cli) - working-directory: src-tauri - run: cargo clippy -p sagascript-cli --all-targets -- -D warnings - - # Core defaults to no features. CLI defaults already include diarization - # and are covered above; core needs this explicit feature matrix. - - name: Cargo check (sagascript-core, diarization) - working-directory: src-tauri - run: cargo check -p sagascript-core --features diarization - - - name: Cargo test (sagascript-core, diarization) - working-directory: src-tauri - run: cargo test -p sagascript-core --features diarization - - - name: Cargo clippy (sagascript-core, diarization) - working-directory: src-tauri - run: cargo clippy -p sagascript-core --features diarization --all-targets -- -D warnings - - - name: Svelte check - run: npx svelte-check --tsconfig ./tsconfig.json - - - name: Build Core ML engine host and check its revision + - name: Cargo gates + run: node scripts/ci/cargo-gates.mjs --os macos + # Model-free subset of the engine-host conformance (#367); the full run needs a Pianissimo + # model and is in the release gate (release-build-macos.yml). + - name: Build the Core ML engine host, check its revision, run the conformance subset run: | ./scripts/stage-engine-host.sh ./scripts/check-engine-host-identity.sh build/engine-host/sagascript-engine-host "$GITHUB_SHA" - + python scripts/engine-host-conformance.py build/engine-host/sagascript-engine-host --no-model + # The tests use Swift Testing (Xcode 16+); the runner's default Xcode may be older. - name: Swift test (Core ML engine host) working-directory: src-tauri/engine-host/coreml - # The tests use Swift Testing (`import Testing`), which ships with Xcode 16+; - # the runner's default Xcode may be older, so run them with the newest one installed. run: | newest=$(find /Applications -maxdepth 1 -name "Xcode_*.app" | sort -V | tail -1) if [ -z "$newest" ]; then echo "No Xcode found in /Applications" >&2; exit 1; fi @@ -237,152 +138,29 @@ jobs: swift --version swift test - # Engine-host protocol conformance (#367), model-free subset, against the - # host staged above: hello, ping, status, the errors the client relies on - # (unsupported, model_missing, not_loaded), shutdown and EOF exit. The full - # run (load, windows, cancel) needs a Pianissimo model and is in the release - # gate (release-build-macos.yml, verify-release-draft.sh; docs/engine-host-protocol.md, - # Conformance). - - name: Engine-host conformance, model-free subset (Core ML host) - run: python scripts/engine-host-conformance.py build/engine-host/sagascript-engine-host --no-model - build-macos: needs: scope if: ${{ needs.scope.result == 'success' && needs.scope.outputs.docs_only != 'true' }} runs-on: macos-14 timeout-minutes: 60 - steps: - - name: Checkout - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0 - - - name: Select evaluation Python runtime - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6 - with: - python-version: "3.12" - - - name: Install Rust - uses: dtolnay/rust-toolchain@e2a55d2ffb04f378e9626c28d38b36d230d1e12f # v1; toolchain must match rust-toolchain.toml - with: - toolchain: 1.99.0 - - - name: Install Node.js - uses: actions/setup-node@249970729cb0ef3589644e2896645e5dc5ba9c38 # v6.5.0 - with: - node-version: 22 - cache: npm - - - name: Cache Rust dependencies - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 - with: - workspaces: src-tauri - shared-key: sagascript-ci-native - - - name: Cache downloaded models - uses: actions/cache@caa296126883cff596d87d8935842f9db880ef25 # v5.1.0 + - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0 + - name: Set up toolchain + uses: ./.github/actions/setup with: - path: ~/Library/Application Support/Sagascript/Models - key: macos-models-${{ hashFiles('src-tauri/crates/sagascript-core/src/settings/manager.rs', 'src-tauri/crates/sagascript-core/src/transcription/model.rs', 'src-tauri/crates/sagascript-core/src/diarization/model.rs') }} - restore-keys: | - macos-models- - - - name: Install npm dependencies - run: npm ci - + cache-key: sagascript-ci-native + models: "true" - name: Build frontend run: npm run build - - # PR validation needs the native release binary for the smoke tests, not - # an installer. Full packaging, signing and notarization remain gated in - # release.yml, so skipping bundling here removes duplicate work without - # weakening native macOS coverage. - + # The smoke tests need the binary, not an installer (packaging: release.yml). - name: Build native release binary (no installer) env: SAGASCRIPT_UPDATER_PUBKEY: ${{ vars.SAGASCRIPT_UPDATER_PUBKEY }} run: npx tauri build --config scripts/tauri-ci-prebuilt.json --no-bundle - - - name: Verify native release binary - run: | - BINARY=src-tauri/target/release/sagascript - [[ -x "$BINARY" ]] - file "$BINARY" - - # --- Tier 1: CLI smoke tests (no model needed) --- - - - name: Smoke test — help and version + - name: Smoke tests (no model; model download, transcription, JSON and JSONL streams) run: | - BINARY=src-tauri/target/release/sagascript - $BINARY --help - $BINARY transcribe --help - $BINARY record --help - - - name: Smoke test — config management - run: | - BINARY=src-tauri/target/release/sagascript - $BINARY config list - $BINARY config get language - $BINARY config set language sv - $BINARY config get language | grep -q sv - $BINARY config reset language - $BINARY config get language | grep -q en - $BINARY config path - - - name: Smoke test — model listing and formats - run: | - BINARY=src-tauri/target/release/sagascript - $BINARY list-models - $BINARY formats - - - name: Smoke test — shell completions - run: | - BINARY=src-tauri/target/release/sagascript - $BINARY completions zsh > /dev/null - $BINARY completions bash > /dev/null - - # --- Tier 2: Model download + transcription --- - - - name: Smoke test — download model and transcribe - run: | - BINARY=src-tauri/target/release/sagascript - $BINARY download-model nb-whisper-tiny - RESULT=$($BINARY transcribe --language no --model nb-whisper-tiny test-audio/norwegian-short-3s.mp3) - echo "Transcription: $RESULT" - echo "$RESULT" | grep -iq "stortinget" - - - name: Smoke test — transcribe with JSON output - run: | - BINARY=src-tauri/target/release/sagascript - STDOUT="$RUNNER_TEMP/transcription-smoke.json" - STDERR="$RUNNER_TEMP/transcription-smoke.stderr" - $BINARY transcribe --json --language no --model nb-whisper-tiny \ - test-audio/norwegian-short-3s.mp3 > "$STDOUT" 2> "$STDERR" - python3 scripts/verify-json-cli-streams.py "$STDOUT" "$STDERR" - - # The output contract of #364 with a real model: JSONL items, a reader - # that stops after the first item, and the error object of a failed --json run. - - name: Smoke test — JSONL, closed pipe and JSON error object - run: | - set -o pipefail - BINARY=src-tauri/target/release/sagascript - INPUTS="$RUNNER_TEMP/jsonl-inputs" - mkdir -p "$INPUTS" - cp test-audio/norwegian-short-3s.mp3 "$INPUTS/a.mp3" - cp test-audio/norwegian-medium-8s.mp3 "$INPUTS/b.mp3" - $BINARY transcribe --jsonl --language no --model nb-whisper-tiny "$INPUTS" \ - > "$RUNNER_TEMP/transcription-smoke.jsonl" 2> "$RUNNER_TEMP/transcription-smoke-jsonl.stderr" - python3 scripts/verify-json-cli-streams.py --jsonl \ - "$RUNNER_TEMP/transcription-smoke.jsonl" "$RUNNER_TEMP/transcription-smoke-jsonl.stderr" - # A smoke check with the real model (pipefail turns a panic or a failing exit into - # a red step). The deterministic check of the closed-pipe contract is - # `run_into_closed_pipe` in src-tauri/crates/sagascript-cli/tests/output_contract.rs. - $BINARY transcribe --jsonl --language no --model nb-whisper-tiny "$INPUTS" | head -1 > /dev/null - STATUS=0 - $BINARY transcribe --json --language no --model nb-whisper-tiny "$RUNNER_TEMP/missing.wav" \ - > "$RUNNER_TEMP/transcription-error.json" 2> "$RUNNER_TEMP/transcription-error.stderr" || STATUS=$? - test "$STATUS" -eq 1 - python3 scripts/verify-json-cli-streams.py --error \ - "$RUNNER_TEMP/transcription-error.json" "$RUNNER_TEMP/transcription-error.stderr" + node scripts/ci/smoke-cli.mjs basic + node scripts/ci/smoke-cli.mjs model check-macos: needs: [scope, test-macos, build-macos] @@ -390,172 +168,59 @@ jobs: runs-on: ubuntu-24.04 timeout-minutes: 20 env: - SCOPE_RESULT: ${{ needs.scope.result }} - SCOPE_TRUSTED: ${{ needs.scope.outputs.scope_trusted }} - DOCS_ONLY: ${{ needs.scope.outputs.docs_only }} - TEST_RESULT: ${{ needs.test-macos.result }} - BUILD_RESULT: ${{ needs.build-macos.result }} + NEEDS: ${{ toJSON(needs) }} steps: + - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0 - name: Assert macOS CI lanes passed - run: | - test "$SCOPE_RESULT" = success - test "$SCOPE_TRUSTED" = true - if [ "$DOCS_ONLY" = true ]; then - test "$TEST_RESULT" = skipped - test "$BUILD_RESULT" = skipped - else - test "$DOCS_ONLY" = false - test "$TEST_RESULT" = success - test "$BUILD_RESULT" = success - fi + run: node scripts/ci/assert-lanes.mjs test-linux: needs: scope if: ${{ needs.scope.result == 'success' && needs.scope.outputs.docs_only != 'true' }} runs-on: ubuntu-24.04 timeout-minutes: 60 - steps: - - name: Checkout - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0 - - - name: Select evaluation Python runtime - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6 - with: - python-version: "3.12" - - - name: Test offline dictation evaluation tooling - run: python -m unittest discover -s scripts/dictation_eval -p 'test_*.py' - - - name: Install Tauri system dependencies - run: | - sudo apt-get update - sudo apt-get install -y \ - libwebkit2gtk-4.1-dev \ - libayatana-appindicator3-dev \ - librsvg2-dev \ - libasound2-dev \ - build-essential \ - file \ - xdotool - - - name: Install Rust - uses: dtolnay/rust-toolchain@e2a55d2ffb04f378e9626c28d38b36d230d1e12f # v1; toolchain must match rust-toolchain.toml - with: - toolchain: 1.99.0 - components: clippy, rustfmt - - # Formatting gate (#352). Both Cargo workspaces must be rustfmt-clean: - # the main one and the standalone ORT engine host. Runs before any - # compilation, so an unformatted PR fails early. - - name: Check Rust formatting - working-directory: src-tauri - run: | - cargo fmt --all -- --check - cargo fmt --manifest-path engine-host/ort/Cargo.toml --all -- --check - - - name: Install Node.js - uses: actions/setup-node@249970729cb0ef3589644e2896645e5dc5ba9c38 # v6.5.0 + - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0 + - name: Set up toolchain + uses: ./.github/actions/setup with: - node-version: 22 - cache: npm - - # Same rust-cache scheme as the macOS job (#178); the shared key spans - # branches while rust-cache keeps OS/target entries separate. The - # standalone ORT engine host is its own Cargo workspace (#357). - - name: Cache Rust dependencies - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 - with: - workspaces: | + rust-components: clippy, rustfmt + linux-deps: "true" + cache-key: sagascript-ci + cache-workspaces: | src-tauri src-tauri/engine-host/ort - shared-key: sagascript-ci - - # Dependency policy (#355), before any compilation: RustSec advisories, - # licenses, banned or duplicated crates and allowed sources, as set in - # src-tauri/deny.toml, for both Cargo workspaces. The advisory database - # is fetched at run time. rust-cache keeps the installed binary. The ORT - # host run allows the policy entries only the main workspace uses. + - name: Test offline dictation evaluation tooling + run: python -m unittest discover -s scripts/dictation_eval -p 'test_*.py' + # The cargo-deny gates run first, before any compilation; rust-cache keeps the binary. - name: Install cargo-deny run: cargo install --locked cargo-deny --version 0.20.2 - - - name: Check Rust dependency policy (cargo-deny) - working-directory: src-tauri - run: | - cargo deny --locked --workspace check - cargo deny --locked --manifest-path engine-host/ort/Cargo.toml --workspace check -A license-not-encountered -A license-exception-not-encountered -A unmatched-source - - - name: Install npm dependencies - run: npm ci - + # tauri::generate_context! embeds ../dist, so the app crate needs it. - name: Build frontend run: npm run build - - # GUI build: compiles the cfg(target_os = "linux") Tauri path so it can't - # silently break on a future PR (the macOS/Windows runners never see it). - # The bare invocation covers the root (app) package, which pulls in - # sagascript-core with the `record` feature. - - name: Cargo check (GUI app) - working-directory: src-tauri - run: cargo check - - - name: Cargo clippy (GUI app) - working-directory: src-tauri - run: cargo clippy --all-targets -- -D warnings - - # Headless CLI (no Tauri) — the sagascript-cli member crate. Default - # features include `record` (cpal/ALSA); the --no-default-features runs - # cover the pure batch-transcribe build, which must need no ALSA. - - name: Cargo check (headless CLI, with record) - working-directory: src-tauri - run: cargo check -p sagascript-cli - - - name: Cargo check (headless CLI, batch-only) - working-directory: src-tauri - run: cargo check -p sagascript-cli --no-default-features - - - name: Cargo clippy (headless CLI, batch-only) - working-directory: src-tauri - run: cargo clippy -p sagascript-cli --all-targets --no-default-features -- -D warnings - - # Clippy policy lints (#356): unwrap/expect/panic/print/size counts in - # production code, per lint and crate, must not rise above - # src-tauri/clippy-policy-baseline.json. It re-lints only the workspace - # crates (workspace + batch-only CLI); dependencies come from the steps - # above. Measures Linux code only. See CONTRIBUTING.md. + - name: Cargo gates + run: node scripts/ci/cargo-gates.mjs --os linux + # Clippy policy lints (#356): counts must not rise above src-tauri/clippy-policy-baseline.json. + # Re-lints only the workspace crates, after the gates above compiled the dependencies. - name: Clippy policy lints (counts must not rise) run: node scripts/clippy-policy.mjs - - - name: Cargo test (headless CLI, batch-only) - working-directory: src-tauri - run: cargo test -p sagascript-cli --no-default-features - - - name: Build batch CLI for evaluation contracts - working-directory: src-tauri - run: cargo build -p sagascript-cli --no-default-features - - - name: Install offline diarization evaluation test dependencies - run: python -m pip install -r scripts/diarization_sv/requirements-evaluation.txt - - name: Test diarization reference and evidence contracts with native identity parity env: SAGASCRIPT_REFERENCE_CLI: ${{ github.workspace }}/src-tauri/target/debug/sagascript run: | + python -m pip install -r scripts/diarization_sv/requirements-evaluation.txt test -x "$SAGASCRIPT_REFERENCE_CLI" python -m unittest discover -s scripts/diarization_sv -p 'test_*.py' - - name: Cross-check native DER and JER against independent scorers run: | curl --fail --location --retry 2 \ https://raw.githubusercontent.com/nryant/dscore/e02f949ac6592279300a2c33d03daf9e0c12fd27/scorelib/metrics.py \ --output "$RUNNER_TEMP/dscore-metrics.py" - # The cross-check verifies the pinned source SHA-256 before loading - # only its numeric JER function. No Perl runner or model inference. + # Verifies the pinned source SHA-256, loads only its numeric JER function. python scripts/diarization_sv/crosscheck_metrics.py \ --binary "$GITHUB_WORKSPACE/src-tauri/target/debug/sagascript" \ --scratch "$RUNNER_TEMP/diarization-metric-crosscheck" \ --dscore-source "$RUNNER_TEMP/dscore-metrics.py" - - name: Upload diarization metric receipts if: always() uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 @@ -564,131 +229,30 @@ jobs: path: ${{ runner.temp }}/diarization-metric-crosscheck if-no-files-found: ignore - # Run core's own tests on Linux too — cargo does not run dependency - # crates' tests, so without these steps core is only compile-checked - # here (its tests would run on macOS/Windows only). - - name: Cargo test (sagascript-core, no features) - working-directory: src-tauri - run: cargo test -p sagascript-core - - - name: Cargo test (sagascript-core, with record) - working-directory: src-tauri - run: cargo test -p sagascript-core --features record - - # Regression guard for the no-ALSA property (#66): the batch-only CLI - # dependency tree must contain neither cpal nor alsa. (grep -c would - # exit 1 on zero matches and fail the step, hence the if/grep -q form.) - - name: Assert batch-only CLI has no ALSA/cpal dependency - working-directory: src-tauri - run: | - if cargo tree -p sagascript-cli --no-default-features -e normal | grep -qiE 'alsa|cpal'; then - echo "ERROR: cpal/ALSA leaked into the batch-only dependency tree:" - cargo tree -p sagascript-cli --no-default-features -e normal | grep -iE 'alsa|cpal' - exit 1 - fi - echo "OK: no cpal/ALSA in batch-only tree" - - # The Rust<->Swift engine-host wire format (#357). No CI lane runs - # `cargo test --workspace` and cargo never runs a dependency's tests, so - # these otherwise ran only in the release gate and windows-package.yml. - - name: Cargo test (sagascript-engine-protocol) - working-directory: src-tauri - run: cargo test -p sagascript-engine-protocol - - # The ONNX Runtime engine host is a separate Cargo workspace (#357), so - # the workspace steps above never reach it; windows-package.yml only - # builds it. Its tests include the context-biasing parity vectors shared - # with the Swift host. ort's `load-dynamic` means neither step needs an - # ONNX Runtime binary. rustfmt is gated in "Check Rust formatting". - - name: Cargo test (ORT engine host) - working-directory: src-tauri/engine-host/ort - run: cargo test --locked - - - name: Cargo clippy (ORT engine host) - working-directory: src-tauri/engine-host/ort - run: cargo clippy --all-targets --locked -- -D warnings - - # Release-profile guard (#357): `release_build_ignores_the_real_environment` - # proves a release build ignores SAGASCRIPT_ENGINE_HOST and - # SAGASCRIPT_PIANISSIMO_MODEL_DIR. It is compiled only without - # debug_assertions and the `dev-overrides` feature, so every debug - # `cargo test` above leaves it out. - - name: Cargo test (sagascript-core, release-profile dev-override guard) - working-directory: src-tauri - run: | - set -o pipefail - cargo test --release -p sagascript-core --lib dev_overrides 2>&1 | tee "$RUNNER_TEMP/release-guard.log" - # cargo test exits 0 when the filter matches nothing: prove the guard ran. - grep -q 'release_build_ignores_the_real_environment ... ok' "$RUNNER_TEMP/release-guard.log" - - # Minimum supported Rust version (#353). `rust-version` in - # src-tauri/Cargo.toml (and engine-host/ort) is a promise to people building - # from source; compile with exactly that toolchain so it stays true. Linux is - # the cheapest lane that builds every workspace member. RUSTUP_TOOLCHAIN - # overrides the newer development pin in rust-toolchain.toml. + # Minimum supported Rust version (#353): `rust-version` is a promise to people building from + # source, so compile with exactly that toolchain. The setup action installs it and exports + # RUSTUP_TOOLCHAIN (over the pin in rust-toolchain.toml); the msrv profile refuses to run without it. msrv-linux: needs: scope if: ${{ needs.scope.result == 'success' && needs.scope.outputs.docs_only != 'true' }} runs-on: ubuntu-24.04 timeout-minutes: 60 - steps: - - name: Checkout - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0 - - - name: Install Tauri system dependencies - run: | - sudo apt-get update - sudo apt-get install -y \ - libwebkit2gtk-4.1-dev \ - libayatana-appindicator3-dev \ - librsvg2-dev \ - libasound2-dev \ - build-essential \ - file - - - name: Install the declared MSRV toolchain - run: | - msrv=$(sed -n 's/^rust-version = "\(.*\)"$/\1/p' src-tauri/Cargo.toml) - ort_msrv=$(sed -n 's/^rust-version = "\(.*\)"$/\1/p' src-tauri/engine-host/ort/Cargo.toml) - test -n "$msrv" - test "$msrv" = "$ort_msrv" - rustup toolchain install "$msrv" --profile minimal - echo "RUSTUP_TOOLCHAIN=$msrv" >> "$GITHUB_ENV" - echo "Checking against Rust $msrv" - - - name: Install Node.js - uses: actions/setup-node@249970729cb0ef3589644e2896645e5dc5ba9c38 # v6.5.0 - with: - node-version: 22 - cache: npm - - - name: Cache Rust dependencies - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 + - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0 + - name: Set up toolchain + uses: ./.github/actions/setup with: - workspaces: | + python: "false" + rust: msrv + linux-deps: "true" + cache-key: sagascript-msrv + cache-workspaces: | src-tauri src-tauri/engine-host/ort - shared-key: sagascript-msrv - - - name: Install npm dependencies - run: npm ci - - # tauri::generate_context! embeds ../dist, so the app crate needs it. - name: Build frontend run: npm run build - - - name: Cargo check (workspace) on MSRV - working-directory: src-tauri - run: cargo check --workspace --locked - - - name: Cargo check (headless CLI, batch-only) on MSRV - working-directory: src-tauri - run: cargo check -p sagascript-cli --no-default-features --locked - - - name: Cargo check (ORT engine host) on MSRV - working-directory: src-tauri/engine-host/ort - run: cargo check --locked + - name: Cargo gates (MSRV profile) + run: node scripts/ci/cargo-gates.mjs --profile msrv check-linux: needs: [scope, test-linux, msrv-linux] @@ -696,259 +260,81 @@ jobs: runs-on: ubuntu-24.04 timeout-minutes: 20 env: - SCOPE_RESULT: ${{ needs.scope.result }} - SCOPE_TRUSTED: ${{ needs.scope.outputs.scope_trusted }} - DOCS_ONLY: ${{ needs.scope.outputs.docs_only }} - LINUX_RESULT: ${{ needs.test-linux.result }} - MSRV_RESULT: ${{ needs.msrv-linux.result }} + NEEDS: ${{ toJSON(needs) }} steps: + - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0 - name: Assert Linux CI lanes passed - run: | - test "$SCOPE_RESULT" = success - test "$SCOPE_TRUSTED" = true - if [ "$DOCS_ONLY" = true ]; then - test "$LINUX_RESULT" = skipped - test "$MSRV_RESULT" = skipped - else - test "$DOCS_ONLY" = false - test "$LINUX_RESULT" = success - test "$MSRV_RESULT" = success - fi + run: node scripts/ci/assert-lanes.mjs test-windows: needs: scope if: ${{ needs.scope.result == 'success' && needs.scope.outputs.docs_only != 'true' }} runs-on: windows-latest timeout-minutes: 60 - steps: - - name: Checkout - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0 - - - name: Select evaluation Python runtime - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6 + - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0 + - name: Set up toolchain + uses: ./.github/actions/setup with: - python-version: "3.12" - - - name: Test offline dictation evaluation tooling - run: python -m unittest discover -s scripts/dictation_eval -p 'test_*.py' - - - name: Install Rust - uses: dtolnay/rust-toolchain@e2a55d2ffb04f378e9626c28d38b36d230d1e12f # v1; toolchain must match rust-toolchain.toml - with: - toolchain: 1.99.0 - components: clippy - - - name: Install Node.js - uses: actions/setup-node@249970729cb0ef3589644e2896645e5dc5ba9c38 # v6.5.0 - with: - node-version: 22 - cache: npm - - # The standalone ORT engine host is its own Cargo workspace (#357); its - # tests run here too (last step below). - - name: Cache Rust dependencies - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 - with: - workspaces: | + rust-components: clippy + cache-key: sagascript-ci-checks + cache-workspaces: | src-tauri src-tauri/engine-host/ort - shared-key: sagascript-ci-checks - - - name: Test Windows build identity gates - run: node --test scripts/test-ci-build-identity.mjs scripts/test-windows-release-identity.mjs scripts/test-windows-package-workflow.mjs scripts/test-release-promotion.mjs - + npm-ci: "false" + # Before npm ci, so a build identity regression fails before any npm or Cargo work. + - name: Test offline dictation evaluation tooling and Windows build identity gates + run: | + python -m unittest discover -s scripts/dictation_eval -p 'test_*.py' + node --test scripts/test-ci-build-identity.mjs scripts/test-windows-release-identity.mjs scripts/test-windows-package-workflow.mjs scripts/test-release-promotion.mjs - name: Install npm dependencies run: npm ci - - # The whole Node test suite on a Windows checkout (#357), not only the - # build identity gates above (which stay first so they fail before any - # npm or Cargo work). Platform-only cases skip themselves. - - name: Test frontend suite - run: npm run test:frontend - - - name: Unit-test frontend modules - run: npm run test:unit - - # The checkout has CRLF line endings (core.autocrlf); prettier.config.js - # accepts them on Windows, test-macos holds the repository to LF. - - name: Check frontend formatting - run: npm run format:check - + # The whole Node suite on a Windows checkout (#357). The checkout has CRLF line endings + # (core.autocrlf); prettier.config.js accepts them here, test-macos holds the repo to LF. + - name: Frontend checks (Node tests, unit tests, format) + run: | + npm run test:frontend + npm run test:unit + npm run format:check - name: Build frontend run: npm run build - - - name: Cargo check - working-directory: src-tauri - run: cargo check - - - name: Cargo test - working-directory: src-tauri - run: cargo test - - - name: Cargo clippy - working-directory: src-tauri - run: cargo clippy --all-targets -- -D warnings - - # Separate invocations preserve the core no-features build; combining - # package flags would unify features with the CLI defaults. - - name: Cargo test (sagascript-core, no features) - working-directory: src-tauri - run: cargo test -p sagascript-core - - - name: Cargo test (sagascript-cli) - working-directory: src-tauri - run: cargo test -p sagascript-cli - - - name: Cargo clippy (sagascript-core, no features) - working-directory: src-tauri - run: cargo clippy -p sagascript-core --all-targets -- -D warnings - - - name: Cargo clippy (sagascript-cli) - working-directory: src-tauri - run: cargo clippy -p sagascript-cli --all-targets -- -D warnings - - # The ONNX Runtime engine host ships for Windows, and its golden protocol - # replay moves the POSIX vector paths (/vectors/...) under an absolute - # Windows directory (#367), which only a Windows run exercises. ort's - # `load-dynamic` means no ONNX Runtime binary is needed. - - name: Cargo test (ORT engine host) - working-directory: src-tauri/engine-host/ort - run: cargo test --locked + - name: Cargo gates + run: node scripts/ci/cargo-gates.mjs --os windows build-windows: needs: scope if: ${{ needs.scope.result == 'success' && needs.scope.outputs.docs_only != 'true' }} runs-on: windows-latest timeout-minutes: 60 - steps: - - name: Checkout - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0 - - - name: Install Rust - uses: dtolnay/rust-toolchain@e2a55d2ffb04f378e9626c28d38b36d230d1e12f # v1; toolchain must match rust-toolchain.toml - with: - toolchain: 1.99.0 - - - name: Install Node.js - uses: actions/setup-node@249970729cb0ef3589644e2896645e5dc5ba9c38 # v6.5.0 + - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0 + - name: Set up toolchain + uses: ./.github/actions/setup with: - node-version: 22 - cache: npm - - - name: Cache Rust dependencies - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 - with: - workspaces: src-tauri - shared-key: sagascript-ci-native - - - name: Cache downloaded models - uses: actions/cache@caa296126883cff596d87d8935842f9db880ef25 # v5.1.0 - with: - path: ~\AppData\Roaming\Sagascript\Models - key: windows-models-${{ hashFiles('src-tauri/crates/sagascript-core/src/settings/manager.rs', 'src-tauri/crates/sagascript-core/src/transcription/model.rs', 'src-tauri/crates/sagascript-core/src/diarization/model.rs') }} - restore-keys: | - windows-models- - - - name: Install npm dependencies - run: npm ci - + cache-key: sagascript-ci-native + models: "true" - name: Build frontend run: npm run build - - # PR validation needs the native release binary for the smoke tests, not - # an installer. Full packaging, signing and notarization remain gated in - # release.yml, so skipping bundling here removes duplicate work without - # weakening native Windows coverage. - + # The smoke tests need the binary, not an installer; packaging stays gated in release.yml. - name: Build native release binary (no installer) run: npx tauri build --config scripts/tauri-ci-prebuilt.json --no-bundle - - - name: Verify native release binary - run: | - $binary = "src-tauri\target\release\sagascript.exe" - if (-not (Test-Path -PathType Leaf $binary)) { - throw "Missing native release binary: $binary" - } - - # --- CLI smoke tests (Windows / PowerShell) --- - - - name: Smoke test — help and version - run: | - $env:BINARY = "src-tauri\target\release\sagascript.exe" - & $env:BINARY --help - & $env:BINARY transcribe --help - & $env:BINARY record --help - - - name: Smoke test — config management - run: | - $env:BINARY = "src-tauri\target\release\sagascript.exe" - & $env:BINARY config list - & $env:BINARY config get language - & $env:BINARY config set language sv - $result = & $env:BINARY config get language - if ($result -notmatch "sv") { throw "Expected 'sv'" } - & $env:BINARY config reset language - $result = & $env:BINARY config get language - if ($result -notmatch "en") { throw "Expected 'en'" } - & $env:BINARY config path - - - name: Smoke test — model listing and formats - run: | - $env:BINARY = "src-tauri\target\release\sagascript.exe" - & $env:BINARY list-models - & $env:BINARY formats - - - name: Smoke test — shell completions - run: | - $env:BINARY = "src-tauri\target\release\sagascript.exe" - & $env:BINARY completions powershell | Out-Null - - # The two model smoke steps stay non-blocking (#357), but no longer - # silently: a step's `conclusion` is reported AFTER continue-on-error, so - # it read "success" while the transcription actually died after - # "Encoding audio..." in about half of the runs since 2026-10-09 (masked - # exit 1; suspected non-portable whisper.cpp build in the shared native - # cache). The report step below turns every masked failure into a - # warning annotation and a job-summary line until the cause is fixed. - - name: Smoke test — download model and transcribe - id: smoke_transcribe - continue-on-error: true - run: | - $env:BINARY = "src-tauri\target\release\sagascript.exe" - & $env:BINARY download-model nb-whisper-tiny - $result = & $env:BINARY transcribe --language no --model nb-whisper-tiny test-audio/norwegian-short-3s.mp3 - Write-Host "Transcription: $result" - if ($result -notmatch "(?i)stortinget") { throw "Expected 'stortinget' in output" } - - - name: Smoke test — transcribe with JSON output - id: smoke_json + - name: Smoke test, tier 1 (no model) + run: node scripts/ci/smoke-cli.mjs basic + # Tier 2 is the only non-blocking step (#357): the transcription dies in about half of the + # runs. continue-on-error masks the step's `conclusion`, so the report step below turns its + # real `outcome` into a warning annotation and a job-summary line. + - name: Smoke test, tier 2 (model download, transcription, JSON and JSONL streams) + id: smoke_model continue-on-error: true - shell: cmd - run: | - set BINARY=src-tauri\target\release\sagascript.exe - %BINARY% transcribe --json --language no --model nb-whisper-tiny test-audio/norwegian-short-3s.mp3 > json_output.txt 2>nul - type json_output.txt - findstr /C:"text" json_output.txt - findstr /C:"duration_seconds" json_output.txt - findstr /C:"language" json_output.txt - + run: node scripts/ci/smoke-cli.mjs model - name: Report masked smoke test failures if: ${{ always() }} shell: bash env: - TRANSCRIBE_OUTCOME: ${{ steps.smoke_transcribe.outcome }} - JSON_OUTCOME: ${{ steps.smoke_json.outcome }} + MODEL_OUTCOME: ${{ steps.smoke_model.outcome }} run: | - for pair in "download model and transcribe=$TRANSCRIBE_OUTCOME" "transcribe with JSON output=$JSON_OUTCOME"; do - name=${pair%%=*} - outcome=${pair##*=} - echo "Windows smoke test — $name: $outcome" >> "$GITHUB_STEP_SUMMARY" - if [ "$outcome" != success ]; then - echo "::warning title=Windows smoke test failed (non-blocking)::Smoke test — $name: $outcome (masked by continue-on-error, see #357)" - fi - done + echo "Windows smoke test, tier 2 (model): $MODEL_OUTCOME" >> "$GITHUB_STEP_SUMMARY" + [ "$MODEL_OUTCOME" = success ] || echo "::warning title=Windows smoke test failed (non-blocking)::Tier 2 (model): $MODEL_OUTCOME, masked by continue-on-error (#357)" check-windows: needs: [scope, test-windows, build-windows] @@ -956,21 +342,8 @@ jobs: runs-on: ubuntu-24.04 timeout-minutes: 20 env: - SCOPE_RESULT: ${{ needs.scope.result }} - SCOPE_TRUSTED: ${{ needs.scope.outputs.scope_trusted }} - DOCS_ONLY: ${{ needs.scope.outputs.docs_only }} - TEST_RESULT: ${{ needs.test-windows.result }} - BUILD_RESULT: ${{ needs.build-windows.result }} + NEEDS: ${{ toJSON(needs) }} steps: + - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0 - name: Assert Windows CI lanes passed - run: | - test "$SCOPE_RESULT" = success - test "$SCOPE_TRUSTED" = true - if [ "$DOCS_ONLY" = true ]; then - test "$TEST_RESULT" = skipped - test "$BUILD_RESULT" = skipped - else - test "$DOCS_ONLY" = false - test "$TEST_RESULT" = success - test "$BUILD_RESULT" = success - fi + run: node scripts/ci/assert-lanes.mjs diff --git a/.github/workflows/windows-package.yml b/.github/workflows/windows-package.yml index 49159201..0cb43f4b 100644 --- a/.github/workflows/windows-package.yml +++ b/.github/workflows/windows-package.yml @@ -251,19 +251,30 @@ jobs: npm run build npm run check + # Ownership of the Windows tests (#358): ci.yml's test-windows, a required check, owns the + # x64 tests of every pull request and runs these same workspace commands + # (scripts/ci/cargo-gates.mjs), so x64 does not repeat them on a pull request. They stay for + # the native ARM64 runner, the only place the tests execute on ARM64, and for releases + # (workflow_call from prebuild-release.yml) and manual runs, whose candidates are gated here. - name: Test and lint Rust workspace + if: github.event_name != 'pull_request' || matrix.architecture == 'arm64' working-directory: src-tauri run: | # `cargo check --workspace` is omitted: clippy --all-targets below is a strict superset. cargo test --workspace cargo clippy --workspace --all-targets -- -D warnings - cargo build -p sagascript-cli --no-default-features + + # The debug tree the CPU policy checks below read, on every architecture. + - name: Build lean CLI (debug) + working-directory: src-tauri + run: cargo build -p sagascript-cli --no-default-features - name: Gate native Windows clipboard transactions shell: pwsh run: | # The host triple was verified above. Reuse the workspace test binary - # instead of recompiling a duplicate explicit-target debug tree. + # instead of recompiling a duplicate explicit-target debug tree (on an x64 + # pull request there is none: this step then compiles the app's test binary). cargo test --manifest-path src-tauri/Cargo.toml -p sagascript native_runner_clipboard_transaction_smoke -- --ignored --test-threads=1 if ($LASTEXITCODE -ne 0) { throw "Native clipboard transaction gate failed" } diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 1323849a..6a05a939 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -82,12 +82,12 @@ npm run verify -- --list # every check with its commands, budget | Area (`--only`) | Checks | What runs (working directory) | Needs | CI lane | |---|---|---|---|---| | `fmt` | `fmt`, `fmt-ort` | `cargo fmt --all -- --check` and `cargo fmt --manifest-path engine-host/ort/Cargo.toml --all -- --check` (`src-tauri`) | Rust | test-linux | -| `clippy` | `clippy-workspace`, `clippy-cli-lean`, `clippy-policy` (Linux); `clippy-app` (fast set) | `cargo clippy --workspace --all-targets -- -D warnings`; `cargo clippy -p sagascript-cli --all-targets --no-default-features -- -D warnings` (`src-tauri`); `node scripts/clippy-policy.mjs` (policy counts, see below) | Rust; Linux libraries; `dist/` (workspace, app, policy) | test-macos, test-linux, test-windows (per crate); policy: test-linux | +| `clippy` | `clippy-workspace`, `clippy-cli-lean`, `clippy-policy` (Linux); `clippy-app` (fast set) | `cargo clippy --workspace --all-targets -- -D warnings`; `cargo clippy -p sagascript-cli --all-targets --no-default-features -- -D warnings` (`src-tauri`); `node scripts/clippy-policy.mjs` (policy counts, see below) | Rust; Linux libraries; `dist/` (workspace, app, policy) | workspace and lean CLI: test-macos, test-linux, test-windows; policy: test-linux | | `rust-deps` | `cli-lean-deps` | `cargo tree -p sagascript-cli --no-default-features -e normal` contains no cpal/ALSA | Rust | test-linux | -| `rust-test` | `test-workspace`, `test-cli-lean` | `cargo test --workspace`; `cargo test -p sagascript-cli --no-default-features` | Rust; Linux libraries; `dist/` (workspace) | workspace: test-macos, test-windows (`cargo test` and per crate); lean CLI: test-macos, test-linux | -| `core-features` | `core-clippy`, `core-clippy-diarization`, `core-test`, `core-test-record`, `core-test-diarization` | `cargo clippy`/`cargo test -p sagascript-core` without features, with `record`, with `diarization` | Rust | test-macos, test-linux, test-windows | -| `engine-protocol` | `engine-protocol` | `cargo test -p sagascript-engine-protocol` | Rust | test-linux | -| `ort-host` | `ort-host-clippy`, `ort-host-test` | `cargo clippy --all-targets --locked -- -D warnings`, `cargo test --locked` (`src-tauri/engine-host/ort`) | Rust | test-linux; tests also test-windows | +| `rust-test` | `test-workspace`, `test-cli-lean` | `cargo test --workspace`; `cargo test -p sagascript-cli --no-default-features` | Rust; Linux libraries; `dist/` (workspace) | workspace and lean CLI: test-macos, test-linux, test-windows | +| `core-features` | `core-clippy`, `core-clippy-diarization`, `core-test`, `core-test-record`, `core-test-diarization` | `cargo clippy`/`cargo test -p sagascript-core` without features, with `record`, with `diarization` | Rust | without features: test-macos, test-linux, test-windows; with a feature: test-linux | +| `engine-protocol` | `engine-protocol` | `cargo test -p sagascript-engine-protocol` | Rust | every test lane, through `cargo test --workspace` | +| `ort-host` | `ort-host-clippy`, `ort-host-test` | `cargo clippy --all-targets --locked -- -D warnings`, `cargo test --locked` (`src-tauri/engine-host/ort`) | Rust | test-linux, test-windows | | `deps` | `deny`, `npm-audit` | `cargo deny --locked --workspace check` and the same for the ORT host (`src-tauri`; policy and what to do about a new advisory: `src-tauri/deny.toml`); `npm audit --audit-level=high` (root lockfile). Both fetch advisory data over the network | `cargo install --locked cargo-deny --version 0.20.2` | deny: test-linux; audit: test-macos | | `release-guard` | `release-guard` | `cargo test --release -p sagascript-core --lib dev_overrides` | Rust | test-linux | | `svelte-check` | `svelte-check` | `npx svelte-check --tsconfig ./tsconfig.json` | `npm ci` | test-macos | @@ -101,9 +101,10 @@ npm run verify -- --list # every check with its commands, budget | `msrv` | `msrv` (opt-in; run it for dependency bumps or newer language features) | `cargo check --workspace --locked`, the lean CLI and the ORT host on the `rust-version` toolchain | `rustup toolchain install 1.88 --profile minimal`; `dist/` | msrv-linux | | `browser` | `browser` (opt-in) | `npm run test:browser` (Playwright against the mocked IPC) | `npx playwright install chromium` | test-macos | -CI splits the cargo commands per crate and feature set; `--list` shows which CI commands each -check stands for, and `scripts/test-verify-ci-parity.mjs` fails when ci.yml's test lanes and -this list drift apart. Not part of verify: release builds and CLI smoke tests (build-macos, +CI keeps its cargo commands in one list, `scripts/ci/cargo-gates.mjs`, with a reason for every gate that +is not run on all three OSes; `--list` shows which CI commands each check stands for, and +`scripts/test-verify-ci-parity.mjs` fails when ci.yml's test lanes (gate list and setup action +included) and this list drift apart. Not part of verify: release builds and CLI smoke tests (build-macos, build-windows), engine-host staging, the DER/JER cross-check against a downloaded scorer, and the release checks `npm run release:check` and `npm run licenses:check`. Where every suite lives is in [docs/ARCHITECTURE.md](docs/ARCHITECTURE.md#5-where-tests-live-and-how-ci-runs-them). diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index cef19cd9..3589e55e 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -381,11 +381,11 @@ Shared fixtures: `sagascript_testsupport` (`TempDir`, `fake_settings`, `sha256_h | Layer | Location | Local command | CI | |---|---|---|---| -| Rust unit, app | sibling `foo_tests.rs` files, declared last in `foo.rs` as `#[cfg(test)] #[path = "foo_tests.rs"] mod tests;` (`commands_tests.rs`, `main_tests.rs`, `meeting_jobs_tests.rs`, `paste/windows_clipboard_tests.rs`); inline `#[cfg(test)]` modules in smaller files | `cargo test` (app only) | macOS, Windows | -| Rust unit, core/CLI/protocol | inline, or sibling `foo_tests.rs` as in the app: core `download_tests.rs`, `settings/manager_tests.rs`, `settings/store_tests.rs`, `transcription/whisper_backend_tests.rs`; CLI `transcribe_tests.rs`, `meeting_reprocessing_tests.rs`; `sagascript-engine-protocol/src/tests.rs` | `cargo test -p `; release-only guard: `cargo test --release -p sagascript-core --lib dev_overrides` | per-crate steps; protocol and the release guard in test-linux | +| Rust unit, app | sibling `foo_tests.rs` files, declared last in `foo.rs` as `#[cfg(test)] #[path = "foo_tests.rs"] mod tests;` (`commands_tests.rs`, `main_tests.rs`, `meeting_jobs_tests.rs`, `paste/windows_clipboard_tests.rs`); inline `#[cfg(test)]` modules in smaller files | `cargo test` (app only), `cargo test --workspace` (everything) | every OS lane (`cargo test --workspace`) | +| Rust unit, core/CLI/protocol | inline, or sibling `foo_tests.rs` as in the app: core `download_tests.rs`, `settings/manager_tests.rs`, `settings/store_tests.rs`, `transcription/whisper_backend_tests.rs`; CLI `transcribe_tests.rs`, `meeting_reprocessing_tests.rs`; `sagascript-engine-protocol/src/tests.rs` | `cargo test -p `; release-only guard: `cargo test --release -p sagascript-core --lib dev_overrides` | `cargo test --workspace` and the per-crate feature-subset gates on every OS; the release guard in test-linux | | Rust integration | `sagascript-core/tests/*.rs` (engine host against the fake host, chunk-merge golden, Pianissimo adapter); `sagascript-cli/tests/*.rs` (spawn `CARGO_BIN_EXE_sagascript`) | `cargo test -p sagascript-core`, `-p sagascript-cli` | macOS, Linux, Windows | -| ORT host | `src-tauri/engine-host/ort/src/**` | `cargo test` and `cargo clippy --all-targets -- -D warnings` in `engine-host/ort/` | test-linux (fmt, test, clippy); test-windows (test) | -| Engine-host protocol | golden transcripts `src-tauri/engine-host/test-vectors/protocol/*.jsonl` with the scripted `fake-engine.json` ([contract](engine-host-protocol.md#golden-transcripts)); replayed by `sagascript-engine-protocol/src/vector_tests.rs`, `engine-host/ort/src/server/golden_tests.rs` and `ProtocolServerTests.swift` | `cargo test -p sagascript-engine-protocol`; `cargo test --locked` in `engine-host/ort/`; `swift test` | test-linux (Rust, ORT host); test-windows (ORT host); test-macos (Swift) | +| ORT host | `src-tauri/engine-host/ort/src/**` | `cargo test` and `cargo clippy --all-targets -- -D warnings` in `engine-host/ort/` | test-linux (fmt, test, clippy); test-windows (test, clippy) | +| Engine-host protocol | golden transcripts `src-tauri/engine-host/test-vectors/protocol/*.jsonl` with the scripted `fake-engine.json` ([contract](engine-host-protocol.md#golden-transcripts)); replayed by `sagascript-engine-protocol/src/vector_tests.rs`, `engine-host/ort/src/server/golden_tests.rs` and `ProtocolServerTests.swift` | `cargo test -p sagascript-engine-protocol`; `cargo test --locked` in `engine-host/ort/`; `swift test` | `cargo test --workspace` in every OS lane (protocol crate); test-linux, test-windows (ORT host); test-macos (Swift) | | MSRV | whole workspace, lean CLI, ORT host | `RUSTUP_TOOLCHAIN=1.88 cargo check --workspace --locked` (see CONTRIBUTING.md) | msrv-linux | | Swift | `src-tauri/engine-host/coreml/Tests/EngineHostCoreTests/` | `scripts/build-engine-host.sh`, then `swift test` (`npm run verify -- --only swift`) | test-macos | | Frontend unit and component (Vitest) | `src/**/*.test.ts` next to the module or component, in happy-dom; typed IPC mock, setup and component backends in `src/test/` ([docs/testing/frontend.md](testing/frontend.md)) | `npm run test:unit` (`test:unit:coverage` for a report) | test-macos, test-windows; release-quality-gate, windows-package | @@ -397,29 +397,38 @@ Shared fixtures: `sagascript_testsupport` (`TempDir`, `fake_settings`, `sha256_h | Transport contract | Rust: `transport_contract*` tests in `events_tests.rs`, `commands_tests.rs`, `meeting_jobs_tests.rs` and `contract/` write or compare `contracts/transport/*.json` and `src/lib/ipc-contract.generated.ts`; TS: `src/test/transport.test.ts`, `src/lib/api.test.ts` ([contracts/README.md](../contracts/README.md)) | `cargo test contract` in `src-tauri/` (`SAGASCRIPT_BLESS_CONTRACTS=1` regenerates), `npm run test:unit`, svelte-check | with the app tests and the frontend suites | | Formatting | Prettier over `src/`, `scripts/`, `tests/` (not `.svelte` yet; `.prettierignore`) | `npm run format:check` (`npm run format` fixes) | test-macos, test-windows | | Lint | ESLint over `src/` (TypeScript, Svelte), `scripts/`, `tests/` (`eslint.config.js`, baseline in `eslint-suppressions.json`) | `npm run lint` | test-macos | -| CLI smoke | release binary in `ci.yml` build jobs: help, `config`, `list-models`, `formats`, completions, download + transcribe, `--json` streams (`scripts/verify-json-cli-streams.py`) | — | build-macos, build-windows | +| CLI smoke | release binary in `ci.yml` build jobs, `scripts/ci/smoke-cli.mjs` (one script for every OS; `basic` tier without a model: help, `config`, `list-models`, `formats`, completions of every shell; `model` tier: download + transcribe, `--json`, `--jsonl`, a closed pipe and the error object through `scripts/verify-json-cli-streams.py`) | `node scripts/ci/smoke-cli.mjs basic --binary ` | build-macos, build-windows (Windows tolerates the `model` tier, #357) | | Conformance | `scripts/engine-host-conformance.py --no-model` against a real host process; with `--model-dir ` the full run | model-free: `npm run verify -- --only engine-host-conformance` (macOS); full run by hand with a downloaded model | model-free subset: the staged Core ML host in test-macos and the signed host in `release-build-macos.yml`; full run (release gate, each time right after a Pianissimo smoke): `release-build-macos.yml` against the host in the installed signed app (prebuild and fallback paths), `scripts/verify-release-draft.sh` against the host in the downloaded draft (promote path in CI, fallback path by hand) and the manual `test-build.yml` ([spec](engine-host-protocol.md#conformance)) | -CI (`.github/workflows/ci.yml`) splits cargo per crate and feature set: test-linux runs both -`cargo fmt --check` gates, `cargo deny check` for both Cargo workspaces (`src-tauri/deny.toml`: -advisories, licenses, a single `reqwest`, allowed sources), app `check`/`clippy`, the CLI with -and without default features, core with no features and with `record`, the protocol crate, ORT -host test/clippy, the release-profile guard, the no-ALSA tree check and the clippy policy -counts (`scripts/clippy-policy.mjs`, which must not rise); msrv-linux runs `cargo -check --workspace --locked`, the lean CLI and the ORT host on the `rust-version` toolchain -(1.88), and check-linux requires it; test-macos runs `npm audit --audit-level=high`, app -`check`/`test`/`clippy`, core with no features and with `diarization`, the CLI both ways, Swift -tests and the frontend suites (Prettier, ESLint, Vitest, Node); test-windows runs the same -frontend suites except ESLint, app, core and CLI tests and clippy, and the ORT host tests. -`cargo test --workspace` (and the Vitest suite) runs in `windows-package.yml` (on PRs touching -`src/`, `src-tauri/`, `scripts/`, package files, `.gitattributes` or the workflow itself) and -`release-quality-gate.yml`. A PR that only touches +CI (`.github/workflows/ci.yml`) has a test lane per OS (test-macos, test-linux, test-windows), a native +build lane with the CLI smoke tests on macOS and Windows, an MSRV lane on Linux, and the `check-*` +aggregators that branch protection reads (`check-macos` and `check-windows` are required). The parts +every lane shares live once, outside the workflow (#358): `.github/actions/setup` (Python, the pinned +Rust toolchain, Node.js, the Rust cache, `npm ci`), `scripts/ci/cargo-gates.mjs` (the one list of cargo +commands, run by every lane with `--os`; the MSRV lane runs its `msrv` profile), `scripts/ci/smoke-cli.mjs` +and `scripts/ci/assert-lanes.mjs` (what an aggregator asserts about the lanes it needs). The same cargo +commands run on every OS except the gates marked `os:` in the list, each with its reason: formatting, +`cargo deny`, the core feature subsets (`diarization`, `record`), the lean-CLI build for the Python +suites, the ALSA tree check and the release-profile guard run on Linux only, the ORT host gates on +Linux and Windows. Everything else is `cargo clippy` and `cargo test` over the workspace, over core +and over the lean CLI (`--no-default-features`). Besides the cargo gates test-linux runs the clippy policy +counts (`scripts/clippy-policy.mjs`, which must not rise) and the offline diarization evaluation; +msrv-linux runs `cargo check --workspace --locked`, the lean CLI and the ORT host on the `rust-version` +toolchain (1.88), and check-linux requires it; test-macos runs `npm audit --audit-level=high`, the +frontend suites (Prettier, ESLint, Vitest, Node), Svelte types, the Swift tests and the engine-host +conformance subset; test-windows runs the same frontend suites except ESLint and the Windows build +identity gates. test-windows owns the x64 Windows tests of every pull request: `windows-package.yml` +(on PRs touching `src/`, `src-tauri/`, `scripts/`, package files, `.gitattributes` or the workflow +itself) repeats `cargo test --workspace` and clippy only on its native ARM64 runner on a pull request, +and on both architectures for releases (`prebuild-release.yml`) and manual runs; +`release-quality-gate.yml` runs them for a release. A PR that only touches `README.md`, `docs/release-readiness.md` or `docs/research/**/*.md` skips the native lanes (`scripts/ci-docs-scope.mjs`). Locally, `npm run verify` runs the test and lint suites above (`scripts/verify.mjs`, documented in [CONTRIBUTING.md](../CONTRIBUTING.md#running-tests)); -`scripts/test-verify-ci-parity.mjs` keeps its check list in step with ci.yml's test lanes. -**Planned:** [#358](https://github.com/Magnus-Gille/sagascript/issues/358) lets CI call the -verify groups directly; [#361](https://github.com/Magnus-Gille/sagascript/issues/361) phase B +`scripts/test-verify-ci-parity.mjs` keeps its check list in step with the commands the test lanes run, +gate list and setup action included. The `scripts/test-ci-*.mjs` tests read the workflows as parsed +YAML (`scripts/lib/ci-workflow.mjs`), not as text. +**Planned:** [#361](https://github.com/Magnus-Gille/sagascript/issues/361) phase B replaces the remaining source-text tests in the `ui` group with component tests. ## 6. Known drift and planned changes diff --git a/docs/adr/0001-cli-first.md b/docs/adr/0001-cli-first.md index 7fdf2631..193e683b 100644 --- a/docs/adr/0001-cli-first.md +++ b/docs/adr/0001-cli-first.md @@ -22,7 +22,7 @@ The code follows it structurally. The desktop binary dispatches any subcommand t `sagascript_cli::run`; a headless build without Tauri exists for Linux and batch use (`cargo build -p sagascript-cli --no-default-features`); even GUI recovery has CLI entry points (`sagascript open`, `sagascript update`). CI exercises the release binary through the -CLI (`config`, `list-models`, `transcribe --json`, in `.github/workflows/ci.yml`), and +CLI (`config`, `list-models`, `transcribe --json`, in `scripts/ci/smoke-cli.mjs`, which `.github/workflows/ci.yml` runs), and `valid_keys_count_matches_settings_struct` in `crates/sagascript-cli/src/config.rs` fails when a `Settings` field has no `config` key. diff --git a/docs/recipes/add-cli-subcommand.md b/docs/recipes/add-cli-subcommand.md index d1ce7d09..f6a746f4 100644 --- a/docs/recipes/add-cli-subcommand.md +++ b/docs/recipes/add-cli-subcommand.md @@ -134,11 +134,13 @@ cargo run -q -p sagascript-cli -- example --json ## 6. CI smoke (model-free commands) -If the command is cheap and needs no model, add it to the "Tier 1: CLI smoke tests" steps -of `build-macos` (bash) and the matching steps of `build-windows` (PowerShell) in -`.github/workflows/ci.yml`. Some `scripts/test-ci-*.mjs` files assert on `ci.yml` text, so -run `npm run test:frontend` after editing it (tests in both the `ci` and `release` groups read -workflow files). +If the command is cheap and needs no model, add it to the `basic` tier of +`scripts/ci/smoke-cli.mjs`: one cross-platform script that `build-macos` and `build-windows` +in `.github/workflows/ci.yml` both run against the release binary (`node scripts/ci/smoke-cli.mjs +basic --binary ` runs it locally). `scripts/test-ci-smoke-cli.mjs` lists the commands each +tier runs, so extend it with the new one. The `scripts/test-ci-*.mjs` tests read the workflows as +parsed YAML; run `npm run test:frontend` after editing a workflow (tests in both the `ci` and +`release` groups read them). ## 7. Docs diff --git a/docs/windows-release.md b/docs/windows-release.md index c83c3b9c..a6a750c9 100644 --- a/docs/windows-release.md +++ b/docs/windows-release.md @@ -33,7 +33,9 @@ described as an official, signed, stable, or fully accepted Windows release. `.github/workflows/windows-package.yml` runs the release metadata, license, frontend, Rust, and real CPU-transcription gates on native `windows-latest` -(x64) and `windows-11-arm` (ARM64) runners. Each runner then +(x64) and `windows-11-arm` (ARM64) runners. (On a pull request the x64 Rust +tests and lints are `ci.yml`'s `test-windows`, a required check, so this +workflow repeats them only on ARM64; releases and manual runs run them on both.) Each runner then builds unsigned NSIS and MSI installers, a portable desktop executable, and a separate console CLI executable. It uploads them as one short-lived GitHub Actions artifact and never creates or modifies a GitHub Release. Its artifacts diff --git a/eslint-suppressions.json b/eslint-suppressions.json index 7caf4f33..e601ac6c 100644 --- a/eslint-suppressions.json +++ b/eslint-suppressions.json @@ -35,11 +35,6 @@ "count": 9 } }, - "scripts/test-ci-parallel-gates.mjs": { - "no-useless-escape": { - "count": 4 - } - }, "scripts/test-code-health-producer.mjs": { "no-control-regex": { "count": 1 diff --git a/scripts/ci/assert-lanes.mjs b/scripts/ci/assert-lanes.mjs new file mode 100644 index 00000000..0cbd98b9 --- /dev/null +++ b/scripts/ci/assert-lanes.mjs @@ -0,0 +1,52 @@ +#!/usr/bin/env node +// The body of the check-macos, check-windows and check-linux aggregators of +// .github/workflows/ci.yml (#358). Branch protection requires check-macos and +// check-windows by name; each job `needs:` the scope job and its OS's lanes and +// runs this script with `NEEDS: ${{ toJSON(needs) }}`. +// +// A lane that did not run is only acceptable when the scope job trusted its own +// classification and found nothing but documentation changed; in every other case +// every lane must have succeeded. Everything else fails closed: a missing or +// malformed input, an untrusted or unsuccessful scope, an unknown docs_only value, +// a lane that failed, was cancelled or was skipped on a full run, and an aggregator +// that waits for no lane at all. +import { pathToFileURL } from "node:url"; + +/** Returns the problems with a `needs` context; an empty list means the lanes are accepted. */ +export function laneProblems(needs) { + if (needs === null || typeof needs !== "object" || Array.isArray(needs)) return ["NEEDS must be the JSON object of the needs context"]; + const problems = []; + const scope = needs.scope; + if (scope?.result !== "success") problems.push(`scope: result is ${JSON.stringify(scope?.result)}, expected "success"`); + if (scope?.outputs?.scope_trusted !== "true") + problems.push(`scope: scope_trusted is ${JSON.stringify(scope?.outputs?.scope_trusted)}, expected "true"`); + const lanes = Object.entries(needs).filter(([id]) => id !== "scope"); + if (lanes.length === 0) problems.push("no lane job in needs: this aggregator would gate nothing"); + const docsOnly = scope?.outputs?.docs_only; + if (docsOnly !== "true" && docsOnly !== "false") { + problems.push(`scope: docs_only is ${JSON.stringify(docsOnly)}, expected "true" or "false"`); + } else { + const expected = docsOnly === "true" ? "skipped" : "success"; + for (const [id, lane] of lanes) { + if (lane?.result !== expected) + problems.push(`${id}: result is ${JSON.stringify(lane?.result)}, expected ${JSON.stringify(expected)} (docs_only=${docsOnly})`); + } + } + return problems; +} + +function main() { + let needs; + try { + needs = JSON.parse(process.env.NEEDS ?? ""); + } catch (error) { + console.error(`NEEDS is not valid JSON: ${error.message}`); + return 1; + } + const problems = laneProblems(needs); + for (const [id, entry] of Object.entries(needs ?? {})) console.log(`${id}: ${entry?.result}`); + for (const problem of problems) console.error(`::error title=CI lanes::${problem}`); + return problems.length === 0 ? 0 : 1; +} + +if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) process.exit(main()); diff --git a/scripts/ci/cargo-gates.mjs b/scripts/ci/cargo-gates.mjs new file mode 100644 index 00000000..5c2d9bfa --- /dev/null +++ b/scripts/ci/cargo-gates.mjs @@ -0,0 +1,257 @@ +#!/usr/bin/env node +// The one list of cargo gates of CI (#358). Every lane of .github/workflows/ci.yml +// runs `node scripts/ci/cargo-gates.mjs --os ` instead of +// writing the cargo commands out per OS, and the MSRV lane runs it with +// `--profile msrv`. scripts/test-ci-cargo-gates.mjs and +// scripts/test-verify-ci-parity.mjs read GATES from here, so the list cannot +// drift between the OS lanes, `npm run verify` (scripts/verify.mjs) and the tests. +// +// A gate runs on every OS unless it has an `os` list, and every `os` list says +// why in `why`: that is the ONE place the per-OS exclusions are documented. +// The rule: a gate whose subject is platform independent (source formatting, a +// Cargo feature subset, a dependency-graph assertion, a build-profile guard) +// runs once, on Linux, the cheapest runner. Everything that executes OS-specific +// code (the workspace tests and lints, the lean batch CLI, the ONNX host where +// it ships) runs on every OS it applies to. +// +// Why a plain `cargo clippy --workspace` / `cargo test --workspace` instead of a +// bare `cargo test` plus `-p sagascript-cli`: the workspace gate is a superset +// (app, CLI with its default features, protocol crate, test support) and, with +// resolver 2, builds core with exactly the features the app and CLI default to. +// The feature subsets below it (core alone, core with one feature, the lean CLI) +// stay separate invocations: combining package flags would unify their features. +// `cargo check` is not listed for the main profile because `cargo clippy +// --all-targets` compiles everything `cargo check` would, and more strictly. +// +// node scripts/ci/cargo-gates.mjs --os linux run, stop at the first failure +// node scripts/ci/cargo-gates.mjs --os linux --keep-going run all, fail at the end +// node scripts/ci/cargo-gates.mjs --os windows --list print the commands, run nothing +// node scripts/ci/cargo-gates.mjs --profile msrv needs RUSTUP_TOOLCHAIN (the lane sets it) +import { spawn } from "node:child_process"; +import { appendFileSync } from "node:fs"; +import { dirname, join } from "node:path"; +import { fileURLToPath, pathToFileURL } from "node:url"; + +export const REPO_ROOT = dirname(dirname(dirname(fileURLToPath(import.meta.url)))); +export const OSES = ["macos", "linux", "windows"]; +export const PROFILES = ["main", "msrv"]; +const OS_OF_PLATFORM = { darwin: "macos", linux: "linux", win32: "windows" }; + +const APP = "src-tauri"; +const ORT = "src-tauri/engine-host/ort"; +const LINUX = ["linux"]; +const gate = (id, cwd, args, extra = {}) => ({ id, cwd, args, profile: "main", ...extra }); + +// The release-profile guard exists only without debug assertions, and cargo +// test exits 0 when its name filter matches nothing: prove the test ran. +export const RELEASE_GUARD_PROOF = /release_build_ignores_the_real_environment \.\.\. ok/; + +/** + * Each gate: { id, cwd (repository relative), args (cargo arguments), profile, + * os? + why? (restriction and its reason), forbidOutput?/requireOutput? + * ({ pattern, message }), quiet? (show the output only on failure) }. + */ +export const GATES = [ + // Dependency policy (#355), before any compilation: RustSec advisories, licenses, banned or duplicated + // crates and allowed sources, as set in src-tauri/deny.toml, for both Cargo workspaces. The advisory + // database is fetched at run time. The lane installs cargo-deny first (ci.yml pins the version); the ORT + // host run allows only the policy entries the main workspace alone uses. + gate("deny", APP, ["deny", "--locked", "--workspace", "check"], { + os: LINUX, + why: "advisories, licenses and sources are properties of the lockfiles, not of the OS; deny.toml already judges every shipped target in one run", + }), + gate( + "deny-ort", + APP, + [ + ...["deny", "--locked", "--manifest-path", "engine-host/ort/Cargo.toml", "--workspace", "check"], + ...["-A", "license-not-encountered", "-A", "license-exception-not-encountered", "-A", "unmatched-source"], + ], + { os: LINUX, why: "same as deny, for the ORT host's own lockfile" }, + ), + // Formatting next, so an unformatted PR fails before anything compiles. + gate("fmt", APP, ["fmt", "--all", "--", "--check"], { + os: LINUX, + why: "rustfmt reads source text only; the result is identical on every OS (#352)", + }), + gate("fmt-ort", APP, ["fmt", "--manifest-path", "engine-host/ort/Cargo.toml", "--all", "--", "--check"], { + os: LINUX, + why: "rustfmt reads source text only; the standalone ORT host is its own Cargo workspace (#352)", + }), + + gate("clippy-workspace", APP, ["clippy", "--workspace", "--all-targets", "--", "-D", "warnings"]), + gate("test-workspace", APP, ["test", "--workspace"]), + + // Feature subsets. Separate invocations preserve the no-features builds. + gate("clippy-core", APP, ["clippy", "-p", "sagascript-core", "--all-targets", "--", "-D", "warnings"]), + gate("test-core", APP, ["test", "-p", "sagascript-core"]), + gate("clippy-cli-lean", APP, ["clippy", "-p", "sagascript-cli", "--all-targets", "--no-default-features", "--", "-D", "warnings"]), + gate("test-cli-lean", APP, ["test", "-p", "sagascript-cli", "--no-default-features"]), + gate( + "clippy-core-diarization", + APP, + ["clippy", "-p", "sagascript-core", "--features", "diarization", "--all-targets", "--", "-D", "warnings"], + { + os: LINUX, + why: "core's `diarization` and `record` features alone are a Cargo feature-graph property, not OS code; the workspace gates already build and test core with both on every OS", + }, + ), + gate("test-core-diarization", APP, ["test", "-p", "sagascript-core", "--features", "diarization"], { + os: LINUX, + why: "same as clippy-core-diarization: a Cargo feature subset, not OS code", + }), + gate("test-core-record", APP, ["test", "-p", "sagascript-core", "--features", "record"], { + os: LINUX, + why: "same as clippy-core-diarization (needs the ALSA headers the Linux lane installs)", + }), + + // The ONNX Runtime engine host is a separate Cargo workspace (#357). + gate("clippy-ort", ORT, ["clippy", "--all-targets", "--locked", "--", "-D", "warnings"], { + os: ["linux", "windows"], + why: "the ORT host ships for Windows and Linux; macOS ships the Core ML host (Swift tests, test-macos)", + }), + gate("test-ort", ORT, ["test", "--locked"], { + os: ["linux", "windows"], + why: "the ORT host ships for Windows and Linux; macOS ships the Core ML host (Swift tests, test-macos)", + }), + + // The debug lean CLI the offline diarization evaluation tests drive + // (SAGASCRIPT_REFERENCE_CLI=target/debug/sagascript). test-cli-lean built it already. + gate("build-cli-lean", APP, ["build", "-p", "sagascript-cli", "--no-default-features"], { + os: LINUX, + why: "only the Linux lane runs the Python diarization suite that drives this binary", + }), + // Regression guard for the no-ALSA property (#66): the lean CLI's dependency tree + // holds neither cpal nor alsa. + gate("no-alsa", APP, ["tree", "-p", "sagascript-cli", "--no-default-features", "-e", "normal"], { + os: LINUX, + why: "a dependency-graph assertion; cargo resolves the same graph for cpal on every OS, ALSA itself only on Linux", + quiet: true, + forbidOutput: { pattern: /alsa|cpal/i, message: "cpal/ALSA leaked into the batch-only dependency tree" }, + }), + // Release-profile guard (#357): a release build ignores SAGASCRIPT_ENGINE_HOST and + // SAGASCRIPT_PIANISSIMO_MODEL_DIR. Compiled only without debug_assertions and + // the `dev-overrides` feature, so every debug test run leaves it out. + gate("release-guard", APP, ["test", "--release", "-p", "sagascript-core", "--lib", "dev_overrides"], { + os: LINUX, + why: "a build-profile property of core, not OS code; one release compile is enough", + requireOutput: { pattern: RELEASE_GUARD_PROOF, message: "the release-profile guard test did not run" }, + }), + + // Minimum supported Rust version (#353): `rust-version` is a promise to people + // building from source. The lane exports RUSTUP_TOOLCHAIN=. + gate("msrv-workspace", APP, ["check", "--workspace", "--locked"], { profile: "msrv" }), + gate("msrv-cli-lean", APP, ["check", "-p", "sagascript-cli", "--no-default-features", "--locked"], { profile: "msrv" }), + gate("msrv-ort", ORT, ["check", "--locked"], { profile: "msrv" }), +]; + +export const commandOf = (item) => ["cargo", ...item.args].join(" "); + +/** The gates of one lane, in run order. */ +export function gatesFor({ os, profile = "main" }) { + if (!OSES.includes(os)) throw new Error(`unknown OS ${JSON.stringify(os)}; expected one of ${OSES.join(", ")}`); + if (!PROFILES.includes(profile)) throw new Error(`unknown profile ${JSON.stringify(profile)}; expected one of ${PROFILES.join(", ")}`); + return GATES.filter((item) => item.profile === profile && (item.os === undefined || item.os.includes(os))); +} + +/** + * Runs the gates through `execute(gate) -> { code, output }`, stopping at the + * first failure unless `keepGoing`. A gate also fails when the output it must + * not contain appears, or the output it must contain is absent. + */ +export async function runGates(gates, { execute, keepGoing = false, log = () => {} }) { + const results = []; + for (const item of gates) { + const started = Date.now(); + log(`::group::${commandOf(item)} (${item.cwd})`); + const { code, output } = await execute(item); + let failure = code === 0 ? null : `exited with status ${code}`; + if (failure === null && item.forbidOutput?.pattern.test(output)) failure = item.forbidOutput.message; + if (failure === null && item.requireOutput && !item.requireOutput.pattern.test(output)) failure = item.requireOutput.message; + if (failure !== null && item.quiet) log(output); + log("::endgroup::"); + if (failure !== null) log(`::error title=Cargo gate failed::${commandOf(item)} (${item.cwd}) ${failure}`); + results.push({ gate: item, failure, seconds: (Date.now() - started) / 1000 }); + if (failure !== null && !keepGoing) break; + } + return results; +} + +function executeWithCargo(item) { + return new Promise((resolve, reject) => { + const watched = item.forbidOutput !== undefined || item.requireOutput !== undefined; + const child = spawn("cargo", item.args, { cwd: join(REPO_ROOT, item.cwd), stdio: watched ? ["inherit", "pipe", "pipe"] : "inherit" }); + let output = ""; + for (const [stream, sink] of [ + [child.stdout, process.stdout], + [child.stderr, process.stderr], + ]) { + stream?.on("data", (chunk) => { + output += chunk; + if (!item.quiet) sink.write(chunk); + }); + } + child.on("error", reject); + child.on("close", (code, signal) => resolve({ code: code ?? (signal ? 128 : 1), output })); + }); +} + +function parseArguments(argv) { + const options = { os: OS_OF_PLATFORM[process.platform], profile: "main", list: false, keepGoing: false, explicitOs: false }; + for (let index = 0; index < argv.length; index += 1) { + const argument = argv[index]; + if (argument === "--os" || argument === "--profile") { + const value = argv[(index += 1)]; + if (value === undefined) throw new Error(`${argument} needs a value`); + if (argument === "--os") Object.assign(options, { os: value, explicitOs: true }); + else options.profile = value; + } else if (argument === "--list") options.list = true; + else if (argument === "--keep-going") options.keepGoing = true; + else throw new Error(`unknown argument ${argument}`); + } + return options; +} + +async function main(argv) { + const options = parseArguments(argv); + const gates = gatesFor(options); + if (options.list) { + for (const item of gates) console.log(`${item.cwd}: ${commandOf(item)}`); + return 0; + } + // A lane that names another OS would silently run the wrong list. + if (options.explicitOs && OS_OF_PLATFORM[process.platform] !== options.os) { + throw new Error(`refusing to run the ${options.os} gates on a ${OS_OF_PLATFORM[process.platform] ?? process.platform} runner`); + } + if (options.profile === "msrv" && !process.env.RUSTUP_TOOLCHAIN) { + throw new Error( + "the msrv profile needs RUSTUP_TOOLCHAIN set to the declared rust-version; without it cargo would use the development toolchain", + ); + } + const results = await runGates(gates, { execute: executeWithCargo, keepGoing: options.keepGoing, log: console.log }); + const skipped = gates.slice(results.length); + const lines = [ + `Cargo gates (${options.os}, ${options.profile}): ${results.length - results.filter((entry) => entry.failure).length} passed, ` + + `${results.filter((entry) => entry.failure).length} failed, ${skipped.length} not run`, + "", + "| Gate | Result | Seconds |", + "|---|---|---|", + ...results.map( + (entry) => `| \`${commandOf(entry.gate)}\` (${entry.gate.cwd}) | ${entry.failure ?? "passed"} | ${entry.seconds.toFixed(0)} |`, + ), + ...skipped.map((item) => `| \`${commandOf(item)}\` (${item.cwd}) | not run (an earlier gate failed) | |`), + ]; + console.log(lines.join("\n")); + if (process.env.GITHUB_STEP_SUMMARY) appendFileSync(process.env.GITHUB_STEP_SUMMARY, `${lines.join("\n")}\n`); + return results.some((entry) => entry.failure) ? 1 : 0; +} + +if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) { + main(process.argv.slice(2)).then( + (code) => process.exit(code), + (error) => { + console.error(error.message); + process.exit(2); + }, + ); +} diff --git a/scripts/ci/smoke-cli.mjs b/scripts/ci/smoke-cli.mjs new file mode 100644 index 00000000..1eef9105 --- /dev/null +++ b/scripts/ci/smoke-cli.mjs @@ -0,0 +1,244 @@ +#!/usr/bin/env node +// CLI smoke tests of the native release binary, one script for every OS (#358). +// ci.yml's build-macos and build-windows lanes run it after `tauri build --no-bundle` +// instead of carrying one inline bash and one inline PowerShell copy per check. +// +// node scripts/ci/smoke-cli.mjs basic tier 1, no model: the binary exists, help, config, +// list-models, formats, shell completions +// node scripts/ci/smoke-cli.mjs model tier 2: download nb-whisper-tiny, transcribe, and the +// output contract of #364 with a real model: --json, +// --jsonl, a reader that stops after the first item and +// the error object of a failed --json run +// --binary default src-tauri/target/release/sagascript[.exe] +// +// Every check of a tier runs even after a failure, and the script exits 1 when any failed, +// so one run names all of them. The model tier is the one the Windows lane tolerates +// (continue-on-error, #357); that decision lives in ci.yml, not here. +import { spawn, spawnSync } from "node:child_process"; +import { accessSync, constants, copyFileSync, existsSync, mkdirSync, mkdtempSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { dirname, join, resolve } from "node:path"; +import { fileURLToPath, pathToFileURL } from "node:url"; + +export const REPO_ROOT = dirname(dirname(dirname(fileURLToPath(import.meta.url)))); +const WINDOWS = process.platform === "win32"; +export const DEFAULT_BINARY = join("src-tauri", "target", "release", WINDOWS ? "sagascript.exe" : "sagascript"); +const MODEL = "nb-whisper-tiny"; +const SHORT_AUDIO = "test-audio/norwegian-short-3s.mp3"; +const MEDIUM_AUDIO = "test-audio/norwegian-medium-8s.mp3"; +const TRANSCRIBE = ["transcribe", "--language", "no", "--model", MODEL]; + +// Completions for every shell the binary supports on every OS, not one per OS. +export const COMPLETION_SHELLS = ["zsh", "bash", "powershell"]; + +const text = (value) => (typeof value === "string" ? value : ""); + +/** + * Each check: { name, run(harness) }. The harness is what a check may do: run the + * binary, write scratch files, run the stream verifier, and assert. + */ +export const TIERS = { + basic: [ + { + name: "the native release binary exists and is executable", + run(h) { + h.checkBinary(); + }, + }, + { + name: "help and version", + run(h) { + for (const args of [["--version"], ["--help"], ["transcribe", "--help"], ["record", "--help"]]) h.ok(args); + }, + }, + { + name: "config management", + run(h) { + h.ok(["config", "list"]); + h.ok(["config", "get", "language"]); + h.ok(["config", "set", "language", "sv"]); + h.assert(h.ok(["config", "get", "language"]).includes("sv"), "config get language should print sv after set"); + h.ok(["config", "reset", "language"]); + h.assert(h.ok(["config", "get", "language"]).includes("en"), "config get language should print en after reset"); + h.ok(["config", "path"]); + }, + }, + { + name: "model listing and formats", + run(h) { + h.ok(["list-models"]); + h.ok(["formats"]); + }, + }, + { + name: "shell completions", + run(h) { + for (const shell of COMPLETION_SHELLS) { + h.assert(h.ok(["completions", shell], { quiet: true }).trim() !== "", `completions ${shell} printed nothing`); + } + }, + }, + ], + model: [ + { + name: "download model and transcribe", + run(h) { + h.ok(["download-model", MODEL]); + const result = h.ok([...TRANSCRIBE, SHORT_AUDIO]); + console.log(`Transcription: ${result.trim()}`); + h.assert(/stortinget/i.test(result), "expected 'stortinget' in the transcription"); + }, + }, + { + name: "transcribe with JSON output", + run(h) { + const { stdout, stderr } = h.capture("transcription-smoke", [...TRANSCRIBE, "--json", SHORT_AUDIO]); + h.verifyStreams([], stdout, stderr); + }, + }, + { + name: "JSONL output", + run(h) { + const { stdout, stderr } = h.capture("transcription-smoke-jsonl", [...TRANSCRIBE, "--jsonl", h.audioDirectory()]); + h.verifyStreams(["--jsonl"], stdout, stderr); + }, + }, + { + // A smoke check with the real model: a panic or a failing exit on a reader that stops after the + // first item fails the check. The deterministic check of the closed-pipe contract is + // `run_into_closed_pipe` in src-tauri/crates/sagascript-cli/tests/output_contract.rs. + name: "closed pipe after the first JSONL item", + async run(h) { + const { code, sawLine } = await h.readFirstLineThenClose([...TRANSCRIBE, "--jsonl", h.audioDirectory()]); + h.assert(sawLine, "no JSONL item before the output ended"); + h.assert(code === 0, `exited with status ${code} after the reader closed the pipe`); + }, + }, + { + name: "JSON error object of a failed run", + run(h) { + const { code, stdout, stderr } = h.capture("transcription-error", [...TRANSCRIBE, "--json", join(h.scratch, "missing.wav")]); + h.assert(code === 1, `a failed --json run must exit 1, got ${code}`); + h.verifyStreams(["--error"], stdout, stderr); + }, + }, + ], +}; + +function createHarness({ binary, scratch }) { + const harness = { + binary, + scratch, + assert(condition, message) { + if (!condition) throw new Error(message); + }, + checkBinary() { + harness.assert(existsSync(binary), `missing native release binary: ${binary}`); + if (!WINDOWS) { + accessSync(binary, constants.X_OK); + harness.runProgram("file", [binary]); + } + }, + // `quiet` keeps a long stdout (a completion script) out of the log; stderr always shows. + runProgram(program, args, { quiet = false } = {}) { + console.log(`+ ${program} ${args.join(" ")}`); + const result = spawnSync(program, args, { cwd: REPO_ROOT, encoding: "utf8" }); + if (result.error) throw result.error; + if (!quiet) process.stdout.write(text(result.stdout)); + process.stderr.write(text(result.stderr)); + return { code: result.status, stdout: text(result.stdout), stderr: text(result.stderr) }; + }, + // Runs the binary; the exit status must be 0. Returns stdout. + ok(args, options) { + const result = harness.runProgram(binary, args, options); + harness.assert(result.code === 0, `${args.join(" ")} exited with status ${result.code}`); + return result.stdout; + }, + // Runs the binary with stdout and stderr in files (as a verifier needs them). Any exit status is returned. + capture(label, args) { + const result = harness.runProgram(binary, args); + const stdout = join(scratch, `${label}.stdout`); + const stderr = join(scratch, `${label}.stderr`); + writeFileSync(stdout, result.stdout); + writeFileSync(stderr, result.stderr); + return { code: result.code, stdout, stderr }; + }, + verifyStreams(mode, stdout, stderr) { + const python = WINDOWS ? "python" : "python3"; + const result = harness.runProgram(python, ["scripts/verify-json-cli-streams.py", ...mode, stdout, stderr]); + harness.assert(result.code === 0, `verify-json-cli-streams.py ${mode.join(" ")} exited with status ${result.code}`); + }, + // Two audio files in a directory of their own: `transcribe` takes a directory. + audioDirectory() { + const inputs = join(scratch, "jsonl-inputs"); + if (!existsSync(inputs)) { + mkdirSync(inputs, { recursive: true }); + copyFileSync(join(REPO_ROOT, SHORT_AUDIO), join(inputs, "a.mp3")); + copyFileSync(join(REPO_ROOT, MEDIUM_AUDIO), join(inputs, "b.mp3")); + } + return inputs; + }, + // `binary args | head -1` under pipefail: read one line, close the pipe, expect a clean exit. + readFirstLineThenClose(args) { + console.log(`+ ${binary} ${args.join(" ")} | head -1`); + return new Promise((resolve, reject) => { + const child = spawn(binary, args, { cwd: REPO_ROOT, stdio: ["ignore", "pipe", "inherit"] }); + let sawLine = false; + child.stdout.on("data", (chunk) => { + if (!sawLine && chunk.includes(10)) { + sawLine = true; + child.stdout.destroy(); + } + }); + child.on("error", reject); + child.on("close", (code) => resolve({ code, sawLine })); + }); + }, + }; + return harness; +} + +/** Runs every check of a tier, reporting each; resolves to the names of the failed ones. */ +export async function runTier(tier, harness, { log = console.log } = {}) { + const failed = []; + for (const check of TIERS[tier]) { + log(`::group::Smoke test: ${check.name}`); + try { + await check.run(harness); + log("::endgroup::"); + log(`ok: ${check.name}`); + } catch (error) { + log("::endgroup::"); + log(`::error title=Smoke test failed::${check.name}: ${error.message}`); + failed.push(check.name); + } + } + return failed; +} + +async function main(argv) { + const tier = argv[0]; + if (!Object.hasOwn(TIERS, tier)) throw new Error(`usage: smoke-cli.mjs <${Object.keys(TIERS).join("|")}> [--binary ]`); + const binaryIndex = argv.indexOf("--binary"); + const binary = resolve(REPO_ROOT, binaryIndex >= 0 ? argv[binaryIndex + 1] : DEFAULT_BINARY); + const scratch = mkdtempSync(join(process.env.RUNNER_TEMP || tmpdir(), "sagascript-smoke-")); + const failed = await runTier(tier, createHarness({ binary, scratch })); + if (failed.length > 0) { + console.error(`${failed.length} smoke check(s) failed in tier ${tier}: ${failed.join("; ")}`); + return 1; + } + return 0; +} + +if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) { + main(process.argv.slice(2)).then( + (code) => process.exit(code), + (error) => { + console.error(error.message); + process.exit(2); + }, + ); +} + +// Re-exported for the tests that run the script against a fake binary. +export { createHarness }; diff --git a/scripts/dictation_eval/test_evaluate_cli.py b/scripts/dictation_eval/test_evaluate_cli.py index 874c419a..1f9c0d38 100644 --- a/scripts/dictation_eval/test_evaluate_cli.py +++ b/scripts/dictation_eval/test_evaluate_cli.py @@ -2,7 +2,6 @@ import json from pathlib import Path -import re import subprocess import sys import tempfile @@ -15,20 +14,6 @@ class EvaluateCliTests(unittest.TestCase): - def test_ci_selects_supported_python_before_all_evaluator_suites(self): - workflow = SCRIPT.parents[2] / ".github" / "workflows" / "ci.yml" - source = workflow.read_text(encoding="utf-8").replace("\r\n", "\n") - for name in ("test-macos", "test-linux", "test-windows"): - marker = f" {name}:\n" - start = source.index(marker) + len(marker) - next_job = re.search(r"^ [A-Za-z0-9_-]+:\s*$", source[start:], re.MULTILINE) - end = start + next_job.start() if next_job else len(source) - job = source[start:end] - setup = job.index("actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1") - suite = job.index("Test offline dictation evaluation tooling") - self.assertLess(setup, suite) - self.assertIn('python-version: "3.12"', job[setup:suite]) - def run_cli(self, *args): return subprocess.run( [sys.executable, str(SCRIPT), *map(str, args)], diff --git a/scripts/lib/ci-workflow.mjs b/scripts/lib/ci-workflow.mjs new file mode 100644 index 00000000..ad29be95 --- /dev/null +++ b/scripts/lib/ci-workflow.mjs @@ -0,0 +1,128 @@ +// A structural reader for the workflow files under test (#358): the YAML is parsed, never matched +// with regular expressions, so a test asserts what a job `needs`, which step runs what and with +// which inputs, and keeps passing when the file is reformatted, re-commented or reordered +// within what the assertion states. +// +// The parser is the `yaml` package. It is not a dependency of this repository: it is vite's +// optional peer, which npm installs from package-lock.json. The ci test group lists it in +// GROUP_NPM_PACKAGES (scripts/node-test-groups.mjs), so a checkout where it is missing fails +// once with a clear message. If vite ever drops it, add `yaml` to devDependencies. +// +// Not for tests that run before `npm ci` (the Windows build identity gates in ci.yml's +// test-windows): they must read text with node:fs only. +import { readFileSync } from "node:fs"; +import { fileURLToPath } from "node:url"; +import { parse } from "yaml"; + +export const REPO_ROOT = fileURLToPath(new URL("../../", import.meta.url)); +export const readRepoFile = (path) => readFileSync(`${REPO_ROOT}${path}`, "utf8").replace(/\r\n?/g, "\n"); + +// A scalar `run` is a script; CRLF checkouts (core.autocrlf on Windows) must not leak into it. +const script = (value) => (typeof value === "string" ? value.replace(/\r\n?/g, "\n").trim() : ""); +const list = (value) => (value === undefined ? [] : Array.isArray(value) ? value : [value]); + +function normalizeStep(step, index) { + return { + index, + name: step.name, + id: step.id, + uses: step.uses, + run: script(step.run), + with: step.with ?? {}, + env: step.env ?? {}, + workingDirectory: step["working-directory"], + shell: step.shell, + condition: step.if === undefined ? undefined : String(step.if), + continueOnError: step["continue-on-error"], + }; +} + +function normalizeJob(id, job) { + return { + id, + needs: list(job.needs), + condition: job.if === undefined ? undefined : String(job.if), + runsOn: job["runs-on"], + timeoutMinutes: job["timeout-minutes"], + env: job.env ?? {}, + outputs: job.outputs ?? {}, + continueOnError: job["continue-on-error"], + steps: (job.steps ?? []).map(normalizeStep), + }; +} + +/** Parses workflow text into `{ doc, jobs, job(id) }`. */ +export function parseWorkflow(text) { + const doc = parse(text); + const jobs = Object.entries(doc.jobs ?? {}).map(([id, job]) => normalizeJob(id, job)); + return { + doc, + jobs, + jobIds: jobs.map((job) => job.id), + job(id) { + const found = jobs.find((job) => job.id === id); + if (!found) throw new Error(`workflow must define the job ${id}`); + return found; + }, + }; +} + +export const loadWorkflow = (file = "ci.yml") => parseWorkflow(readRepoFile(`.github/workflows/${file}`)); + +/** A local composite action: `{ inputs, steps }`. */ +export function loadAction(name) { + const doc = parse(readRepoFile(`.github/actions/${name}/action.yml`)); + return { doc, inputs: doc.inputs ?? {}, steps: (doc.runs?.steps ?? []).map(normalizeStep) }; +} + +/** The input a composite action step is called with, falling back to the action's default. */ +export function actionInput(action, step, name) { + return String(step.with[name] ?? action.inputs[name]?.default ?? ""); +} + +// Evaluates the only condition form the local actions use: `${{ inputs. == '' }}`. +function conditionHolds(condition, valueOf) { + if (condition === undefined) return true; + const match = /^\$\{\{\s*inputs\.([\w-]+) == '([^']*)'\s*\}\}$/.exec(condition.trim()); + if (!match) throw new Error(`unsupported condition in a local action: ${condition}`); + return valueOf(match[1]) === match[2]; +} + +/** + * The steps of a job with every `uses: ./.github/actions/` step replaced by the `run` steps of + * that action which its inputs enable (marked `viaAction`). Steps that `uses:` another action stay. + */ +export function expandedSteps(job) { + return job.steps.flatMap((step) => { + const local = /^\.\/\.github\/actions\/([\w-]+)$/.exec(step.uses ?? ""); + if (!local) return [step]; + const action = loadAction(local[1]); + return action.steps + .filter((inner) => inner.run !== "" && conditionHolds(inner.condition, (name) => actionInput(action, step, name))) + .map((inner) => ({ ...inner, viaAction: local[1] })); + }); +} + +/** The logical command lines of a step's script: continuations joined, blank and comment lines dropped. */ +export function commandLines(step) { + return step.run + .replace(/\\\n\s*/g, " ") + .split("\n") + .map((line) => line.trim()) + .filter((line) => line !== "" && !line.startsWith("#")); +} + +/** The `--os` and `--profile` a step passes to scripts/ci/cargo-gates.mjs, or null when it does not run it. */ +export function gateInvocation(step) { + const line = commandLines(step).find((candidate) => /^node scripts\/ci\/cargo-gates\.mjs(\s|$)/.test(candidate)); + if (line === undefined) return null; + const words = line.split(/\s+/); + const argument = (flag) => { + const index = words.indexOf(flag); + return index >= 0 ? words[index + 1] : undefined; + }; + return { os: argument("--os"), profile: argument("--profile") ?? "main", line }; +} + +/** Every step of every job, as `{ job, step }`. */ +export const allSteps = (workflow) => workflow.jobs.flatMap((job) => job.steps.map((step) => ({ job, step }))); diff --git a/scripts/node-test-groups.mjs b/scripts/node-test-groups.mjs index d113114e..88942f57 100644 --- a/scripts/node-test-groups.mjs +++ b/scripts/node-test-groups.mjs @@ -26,7 +26,10 @@ export const GROUP_DESCRIPTIONS = { // failure per file. Groups not listed need nothing beyond Node itself. export const GROUP_NPM_PACKAGES = { ui: ["typescript"], - ci: ["vite"], + // `yaml` is vite's optional peer, which npm installs from the lockfile; scripts/lib/ci-workflow.mjs + // parses the workflows with it (#358). + ci: ["vite", "yaml"], + release: ["yaml"], }; export const TEST_GROUPS = { @@ -45,12 +48,14 @@ export const TEST_GROUPS = { "scripts/diarization_sv/test_reference_review.mjs", "scripts/test-browser-mock-commands.mjs", "scripts/test-ci-build-identity.mjs", + "scripts/test-ci-cargo-gates.mjs", "scripts/test-ci-cli-feature-coverage.mjs", "scripts/test-ci-cohort-timings.mjs", "scripts/test-ci-docs-scope.mjs", "scripts/test-ci-docs-workflow.mjs", "scripts/test-ci-parallel-gates.mjs", "scripts/test-ci-run-timings.mjs", + "scripts/test-ci-smoke-cli.mjs", "scripts/test-ci-unrun-suites.mjs", "scripts/test-clippy-policy.mjs", "scripts/test-dependency-policy.mjs", diff --git a/scripts/test-ci-cargo-gates.mjs b/scripts/test-ci-cargo-gates.mjs new file mode 100644 index 00000000..19f65d41 --- /dev/null +++ b/scripts/test-ci-cargo-gates.mjs @@ -0,0 +1,229 @@ +import assert from "node:assert/strict"; +import { spawnSync } from "node:child_process"; +import { existsSync } from "node:fs"; +import test from "node:test"; +import { join } from "node:path"; +import { fileURLToPath } from "node:url"; +import { allSteps, commandLines, gateInvocation, loadWorkflow } from "./lib/ci-workflow.mjs"; +import { GATES, OSES, PROFILES, RELEASE_GUARD_PROOF, REPO_ROOT, commandOf, gatesFor, runGates } from "./ci/cargo-gates.mjs"; + +// The list of cargo gates (scripts/ci/cargo-gates.mjs) is the one source of the cargo commands every +// lane of ci.yml runs (#358). These tests hold the list, its wiring into the lanes and its runner. + +const SCRIPT = fileURLToPath(new URL("./ci/cargo-gates.mjs", import.meta.url)); +const lines = (os, profile) => gatesFor({ os, profile }).map((gate) => `${gate.cwd}: ${commandOf(gate)}`); + +const COMMON = [ + "src-tauri: cargo clippy --workspace --all-targets -- -D warnings", + "src-tauri: cargo test --workspace", + "src-tauri: cargo clippy -p sagascript-core --all-targets -- -D warnings", + "src-tauri: cargo test -p sagascript-core", + "src-tauri: cargo clippy -p sagascript-cli --all-targets --no-default-features -- -D warnings", + "src-tauri: cargo test -p sagascript-cli --no-default-features", +]; +const ORT = [ + "src-tauri/engine-host/ort: cargo clippy --all-targets --locked -- -D warnings", + "src-tauri/engine-host/ort: cargo test --locked", +]; +const DENY = [ + "src-tauri: cargo deny --locked --workspace check", + "src-tauri: cargo deny --locked --manifest-path engine-host/ort/Cargo.toml --workspace check -A license-not-encountered -A license-exception-not-encountered -A unmatched-source", +]; + +// The commands each lane ran before the list existed, plus the ones the list added on purpose, +// spelled out so that dropping or adding a gate has to change this fixture too. +const EXPECTED = { + macos: COMMON, + windows: [...COMMON, ...ORT], + linux: [ + ...DENY, + "src-tauri: cargo fmt --all -- --check", + "src-tauri: cargo fmt --manifest-path engine-host/ort/Cargo.toml --all -- --check", + ...COMMON, + "src-tauri: cargo clippy -p sagascript-core --features diarization --all-targets -- -D warnings", + "src-tauri: cargo test -p sagascript-core --features diarization", + "src-tauri: cargo test -p sagascript-core --features record", + ...ORT, + "src-tauri: cargo build -p sagascript-cli --no-default-features", + "src-tauri: cargo tree -p sagascript-cli --no-default-features -e normal", + "src-tauri: cargo test --release -p sagascript-core --lib dev_overrides", + ], + msrv: [ + "src-tauri: cargo check --workspace --locked", + "src-tauri: cargo check -p sagascript-cli --no-default-features --locked", + "src-tauri/engine-host/ort: cargo check --locked", + ], +}; + +test("every gate is well formed, and every per-OS exclusion states its reason", () => { + assert.equal(new Set(GATES.map((gate) => gate.id)).size, GATES.length, "gate ids are unique"); + for (const gate of GATES) { + assert.ok(PROFILES.includes(gate.profile), `${gate.id}: profile`); + assert.ok(existsSync(join(REPO_ROOT, gate.cwd)), `${gate.id}: ${gate.cwd} exists`); + assert.ok(gate.args.length > 0 && gate.args.every((argument) => typeof argument === "string" && argument !== ""), `${gate.id}: args`); + if (gate.os === undefined) { + assert.equal(gate.why, undefined, `${gate.id} runs everywhere and needs no exclusion reason`); + } else { + assert.ok(gate.os.length > 0 && gate.os.every((os) => OSES.includes(os)), `${gate.id}: os`); + assert.ok(gate.os.length < OSES.length, `${gate.id} lists every OS; drop the restriction`); + assert.ok(typeof gate.why === "string" && gate.why.length >= 20, `${gate.id} excludes an OS and must say why`); + } + } +}); + +test("each lane runs exactly the expected cargo commands, in order", () => { + for (const os of OSES) assert.deepEqual(lines(os), EXPECTED[os], `${os} gates`); + assert.deepEqual(lines("linux", "msrv"), EXPECTED.msrv, "msrv gates"); +}); + +test("the core gates run on every OS; every other difference between the OS lists is an exclusion in the list", () => { + const everywhere = OSES.map((os) => new Set(lines(os))); + for (const command of COMMON) + assert.ok( + everywhere.every((set) => set.has(command)), + `${command} must run on every OS`, + ); + for (const gate of GATES.filter((entry) => entry.profile === "main")) { + const runsOn = OSES.filter((os) => lines(os).includes(`${gate.cwd}: ${commandOf(gate)}`)); + assert.deepEqual( + runsOn, + OSES.filter((os) => gate.os === undefined || gate.os.includes(os)), + `${gate.id} runs where it says`, + ); + } +}); + +test("the release guard proves its test ran and the dependency-tree gate forbids cpal and ALSA", () => { + const guard = GATES.find((gate) => gate.id === "release-guard"); + assert.ok(guard.requireOutput.pattern.test("test dev_overrides::tests::release_build_ignores_the_real_environment ... ok")); + assert.ok(!guard.requireOutput.pattern.test("running 0 tests")); + assert.equal(guard.requireOutput.pattern, RELEASE_GUARD_PROOF); + const tree = GATES.find((gate) => gate.id === "no-alsa"); + assert.ok(tree.forbidOutput.pattern.test("├── cpal v0.15.3") && tree.forbidOutput.pattern.test("alsa-sys")); + assert.ok(!tree.forbidOutput.pattern.test("sagascript-core v1.4.4")); +}); + +test("cargo-deny gates run first, before any gate that compiles", () => { + const linux = gatesFor({ os: "linux" }); + assert.deepEqual( + linux.slice(0, 2).map((gate) => gate.args[0]), + ["deny", "deny"], + ); + assert.ok(linux.slice(2).every((gate) => gate.args[0] !== "deny")); +}); + +test("every OS lane runs the gate script once with its own OS, and the MSRV lane its profile", () => { + const workflow = loadWorkflow(); + const osOf = (runsOn) => (/^macos/.test(runsOn) ? "macos" : /^windows/.test(runsOn) ? "windows" : "linux"); + const lanes = { "test-macos": "macos", "test-linux": "linux", "test-windows": "windows" }; + for (const [jobId, os] of Object.entries(lanes)) { + const job = workflow.job(jobId); + assert.equal(osOf(job.runsOn), os, `${jobId} runs on ${os}`); + const invocations = job.steps.map(gateInvocation).filter(Boolean); + assert.equal(invocations.length, 1, `${jobId} must run the gate script exactly once`); + assert.deepEqual([invocations[0].os, invocations[0].profile], [os, "main"]); + const step = job.steps.find((candidate) => gateInvocation(candidate)); + assert.equal(step.condition, undefined, `${jobId}: the gates are unconditional`); + assert.equal(step.continueOnError, undefined, `${jobId}: the gates are blocking`); + assert.equal(step.workingDirectory, undefined, "the script resolves each gate's directory itself"); + } + const msrv = workflow.job("msrv-linux"); + const [invocation] = msrv.steps.map(gateInvocation).filter(Boolean); + assert.deepEqual([invocation.os, invocation.profile], [undefined, "msrv"]); + for (const { job } of allSteps(workflow).filter(({ step }) => gateInvocation(step))) { + assert.ok([...Object.keys(lanes), "msrv-linux"].includes(job.id), `${job.id} must not run the cargo gates`); + } +}); + +test("ci.yml writes no cargo gate command of its own: the only cargo it runs is the cargo-deny install", () => { + const workflow = loadWorkflow(); + const cargo = allSteps(workflow).flatMap(({ job, step }) => + commandLines(step) + .filter((line) => /^cargo\s/.test(line)) + .map((line) => `${job.id}: ${line}`), + ); + assert.deepEqual(cargo, ["test-linux: cargo install --locked cargo-deny --version 0.20.2"]); +}); + +test("windows-package.yml keeps the workspace commands the gates run", () => { + const workflow = loadWorkflow("windows-package.yml"); + const step = workflow.jobs[0].steps.find((candidate) => candidate.name === "Test and lint Rust workspace"); + const commands = commandLines(step); + for (const id of ["test-workspace", "clippy-workspace"]) { + const gate = GATES.find((candidate) => candidate.id === id); + assert.ok(commands.includes(commandOf(gate)), `${commandOf(gate)} must stay in step with the gate ${id}`); + } +}); + +// -- the runner ---------------------------------------------------------------------------------- + +const fakeGate = (id, extra = {}) => ({ id, cwd: "src-tauri", args: ["test", id], profile: "main", ...extra }); +const executeWith = (outcomes) => async (gate) => outcomes[gate.id] ?? { code: 0, output: "" }; + +test("the runner stops at the first failing gate unless it keeps going", async () => { + const gates = [fakeGate("a"), fakeGate("b"), fakeGate("c")]; + const execute = executeWith({ b: { code: 101, output: "" } }); + const stopped = await runGates(gates, { execute }); + assert.deepEqual( + stopped.map((entry) => [entry.gate.id, entry.failure]), + [ + ["a", null], + ["b", "exited with status 101"], + ], + ); + const all = await runGates(gates, { execute, keepGoing: true }); + assert.deepEqual( + all.map((entry) => entry.gate.id), + ["a", "b", "c"], + ); + assert.equal(all.filter((entry) => entry.failure).length, 1); +}); + +test("a gate also fails on output it must not contain, or lacks the output it must contain", async () => { + const forbid = fakeGate("tree", { forbidOutput: { pattern: /alsa/, message: "alsa leaked" }, quiet: true }); + const require = fakeGate("guard", { requireOutput: { pattern: /ran ok/, message: "guard did not run" } }); + const logged = []; + const log = (line) => logged.push(line); + const [leak] = await runGates([forbid], { execute: executeWith({ tree: { code: 0, output: "alsa-sys" } }), log }); + assert.equal(leak.failure, "alsa leaked"); + assert.ok(logged.includes("alsa-sys"), "a quiet gate shows its output when it fails"); + logged.length = 0; + const [clean] = await runGates([forbid], { execute: executeWith({ tree: { code: 0, output: "cpal-free" } }), log }); + assert.equal(clean.failure, null); + assert.ok(!logged.includes("cpal-free"), "a quiet gate stays quiet when it passes"); + const [missing] = await runGates([require], { execute: executeWith({ guard: { code: 0, output: "running 0 tests" } }) }); + assert.equal(missing.failure, "guard did not run"); + const [proved] = await runGates([require], { execute: executeWith({ guard: { code: 0, output: "it ran ok" } }) }); + assert.equal(proved.failure, null); + const [failed] = await runGates([require], { execute: executeWith({ guard: { code: 1, output: "ran ok" } }) }); + assert.equal(failed.failure, "exited with status 1"); +}); + +// -- the command line ---------------------------------------------------------------------------- + +function cli(args, env = {}) { + const inherited = { ...process.env }; + delete inherited.RUSTUP_TOOLCHAIN; + return spawnSync(process.execPath, [SCRIPT, ...args], { encoding: "utf8", env: { ...inherited, ...env } }); +} + +test("--list prints a lane's commands and runs nothing", () => { + for (const os of OSES) { + const result = cli(["--os", os, "--list"]); + assert.equal(result.status, 0, result.stderr); + assert.deepEqual(result.stdout.trim().split("\n"), EXPECTED[os]); + } + assert.deepEqual(cli(["--profile", "msrv", "--list"]).stdout.trim().split("\n"), EXPECTED.msrv); +}); + +test("the script refuses to run another OS's gates, an unknown option and an MSRV run without a toolchain", () => { + const foreign = OSES.find((os) => os !== { darwin: "macos", linux: "linux", win32: "windows" }[process.platform]); + const wrongOs = cli(["--os", foreign]); + assert.equal(wrongOs.status, 2); + assert.match(wrongOs.stderr, /refusing to run the \w+ gates/); + assert.equal(cli(["--nonsense"]).status, 2); + assert.equal(cli(["--os", "plan9", "--list"]).status, 2); + const noToolchain = cli(["--profile", "msrv"]); + assert.equal(noToolchain.status, 2); + assert.match(noToolchain.stderr, /RUSTUP_TOOLCHAIN/); +}); diff --git a/scripts/test-ci-cli-feature-coverage.mjs b/scripts/test-ci-cli-feature-coverage.mjs index b21c8db7..fb45f685 100644 --- a/scripts/test-ci-cli-feature-coverage.mjs +++ b/scripts/test-ci-cli-feature-coverage.mjs @@ -1,39 +1,101 @@ import assert from "node:assert/strict"; -import { readFile } from "node:fs/promises"; import test from "node:test"; +import { gateInvocation, loadWorkflow, readRepoFile } from "./lib/ci-workflow.mjs"; +import { OSES, commandOf, gatesFor } from "./ci/cargo-gates.mjs"; -const [cliManifest, coreManifest, ciWorkflow] = await Promise.all([ - readFile(new URL("../src-tauri/crates/sagascript-cli/Cargo.toml", import.meta.url), "utf8"), - readFile(new URL("../src-tauri/crates/sagascript-core/Cargo.toml", import.meta.url), "utf8"), - readFile(new URL("../.github/workflows/ci.yml", import.meta.url), "utf8"), -]); +// How CI covers the Cargo features of the CLI and of core (#81, #358). Structural: the gate list +// that every lane runs, not the text of ci.yml. -function macosWorkflowFor(workflow) { - const normalized = workflow.replace(/\r\n/g, "\n"); - return normalized.slice(normalized.indexOf(" test-macos:"), normalized.indexOf(" build-macos:")); +const manifest = (crate) => readRepoFile(`src-tauri/crates/${crate}/Cargo.toml`); +const workflow = loadWorkflow(); +const lanes = { "test-macos": "macos", "test-linux": "linux", "test-windows": "windows" }; + +// The gates the lane really runs: found through the lane's own invocation of the gate script. +function gatesOfLane(jobId) { + const invocation = workflow.job(jobId).steps.map(gateInvocation).find(Boolean); + assert.ok(invocation, `${jobId} runs the gate script`); + return gatesFor({ os: invocation.os, profile: invocation.profile }); } -function assertMacosDiarizationCoverage(workflow) { - const macosWorkflow = macosWorkflowFor(workflow); - assert.match(cliManifest, /default\s*=\s*\[\s*"record"\s*,\s*"diarization"\s*\]/, "CLI default features must include diarization"); - assert.match(coreManifest, /default\s*=\s*\[\s*\]/, "core must retain its no-feature default"); +test("the CLI keeps diarization in its default features and core keeps none", () => { + assert.match( + manifest("sagascript-cli"), + /default\s*=\s*\[\s*"record"\s*,\s*"diarization"\s*\]/, + "CLI default features must include diarization", + ); + assert.match(manifest("sagascript-core"), /default\s*=\s*\[\s*\]/, "core must retain its no-feature default"); +}); - assert.match(macosWorkflow, /run: cargo test -p sagascript-cli\n/); - assert.match(macosWorkflow, /run: cargo clippy -p sagascript-cli --all-targets -- -D warnings\n/); - assert.doesNotMatch(macosWorkflow, /-p sagascript-cli --features diarization/); +test("every OS lint and tests the workspace, which builds the CLI with those defaults, with no package excluded", () => { + for (const [jobId, os] of Object.entries(lanes)) { + const gates = gatesOfLane(jobId); + for (const verb of ["clippy", "test"]) { + const gate = gates.find((candidate) => candidate.args[0] === verb && candidate.args.includes("--workspace")); + assert.ok(gate, `${os}: ${verb} --workspace`); + assert.ok( + !gate.args.some((argument) => ["--exclude", "--features", "--no-default-features", "-p"].includes(argument)), + `${os}: ${commandOf(gate)}`, + ); + } + assert.deepEqual( + gates.filter((gate) => gate.args.includes("--workspace") && ["clippy", "test"].includes(gate.args[0])).map((gate) => gate.args[0]), + ["clippy", "test"], + ); + assert.ok(OSES.includes(os)); + } +}); - assert.equal( - (macosWorkflow.match(/-p sagascript-core --features diarization/g) ?? []).length, - 3, - "core check, test, and all-target clippy gates must remain separate", - ); -} +test("feature sets stay separate invocations: no gate names two packages, and the CLI never gets `--features diarization`", () => { + for (const jobId of Object.keys(lanes)) { + for (const gate of gatesOfLane(jobId)) { + const packages = gate.args.filter((argument, index) => gate.args[index - 1] === "-p"); + assert.ok(packages.length <= 1, `${commandOf(gate)} combines packages, which would unify their features`); + assert.ok( + !(packages.includes("sagascript-cli") && gate.args.includes("--features")), + `${commandOf(gate)}: the CLI's features come from its defaults, not a flag`, + ); + } + } +}); -for (const [label, lineEnding] of [ - ["LF", "\n"], - ["CRLF", "\r\n"], -]) { - test(`macOS CI keeps CLI diarization in defaults and gates core explicitly (${label})`, () => { - assertMacosDiarizationCoverage(ciWorkflow.replace(/\r?\n/g, lineEnding)); - }); -} +test("core is gated explicitly with and without diarization, and the lean CLI without default features", () => { + const all = Object.keys(lanes).flatMap((jobId) => gatesOfLane(jobId)); + const matching = (predicate) => all.filter(predicate).map((gate) => gate.args.join(" ")); + const coreAlone = (gate) => + gate.args.includes("sagascript-core") && !gate.args.includes("--features") && !gate.args.includes("--release"); + const coreDiarization = (gate) => gate.args.includes("sagascript-core") && gate.args.join(" ").includes("--features diarization"); + // Core with no features, on every OS: clippy over all targets and the tests. + for (const jobId of Object.keys(lanes)) { + const gates = gatesOfLane(jobId); + assert.deepEqual( + gates.filter(coreAlone).map((gate) => gate.args[0]), + ["clippy", "test"], + `${jobId}: core without features`, + ); + } + // Core with diarization: a clippy over all targets and a test run, separate from the above. `cargo + // check` is not a third gate: clippy --all-targets compiles everything it would. + assert.deepEqual( + new Set(matching(coreDiarization)), + new Set([ + "clippy -p sagascript-core --features diarization --all-targets -- -D warnings", + "test -p sagascript-core --features diarization", + ]), + ); + assert.ok( + all.every((gate) => gate.args[0] !== "check" || gate.profile === "msrv"), + "no plain cargo check outside the MSRV profile", + ); + // The lean CLI, which strips diarization with record, is linted and tested on every OS. + for (const jobId of Object.keys(lanes)) { + const lean = gatesOfLane(jobId).filter( + (gate) => + ["clippy", "test"].includes(gate.args[0]) && gate.args.includes("sagascript-cli") && gate.args.includes("--no-default-features"), + ); + assert.deepEqual( + lean.map((gate) => gate.args[0]), + ["clippy", "test"], + `${jobId}: lean CLI`, + ); + } +}); diff --git a/scripts/test-ci-docs-workflow.mjs b/scripts/test-ci-docs-workflow.mjs index 95fc320f..31d25f24 100644 --- a/scripts/test-ci-docs-workflow.mjs +++ b/scripts/test-ci-docs-workflow.mjs @@ -4,83 +4,59 @@ import { copyFileSync, mkdtempSync, mkdirSync, readFileSync, rmSync, writeFileSy import { tmpdir } from "node:os"; import { join } from "node:path"; import test from "node:test"; +import { REPO_ROOT, loadWorkflow, parseWorkflow } from "./lib/ci-workflow.mjs"; -const workflow = readFileSync(new URL("../.github/workflows/ci.yml", import.meta.url), "utf8"); -function normalizeWorkflow(source) { - return source.replace(/\r\n?/g, "\n"); -} -function job(id, source = workflow) { - const normalized = normalizeWorkflow(source); - return normalized.split(`\n ${id}:\n`)[1].split(/\n [a-z][a-z0-9-]*:\n/)[0]; -} -function script(id, source = workflow) { - return job(id, source) - .split(" run: |\n")[1] - .split(/\n - /)[0] - .split("\n") - .map((line) => line.replace(/^ /, "")) - .join("\n"); +// The docs-only scope gate of ci.yml and what the aggregators make of it. The workflow is parsed, +// and the scripts under test are the ones its steps run. + +const workflow = loadWorkflow(); +const CHECKS = ["check-macos", "check-windows", "check-linux"]; + +function stepRun(source, jobId, predicate) { + const step = source.job(jobId).steps.find(predicate); + assert.ok(step, `${jobId} has the step`); + return step.run; } -function shell(command, env, cwd) { - return spawnSync("bash", ["-euo", "pipefail", "-c", command], { - cwd, - env: { ...process.env, ...env }, - encoding: "utf8", - }); +const scopeScript = (source = workflow) => stepRun(source, "scope", (step) => step.id === "classify"); +const shell = (command, env, cwd) => + spawnSync("bash", ["-euo", "pipefail", "-c", command], { cwd, env: { ...process.env, ...env }, encoding: "utf8" }); + +// A check job's own step, run as the workflow runs it, against a needs context built from the +// job's own `needs`. +function runCheck(jobId, { docsOnly, trusted = "true", scope = "success", results }) { + const job = workflow.job(jobId); + const needs = { scope: { result: scope, outputs: { docs_only: docsOnly, scope_trusted: trusted, scope_classification: "x" } } }; + const lanes = job.needs.filter((need) => need !== "scope"); + lanes.forEach((lane, index) => (needs[lane] = { result: results[index] ?? results[0], outputs: {} })); + const step = job.steps.find((candidate) => candidate.run.startsWith("node scripts/ci/assert-lanes.mjs")); + assert.equal(job.env.NEEDS, "${{ toJSON(needs) }}", "the step reads the job's needs context"); + return shell(step.run, { NEEDS: JSON.stringify(needs) }, REPO_ROOT); } test("required checks accept only successful full lanes or explicitly classified skipped docs lanes", () => { - const results = ["success", "failure", "cancelled", "skipped", ""]; - for (const id of ["check-macos", "check-windows", "check-linux"]) { - const run = script(id); - assert.match(job(id), /if: \$\{\{ always\(\) \}\}/); - for (const docsOnly of ["true", "false", ""]) { - for (const first of results) { - for (const second of results) { - const env = { - SCOPE_RESULT: "success", - SCOPE_TRUSTED: "true", - DOCS_ONLY: docsOnly, - TEST_RESULT: first, - BUILD_RESULT: second, - LINUX_RESULT: first, - MSRV_RESULT: second, - }; - const expected = - docsOnly === "true" - ? first === "skipped" && second === "skipped" - : docsOnly === "false" && first === "success" && second === "success"; - assert.equal(shell(run, env).status === 0, expected, `${id}: ${JSON.stringify(env)}`); + // One process per case: the matrix of every result is in test-ci-parallel-gates.mjs, in-process. + for (const id of CHECKS) { + for (const docsOnly of ["true", "false"]) { + for (const first of ["success", "failure", "cancelled", "skipped"]) { + for (const second of ["success", "skipped"]) { + const expected = docsOnly === "true" ? first === "skipped" && second === "skipped" : first === "success" && second === "success"; + assert.equal( + runCheck(id, { docsOnly, results: [first, second] }).status === 0, + expected, + `${id}: docs_only ${docsOnly}, ${first}/${second}`, + ); } } } - for (const result of results.filter((value) => value !== "success")) { - assert.notEqual( - shell(run, { - SCOPE_RESULT: result, - SCOPE_TRUSTED: "true", - DOCS_ONLY: "true", - TEST_RESULT: "skipped", - BUILD_RESULT: "skipped", - LINUX_RESULT: "skipped", - MSRV_RESULT: "skipped", - }).status, - 0, - `${id} rejects unsuccessful scope`, - ); + assert.notEqual(runCheck(id, { docsOnly: "", results: ["success"] }).status, 0, `${id} rejects an empty docs_only`); + for (const scope of ["failure", "cancelled", "skipped"]) { + assert.notEqual(runCheck(id, { docsOnly: "true", scope, results: ["skipped"] }).status, 0, `${id} rejects scope ${scope}`); } for (const trusted of ["false", ""]) { assert.notEqual( - shell(run, { - SCOPE_RESULT: "success", - SCOPE_TRUSTED: trusted, - DOCS_ONLY: "true", - TEST_RESULT: "skipped", - BUILD_RESULT: "skipped", - LINUX_RESULT: "skipped", - MSRV_RESULT: "skipped", - }).status, + runCheck(id, { docsOnly: "true", trusted, results: ["skipped"] }).status, 0, + `${id} rejects scope_trusted ${trusted}`, ); } } @@ -116,7 +92,7 @@ test("scope bootstrap and unavailable commits select full CI; known docs and pus const classify = (overrides = {}) => { writeFileSync(output, ""); const result = shell( - script("scope"), + scopeScript(), { ...env, EVENT_NAME: "pull_request", @@ -147,25 +123,28 @@ test("scope bootstrap and unavailable commits select full CI; known docs and pus assert.doesNotMatch(result.output, /docs_only=true/); }); -test("CRLF workflow text still parses and executes scope and aggregate scripts", () => { - const crlfWorkflow = normalizeWorkflow(workflow).replaceAll("\n", "\r\n"); - const scope = script("scope", crlfWorkflow); - const aggregateEnvironment = { - SCOPE_RESULT: "success", - SCOPE_TRUSTED: "true", - DOCS_ONLY: "false", - TEST_RESULT: "success", - BUILD_RESULT: "success", - LINUX_RESULT: "success", - MSRV_RESULT: "success", - }; +test("the scope job needs no script file of the revision it classifies", () => { + // It checks out the PR base, which may predate scripts/ci/ (adoption): its own script must be inline. + const scope = workflow.job("scope"); + assert.deepEqual(scope.needs, []); + const checkout = scope.steps[0]; + assert.equal(checkout.with.ref, "${{ github.event_name == 'pull_request' && github.event.pull_request.base.sha || github.sha }}"); + for (const step of scope.steps) assert.ok(!step.run.includes("scripts/ci/"), "scope must not depend on files added by the PR"); +}); - const scopeResult = shell(scope, { EVENT_NAME: "push", GITHUB_OUTPUT: "" }); +test("CRLF workflow text still parses and executes scope and aggregate scripts", () => { + const crlfWorkflow = parseWorkflow( + readFileSync(new URL("../.github/workflows/ci.yml", import.meta.url), "utf8") + .replace(/\r\n?/g, "\n") + .replaceAll("\n", "\r\n"), + ); + const scopeResult = shell(scopeScript(crlfWorkflow), { EVENT_NAME: "push", GITHUB_OUTPUT: "" }); assert.equal(scopeResult.status, 0, scopeResult.stderr); assert.match(scopeResult.stdout, /push-event-requires-full-ci/); - - for (const id of ["check-macos", "check-linux", "check-windows"]) { - const result = shell(script(id, crlfWorkflow), aggregateEnvironment); + for (const id of CHECKS) { + const result = runCheck(id, { docsOnly: "false", results: ["success"] }); assert.equal(result.status, 0, `${id}: ${result.stderr}`); + const step = crlfWorkflow.job(id).steps.find((candidate) => candidate.run.startsWith("node scripts/ci/assert-lanes.mjs")); + assert.equal(step.run, "node scripts/ci/assert-lanes.mjs"); } }); diff --git a/scripts/test-ci-parallel-gates.mjs b/scripts/test-ci-parallel-gates.mjs index 3318b6f6..c1c4f355 100644 --- a/scripts/test-ci-parallel-gates.mjs +++ b/scripts/test-ci-parallel-gates.mjs @@ -1,369 +1,266 @@ import assert from "node:assert/strict"; -import { execFileSync } from "node:child_process"; -import { readFile } from "node:fs/promises"; +import { spawnSync } from "node:child_process"; +import { readFileSync } from "node:fs"; import test from "node:test"; +import { fileURLToPath } from "node:url"; +import { actionInput, commandLines, gateInvocation, loadAction, loadWorkflow, parseWorkflow } from "./lib/ci-workflow.mjs"; +import { gatesFor } from "./ci/cargo-gates.mjs"; +import { laneProblems } from "./ci/assert-lanes.mjs"; -const workflow = (await readFile(new URL("../.github/workflows/ci.yml", import.meta.url), "utf8")).replace(/\r\n/g, "\n"); +// The parallel lanes of ci.yml (#358): what each one needs, how the shared setup action and the +// aggregators are wired, and that the aggregators fail closed. The workflow is parsed, not matched. -function jobBlock(jobId) { - const start = workflow.indexOf(` ${jobId}:`); - assert.notEqual(start, -1, `workflow must define ${jobId}`); - const remainder = workflow.slice(start); - const body = remainder.slice(remainder.indexOf("\n") + 1); - const nextJob = body.search(/^ [a-z][a-z0-9-]*:/m); - return remainder.slice(0, nextJob === -1 ? remainder.length : remainder.indexOf("\n") + 1 + nextJob); -} - -function scopedAggregator(jobId, expectedNeeds, laneResultNames, stepName) { - const block = jobBlock(jobId); - const needs = /^ needs: \[([^\]]+)\]$/m - .exec(block)?.[1] - .split(",") - .map((value) => value.trim()); - assert.deepEqual(needs, expectedNeeds, `${jobId} must wait for scope and its applicable lane(s)`); - assert.match(block, /^ if: \$\{\{ always\(\) \}\}$/m); - assert.ok(block.includes("SCOPE_RESULT: ${{ needs.scope.result }}")); - assert.ok(block.includes("SCOPE_TRUSTED: ${{ needs.scope.outputs.scope_trusted }}")); - assert.ok(block.includes("DOCS_ONLY: ${{ needs.scope.outputs.docs_only }}")); - for (const [variable, lane] of Object.entries(laneResultNames)) { - assert.ok(block.includes(`${variable}: \${{ needs.${lane}.result }}`)); - } - const run = parseSteps(block).get(stepName); - assert.match(run, /test "\$SCOPE_RESULT" = success/); - assert.match(run, /test "\$SCOPE_TRUSTED" = true/); - assert.match(run, /if \[ "\$DOCS_ONLY" = true \]/); - return run; -} +const workflow = loadWorkflow(); +const ASSERT_LANES = fileURLToPath(new URL("./ci/assert-lanes.mjs", import.meta.url)); +const SCOPE_CONDITION = "${{ needs.scope.result == 'success' && needs.scope.outputs.docs_only != 'true' }}"; +const LANES = ["test-macos", "build-macos", "test-linux", "msrv-linux", "test-windows", "build-windows"]; +const CHECKS = { "check-macos": "macos", "check-linux": "linux", "check-windows": "windows" }; +const setupStep = (job) => job.steps.find((step) => step.uses === "./.github/actions/setup"); -function runScopedAggregator(run, values) { - execFileSync("bash", ["-euo", "pipefail", "-c", run], { - env: { PATH: process.env.PATH, ...values }, - stdio: "pipe", - }); -} +test("the job set is the scope job, the six lanes and the three aggregators", () => { + assert.deepEqual([...workflow.jobIds].sort(), ["scope", ...LANES, ...Object.keys(CHECKS)].sort()); +}); -for (const [jobId, needs, laneResultNames, stepName] of [ - [ - "check-macos", - ["scope", "test-macos", "build-macos"], - { TEST_RESULT: "test-macos", BUILD_RESULT: "build-macos" }, - "Assert macOS CI lanes passed", - ], - [ - "check-windows", - ["scope", "test-windows", "build-windows"], - { TEST_RESULT: "test-windows", BUILD_RESULT: "build-windows" }, - "Assert Windows CI lanes passed", - ], - [ - "check-linux", - ["scope", "test-linux", "msrv-linux"], - { LINUX_RESULT: "test-linux", MSRV_RESULT: "msrv-linux" }, - "Assert Linux CI lanes passed", - ], -]) { - test(`${jobId} accepts only trusted docs-only skips or successful full lanes`, () => { - const run = scopedAggregator(jobId, needs, laneResultNames, stepName); - const laneVariables = Object.keys(laneResultNames); - const correctDocsOnly = Object.fromEntries(laneVariables.map((variable) => [variable, "skipped"])); - const correctFull = Object.fromEntries(laneVariables.map((variable) => [variable, "success"])); +test("every lane waits only for the scope gate and is skipped together with every other lane on a trusted docs-only change", () => { + for (const id of LANES) { + const job = workflow.job(id); + assert.deepEqual(job.needs, ["scope"], `${id} must wait only for the scope gate`); + assert.equal(job.condition, SCOPE_CONDITION, `${id} must run only on a successful, non-docs-only scope`); + } +}); - assert.doesNotThrow(() => - runScopedAggregator(run, { - SCOPE_RESULT: "success", - SCOPE_TRUSTED: "true", - DOCS_ONLY: "true", - ...correctDocsOnly, - }), +test("check-macos and check-windows keep their names, need every lane of their OS and run whatever the lanes did", () => { + for (const [id, os] of Object.entries(CHECKS)) { + const job = workflow.job(id); + assert.equal(workflow.doc.jobs[id].name, undefined, `${id}: the check-run name is the job id; a name: would rename the required check`); + const lanes = LANES.filter((lane) => lane.endsWith(`-${os}`)); + assert.ok(lanes.length >= 2, `${os} has a test lane and a second lane`); + assert.deepEqual([...job.needs].sort(), ["scope", ...lanes].sort(), `${id} must wait for scope and every ${os} lane`); + assert.equal(job.condition, "${{ always() }}", `${id} must report even when a lane failed or was skipped`); + assert.equal(job.env.NEEDS, "${{ toJSON(needs) }}"); + assert.equal(job.timeoutMinutes, 20); + assert.equal(job.steps[0].uses?.split("@")[0], "actions/checkout"); + assert.deepEqual( + job.steps.slice(1).map((step) => step.run), + ["node scripts/ci/assert-lanes.mjs"], ); - assert.doesNotThrow(() => - runScopedAggregator(run, { - SCOPE_RESULT: "success", - SCOPE_TRUSTED: "true", - DOCS_ONLY: "false", - ...correctFull, - }), + assert.ok( + job.steps.every((step) => step.condition === undefined && step.continueOnError === undefined), + `${id}'s steps are unconditional`, ); + } +}); + +// The `needs` context a check job would see: its own needs, with the results given. +function needsContext(id, { scope = "success", trusted = "true", docsOnly = "false", lane = "success", overrides = {} }) { + const needs = { scope: { result: scope, outputs: { docs_only: docsOnly, scope_trusted: trusted, scope_classification: "x" } } }; + for (const name of workflow.job(id).needs.filter((need) => need !== "scope")) { + needs[name] = { result: overrides[name] ?? lane, outputs: {} }; + } + return needs; +} - for (const scopeResult of ["failure", "cancelled", "skipped"]) { +for (const id of Object.keys(CHECKS)) { + test(`${id} accepts only trusted docs-only skips or successful full lanes`, () => { + const accepts = (options) => laneProblems(needsContext(id, options)).length === 0; + assert.ok(accepts({ docsOnly: "true", lane: "skipped" })); + assert.ok(accepts({ docsOnly: "false", lane: "success" })); + for (const scope of ["failure", "cancelled", "skipped"]) { for (const docsOnly of ["true", "false"]) { - assert.throws(() => - runScopedAggregator(run, { - SCOPE_RESULT: scopeResult, - SCOPE_TRUSTED: "true", - DOCS_ONLY: docsOnly, - ...(docsOnly === "true" ? correctDocsOnly : correctFull), - }), + assert.ok( + !accepts({ scope, docsOnly, lane: docsOnly === "true" ? "skipped" : "success" }), + `scope ${scope}, docs_only ${docsOnly}`, ); } } for (const trusted of ["false", "", "untrusted"]) { - assert.throws(() => - runScopedAggregator(run, { - SCOPE_RESULT: "success", - SCOPE_TRUSTED: trusted, - DOCS_ONLY: "true", - ...correctDocsOnly, - }), - ); + assert.ok(!accepts({ trusted, docsOnly: "true", lane: "skipped" }), `scope_trusted ${trusted}`); } - for (const docsOnly of ["", "unknown"]) { - assert.throws(() => - runScopedAggregator(run, { - SCOPE_RESULT: "success", - SCOPE_TRUSTED: "true", - DOCS_ONLY: docsOnly, - ...correctFull, - }), - ); - } - for (const variable of laneVariables) { - for (const result of ["failure", "cancelled", "skipped"]) { - assert.throws(() => - runScopedAggregator(run, { - SCOPE_RESULT: "success", - SCOPE_TRUSTED: "true", - DOCS_ONLY: "false", - ...correctFull, - [variable]: result, - }), + for (const docsOnly of ["", "unknown"]) assert.ok(!accepts({ docsOnly }), `docs_only ${JSON.stringify(docsOnly)}`); + // Every lane the job waits for, one by one, with every result that is not the expected one. + for (const lane of workflow.job(id).needs.filter((need) => need !== "scope")) { + for (const result of ["failure", "cancelled", "skipped", ""]) { + assert.ok(!accepts({ overrides: { [lane]: result } }), `${lane} ${JSON.stringify(result)} on a full run`); + } + for (const result of ["success", "failure", "cancelled", ""]) { + assert.ok( + !accepts({ docsOnly: "true", lane: "skipped", overrides: { [lane]: result } }), + `${lane} ${JSON.stringify(result)} on a docs-only run`, ); } } }); } -function parseSteps(job) { - const lines = job.split("\n"); - const starts = lines.map((line, index) => (/^ - name: (.*)$/.exec(line) ? index : -1)).filter((index) => index >= 0); - const result = new Map(); +test("an aggregator that waits for no lane, or gets no usable needs context, fails", () => { + assert.notEqual(laneProblems({ scope: { result: "success", outputs: { docs_only: "false", scope_trusted: "true" } } }).length, 0); + for (const bad of [null, [], "success", 1]) assert.notEqual(laneProblems(bad).length, 0); + const run = (env) => spawnSync(process.execPath, [ASSERT_LANES], { encoding: "utf8", env: { PATH: process.env.PATH, ...env } }); + assert.equal(run({ NEEDS: JSON.stringify(needsContext("check-macos", {})) }).status, 0); + assert.equal(run({ NEEDS: JSON.stringify(needsContext("check-macos", { overrides: { "build-macos": "failure" } })) }).status, 1); + assert.equal(run({}).status, 1, "no NEEDS"); + assert.equal(run({ NEEDS: "not json" }).status, 1); + assert.match(run({ NEEDS: JSON.stringify(needsContext("check-macos", { lane: "failure" })) }).stderr, /test-macos: result is "failure"/); +}); - for (let i = 0; i < starts.length; i += 1) { - const start = starts[i]; - const end = starts[i + 1] ?? lines.length; - const segment = lines.slice(start, end); - const name = /^ - name: (.*)$/.exec(segment[0])[1]; - const runIndex = segment.findIndex((line) => /^ run:\s*/.test(line)); - if (runIndex < 0) continue; +test("the scope job publishes the outputs the aggregators read", () => { + const scope = workflow.job("scope"); + assert.deepEqual(Object.keys(scope.outputs).sort(), ["docs_only", "scope_classification", "scope_trusted"]); + for (const name of ["docs_only", "scope_trusted"]) assert.equal(scope.outputs[name], `\${{ steps.classify.outputs.${name} }}`); +}); - const runValue = segment[runIndex].replace(/^ run:\s*/, ""); - let command; - if (runValue === "|" || runValue === ">" || runValue === ">-") { - command = segment - .slice(runIndex + 1) - .filter((line) => line.length === 0 || line.startsWith(" ")) - .map((line) => line.replace(/^ /, "")) - .join("\n"); - } else { - command = runValue; +test("the setup action is the only toolchain preamble: every lane uses it with declared inputs and the cache its gates need", () => { + const action = loadAction("setup"); + const KEYS = { + "test-macos": "sagascript-ci-checks", + "test-windows": "sagascript-ci-checks", + "build-macos": "sagascript-ci-native", + "build-windows": "sagascript-ci-native", + "test-linux": "sagascript-ci", + "msrv-linux": "sagascript-msrv", + }; + for (const id of LANES) { + const job = workflow.job(id); + const setups = job.steps.filter((step) => step.uses === "./.github/actions/setup"); + assert.equal(setups.length, 1, `${id} sets up once, through the action`); + const [setup] = setups; + for (const name of Object.keys(setup.with)) assert.ok(Object.hasOwn(action.inputs, name), `${id} passes an undeclared input: ${name}`); + for (const [name, input] of Object.entries(action.inputs)) if (input.required) assert.ok(name in setup.with, `${id} must pass ${name}`); + assert.equal(setup.with["cache-key"], KEYS[id]); + assert.equal(job.steps[0].uses?.split("@")[0], "actions/checkout", `${id}: checkout stays in the job, ahead of the local action`); + assert.equal(job.steps.indexOf(setup), 1, `${id}: the setup follows the checkout directly`); + for (const step of job.steps) { + assert.ok( + !/^(actions\/setup-(python|node)|dtolnay\/rust-toolchain|Swatinem\/rust-cache)@/.test(step.uses ?? ""), + `${id} repeats a step of the setup action`, + ); } - result.set(name, command.trim()); - } - return result; -} -function normalizeCommand(command) { - return command.replace(/\r\n/g, "\n").trim(); -} - -// These are the Cargo/native/smoke run bodies in the pre-split check-macos and -// check-windows jobs at base 1ab1128. Keeping the fixture explicit makes a -// dropped invocation fail this test even if the replacement workflow remains valid YAML. -const BASELINE_GATES = { - macos: { - test: [ - ["Cargo check", "cargo check"], - ["Cargo test", "cargo test"], - ["Cargo clippy", "cargo clippy --all-targets -- -D warnings"], - ["Cargo test (sagascript-core, no features)", "cargo test -p sagascript-core"], - ["Cargo test (sagascript-cli)", "cargo test -p sagascript-cli"], - ["Cargo clippy (sagascript-core, no features)", "cargo clippy -p sagascript-core --all-targets -- -D warnings"], - ["Cargo clippy (sagascript-cli)", "cargo clippy -p sagascript-cli --all-targets -- -D warnings"], - ["Cargo check (sagascript-core, diarization)", "cargo check -p sagascript-core --features diarization"], - ["Cargo test (sagascript-core, diarization)", "cargo test -p sagascript-core --features diarization"], - [ - "Cargo clippy (sagascript-core, diarization)", - "cargo clippy -p sagascript-core --features diarization --all-targets -- -D warnings", - ], - ], - build: [ - ["Build native release binary (no installer)", "npx tauri build --config scripts/tauri-ci-prebuilt.json --no-bundle"], - ["Verify native release binary", 'BINARY=src-tauri/target/release/sagascript\n[[ -x "$BINARY" ]]\nfile "$BINARY"'], - [ - "Smoke test — help and version", - "BINARY=src-tauri/target/release/sagascript\n$BINARY --help\n$BINARY transcribe --help\n$BINARY record --help", - ], - [ - "Smoke test — config management", - "BINARY=src-tauri/target/release/sagascript\n$BINARY config list\n$BINARY config get language\n$BINARY config set language sv\n$BINARY config get language | grep -q sv\n$BINARY config reset language\n$BINARY config get language | grep -q en\n$BINARY config path", - ], - ["Smoke test — model listing and formats", "BINARY=src-tauri/target/release/sagascript\n$BINARY list-models\n$BINARY formats"], - [ - "Smoke test — shell completions", - "BINARY=src-tauri/target/release/sagascript\n$BINARY completions zsh > /dev/null\n$BINARY completions bash > /dev/null", - ], - [ - "Smoke test — download model and transcribe", - 'BINARY=src-tauri/target/release/sagascript\n$BINARY download-model nb-whisper-tiny\nRESULT=$($BINARY transcribe --language no --model nb-whisper-tiny test-audio/norwegian-short-3s.mp3)\necho "Transcription: $RESULT"\necho "$RESULT" | grep -iq "stortinget"', - ], - [ - "Smoke test — transcribe with JSON output", - 'BINARY=src-tauri/target/release/sagascript\nSTDOUT="$RUNNER_TEMP/transcription-smoke.json"\nSTDERR="$RUNNER_TEMP/transcription-smoke.stderr"\n$BINARY transcribe --json --language no --model nb-whisper-tiny \\\n test-audio/norwegian-short-3s.mp3 > "$STDOUT" 2> "$STDERR"\npython3 scripts/verify-json-cli-streams.py "$STDOUT" "$STDERR"', - ], - ], - }, - windows: { - test: [ - ["Cargo check", "cargo check"], - ["Cargo test", "cargo test"], - ["Cargo clippy", "cargo clippy --all-targets -- -D warnings"], - ["Cargo test (sagascript-core, no features)", "cargo test -p sagascript-core"], - ["Cargo test (sagascript-cli)", "cargo test -p sagascript-cli"], - ["Cargo clippy (sagascript-core, no features)", "cargo clippy -p sagascript-core --all-targets -- -D warnings"], - ["Cargo clippy (sagascript-cli)", "cargo clippy -p sagascript-cli --all-targets -- -D warnings"], - ], - build: [ - ["Build native release binary (no installer)", "npx tauri build --config scripts/tauri-ci-prebuilt.json --no-bundle"], - [ - "Verify native release binary", - '$binary = "src-tauri\\target\\release\\sagascript.exe"\nif (-not (Test-Path -PathType Leaf $binary)) {\n throw "Missing native release binary: $binary"\n}', - ], - [ - "Smoke test — help and version", - '$env:BINARY = "src-tauri\\target\\release\\sagascript.exe"\n& $env:BINARY --help\n& $env:BINARY transcribe --help\n& $env:BINARY record --help', - ], - [ - "Smoke test — config management", - '$env:BINARY = "src-tauri\\target\\release\\sagascript.exe"\n& $env:BINARY config list\n& $env:BINARY config get language\n& $env:BINARY config set language sv\n$result = & $env:BINARY config get language\nif ($result -notmatch "sv") { throw "Expected \'sv\'" }\n& $env:BINARY config reset language\n$result = & $env:BINARY config get language\nif ($result -notmatch "en") { throw "Expected \'en\'" }\n& $env:BINARY config path', - ], - [ - "Smoke test — model listing and formats", - '$env:BINARY = "src-tauri\\target\\release\\sagascript.exe"\n& $env:BINARY list-models\n& $env:BINARY formats', - ], - [ - "Smoke test — shell completions", - '$env:BINARY = "src-tauri\\target\\release\\sagascript.exe"\n& $env:BINARY completions powershell | Out-Null', - ], - [ - "Smoke test — download model and transcribe", - '$env:BINARY = "src-tauri\\target\\release\\sagascript.exe"\n& $env:BINARY download-model nb-whisper-tiny\n$result = & $env:BINARY transcribe --language no --model nb-whisper-tiny test-audio/norwegian-short-3s.mp3\nWrite-Host "Transcription: $result"\nif ($result -notmatch "(?i)stortinget") { throw "Expected \'stortinget\' in output" }', - ], - [ - "Smoke test — transcribe with JSON output", - 'set BINARY=src-tauri\\target\\release\\sagascript.exe\n%BINARY% transcribe --json --language no --model nb-whisper-tiny test-audio/norwegian-short-3s.mp3 > json_output.txt 2>nul\ntype json_output.txt\nfindstr /C:"text" json_output.txt\nfindstr /C:"duration_seconds" json_output.txt\nfindstr /C:"language" json_output.txt', - ], - ], - }, -}; - -for (const [platform, jobIds] of Object.entries({ - macos: { test: "test-macos", build: "build-macos" }, - windows: { test: "test-windows", build: "build-windows" }, -})) { - for (const lane of ["test", "build"]) { - test(`${platform} ${lane} lane retains every baseline gate`, () => { - const block = jobBlock(jobIds[lane]); - assert.match(block, /^ needs: scope$/m, "each lane must wait only for the scope gate"); - assert.match(block, /^ if: \$\{\{ needs\.scope\.result == 'success' && needs\.scope\.outputs\.docs_only != 'true' \}\}$/m); - const commands = parseSteps(block); - for (const [name, expected] of BASELINE_GATES[platform][lane]) { - assert.equal(commands.get(name), normalizeCommand(expected), `${jobIds[lane]} must retain the baseline ${name} command`); - } - }); + // Which cargo gates the lane runs decides which Rust components and workspaces it needs. + const invocation = job.steps.map(gateInvocation).find(Boolean); + const gates = invocation ? gatesFor({ os: invocation.os ?? "linux", profile: invocation.profile }) : []; + const components = actionInput(action, setup, "rust-components") + .split(",") + .map((part) => part.trim()); + if (gates.some((gate) => gate.args[0] === "clippy")) assert.ok(components.includes("clippy"), `${id} runs clippy`); + if (gates.some((gate) => gate.args[0] === "fmt")) assert.ok(components.includes("rustfmt"), `${id} runs rustfmt`); + const workspaces = actionInput(action, setup, "cache-workspaces") + .split("\n") + .map((line) => line.trim()); + assert.ok(workspaces.includes("src-tauri")); + if (gates.some((gate) => gate.cwd === "src-tauri/engine-host/ort")) { + assert.ok(workspaces.includes("src-tauri/engine-host/ort"), `${id} runs ORT host gates and must cache its workspace`); + } } -} + const msrv = setupStep(workflow.job("msrv-linux")); + assert.equal(actionInput(action, msrv, "rust"), "msrv"); + assert.equal(actionInput(action, msrv, "python"), "false"); +}); -// The Windows model smoke steps are the only non-blocking exception (#357). -// continue-on-error masks their failures (a step's `conclusion` reads -// "success"), so a report step must surface every one from the real `outcome`. -test("only the two Windows smoke steps are non-blocking, and their failures are reported", () => { - const jobs = workflow.slice(workflow.indexOf("\njobs:\n")); - const jobIds = [...jobs.matchAll(/^ ([a-z][a-z0-9-]*):$/gm)].map((match) => match[1]); - for (const expected of ["test-macos", "build-macos", "test-linux", "msrv-linux", "test-windows", "build-windows"]) { - assert.ok(jobIds.includes(expected), `workflow must define ${expected}`); +test("every lane that runs Python selects the supported runtime first", () => { + // Moved here from scripts/dictation_eval/test_evaluate_cli.py: the workflow is parsed, not read as text. + const action = loadAction("setup"); + const python = action.steps.find((step) => step.uses?.startsWith("actions/setup-python@")); + assert.equal(python.with["python-version"], "3.12"); + assert.equal(python.condition, "${{ inputs.python == 'true' }}"); + assert.equal(action.inputs.python.default, "true"); + for (const id of ["test-macos", "test-linux", "test-windows"]) { + const steps = workflow.job(id).steps; + const setup = steps.findIndex((step) => step.uses === "./.github/actions/setup"); + const suites = steps.findIndex((step) => + commandLines(step).some((line) => line.startsWith("python -m unittest discover -s scripts/dictation_eval")), + ); + assert.equal(actionInput(action, steps[setup], "python"), "true", `${id} selects Python`); + assert.ok(setup >= 0 && suites > setup, `${id} selects Python before the evaluator suites`); } - for (const jobId of jobIds.filter((id) => id !== "build-windows")) { - assert.doesNotMatch(jobBlock(jobId), /continue-on-error:/, `${jobId} must not tolerate failures`); + // The native build lanes run the JSON stream verifier of the smoke tests with it; the MSRV lane runs no Python. + for (const id of ["build-macos", "build-windows"]) assert.equal(actionInput(action, setupStep(workflow.job(id)), "python"), "true"); + assert.equal(actionInput(action, setupStep(workflow.job("msrv-linux")), "python"), "false"); +}); + +test("only the native build lanes cache downloaded models", () => { + const action = loadAction("setup"); + for (const id of LANES) { + assert.equal( + actionInput(action, setupStep(workflow.job(id)), "models"), + id.startsWith("build-") ? "true" : "false", + `${id}: model cache`, + ); } - const windows = jobBlock("build-windows"); - assert.doesNotMatch(windows, /^ continue-on-error:/m, "build-windows itself must stay blocking"); - assert.equal((windows.match(/^ continue-on-error: true$/gm) ?? []).length, 2); - const stepText = (name) => windows.slice(windows.indexOf(` - name: ${name}`)).split(/\n - name:/)[0]; - for (const [name, id] of [ - ["Smoke test — download model and transcribe", "smoke_transcribe"], - ["Smoke test — transcribe with JSON output", "smoke_json"], - ]) { - const step = stepText(name); - assert.match(step, /^ continue-on-error: true$/m, `${name} is a tolerated step`); - assert.match(step, new RegExp(`^ id: ${id}$`, "m"), `${name} needs id ${id} so its outcome can be reported`); + const step = action.steps.find((candidate) => candidate.name === "Cache downloaded models"); + assert.equal(step.condition, "${{ inputs.models == 'true' }}"); + for (const source of ["settings/manager.rs", "transcription/model.rs", "diarization/model.rs"]) { + assert.ok(step.with.key.includes(`sagascript-core/src/${source}`), `the model cache key hashes ${source}`); } - const reportName = "Report masked smoke test failures"; - const report = stepText(reportName); - assert.ok(windows.indexOf(reportName) > windows.indexOf("Smoke test — transcribe with JSON output")); - assert.match(report, /^ if: \$\{\{ always\(\) \}\}$/m); - assert.match(report, /TRANSCRIBE_OUTCOME: \$\{\{ steps\.smoke_transcribe\.outcome \}\}/); - assert.match(report, /JSON_OUTCOME: \$\{\{ steps\.smoke_json\.outcome \}\}/); - assert.match(report, /::warning /); - assert.match(report, /GITHUB_STEP_SUMMARY/); }); -test("model caches are isolated to native lanes", () => { - assert.doesNotMatch(jobBlock("test-macos"), /name: Cache downloaded models/); - assert.doesNotMatch(jobBlock("test-windows"), /name: Cache downloaded models/); - assert.match(jobBlock("build-macos"), /name: Cache downloaded models/); - assert.match(jobBlock("build-windows"), /name: Cache downloaded models/); - assert.match(jobBlock("test-macos"), /shared-key: sagascript-ci-checks/); - assert.match(jobBlock("test-windows"), /shared-key: sagascript-ci-checks/); - assert.match(jobBlock("build-macos"), /shared-key: sagascript-ci-native/); - assert.match(jobBlock("build-windows"), /shared-key: sagascript-ci-native/); +test("the setup action's steps run in the order the lanes depend on", () => { + const action = loadAction("setup"); + const order = action.steps.map((step) => step.name); + const before = (first, second) => + assert.ok(order.indexOf(first) >= 0 && order.indexOf(first) < order.indexOf(second), `${first} before ${second}`); + before("Install Tauri system dependencies", "Select evaluation Python runtime"); + before("Install Rust", "Cache Rust dependencies"); + before("Install the declared MSRV toolchain", "Cache Rust dependencies"); + before("Install Node.js", "Install npm dependencies"); + before("Cache Rust dependencies", "Install npm dependencies"); + // The MSRV toolchain is exported before anything later in the lane looks at rustc. + const lines = commandLines(action.steps.find((step) => step.name === "Install the declared MSRV toolchain")); + assert.ok(lines.includes('echo "RUSTUP_TOOLCHAIN=$msrv" >> "$GITHUB_ENV"')); + assert.ok( + lines.some((line) => line.includes("src-tauri/Cargo.toml")) && lines.some((line) => line.includes("engine-host/ort/Cargo.toml")), + ); }); -function aggregator(jobId, expectedNeeds, testResultName, buildResultName) { - const block = jobBlock(jobId); - const needs = /^ needs: \[([^\]]+)\]$/m - .exec(block)?.[1] - .split(",") - .map((value) => value.trim()); - assert.deepEqual(needs, expectedNeeds, `${jobId} must wait for scope and its applicable lane(s)`); - assert.match(block, /^ if: \$\{\{ always\(\) \}\}$/m); - assert.match(block, new RegExp(`TEST_RESULT: \\\$\\{\\{ needs\\.${testResultName}\\.result \\\}\\}`)); - assert.match(block, new RegExp(`BUILD_RESULT: \\\$\\{\\{ needs\\.${buildResultName}\\.result \\\}\\}`)); - const run = parseSteps(block).get(jobId === "check-macos" ? "Assert macOS CI lanes passed" : "Assert Windows CI lanes passed"); - assert.match(run, /test "\$TEST_RESULT" = success/); - assert.match(run, /test "\$BUILD_RESULT" = success/); - return run; -} +test("test-windows runs the build identity gates before npm ci and any Cargo work, from sources that need no npm packages", () => { + const steps = workflow.job("test-windows").steps; + const find = (predicate) => steps.findIndex((step) => commandLines(step).some(predicate)); + const identityCommand = + "node --test scripts/test-ci-build-identity.mjs scripts/test-windows-release-identity.mjs scripts/test-windows-package-workflow.mjs"; + const identity = find((line) => line.startsWith(identityCommand)); + const setup = steps.findIndex((step) => step.uses === "./.github/actions/setup"); + const install = find((line) => line === "npm ci"); + const gates = find((line) => line.startsWith("node scripts/ci/cargo-gates.mjs")); + assert.ok(setup >= 0 && identity > setup, "after Node is installed"); + assert.equal( + actionInput(loadAction("setup"), steps[setup], "npm-ci"), + "false", + "the action must not run npm ci ahead of the identity gates", + ); + assert.ok(install > identity && gates > install, "npm ci follows the identity gates; Cargo work follows npm ci"); + // They run before node_modules exists, so they and what they import must not need it. + for (const file of [ + "test-ci-build-identity", + "test-windows-release-identity", + "test-windows-package-workflow", + "test-release-promotion", + ]) { + const source = readFileSync(new URL(`./${file}.mjs`, import.meta.url), "utf8"); + assert.ok(!/from "yaml"|ci-workflow\.mjs/.test(source), `${file}.mjs runs before npm ci and must not import the YAML reader`); + } +}); -function runAggregator(run, testResult, buildResult) { - execFileSync("bash", ["-euo", "pipefail", "-c", run], { - env: { - PATH: process.env.PATH, - SCOPE_RESULT: "success", - SCOPE_TRUSTED: "true", - DOCS_ONLY: "false", - TEST_RESULT: testResult, - BUILD_RESULT: buildResult, - }, - stdio: "pipe", - }); -} +test("windows-package.yml leaves the x64 Rust tests of a pull request to test-windows and keeps them for ARM64, releases and manual runs", () => { + const windows = loadWorkflow("windows-package.yml").jobs[0]; + const names = windows.steps.map((step) => step.name); + const tests = windows.steps[names.indexOf("Test and lint Rust workspace")]; + assert.equal(tests.condition, "github.event_name != 'pull_request' || matrix.architecture == 'arm64'"); + // The debug lean CLI that the CPU policy checks read stays built on every architecture. + const build = windows.steps[names.indexOf("Build lean CLI (debug)")]; + assert.ok(build, "the debug build is its own step"); + assert.equal(build.condition, undefined); + assert.deepEqual(commandLines(build), ["cargo build -p sagascript-cli --no-default-features"]); + assert.ok(names.indexOf(build.name) > names.indexOf(tests.name)); + assert.ok(names.indexOf(build.name) < names.indexOf("Verify debug x64 inference CPU policy")); + // test-windows still runs the tests this one no longer repeats for x64. + assert.ok(gatesFor({ os: "windows" }).some((gate) => gate.args.join(" ") === "test --workspace")); +}); -for (const [jobId, needs] of [ - ["check-macos", ["scope", "test-macos", "build-macos"]], - ["check-windows", ["scope", "test-windows", "build-windows"]], -]) { - test(`${jobId} fails closed for every non-success lane result`, () => { - const [, testLane, buildLane] = needs; - const run = aggregator(jobId, needs, testLane, buildLane); - for (const testResult of ["success", "failure", "cancelled", "skipped"]) { - for (const buildResult of ["success", "failure", "cancelled", "skipped"]) { - const invoke = () => runAggregator(run, testResult, buildResult); - if (testResult === "success" && buildResult === "success") { - assert.doesNotThrow(invoke); - } else { - assert.throws(invoke, `${jobId} must reject ${testResult}/${buildResult}`); - } - } - } - }); -} +test("the workflow parses the same with CRLF line endings, as on a Windows checkout", () => { + const source = readFileSync(new URL("../.github/workflows/ci.yml", import.meta.url), "utf8").replace(/\r\n?/g, "\n"); + const lf = parseWorkflow(source); + const crlf = parseWorkflow(source.replaceAll("\n", "\r\n")); + assert.deepEqual(crlf.jobs, lf.jobs); + const classify = crlf.job("scope").steps.find((step) => step.id === "classify"); + assert.ok(!classify.run.includes("\r"), "scripts stay free of carriage returns"); +}); diff --git a/scripts/test-ci-smoke-cli.mjs b/scripts/test-ci-smoke-cli.mjs new file mode 100644 index 00000000..1500d24c --- /dev/null +++ b/scripts/test-ci-smoke-cli.mjs @@ -0,0 +1,244 @@ +import assert from "node:assert/strict"; +import { spawnSync } from "node:child_process"; +import { chmodSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import test from "node:test"; +import { fileURLToPath } from "node:url"; +import { allSteps, commandLines, loadWorkflow } from "./lib/ci-workflow.mjs"; +import { COMPLETION_SHELLS, DEFAULT_BINARY, TIERS, runTier } from "./ci/smoke-cli.mjs"; + +// One smoke script for every OS (scripts/ci/smoke-cli.mjs, #358) replaces the inline bash and +// PowerShell blocks of the two native build lanes. These tests hold what it checks, how the +// lanes call it, and its behaviour against a fake binary. + +const SCRIPT = fileURLToPath(new URL("./ci/smoke-cli.mjs", import.meta.url)); + +// A harness that records the commands a tier runs instead of running anything. `language` plays +// the persisted setting `config set` / `config reset` change. +function recordingHarness() { + const calls = []; + let language = "en"; + return { + calls, + binary: "sagascript", + scratch: "/scratch", + assert(condition, message) { + if (!condition) throw new Error(message); + }, + checkBinary() { + calls.push("check binary"); + }, + runProgram(program, args) { + calls.push([program, ...args].join(" ")); + return { code: 0, stdout: "", stderr: "" }; + }, + ok(args) { + calls.push(args.join(" ")); + if (args[0] === "config" && args[1] === "set") language = args[3]; + if (args[0] === "config" && args[1] === "reset") language = "en"; + if (args[0] === "config" && args[1] === "get") return `${language}\n`; + if (args[0] === "transcribe") return "Stortinget har vedtatt\n"; + if (args[0] === "completions") return "# completion script\n"; + return ""; + }, + capture(label, args) { + calls.push(args.join(" ")); + return { code: args.at(-1).endsWith("missing.wav") ? 1 : 0, stdout: `${label}.stdout`, stderr: `${label}.stderr` }; + }, + verifyStreams(mode) { + calls.push(`verify ${mode.join(" ") || "--json"}`); + }, + audioDirectory: () => "/scratch/jsonl-inputs", + async readFirstLineThenClose(args) { + calls.push(`${args.join(" ")} | head -1`); + return { code: 0, sawLine: true }; + }, + }; +} + +const quiet = { log: () => {} }; + +test("the basic tier covers the binary, help, config, model listing, formats and completions of every shell", async () => { + const harness = recordingHarness(); + const failed = await runTier("basic", harness, quiet); + assert.deepEqual(failed, []); + assert.deepEqual(harness.calls, [ + "check binary", + "--version", + "--help", + "transcribe --help", + "record --help", + "config list", + "config get language", + "config set language sv", + "config get language", + "config reset language", + "config get language", + "config path", + "list-models", + "formats", + ...COMPLETION_SHELLS.map((shell) => `completions ${shell}`), + ]); + assert.deepEqual(COMPLETION_SHELLS, ["zsh", "bash", "powershell"], "completions for every shell on every OS"); +}); + +test("the model tier downloads, transcribes and verifies the JSON, JSONL and error streams of the output contract", async () => { + const harness = recordingHarness(); + const failed = await runTier("model", harness, quiet); + assert.deepEqual(failed, []); + const model = "--language no --model nb-whisper-tiny"; + assert.deepEqual(harness.calls, [ + "download-model nb-whisper-tiny", + `transcribe ${model} test-audio/norwegian-short-3s.mp3`, + `transcribe ${model} --json test-audio/norwegian-short-3s.mp3`, + "verify --json", + `transcribe ${model} --jsonl /scratch/jsonl-inputs`, + "verify --jsonl", + `transcribe ${model} --jsonl /scratch/jsonl-inputs | head -1`, + `transcribe ${model} --json /scratch/missing.wav`, + "verify --error", + ]); +}); + +test("every check of a tier runs even after one fails, and each failure is reported by name", async () => { + const harness = recordingHarness(); + const original = harness.ok; + harness.ok = (args) => { + if (args[0] === "list-models") throw new Error("list-models exited with status 1"); + return original(args); + }; + const logged = []; + const failed = await runTier("basic", harness, { log: (line) => logged.push(line) }); + assert.deepEqual(failed, ["model listing and formats"]); + assert.ok(harness.calls.includes("completions powershell"), "later checks still ran"); + assert.ok(logged.some((line) => line.startsWith("::error title=Smoke test failed::model listing and formats"))); +}); + +test("the lanes run the smoke script after the native build, and only the Windows model tier is non-blocking", () => { + const workflow = loadWorkflow(); + for (const jobId of ["build-macos", "build-windows"]) { + const steps = workflow.job(jobId).steps; + const index = (predicate) => steps.findIndex((step) => commandLines(step).some(predicate)); + const build = index((line) => line.startsWith("npx tauri build")); + const basic = index((line) => line === "node scripts/ci/smoke-cli.mjs basic"); + const model = index((line) => line === "node scripts/ci/smoke-cli.mjs model"); + assert.ok(build >= 0 && basic > build && model >= basic, `${jobId}: build, then the basic tier, then the model tier`); + assert.equal(steps[basic].continueOnError, undefined, `${jobId}: the basic tier blocks`); + assert.equal(steps[model].continueOnError, jobId === "build-windows" ? true : undefined, `${jobId}: model tier tolerance`); + } + const tolerant = allSteps(workflow).filter(({ step }) => step.continueOnError !== undefined); + assert.deepEqual( + tolerant.map(({ job, step }) => `${job.id}:${step.id}`), + ["build-windows:smoke_model"], + "the Windows model tier is the only step that tolerates failure (#357)", + ); + assert.ok( + tolerant.every(({ job }) => job.continueOnError === undefined), + "no job tolerates failure", + ); +}); + +test("a masked Windows smoke failure is reported from the step's real outcome", () => { + const steps = loadWorkflow().job("build-windows").steps; + const model = steps.findIndex((step) => step.id === "smoke_model"); + const report = steps.findIndex((step) => step.name === "Report masked smoke test failures"); + assert.ok(report > model, "the report follows the tolerated step"); + assert.equal(steps[report].condition, "${{ always() }}"); + assert.equal(steps[report].env.MODEL_OUTCOME, "${{ steps.smoke_model.outcome }}"); + const lines = commandLines(steps[report]).join("\n"); + assert.ok(lines.includes("GITHUB_STEP_SUMMARY") && lines.includes("::warning "), "the summary and a warning annotation"); +}); + +// -- the script against a fake binary ---------------------------------------------------------- + +// A stand-in for `sagascript`: keeps `language` in a file, prints what the real CLI prints in the +// places the checks look. FAKE_FAIL names a behaviour to break. +const FAKE_BINARY = `#!/usr/bin/env node +const fs = require("node:fs"); +const path = require("node:path"); +const state = path.join(process.env.FAKE_STATE, "language"); +const args = process.argv.slice(2); +const fail = process.env.FAKE_FAIL; +const read = () => (fs.existsSync(state) ? fs.readFileSync(state, "utf8") : "en"); +const [command, sub] = args; +if (command === "config" && sub === "set") fs.writeFileSync(state, args[3]); +else if (command === "config" && sub === "reset") fs.rmSync(state, { force: true }); +else if (command === "config" && sub === "get") console.log(fail === "config" ? "xx" : read()); +else if (command === "completions") process.stdout.write(fail === "completions" ? "" : "# " + sub + " completion\\n"); +else if (command === "transcribe" && args.includes("--help")) console.log("transcribe help"); +else if (command === "record" && args.includes("--help")) console.log("record help"); +else if (command === "transcribe") { + const input = args.at(-1); + if (input.endsWith("missing.wav")) { + console.log(JSON.stringify({ status: "error", code: "file_decode_error", error: "no such file" })); + console.error("Error: Transcription failed"); + process.exit(1); + } else if (args.includes("--jsonl")) { + for (const name of ["a", "b"]) console.log(JSON.stringify({ status: "ok", source: name + ".mp3", result: { text: "Stortinget " + name } })); + } else if (args.includes("--json")) { + console.log(JSON.stringify({ text: "Stortinget har vedtatt", language: "no" })); + } else { + console.log("Stortinget har vedtatt"); + } +} else if (!["--version", "--help", "list-models", "formats", "config", "download-model"].includes(command)) { + process.exit(2); +} else if (command === "config" && !["list", "path"].includes(sub)) { + process.exit(2); +} +`; + +const python = spawnSync(process.platform === "win32" ? "python" : "python3", ["--version"]).status === 0; + +function withFakeBinary(callback) { + const directory = mkdtempSync(join(tmpdir(), "sagascript-smoke-test-")); + try { + const binary = join(directory, "sagascript"); + writeFileSync(binary, FAKE_BINARY); + chmodSync(binary, 0o755); + return callback(binary, directory); + } finally { + rmSync(directory, { recursive: true, force: true }); + } +} + +function runScript(tier, binary, directory, env = {}) { + return spawnSync(process.execPath, [SCRIPT, tier, "--binary", binary], { + encoding: "utf8", + env: { ...process.env, FAKE_STATE: directory, RUNNER_TEMP: directory, ...env }, + }); +} + +test("both tiers pass against a fake binary that keeps the contract", { skip: process.platform === "win32" || !python }, () => { + withFakeBinary((binary, directory) => { + for (const tier of Object.keys(TIERS)) { + const result = runScript(tier, binary, directory); + assert.equal(result.status, 0, `${tier}: ${result.stdout}${result.stderr}`); + } + }); +}); + +test( + "the script exits 1 and names the check when the binary breaks the contract", + { skip: process.platform === "win32" || !python }, + () => { + withFakeBinary((binary, directory) => { + for (const [fault, tier, check] of [ + ["config", "basic", "config management"], + ["completions", "basic", "shell completions"], + ]) { + const result = runScript(tier, binary, directory, { FAKE_FAIL: fault }); + assert.equal(result.status, 1, `${fault}: ${result.stdout}${result.stderr}`); + assert.match(result.stdout, new RegExp(`::error title=Smoke test failed::${check}`)); + } + const missing = runScript("basic", join(directory, "no-such-binary"), directory); + assert.equal(missing.status, 1); + assert.match(missing.stdout, /the native release binary exists and is executable/); + assert.equal(runScript("nonsense", binary, directory).status, 2); + }); + }, +); + +test("the default binary is the native release build the lanes just made", () => { + assert.equal(DEFAULT_BINARY.replaceAll("\\", "/").replace(/\.exe$/, ""), "src-tauri/target/release/sagascript"); +}); diff --git a/scripts/test-ci-unrun-suites.mjs b/scripts/test-ci-unrun-suites.mjs index 9febc4ba..a4e0f4bb 100644 --- a/scripts/test-ci-unrun-suites.mjs +++ b/scripts/test-ci-unrun-suites.mjs @@ -1,69 +1,86 @@ import assert from "node:assert/strict"; -import { readFile } from "node:fs/promises"; +import { readFileSync, readdirSync } from "node:fs"; import test from "node:test"; +import { fileURLToPath } from "node:url"; +import { commandOf, gatesFor } from "./ci/cargo-gates.mjs"; +import { actionInput, commandLines, loadAction, loadWorkflow, readRepoFile } from "./lib/ci-workflow.mjs"; -// #357: suites that used to run in no PR lane (or only in the release gate) -// are now part of PR CI. These assertions keep them there. +// #357: suites that used to run in no PR lane (or only in the release gate) are now part of PR CI. +// These assertions keep them there. The gate list (scripts/ci/cargo-gates.mjs) says which cargo +// commands each OS runs; the workflow says which lane runs the list, and when. -const read = async (path) => (await readFile(new URL(`../${path}`, import.meta.url), "utf8")).replace(/\r\n?/g, "\n"); +const workflow = loadWorkflow(); +const action = loadAction("setup"); +const devOverrides = readRepoFile("src-tauri/crates/sagascript-core/src/transcription/dev_overrides.rs"); +const workspaceManifest = readRepoFile("src-tauri/Cargo.toml"); +const ortManifest = readRepoFile("src-tauri/engine-host/ort/Cargo.toml"); -const [workflow, devOverrides, workspaceManifest, ortManifest] = await Promise.all([ - read(".github/workflows/ci.yml"), - read("src-tauri/crates/sagascript-core/src/transcription/dev_overrides.rs"), - read("src-tauri/Cargo.toml"), - read("src-tauri/engine-host/ort/Cargo.toml"), -]); - -function jobBlock(jobId) { - const start = workflow.indexOf(`\n ${jobId}:\n`); - assert.notEqual(start, -1, `workflow must define ${jobId}`); - const body = workflow.slice(start + 1); - const next = body.slice(1).search(/^ [a-z][a-z0-9-]*:$/m); - return next === -1 ? body : body.slice(0, next + 1); -} - -// Each step as { name, raw, run, workingDirectory }, in workflow order. -function steps(jobId) { - return jobBlock(jobId) - .split(/\n(?= - name: )/) - .slice(1) - .map((raw) => { - const name = /^ - name: (.*)$/m.exec(raw)[1]; - const inline = /^ run: (?![|>])(.*)$/m.exec(raw)?.[1]; - const block = /^ run: [|>]-?\n((?: .*\n?|\n)*)/m - .exec(raw)?.[1] - ?.split("\n") - .map((line) => line.replace(/^ /, "")) - .join("\n"); - const workingDirectory = /^ working-directory: (.*)$/m.exec(raw)?.[1]; - return { name, raw, run: (inline ?? block ?? "").trim(), workingDirectory }; - }); +const gateLines = (os) => gatesFor({ os }).map((gate) => `${gate.cwd}: ${commandOf(gate)}`); +const stepIndex = (jobId, predicate) => workflow.job(jobId).steps.findIndex(predicate); +const runs = (command) => (step) => commandLines(step).includes(command); +const gatesStep = (os) => (step) => commandLines(step).includes(`node scripts/ci/cargo-gates.mjs --os ${os}`); +// The step that runs `npm ci`: the setup action when it is asked to, otherwise an explicit step. +function npmInstall(jobId) { + return stepIndex( + jobId, + (step) => (step.uses === "./.github/actions/setup" && actionInput(action, step, "npm-ci") === "true") || runs("npm ci")(step), + ); } - -function stepRunning(jobId, command, workingDirectory) { - const matches = steps(jobId).filter((step) => step.run === command); - assert.equal(matches.length, 1, `${jobId} must run \`${command}\` exactly once as its own step`); - const [step] = matches; - assert.equal(step.workingDirectory, workingDirectory, `\`${command}\` must run in ${workingDirectory}`); - assert.doesNotMatch(step.raw, /^ (if|continue-on-error):/m, `\`${command}\` must be unconditional and blocking`); - return step; +function assertBlocking(jobId, index, what) { + const step = workflow.job(jobId).steps[index]; + assert.ok(step, `${jobId} must have ${what}`); + assert.equal(step.condition, undefined, `${what} must be unconditional`); + assert.equal(step.continueOnError, undefined, `${what} must be blocking`); } -test("test-linux runs the engine-protocol wire-format tests", () => { - stepRunning("test-linux", "cargo test -p sagascript-engine-protocol", "src-tauri"); +test("every lane runs the engine-protocol wire-format tests through the workspace tests", () => { + const members = /^members = \[(.*)\]$/m.exec(workspaceManifest)?.[1] ?? ""; + assert.ok(members.includes('"crates/*"'), "the workspace takes every crate under crates/"); + assert.ok( + readdirSync(fileURLToPath(new URL("../src-tauri/crates/", import.meta.url))).includes("sagascript-engine-protocol"), + "the protocol crate is one of them", + ); + for (const os of ["macos", "linux", "windows"]) { + assert.ok( + gateLines(os).includes("src-tauri: cargo test --workspace"), + `${os} runs cargo test --workspace, which reaches the protocol crate`, + ); + } + for (const [jobId, os] of [ + ["test-macos", "macos"], + ["test-linux", "linux"], + ["test-windows", "windows"], + ]) { + assertBlocking(jobId, stepIndex(jobId, gatesStep(os)), "the cargo gates"); + } }); -test("test-linux tests and lints the standalone ORT engine host workspace", () => { +test("test-linux and test-windows test and lint the standalone ORT engine host workspace", () => { assert.match(ortManifest, /^\[workspace\]$/m, "the ORT host is its own Cargo workspace"); - stepRunning("test-linux", "cargo test --locked", "src-tauri/engine-host/ort"); - stepRunning("test-linux", "cargo clippy --all-targets --locked -- -D warnings", "src-tauri/engine-host/ort"); - const fmt = steps("test-linux").find((step) => step.name === "Check Rust formatting"); - assert.ok(fmt, "test-linux must keep its rustfmt gate"); - assert.match(fmt.run, /^cargo fmt --manifest-path engine-host\/ort\/Cargo\.toml --all -- --check$/m); - - const cache = steps("test-linux").find((step) => step.name === "Cache Rust dependencies"); - assert.ok(cache, "test-linux must keep its Rust cache"); - assert.match(cache.raw, /^ workspaces: \|\n src-tauri\n src-tauri\/engine-host\/ort\n/m); + for (const os of ["linux", "windows"]) { + const lines = gateLines(os); + assert.ok(lines.includes("src-tauri/engine-host/ort: cargo test --locked"), `${os} tests the ORT host`); + assert.ok(lines.includes("src-tauri/engine-host/ort: cargo clippy --all-targets --locked -- -D warnings"), `${os} lints the ORT host`); + } + assert.ok( + !gateLines("macos").some((line) => line.startsWith("src-tauri/engine-host/ort")), + "macOS ships the Core ML host, not the ORT host", + ); + assert.ok( + gateLines("linux").includes("src-tauri: cargo fmt --manifest-path engine-host/ort/Cargo.toml --all -- --check"), + "test-linux must keep its rustfmt gate for the ORT host", + ); + for (const jobId of ["test-linux", "test-windows"]) { + const setup = workflow.job(jobId).steps.find((step) => step.uses === "./.github/actions/setup"); + assert.deepEqual( + actionInput(action, setup, "cache-workspaces") + .split("\n") + .map((line) => line.trim()) + .filter(Boolean), + ["src-tauri", "src-tauri/engine-host/ort"], + `${jobId} must cache both Cargo workspaces`, + ); + } }); test("test-linux runs the release-only dev-override guard in the release profile", () => { @@ -75,52 +92,78 @@ test("test-linux runs the release-only dev-override guard in the release profile // ...and its doc comment names the command CI must run. const command = "cargo test --release -p sagascript-core --lib dev_overrides"; assert.ok(devOverrides.includes(`(\`${command}\`)`), "the test's doc comment must name the CI command"); - const guard = steps("test-linux").filter((step) => step.run.split("\n").some((line) => line.startsWith(command))); - assert.equal(guard.length, 1, "test-linux must run the release guard exactly once"); - assert.equal(guard[0].workingDirectory, "src-tauri"); - assert.doesNotMatch(guard[0].raw, /^ (if|continue-on-error):/m, "the release guard must be unconditional and blocking"); - // cargo test exits 0 when the filter matches nothing; the step must prove - // the guard test itself ran, and must not lose cargo's exit code in a pipe. - assert.match(guard[0].run, /^set -o pipefail$/m); - assert.match(guard[0].run, /^grep -q 'release_build_ignores_the_real_environment \.\.\. ok' /m); + const guards = gatesFor({ os: "linux" }).filter((gate) => commandOf(gate) === command); + assert.equal(guards.length, 1, "test-linux must run the release guard exactly once"); + const [guard] = guards; + assert.equal(guard.cwd, "src-tauri"); + // cargo test exits 0 when the filter matches nothing; the gate must prove the guard test itself ran. + assert.ok( + guard.requireOutput?.pattern.test("release_build_ignores_the_real_environment ... ok"), + "the gate must require the guard's own pass line", + ); + assert.ok(!guard.requireOutput.pattern.test("running 0 tests")); + assert.ok( + ["macos", "windows"].every((os) => !gateLines(os).some((line) => line.includes("dev_overrides"))), + "the guard compiles core in the release profile once, on Linux", + ); // A release profile with debug assertions would compile the guard out and // leave the step green with zero tests. const release = /^\[profile\.release\]\n([\s\S]*?)(?=^\[|(?![\s\S]))/m.exec(workspaceManifest)?.[1]; assert.ok(release !== undefined, "src-tauri/Cargo.toml must define [profile.release]"); assert.doesNotMatch(release, /^debug-assertions\s*=\s*true/m); - assert.doesNotMatch(workflow, /CARGO_PROFILE_RELEASE_DEBUG_ASSERTIONS/); + const environments = [ + workflow.doc.env, + action.doc.runs.env, + ...workflow.jobs.flatMap((job) => [job.env, ...job.steps.map((step) => step.env)]), + ...action.steps.map((step) => step.env), + ]; + for (const environment of environments) { + assert.ok( + !Object.keys(environment ?? {}).includes("CARGO_PROFILE_RELEASE_DEBUG_ASSERTIONS"), + "no lane may switch debug assertions on in release", + ); + } + for (const name of readdirSync(fileURLToPath(new URL("../.github/workflows/", import.meta.url)))) { + assert.ok( + !readFileSync(new URL(`../.github/workflows/${name}`, import.meta.url), "utf8").includes("CARGO_PROFILE_RELEASE_DEBUG_ASSERTIONS"), + name, + ); + } }); -test("test-windows runs the full frontend suite after npm ci", () => { - const names = steps("test-windows").map((step) => step.name); - const suite = stepRunning("test-windows", "npm run test:frontend", undefined); - const index = names.indexOf(suite.name); - assert.ok(names.indexOf("Install npm dependencies") >= 0, "test-windows must install npm dependencies"); - assert.ok(index > names.indexOf("Install npm dependencies"), "the suite needs node_modules"); - assert.ok(index < names.indexOf("Cargo check"), "the suite must fail before Cargo work"); - const identity = names.indexOf("Test Windows build identity gates"); - assert.ok(identity >= 0 && identity < names.indexOf("Install npm dependencies"), "the early build identity gates stay ahead of npm ci"); +test("test-windows runs the full frontend suite after npm ci and before Cargo work", () => { + const suite = stepIndex("test-windows", runs("npm run test:frontend")); + const install = npmInstall("test-windows"); + assert.ok(install >= 0, "test-windows must install npm dependencies"); + assert.ok(suite > install, "the suite needs node_modules"); + assert.ok(suite < stepIndex("test-windows", gatesStep("windows")), "the suite must fail before Cargo work"); + assertBlocking("test-windows", suite, "`npm run test:frontend`"); + const identity = stepIndex("test-windows", (step) => + commandLines(step).some((line) => line.startsWith("node --test scripts/test-ci-build-identity.mjs")), + ); + assert.ok(identity >= 0 && identity < install, "the early build identity gates stay ahead of npm ci"); }); test("test-macos runs ESLint after npm ci and before Cargo work", () => { - const names = steps("test-macos").map((step) => step.name); - const step = stepRunning("test-macos", "npm run lint", undefined); - const index = names.indexOf(step.name); - assert.ok(index > names.indexOf("Install npm dependencies"), "`npm run lint` needs node_modules"); - assert.ok(index < names.indexOf("Cargo check"), "`npm run lint` must fail before Cargo work"); + const lint = stepIndex("test-macos", runs("npm run lint")); + assert.ok(lint > npmInstall("test-macos"), "`npm run lint` needs node_modules"); + assert.ok(lint < stepIndex("test-macos", gatesStep("macos")), "`npm run lint` must fail before Cargo work"); + assertBlocking("test-macos", lint, "`npm run lint`"); }); test("test-macos and test-windows run the Vitest unit tests and the Prettier check after npm ci", () => { - for (const job of ["test-macos", "test-windows"]) { - const names = steps(job).map((step) => step.name); - const install = names.indexOf("Install npm dependencies"); - assert.ok(install >= 0, `${job} must install npm dependencies`); + for (const [jobId, os] of [ + ["test-macos", "macos"], + ["test-windows", "windows"], + ]) { + const install = npmInstall(jobId); + assert.ok(install >= 0, `${jobId} must install npm dependencies`); for (const command of ["npm run test:unit", "npm run format:check"]) { - const step = stepRunning(job, command, undefined); - const index = names.indexOf(step.name); - assert.ok(index > install, `${job}: \`${command}\` needs node_modules`); - assert.ok(index < names.indexOf("Cargo check"), `${job}: \`${command}\` must fail before Cargo work`); + const index = stepIndex(jobId, runs(command)); + assert.ok(index > install, `${jobId}: \`${command}\` needs node_modules`); + assert.ok(index < stepIndex(jobId, gatesStep(os)), `${jobId}: \`${command}\` must fail before Cargo work`); + assertBlocking(jobId, index, `\`${command}\``); } } }); diff --git a/scripts/test-clippy-policy.mjs b/scripts/test-clippy-policy.mjs index 55a0c6db..c9417651 100644 --- a/scripts/test-clippy-policy.mjs +++ b/scripts/test-clippy-policy.mjs @@ -8,6 +8,8 @@ import { join } from "node:path"; import test from "node:test"; import { fileURLToPath } from "node:url"; +import { gatesFor } from "./ci/cargo-gates.mjs"; +import { commandLines, loadWorkflow } from "./lib/ci-workflow.mjs"; import { POLICY_LINTS, RUNS, @@ -409,17 +411,25 @@ test("clippy.toml exempts tests and sets the size thresholds", () => { }); test("test-linux runs the policy check after the clippy gates", () => { - const ci = read(".github/workflows/ci.yml"); - const start = ci.indexOf("\n test-linux:\n"); - const end = ci.indexOf("\n msrv-linux:\n"); - assert.ok(start >= 0 && end > start, "ci.yml must define test-linux before msrv-linux"); - const job = ci.slice(start, end); - const steps = job.split(/\n(?= - name: )/).slice(1); - const names = steps.map((step) => /^ - name: (.*)$/m.exec(step)[1]); - const index = steps.findIndex((step) => /^ run: node scripts\/clippy-policy\.mjs$/m.test(step)); - assert.ok(index >= 0, "test-linux must run `node scripts/clippy-policy.mjs` as its own step"); - assert.doesNotMatch(steps[index], /^ (if|continue-on-error|working-directory):/m); - assert.ok(index > names.indexOf("Install Node.js"), "the check needs Node.js"); - assert.ok(index > names.indexOf("Cargo clippy (GUI app)"), "the regular clippy gate runs first"); - assert.ok(index > names.indexOf("Cargo clippy (headless CLI, batch-only)"), "the regular clippy gate runs first"); + const steps = loadWorkflow().job("test-linux").steps; + const index = steps.findIndex((step) => commandLines(step).includes("node scripts/clippy-policy.mjs")); + assert.ok(index >= 0, "test-linux must run `node scripts/clippy-policy.mjs`"); + assert.deepEqual(commandLines(steps[index]), ["node scripts/clippy-policy.mjs"], "as its own step"); + assert.equal(steps[index].condition, undefined); + assert.equal(steps[index].continueOnError, undefined); + assert.equal(steps[index].workingDirectory, undefined); + const setup = steps.findIndex((step) => step.uses === "./.github/actions/setup"); + assert.ok(index > setup, "the check needs Node.js, which the setup action installs"); + // The regular clippy gates run first: they compile the dependencies the policy run reuses. + const gates = steps.findIndex((step) => commandLines(step).includes("node scripts/ci/cargo-gates.mjs --os linux")); + assert.ok(index > gates, "the regular clippy gates run first"); + const clippy = gatesFor({ os: "linux" }).filter((gate) => gate.args[0] === "clippy"); + assert.ok( + clippy.some((gate) => gate.args.includes("--workspace")), + "the workspace clippy gate", + ); + assert.ok( + clippy.some((gate) => gate.args.includes("--no-default-features") && gate.args.includes("sagascript-cli")), + "the lean CLI clippy gate", + ); }); diff --git a/scripts/test-dependency-policy.mjs b/scripts/test-dependency-policy.mjs index 80352628..c3406bad 100644 --- a/scripts/test-dependency-policy.mjs +++ b/scripts/test-dependency-policy.mjs @@ -6,11 +6,12 @@ import assert from "node:assert/strict"; import { readdirSync } from "node:fs"; import { readFile } from "node:fs/promises"; import test from "node:test"; +import { commandOf, gatesFor } from "./ci/cargo-gates.mjs"; +import { commandLines, loadWorkflow } from "./lib/ci-workflow.mjs"; const read = async (path) => (await readFile(new URL(`../${path}`, import.meta.url), "utf8")).replace(/\r\n?/g, "\n"); -const [workflow, deny, notices, ...manifests] = await Promise.all([ - read(".github/workflows/ci.yml"), +const [deny, notices, ...manifests] = await Promise.all([ read("src-tauri/deny.toml"), read("scripts/generate-third-party-notices.mjs"), ...[ @@ -20,40 +21,13 @@ const [workflow, deny, notices, ...manifests] = await Promise.all([ ].map(read), ]); -function jobBlock(jobId) { - const start = workflow.indexOf(`\n ${jobId}:\n`); - assert.notEqual(start, -1, `workflow must define ${jobId}`); - const body = workflow.slice(start + 1); - const next = body.slice(1).search(/^ [a-z][a-z0-9-]*:$/m); - return next === -1 ? body : body.slice(0, next + 1); -} - -// Each step as { name, raw, commands, workingDirectory }, in workflow order. -function steps(jobId) { - return jobBlock(jobId) - .split(/\n(?= - name: )/) - .slice(1) - .map((raw) => { - const name = /^ - name: (.*)$/m.exec(raw)[1]; - const inline = /^ run: (?![|>])(.*)$/m.exec(raw)?.[1]; - const block = /^ run: [|>]-?\n((?: .*\n?|\n)*)/m.exec(raw)?.[1]; - const commands = (inline ?? block ?? "") - .split("\n") - .map((line) => line.trim()) - .filter(Boolean); - const workingDirectory = /^ working-directory: (.*)$/m.exec(raw)?.[1]; - return { name, raw, commands, workingDirectory }; - }); -} - -function blockingStep(jobId, command, workingDirectory) { - const all = steps(jobId); - const matches = all.filter((step) => step.commands.includes(command)); - assert.equal(matches.length, 1, `${jobId} must run \`${command}\` exactly once`); - const [step] = matches; - assert.equal(step.workingDirectory, workingDirectory, `\`${command}\` must run in ${workingDirectory ?? "the repository root"}`); - assert.doesNotMatch(step.raw, /^ (if|continue-on-error):/m, `\`${command}\` must be unconditional and blocking`); - return all.indexOf(step); +const workflow = loadWorkflow(); +const stepIndex = (jobId, predicate) => workflow.job(jobId).steps.findIndex(predicate); +const runs = (command) => (step) => commandLines(step).includes(command); +// A step that gates unconditionally: no `if`, no `continue-on-error`. +function assertBlocking(step, what) { + assert.equal(step.condition, undefined, `${what} must be unconditional`); + assert.equal(step.continueOnError, undefined, `${what} must be blocking`); } // The string items of a TOML array `key = [ ... ]` inside `[section]`. @@ -66,37 +40,50 @@ function tomlStrings(text, section, key) { } test("test-linux gates both Cargo workspaces on cargo-deny before compiling", () => { - const names = steps("test-linux").map((step) => step.name); - const install = blockingStep("test-linux", "cargo install --locked cargo-deny --version 0.20.2", undefined); - const main = blockingStep("test-linux", "cargo deny --locked --workspace check", "src-tauri"); + const INSTALL = "cargo install --locked cargo-deny --version 0.20.2"; + const linux = gatesFor({ os: "linux" }); // The ORT host run may only silence the "policy entry not used here" notes, // not skip a check or allow a finding. assert.deepEqual( - steps("test-linux")[main].commands, + linux.slice(0, 2).map((gate) => [gate.cwd, commandOf(gate)]), [ - "cargo deny --locked --workspace check", - "cargo deny --locked --manifest-path engine-host/ort/Cargo.toml --workspace check -A license-not-encountered -A license-exception-not-encountered -A unmatched-source", + ["src-tauri", "cargo deny --locked --workspace check"], + [ + "src-tauri", + "cargo deny --locked --manifest-path engine-host/ort/Cargo.toml --workspace check -A license-not-encountered -A license-exception-not-encountered -A unmatched-source", + ], ], - "the cargo-deny step must check both workspaces, every check kind, against the committed lockfiles", + "the cargo-deny gates must check both workspaces, every check kind, against the committed lockfiles, ahead of every gate that compiles", + ); + assert.ok( + linux.slice(2).every((gate) => gate.args[0] !== "deny"), + "no other gate runs cargo-deny", ); - assert.ok(install < main, "cargo-deny must be installed before it runs"); - assert.ok(main > names.indexOf("Cache Rust dependencies"), "the cached cargo-deny binary must be restored first"); - assert.ok(main < names.indexOf("Cargo check (GUI app)"), "the policy must fail before any compilation"); + const steps = workflow.job("test-linux").steps; + const setup = stepIndex("test-linux", (step) => step.uses === "./.github/actions/setup"); + const install = stepIndex("test-linux", runs(INSTALL)); + const gates = stepIndex("test-linux", (step) => commandLines(step).includes("node scripts/ci/cargo-gates.mjs --os linux")); + assert.ok(setup >= 0 && install >= 0 && gates >= 0); + assertBlocking(steps[install], "the cargo-deny install"); + assertBlocking(steps[gates], "the cargo gates"); + assert.ok(setup < install, "the cached cargo-deny binary must be restored first"); + assert.ok(install < gates, "cargo-deny must be installed before it runs"); }); test("npm run verify runs the same cargo-deny release CI installs", async () => { const { CARGO_DENY_VERSION } = await import("./verify.mjs"); assert.ok( - steps("test-linux").some((step) => step.commands.includes(`cargo install --locked cargo-deny --version ${CARGO_DENY_VERSION}`)), + workflow.job("test-linux").steps.some(runs(`cargo install --locked cargo-deny --version ${CARGO_DENY_VERSION}`)), `verify's preflight asks for cargo-deny ${CARGO_DENY_VERSION}; test-linux must install the same release`, ); }); test("test-macos gates the npm lockfile on npm audit", () => { - const names = steps("test-macos").map((step) => step.name); - const audit = blockingStep("test-macos", "npm audit --audit-level=high", undefined); - assert.ok(audit > names.indexOf("Install npm dependencies"), "audit after npm ci"); - assert.ok(audit < names.indexOf("Build frontend"), "audit before the bundle is built"); + const audit = stepIndex("test-macos", runs("npm audit --audit-level=high")); + assert.ok(audit >= 0, "test-macos must run npm audit"); + assertBlocking(workflow.job("test-macos").steps[audit], "`npm audit --audit-level=high`"); + assert.ok(audit > stepIndex("test-macos", (step) => step.uses === "./.github/actions/setup"), "audit after npm ci (the setup action)"); + assert.ok(audit < stepIndex("test-macos", runs("npm run build")), "audit before the bundle is built"); }); // SPDX expression satisfiability: AND binds tighter than OR, the legacy "/" diff --git a/scripts/test-frontend-test-groups.mjs b/scripts/test-frontend-test-groups.mjs index 3957b807..80305e9e 100644 --- a/scripts/test-frontend-test-groups.mjs +++ b/scripts/test-frontend-test-groups.mjs @@ -2,7 +2,7 @@ import assert from "node:assert/strict"; import { spawn } from "node:child_process"; import { chmodSync, existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; -import { delimiter, dirname, join } from "node:path"; +import { delimiter, dirname, join, posix } from "node:path"; import { fileURLToPath } from "node:url"; import test from "node:test"; import { isProcessAlive } from "./lib/process-tree.mjs"; @@ -92,16 +92,25 @@ test("the real manifest classifies every Node test file exactly once", () => { test("each group declares the npm packages its tests import", () => { // Static, side-effect and dynamic imports of a bare package specifier. const bareImport = /\bfrom\s*["']([@a-z][\w@./:-]*)["']|\bimport\s*\(?\s*["']([@a-z][\w@./:-]*)["']/g; + const relativeImport = /\bfrom\s*["'](\.{1,2}\/[^"']+\.mjs)["']/g; + // A test needs the packages of the repository modules it imports as well (scripts/lib/ci-workflow.mjs + // reads the workflows with `yaml`, #358), so relative imports are followed. + function collect(file, imported, seen) { + if (seen.has(file)) return; + seen.add(file); + // Comments that mention imports (`import "name"`) are not imports. + const source = readFileSync(join(repoRoot, file), "utf8").replace(/\/\*[\s\S]*?\*\/|^\s*\/\/.*$/gm, ""); + for (const match of source.matchAll(bareImport)) { + const specifier = match[1] ?? match[2]; + if (specifier.startsWith("node:")) continue; + imported.add(specifier.startsWith("@") ? specifier.split("/").slice(0, 2).join("/") : specifier.split("/")[0]); + } + for (const match of source.matchAll(relativeImport)) + collect(posix.normalize(posix.join(posix.dirname(file), match[1])), imported, seen); + } for (const [group, files] of Object.entries(TEST_GROUPS)) { const imported = new Set(); - for (const file of files) { - const source = readFileSync(join(repoRoot, file), "utf8"); - for (const match of source.matchAll(bareImport)) { - const specifier = match[1] ?? match[2]; - if (specifier.startsWith("node:")) continue; - imported.add(specifier.startsWith("@") ? specifier.split("/").slice(0, 2).join("/") : specifier.split("/")[0]); - } - } + for (const file of files) collect(file, imported, new Set()); assert.deepEqual([...imported].sort(), [...(GROUP_NPM_PACKAGES[group] ?? [])].sort(), `group ${group}`); } }); diff --git a/scripts/test-pianissimo-requirements.mjs b/scripts/test-pianissimo-requirements.mjs index 763355cb..57e093f5 100644 --- a/scripts/test-pianissimo-requirements.mjs +++ b/scripts/test-pianissimo-requirements.mjs @@ -2,6 +2,7 @@ import assert from "node:assert/strict"; import { existsSync, readFileSync } from "node:fs"; import { test } from "node:test"; import { fileURLToPath } from "node:url"; +import { commandLines, loadWorkflow } from "./lib/ci-workflow.mjs"; const read = (path) => readFileSync(fileURLToPath(new URL(`../${path}`, import.meta.url)), "utf8"); const exists = (path) => existsSync(fileURLToPath(new URL(`../${path}`, import.meta.url))); @@ -74,10 +75,23 @@ test("release verifier checks the bundled host", () => { }); test("CI builds the host, runs swift test, and checks the revision", () => { - const content = read(".github/workflows/ci.yml"); - assert.match(content, /scripts\/stage-engine-host\.sh/); - assert.match(content, /swift test/); - assert.match(content, /check-engine-host-identity\.sh[^\n]*GITHUB_SHA/); + const steps = loadWorkflow().job("test-macos").steps; + const stepWith = (predicate) => steps.findIndex((step) => commandLines(step).some(predicate)); + const stage = stepWith((line) => line === "./scripts/stage-engine-host.sh"); + const identity = stepWith( + (line) => line === './scripts/check-engine-host-identity.sh build/engine-host/sagascript-engine-host "$GITHUB_SHA"', + ); + const swift = stepWith((line) => line === "swift test"); + for (const [what, index] of [ + ["stages the host", stage], + ["checks its revision against GITHUB_SHA", identity], + ["runs swift test", swift], + ]) { + assert.ok(index >= 0, `test-macos ${what}`); + assert.equal(steps[index].continueOnError, undefined, `test-macos ${what} and blocks on it`); + } + assert.ok(stage <= identity, "the host is staged before its revision is checked"); + assert.equal(steps[swift].workingDirectory, "src-tauri/engine-host/coreml"); }); test("Pianissimo smoke covers status, download, doctor and transcribe", () => { diff --git a/scripts/test-pipeline-cache.mjs b/scripts/test-pipeline-cache.mjs index 594f15ec..2b73c5ac 100644 --- a/scripts/test-pipeline-cache.mjs +++ b/scripts/test-pipeline-cache.mjs @@ -1,11 +1,11 @@ import assert from "node:assert/strict"; import { readFile } from "node:fs/promises"; import test from "node:test"; +import { allSteps, loadAction, loadWorkflow } from "./lib/ci-workflow.mjs"; const rustCacheRef = "Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6"; -const [ciWorkflow, testBuildWorkflow, windowsWorkflow] = await Promise.all([ - readFile(new URL("../.github/workflows/ci.yml", import.meta.url), "utf8"), +const [testBuildWorkflow, windowsWorkflow] = await Promise.all([ readFile(new URL("../.github/workflows/test-build.yml", import.meta.url), "utf8"), readFile(new URL("../.github/workflows/windows-package.yml", import.meta.url), "utf8"), ]); @@ -18,12 +18,15 @@ function section(content, startMarker, endMarker) { } test("CI rust caches use the reviewed immutable action revision", () => { - const refs = ciWorkflow.match(/uses: Swatinem\/rust-cache@[^\s]+/g) ?? []; - // test-macos, build-macos, test-linux, msrv-linux (#353), test-windows, build-windows - assert.equal(refs.length, 6); + const cacheUses = (steps) => steps.map((step) => step.uses).filter((uses) => uses?.startsWith("Swatinem/rust-cache@")); + // The cache is part of the setup action (#358), the one place that names the revision; ci.yml's + // six lanes (test-macos, build-macos, test-linux, msrv-linux #353, test-windows, build-windows) + // reach it through the action, which scripts/test-ci-parallel-gates.mjs checks lane by lane. + assert.deepEqual(cacheUses(loadAction("setup").steps), [rustCacheRef]); assert.deepEqual( - refs, - Array.from({ length: 6 }, () => `uses: ${rustCacheRef}`), + cacheUses(allSteps(loadWorkflow()).map(({ step }) => step)), + [], + "a ci.yml lane that names its own rust-cache would bypass the shared setup", ); }); diff --git a/scripts/test-release-promotion.mjs b/scripts/test-release-promotion.mjs index 330bf823..99b8cab8 100644 --- a/scripts/test-release-promotion.mjs +++ b/scripts/test-release-promotion.mjs @@ -97,7 +97,16 @@ test("windows-package keeps the console CLI, installed-CLI and portable zip step assert.match(w, /Verify silent per-user install runs the installed ARM64 CLI/); assert.match(w, /Build and verify ARM64 portable zip/); assert.match(w, /Verify installers carry the console CLI/); - assert.doesNotMatch(w, /if:[^\n]*github\.event_name/); + // The one step that depends on the event is the Rust workspace test (#358): ci.yml's test-windows + // owns the x64 tests of a pull request, so this workflow skips them for x64 there. Releases come in + // through workflow_call from a push or a dispatch, never a pull request, and still run them. + const conditional = w + .split("\n - name: ") + .slice(1) + .filter((step) => /if:[^\n]*github\.event_name/.test(step)) + .map((step) => step.split("\n")[0]); + assert.deepEqual(conditional, ["Test and lint Rust workspace"]); + assert.match(w, /if: github\.event_name != 'pull_request' \|\| matrix\.architecture == 'arm64'\n/); }); test("prebuild runs on main pushes only, never on pull requests, and never cancels", () => { diff --git a/scripts/test-rust-toolchain-pin.mjs b/scripts/test-rust-toolchain-pin.mjs index e8a8ff88..f74434f5 100644 --- a/scripts/test-rust-toolchain-pin.mjs +++ b/scripts/test-rust-toolchain-pin.mjs @@ -5,17 +5,25 @@ import assert from "node:assert/strict"; import { readFileSync, readdirSync } from "node:fs"; import { fileURLToPath } from "node:url"; import test from "node:test"; +import { commandOf, gatesFor } from "./ci/cargo-gates.mjs"; +import { commandLines, loadAction, loadWorkflow } from "./lib/ci-workflow.mjs"; const read = (path) => readFileSync(fileURLToPath(new URL(`../${path}`, import.meta.url)), "utf8").replace(/\r\n?/g, "\n"); const toolchainFile = read("rust-toolchain.toml"); const channel = /^channel = "([^"]+)"\s*(?:#.*)?$/m.exec(toolchainFile)?.[1]; const nodeMajor = read(".nvmrc").trim(); -const workflows = Object.fromEntries( - readdirSync(fileURLToPath(new URL("../.github/workflows/", import.meta.url))) +// Every workflow, and the local composite actions (#358): ci.yml installs its toolchains through +// .github/actions/setup, so the pins live in the action file. +const workflows = Object.fromEntries([ + ...readdirSync(fileURLToPath(new URL("../.github/workflows/", import.meta.url))) .filter((name) => name.endsWith(".yml")) .map((name) => [name, read(`.github/workflows/${name}`)]), -); + ...readdirSync(fileURLToPath(new URL("../.github/actions/", import.meta.url))).map((name) => [ + `actions/${name}/action.yml`, + read(`.github/actions/${name}/action.yml`), + ]), +]); const rustVersion = (manifest) => /^rust-version = "([^"]+)"$/m.exec(read(manifest))?.[1]; const msrv = rustVersion("src-tauri/Cargo.toml"); @@ -49,7 +57,7 @@ test("every workflow Rust install is SHA-pinned and uses the rust-toolchain.toml assert.equal(withInputs(text, match.index).toolchain, channel, `${name}: toolchain must equal rust-toolchain.toml`); } } - assert.ok(installs >= 8, `expected the CI, release and packaging installs, found ${installs}`); + assert.ok(installs >= 6, `expected the setup action and the release, packaging and gate installs, found ${installs}`); }); test("a workflow that installs a different Rust release overrides the toolchain file explicitly", () => { @@ -80,12 +88,22 @@ test("the MSRV is declared once, inherited by every crate and checked in CI", () for (const crate of ["src-tauri", ...members]) { assert.match(read(`${crate}/Cargo.toml`), /^rust-version\.workspace = true$/m, `${crate} must inherit rust-version`); } - const ci = workflows["ci.yml"]; - const msrvJob = ci.slice(ci.indexOf("\n msrv-linux:\n"), ci.indexOf("\n check-linux:\n")); - assert.match(msrvJob, /rust-version = /, "msrv-linux must read the declared rust-version, not hardcode it"); - assert.match(msrvJob, /echo "RUSTUP_TOOLCHAIN=\$msrv" >> "\$GITHUB_ENV"/); - assert.match(msrvJob, /cargo check --workspace --locked/); - assert.match(ci, /^ check-linux:\n needs: \[[^\]]*msrv-linux[^\]]*\]$/m, "check-linux must gate on msrv-linux"); + // The MSRV lane installs the declared `rust-version` through the setup action, which exports + // RUSTUP_TOOLCHAIN, and compiles with the gates' msrv profile. + const workflow = loadWorkflow(); + const msrvLane = workflow.job("msrv-linux"); + const setup = msrvLane.steps.find((step) => step.uses === "./.github/actions/setup"); + assert.equal(setup?.with.rust, "msrv", "msrv-linux must install the declared MSRV, not a pinned release"); + const install = loadAction("setup").steps.find((step) => step.name === "Install the declared MSRV toolchain"); + const lines = commandLines(install); + assert.ok( + lines.some((line) => line.includes("rust-version = ")), + "the action must read the declared rust-version, not hardcode it", + ); + assert.ok(lines.includes('echo "RUSTUP_TOOLCHAIN=$msrv" >> "$GITHUB_ENV"')); + assert.ok(msrvLane.steps.some((step) => commandLines(step).includes("node scripts/ci/cargo-gates.mjs --profile msrv"))); + assert.ok(gatesFor({ os: "linux", profile: "msrv" }).some((gate) => commandOf(gate) === "cargo check --workspace --locked")); + assert.ok(workflow.job("check-linux").needs.includes("msrv-linux"), "check-linux must gate on msrv-linux"); }); test("Node.js is pinned to one major everywhere", () => { diff --git a/scripts/test-verify-ci-parity.mjs b/scripts/test-verify-ci-parity.mjs index 861c9a05..bb29da9d 100644 --- a/scripts/test-verify-ci-parity.mjs +++ b/scripts/test-verify-ci-parity.mjs @@ -4,28 +4,36 @@ import { fileURLToPath } from "node:url"; import test from "node:test"; import { TEST_GROUPS } from "./node-test-groups.mjs"; import { CHECKS } from "./verify.mjs"; +import { GATES, commandOf, gatesFor } from "./ci/cargo-gates.mjs"; +import { expandedSteps, gateInvocation, parseWorkflow } from "./lib/ci-workflow.mjs"; // Keeps scripts/verify.mjs (the local check list, #350) and the test lanes of -// .github/workflows/ci.yml from drifting apart until CI calls verify itself -// (#358): +// .github/workflows/ci.yml from drifting apart: // - every ci.yml job is either a test lane or a listed non-test job; -// - every command in a test lane's run scripts (named or nameless steps, any -// program, not only cargo/npm/node) is covered by a verify check, the -// Node test-group manifest, an allowlisted difference or a listed shell -// word that is not a check (each with its reason); +// - every command in a test lane (named or nameless steps, any program, not +// only cargo/npm/node) is covered by a verify check, the Node test-group +// manifest, an allowlisted difference or a listed shell word that is not a +// check (each with its reason); // - every CI command a verify check claims exists in a test lane. // +// What a lane runs is read from the parsed workflow (#358), through the two +// places the lanes keep commands outside their own `run:` scripts: the cargo +// gate list (a step running scripts/ci/cargo-gates.mjs stands for that OS's +// gates, in scripts/ci/cargo-gates.mjs) and the local setup action (a step +// using it stands for the action's `run` steps its inputs enable). +// // Limits: environment set elsewhere (job `env:`, $GITHUB_ENV, e.g. the MSRV // lane's RUSTUP_TOOLCHAIN or test-macos's DEVELOPER_DIR), `if:` conditions, -// `uses:` actions, commands inside $(…), PowerShell syntax and shell control -// flow beyond splitting at `;`, `&&`, `||` and `|` are not compared. +// third-party `uses:` actions, commands inside $(…), PowerShell syntax and +// shell control flow beyond splitting at `;`, `&&`, `||` and `|` are not +// compared. // // SAGASCRIPT_PARITY_CI_YML= checks another copy of ci.yml (used to show // the detections on mutated copies; never set in CI). const ciPath = process.env.SAGASCRIPT_PARITY_CI_YML ?? fileURLToPath(new URL("../.github/workflows/ci.yml", import.meta.url)); const read = (path) => readFileSync(path, "utf8").replace(/\r\n?/g, "\n"); -const workflow = read(ciPath); +const workflow = parseWorkflow(read(ciPath)); const packageJson = read(fileURLToPath(new URL("../package.json", import.meta.url))); const TEST_LANES = ["test-macos", "test-linux", "msrv-linux", "test-windows"]; @@ -92,49 +100,7 @@ const ALLOWLIST = [ ]; // --------------------------------------------------------------------------- -// A small reader for the subset of YAML and shell that ci.yml uses. - -function jobsSection(text) { - const start = text.indexOf("\njobs:\n"); - assert.notEqual(start, -1, "ci.yml has a jobs: section"); - return text.slice(start + "\njobs:\n".length - 1); -} - -function jobIds(text) { - return [...jobsSection(text).matchAll(/^ {2}([A-Za-z0-9_-]+):\s*$/gm)].map((match) => match[1]); -} - -function jobBlock(text, jobId) { - const jobs = jobsSection(text); - const start = jobs.indexOf(`\n ${jobId}:\n`); - assert.notEqual(start, -1, `ci.yml must define ${jobId}`); - const body = jobs.slice(start + 1); - const next = body.slice(1).search(/^ {2}[A-Za-z0-9_-]+:\s*$/m); - return next === -1 ? body : body.slice(0, next + 1); -} - -// Steps of a job as { name, run, cwd, env }, named or not. -function stepsOf(text, jobId) { - return jobBlock(text, jobId) - .split(/\n(?= {6}- )/) - .slice(1) - .map((chunk) => { - const raw = chunk.replace(/^ {6}- /, " "); - const inline = /^ {8}run: (?![|>])(.*)$/m.exec(raw)?.[1]; - const block = /^ {8}run: [|>]-?\n((?: {10}.*\n?|\n)*)/m - .exec(raw)?.[1] - ?.split("\n") - .map((line) => line.replace(/^ {10}/, "")) - .join("\n"); - const envBlock = /^ {8}env:\n((?: {10}.*\n?)*)/m.exec(raw)?.[1] ?? ""; - return { - name: /^ {8}name: (.*)$/m.exec(raw)?.[1], - run: inline ?? block ?? "", - cwd: /^ {8}working-directory: (.*)$/m.exec(raw)?.[1], - env: Object.fromEntries([...envBlock.matchAll(/^ {10}([A-Za-z_][A-Za-z0-9_]*): (.*)$/gm)].map((match) => [match[1], match[2]])), - }; - }); -} +// A small reader for the subset of shell that the lanes' run scripts use. // Split a script into simple commands, honouring quotes and $(…): each is a // list of words with leading VAR=value assignments and redirections removed. @@ -207,9 +173,21 @@ function commandsIn(script) { .map((words) => words.join(" ")); } -const ciCommands = TEST_LANES.flatMap((job) => - stepsOf(workflow, job).flatMap((step) => commandsIn(step.run).map((run) => ({ job, run, cwd: step.cwd ?? "." }))), -); +const OS_OF_RUNNER = (runsOn) => (/^macos/.test(runsOn) ? "macos" : /^windows/.test(runsOn) ? "windows" : "linux"); + +// The commands of a lane as { job, run, cwd, step }, gates and action steps expanded. +function laneCommands(jobId) { + const job = workflow.job(jobId); + return expandedSteps(job).flatMap((step) => { + const invocation = gateInvocation(step); + if (invocation) { + const gates = gatesFor({ os: invocation.os ?? OS_OF_RUNNER(job.runsOn), profile: invocation.profile }); + return gates.map((gate) => ({ job: jobId, run: commandOf(gate), cwd: gate.cwd, step })); + } + return commandsIn(step.run).map((run) => ({ job: jobId, run, cwd: step.workingDirectory ?? ".", step })); + }); +} +const ciCommands = TEST_LANES.flatMap(laneCommands); const key = ({ run, cwd }) => `${cwd ?? "."}: ${run}`; const classified = new Set(Object.values(TEST_GROUPS).flat()); @@ -260,10 +238,11 @@ jobs: steps: - run: echo hi `; - assert.deepEqual(jobIds(synthetic), ["lane", "other"]); - const steps = stepsOf(synthetic, "lane"); + const parsed = parseWorkflow(synthetic); + assert.deepEqual(parsed.jobIds, ["lane", "other"]); + const steps = parsed.job("lane").steps; assert.deepEqual( - steps.map((step) => [step.name, step.cwd]), + steps.map((step) => [step.name, step.workingDirectory]), [ [undefined, undefined], [undefined, undefined], @@ -282,7 +261,7 @@ jobs: }); test("every ci.yml job is a test lane or a listed non-test job", () => { - const jobs = jobIds(workflow); + const jobs = workflow.jobIds; const known = new Set([...TEST_LANES, ...Object.keys(NON_TEST_JOBS)]); assert.deepEqual( jobs.filter((job) => !known.has(job)), @@ -311,6 +290,21 @@ test("the parser finds the commands of the CI test lanes", () => { ciCommands.some((command) => command.job === job), `${job} has commands`, ); + // The commands the lanes take from the gate list and from the setup action are visible too. + for (const expected of ["src-tauri: cargo test --workspace", ".: npm ci"]) assert.ok(keys.has(expected), `parsed ${expected}`); + assert.ok( + ciCommands.some((command) => command.job === "test-linux" && command.run.startsWith("sudo apt-get")), + "the setup action's apt step", + ); + assert.ok( + ciCommands.some((command) => command.job === "msrv-linux" && command.run.startsWith("rustup toolchain install")), + "the MSRV install", + ); +}); + +test("every gate of the list is run by some test lane, so the list holds no command the parity test cannot see", () => { + const seen = new Set(ciCommands.map(key)); + for (const gate of GATES) assert.ok(seen.has(key({ run: commandOf(gate), cwd: gate.cwd })), `${gate.id} is run by no test lane`); }); test("every command in the CI test lanes has a verify check, a manifest entry or an allowlisted reason", () => { @@ -378,14 +372,23 @@ test("workflows outside ci.yml named in a check's lanes run its command", () => test("the release guard proves its test ran, locally as in CI", () => { // cargo test exits 0 when the name filter matches nothing. - const proof = /grep -q '(release_build_ignores_the_real_environment \.\.\. ok)'/.exec(workflow)?.[1]; - assert.ok(proof, "test-linux greps the release guard's output"); + const proof = GATES.find((gate) => gate.id === "release-guard").requireOutput?.pattern; + assert.ok(proof, "the gate list requires the release guard's output"); const [step] = CHECKS.find((check) => check.id === "release-guard").steps; - assert.ok(step.requireOutput?.pattern.test(proof), "verify's release-guard requires the same line"); + assert.ok( + step.requireOutput?.pattern.test("release_build_ignores_the_real_environment ... ok"), + "verify's release-guard requires the same line", + ); + assert.ok(proof.test("test tests::release_build_ignores_the_real_environment ... ok")); + assert.deepEqual( + [proof.source, step.requireOutput.pattern.source], + [proof.source, proof.source], + "verify and CI require the same output", + ); }); test("the browser check gives the QA scripts an output directory, as CI does", () => { - const ciStep = stepsOf(workflow, "test-macos").find((step) => commandsIn(step.run).includes("npm run test:browser")); + const ciStep = expandedSteps(workflow.job("test-macos")).find((step) => commandsIn(step.run).includes("npm run test:browser")); assert.ok(ciStep?.env.QA_OUTPUT_DIR, "test-macos sets QA_OUTPUT_DIR for npm run test:browser"); const [step] = CHECKS.find((check) => check.id === "browser").steps; assert.ok(step.env?.QA_OUTPUT_DIR, "verify's browser check sets QA_OUTPUT_DIR"); diff --git a/scripts/test-windows-package-workflow.mjs b/scripts/test-windows-package-workflow.mjs index f3f383b2..38676754 100644 --- a/scripts/test-windows-package-workflow.mjs +++ b/scripts/test-windows-package-workflow.mjs @@ -4,18 +4,8 @@ import { readFile } from "node:fs/promises"; import test from "node:test"; import { fileURLToPath } from "node:url"; -test("Windows PR CI runs build identity regression tests before compilation", async () => { - const ci = (await readFile(new URL("../.github/workflows/ci.yml", import.meta.url), "utf8")).replace(/\r\n?/g, "\n"); - const testWindowsStart = ci.indexOf(" test-windows:"); - const buildWindowsStart = ci.indexOf(" build-windows:", testWindowsStart); - assert.ok(testWindowsStart >= 0 && buildWindowsStart > testWindowsStart); - const windows = ci.slice(testWindowsStart, buildWindowsStart); - const command = - "node --test scripts/test-ci-build-identity.mjs scripts/test-windows-release-identity.mjs scripts/test-windows-package-workflow.mjs"; - const checks = windows.indexOf(command); - assert.ok(checks > windows.indexOf("name: Install Node.js")); - assert.ok(checks < windows.indexOf("name: Cargo check")); -}); +// The ordering of ci.yml's Windows build identity gates (they run before `npm ci`, so this file must +// stay free of npm packages) is asserted in scripts/test-ci-parallel-gates.mjs. test("Windows candidates pin and verify source identity around release compilation", async () => { const workflow = await readFile(new URL("../.github/workflows/windows-package.yml", import.meta.url), "utf8"); diff --git a/scripts/test-workflow-hardening.mjs b/scripts/test-workflow-hardening.mjs index 1f5da7be..2547439c 100644 --- a/scripts/test-workflow-hardening.mjs +++ b/scripts/test-workflow-hardening.mjs @@ -7,6 +7,7 @@ import assert from "node:assert/strict"; import { readFileSync, readdirSync } from "node:fs"; import test from "node:test"; import { fileURLToPath } from "node:url"; +import { parse } from "yaml"; const directory = fileURLToPath(new URL("../.github/workflows/", import.meta.url)); const workflows = Object.fromEntries( @@ -81,6 +82,60 @@ function writeScopes(scopes) { }); } +// Local composite actions (#358) hold steps a workflow used to hold: their `uses:` pins and tool +// installs are held to the same rules. +const actionsDirectory = fileURLToPath(new URL("../.github/actions/", import.meta.url)); +const localActions = Object.fromEntries( + readdirSync(actionsDirectory) + .sort() + .map((name) => [`actions/${name}/action.yml`, readFileSync(`${actionsDirectory}${name}/action.yml`, "utf8").replace(/\r\n?/g, "\n")]), +); + +function assertActionsPinned(name, text) { + let actions = 0; + for (const [line, ref, comment] of text.matchAll(/^\s*(?:-\s+)?uses:\s*(\S+)(.*)$/gm)) { + if (ref.startsWith("./")) continue; + actions += 1; + const pinned = /^([^@\s]+)@([0-9a-f]{40})$/.exec(ref); + assert.ok(pinned, `${name}: ${line.trim()} must be pinned to a full commit SHA`); + assert.match(comment, /^\s+# v\d+(?:\.\d+)*\b/, `${name}: ${line.trim()} must name the release it resolves to (# vX.Y.Z)`); + } + assert.ok(actions > 0, `${name} uses no actions; the pin check would be vacuous`); +} + +function assertToolsPinned(name, text) { + for (const [command] of text.matchAll(/^.*\bcargo install\b.*$/gm)) { + assert.match(command, /--locked\b/, `${name}: ${command.trim()} must use --locked`); + assert.match(command, /--version \d+\.\d+\.\d+\b/, `${name}: ${command.trim()} must pin --version`); + } + for (const [command, args] of text.matchAll(/^.*\bnpm (?:install|i) (?:-g|--global)\b(.*)$/gm)) { + const packages = args + .trim() + .split(/\s+/) + .filter((arg) => arg && !arg.startsWith("-")); + assert.ok(packages.length > 0, `${name}: ${command.trim()} installs nothing`); + for (const spec of packages) { + assert.match(spec, /^(?:@[^@/\s]+\/)?[^@\s]+@\d+\.\d+\.\d+$/, `${name}: ${spec} must be pinned to an exact version`); + } + } +} + +test("Dependabot keeps the pins of every local composite action current", () => { + const dependabot = parse(readFileSync(new URL("../.github/dependabot.yml", import.meta.url), "utf8")); + const actions = dependabot.updates.find((update) => update["package-ecosystem"] === "github-actions"); + assert.ok(actions, "Dependabot updates GitHub Actions"); + const directories = actions.directories ?? [actions.directory]; + assert.ok(directories.includes("/"), "the workflows"); + for (const name of Object.keys(localActions)) { + assert.ok(directories.includes(`/.github/${name.replace(/\/action\.yml$/, "")}`), `${name} is not watched by Dependabot`); + } +}); + +for (const [name, text] of Object.entries(localActions)) { + test(`${name} pins every action to a commit SHA with its release in a comment`, () => assertActionsPinned(name, text)); + test(`${name} installs command-line tools at exact versions`, () => assertToolsPinned(name, text)); +} + for (const [name, text] of Object.entries(workflows)) { test(`${name} declares a least-privilege top-level permissions block`, () => { const scopes = topLevelPermissions(text); @@ -110,17 +165,7 @@ for (const [name, text] of Object.entries(workflows)) { } }); - test(`${name} pins every action to a commit SHA with its release in a comment`, () => { - let actions = 0; - for (const [line, ref, comment] of text.matchAll(/^\s*(?:-\s+)?uses:\s*(\S+)(.*)$/gm)) { - if (ref.startsWith("./")) continue; - actions += 1; - const pinned = /^([^@\s]+)@([0-9a-f]{40})$/.exec(ref); - assert.ok(pinned, `${name}: ${line.trim()} must be pinned to a full commit SHA`); - assert.match(comment, /^\s+# v\d+(?:\.\d+)*\b/, `${name}: ${line.trim()} must name the release it resolves to (# vX.Y.Z)`); - } - assert.ok(actions > 0, `${name} uses no actions; the pin check would be vacuous`); - }); + test(`${name} pins every action to a commit SHA with its release in a comment`, () => assertActionsPinned(name, text)); test(`${name} grants write scopes only to reviewed publishing jobs`, () => { for (const job of jobs(text)) { @@ -131,22 +176,7 @@ for (const [name, text] of Object.entries(workflows)) { } }); - test(`${name} installs command-line tools at exact versions`, () => { - for (const [command] of text.matchAll(/^.*\bcargo install\b.*$/gm)) { - assert.match(command, /--locked\b/, `${name}: ${command.trim()} must use --locked`); - assert.match(command, /--version \d+\.\d+\.\d+\b/, `${name}: ${command.trim()} must pin --version`); - } - for (const [command, args] of text.matchAll(/^.*\bnpm (?:install|i) (?:-g|--global)\b(.*)$/gm)) { - const packages = args - .trim() - .split(/\s+/) - .filter((arg) => arg && !arg.startsWith("-")); - assert.ok(packages.length > 0, `${name}: ${command.trim()} installs nothing`); - for (const spec of packages) { - assert.match(spec, /^(?:@[^@/\s]+\/)?[^@\s]+@\d+\.\d+\.\d+$/, `${name}: ${spec} must be pinned to an exact version`); - } - } - }); + test(`${name} installs command-line tools at exact versions`, () => assertToolsPinned(name, text)); } test("every reviewed writer still exists and still needs its scope", () => { diff --git a/scripts/verify.mjs b/scripts/verify.mjs index e032482f..f00636d7 100644 --- a/scripts/verify.mjs +++ b/scripts/verify.mjs @@ -3,7 +3,8 @@ // CHECKS below is the source of truth for what "verify my change" means: // CONTRIBUTING.md documents it, `--list` prints it, and // scripts/test-verify-ci-parity.mjs keeps it in step with the test lanes of -// .github/workflows/ci.yml. Before anything runs, a preflight checks the +// .github/workflows/ci.yml, whose cargo gates are listed once, in +// scripts/ci/cargo-gates.mjs. Before anything runs, a preflight checks the // tools and packages the selected checks need and reports everything that is // missing in one error; it never installs anything. Every check has a time // budget; on timeout, Ctrl-C, SIGTERM or SIGHUP its whole process tree is @@ -218,12 +219,7 @@ export const CHECKS = [ platforms: ALL, requires: ["rust", "linux-libs", "frontend-dist"], timeoutMinutes: 45, - ci: [ - ci("cargo clippy --all-targets -- -D warnings", "src-tauri"), - ci("cargo clippy -p sagascript-cli --all-targets -- -D warnings", "src-tauri"), - ci("cargo check", "src-tauri"), - ci("cargo check -p sagascript-cli", "src-tauri"), - ], + ci: [ci("cargo clippy --workspace --all-targets -- -D warnings", "src-tauri")], lanes: ["test-macos", "test-linux", "test-windows"], }, { @@ -235,8 +231,9 @@ export const CHECKS = [ platforms: ALL, requires: ["rust", "linux-libs", "frontend-dist"], timeoutMinutes: 45, - ci: [ci("cargo clippy --all-targets -- -D warnings", "src-tauri")], - lanes: ["test-macos", "test-linux", "test-windows"], + // CI runs the workspace gate, which covers the app crate. + ci: [], + lanes: [], }, { id: "clippy-cli-lean", @@ -247,11 +244,8 @@ export const CHECKS = [ platforms: ALL, requires: ["rust"], timeoutMinutes: 30, - ci: [ - ci("cargo clippy -p sagascript-cli --all-targets --no-default-features -- -D warnings", "src-tauri"), - ci("cargo check -p sagascript-cli --no-default-features", "src-tauri"), - ], - lanes: ["test-linux"], + ci: [ci("cargo clippy -p sagascript-cli --all-targets --no-default-features -- -D warnings", "src-tauri")], + lanes: ["test-macos", "test-linux", "test-windows"], }, { id: "clippy-policy", @@ -317,7 +311,7 @@ export const CHECKS = [ requires: ["rust"], timeoutMinutes: 30, ci: [ci("cargo clippy -p sagascript-core --all-targets -- -D warnings", "src-tauri")], - lanes: ["test-macos", "test-windows"], + lanes: ["test-macos", "test-linux", "test-windows"], }, { id: "core-clippy-diarization", @@ -329,7 +323,7 @@ export const CHECKS = [ requires: ["rust"], timeoutMinutes: 30, ci: [ci("cargo clippy -p sagascript-core --features diarization --all-targets -- -D warnings", "src-tauri")], - lanes: ["test-macos"], + lanes: ["test-linux"], }, { id: "test-workspace", @@ -340,8 +334,8 @@ export const CHECKS = [ platforms: ALL, requires: ["rust", "linux-libs", "frontend-dist"], timeoutMinutes: 60, - ci: [ci("cargo test", "src-tauri"), ci("cargo test -p sagascript-cli", "src-tauri")], - lanes: ["test-macos", "test-windows"], + ci: [ci("cargo test --workspace", "src-tauri")], + lanes: ["test-macos", "test-linux", "test-windows"], }, { id: "test-cli-lean", @@ -353,7 +347,7 @@ export const CHECKS = [ requires: ["rust"], timeoutMinutes: 30, ci: [ci("cargo test -p sagascript-cli --no-default-features", "src-tauri")], - lanes: ["test-macos", "test-linux"], + lanes: ["test-macos", "test-linux", "test-windows"], }, { id: "engine-protocol", @@ -364,8 +358,9 @@ export const CHECKS = [ platforms: ALL, requires: ["rust"], timeoutMinutes: 15, - ci: [ci("cargo test -p sagascript-engine-protocol", "src-tauri")], - lanes: ["test-linux"], + // CI reaches the protocol crate through `cargo test --workspace` on every OS. + ci: [], + lanes: [], }, { id: "core-test", @@ -400,11 +395,8 @@ export const CHECKS = [ platforms: ALL, requires: ["rust"], timeoutMinutes: 30, - ci: [ - ci("cargo test -p sagascript-core --features diarization", "src-tauri"), - ci("cargo check -p sagascript-core --features diarization", "src-tauri"), - ], - lanes: ["test-macos"], + ci: [ci("cargo test -p sagascript-core --features diarization", "src-tauri")], + lanes: ["test-linux"], }, { id: "ort-host-clippy", @@ -416,7 +408,7 @@ export const CHECKS = [ requires: ["rust"], timeoutMinutes: 30, ci: [ci("cargo clippy --all-targets --locked -- -D warnings", "src-tauri/engine-host/ort")], - lanes: ["test-linux"], + lanes: ["test-linux", "test-windows"], }, { id: "ort-host-test", From d3c5e7b989c65175d7c06f385064b7ed7640593a Mon Sep 17 00:00:00 2001 From: Magnus Gille Date: Wed, 14 Oct 2026 22:58:25 +0000 Subject: [PATCH 2/7] ci: run multi-command Windows steps under bash (#358) pwsh reports only the last native command's exit code, so a failing npm run test:frontend ahead of a passing npm run format:check went green in test-windows. The step now runs under bash, a structural test requires it for every multi-command step of a Windows lane, and the smoke test compares recorded commands in one path spelling. windows-package.yml keeps the old step for every path but the x64 pull request, so release behaviour is unchanged. Co-Authored-By: Claude Sonnet 5.5 --- .github/workflows/ci.yml | 21 ++++++++++----------- .github/workflows/windows-package.yml | 6 +++++- scripts/test-ci-parallel-gates.mjs | 22 +++++++++++++++++++--- scripts/test-ci-smoke-cli.mjs | 26 +++++++++++++++----------- scripts/test-release-promotion.mjs | 9 +++++---- 5 files changed, 54 insertions(+), 30 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c03208d7..a7b38839 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,10 +1,9 @@ name: CI # Shared parts live once, outside this file (#358): .github/actions/setup (toolchain preamble), -# scripts/ci/cargo-gates.mjs (THE list of cargo gates, run by every lane; per-OS exclusions are -# documented there), scripts/ci/smoke-cli.mjs (CLI smoke tests), scripts/ci/assert-lanes.mjs (the -# check-* aggregators). Branch protection requires check-macos and check-windows by name: keep both -# and keep them `needs:` every lane of their OS. +# scripts/ci/cargo-gates.mjs (THE list of cargo gates; per-OS exclusions are documented there), +# scripts/ci/smoke-cli.mjs and scripts/ci/assert-lanes.mjs (the check-* aggregators). Branch +# protection requires check-macos and check-windows by name: keep both, `needs:` every lane of their OS. # # Windows tests have one owner per architecture: test-windows owns the x64 tests of every pull # request (the required lane). windows-package.yml does not repeat `cargo test --workspace` and @@ -283,16 +282,16 @@ jobs: src-tauri/engine-host/ort npm-ci: "false" # Before npm ci, so a build identity regression fails before any npm or Cargo work. - - name: Test offline dictation evaluation tooling and Windows build identity gates - run: | - python -m unittest discover -s scripts/dictation_eval -p 'test_*.py' - node --test scripts/test-ci-build-identity.mjs scripts/test-windows-release-identity.mjs scripts/test-windows-package-workflow.mjs scripts/test-release-promotion.mjs + - name: Test Windows build identity gates + run: node --test scripts/test-ci-build-identity.mjs scripts/test-windows-release-identity.mjs scripts/test-windows-package-workflow.mjs scripts/test-release-promotion.mjs - name: Install npm dependencies run: npm ci - # The whole Node suite on a Windows checkout (#357). The checkout has CRLF line endings - # (core.autocrlf); prettier.config.js accepts them here, test-macos holds the repo to LF. - - name: Frontend checks (Node tests, unit tests, format) + # The whole Node suite on a Windows checkout (#357; CRLF line endings, which prettier.config.js + # accepts here). bash, not pwsh: pwsh would report only the LAST native command's exit code. + - name: Python and frontend checks (dictation tooling, Node tests, unit tests, format) + shell: bash run: | + python -m unittest discover -s scripts/dictation_eval -p 'test_*.py' npm run test:frontend npm run test:unit npm run format:check diff --git a/.github/workflows/windows-package.yml b/.github/workflows/windows-package.yml index 0cb43f4b..773b217b 100644 --- a/.github/workflows/windows-package.yml +++ b/.github/workflows/windows-package.yml @@ -256,6 +256,8 @@ jobs: # (scripts/ci/cargo-gates.mjs), so x64 does not repeat them on a pull request. They stay for # the native ARM64 runner, the only place the tests execute on ARM64, and for releases # (workflow_call from prebuild-release.yml) and manual runs, whose candidates are gated here. + # Note that in this pwsh step only the LAST command's exit code counts: a failing test or + # lint is reported by test-windows, not by this step. - name: Test and lint Rust workspace if: github.event_name != 'pull_request' || matrix.architecture == 'arm64' working-directory: src-tauri @@ -263,9 +265,11 @@ jobs: # `cargo check --workspace` is omitted: clippy --all-targets below is a strict superset. cargo test --workspace cargo clippy --workspace --all-targets -- -D warnings + cargo build -p sagascript-cli --no-default-features - # The debug tree the CPU policy checks below read, on every architecture. + # An x64 pull request skips the step above; the CPU policy check below still reads its debug tree. - name: Build lean CLI (debug) + if: github.event_name == 'pull_request' && matrix.architecture == 'x64' working-directory: src-tauri run: cargo build -p sagascript-cli --no-default-features diff --git a/scripts/test-ci-parallel-gates.mjs b/scripts/test-ci-parallel-gates.mjs index c1c4f355..4beb1827 100644 --- a/scripts/test-ci-parallel-gates.mjs +++ b/scripts/test-ci-parallel-gates.mjs @@ -245,10 +245,12 @@ test("windows-package.yml leaves the x64 Rust tests of a pull request to test-wi const names = windows.steps.map((step) => step.name); const tests = windows.steps[names.indexOf("Test and lint Rust workspace")]; assert.equal(tests.condition, "github.event_name != 'pull_request' || matrix.architecture == 'arm64'"); - // The debug lean CLI that the CPU policy checks read stays built on every architecture. + // The debug lean CLI that the CPU policy checks read stays built on every architecture: inside the + // step above where it runs, in a step of its own exactly where that step is skipped. + assert.ok(commandLines(tests).includes("cargo build -p sagascript-cli --no-default-features")); const build = windows.steps[names.indexOf("Build lean CLI (debug)")]; - assert.ok(build, "the debug build is its own step"); - assert.equal(build.condition, undefined); + assert.ok(build, "the debug build has a step of its own for the skipped case"); + assert.equal(build.condition, "github.event_name == 'pull_request' && matrix.architecture == 'x64'"); assert.deepEqual(commandLines(build), ["cargo build -p sagascript-cli --no-default-features"]); assert.ok(names.indexOf(build.name) > names.indexOf(tests.name)); assert.ok(names.indexOf(build.name) < names.indexOf("Verify debug x64 inference CPU policy")); @@ -264,3 +266,17 @@ test("the workflow parses the same with CRLF line endings, as on a Windows check const classify = crlf.job("scope").steps.find((step) => step.id === "classify"); assert.ok(!classify.run.includes("\r"), "scripts stay free of carriage returns"); }); + +test("a multi-command step of a Windows lane runs under bash: pwsh reports only the last native command's exit code", () => { + // `npm run test:frontend` failing in front of a passing `npm run format:check` went green in a first draft of this file. + const windowsJobs = workflow.jobs.filter((job) => /^windows/.test(job.runsOn)); + assert.deepEqual(windowsJobs.map((job) => job.id).sort(), ["build-windows", "test-windows"]); + for (const job of windowsJobs) { + for (const step of job.steps.filter((candidate) => candidate.run !== "")) { + if (commandLines(step).length > 1) assert.equal(step.shell, "bash", `${job.id}: "${step.name}" runs several commands`); + } + } + for (const step of loadAction("setup").steps.filter((candidate) => candidate.run !== "")) { + assert.equal(step.shell, "bash", `the setup action's "${step.name}" must name its shell (and runs on Windows)`); + } +}); diff --git a/scripts/test-ci-smoke-cli.mjs b/scripts/test-ci-smoke-cli.mjs index 1500d24c..12ddca3d 100644 --- a/scripts/test-ci-smoke-cli.mjs +++ b/scripts/test-ci-smoke-cli.mjs @@ -88,17 +88,21 @@ test("the model tier downloads, transcribes and verifies the JSON, JSONL and err const failed = await runTier("model", harness, quiet); assert.deepEqual(failed, []); const model = "--language no --model nb-whisper-tiny"; - assert.deepEqual(harness.calls, [ - "download-model nb-whisper-tiny", - `transcribe ${model} test-audio/norwegian-short-3s.mp3`, - `transcribe ${model} --json test-audio/norwegian-short-3s.mp3`, - "verify --json", - `transcribe ${model} --jsonl /scratch/jsonl-inputs`, - "verify --jsonl", - `transcribe ${model} --jsonl /scratch/jsonl-inputs | head -1`, - `transcribe ${model} --json /scratch/missing.wav`, - "verify --error", - ]); + // A Windows path joins with backslashes; the commands are compared in one spelling. + assert.deepEqual( + harness.calls.map((call) => call.replaceAll("\\", "/")), + [ + "download-model nb-whisper-tiny", + `transcribe ${model} test-audio/norwegian-short-3s.mp3`, + `transcribe ${model} --json test-audio/norwegian-short-3s.mp3`, + "verify --json", + `transcribe ${model} --jsonl /scratch/jsonl-inputs`, + "verify --jsonl", + `transcribe ${model} --jsonl /scratch/jsonl-inputs | head -1`, + `transcribe ${model} --json /scratch/missing.wav`, + "verify --error", + ], + ); }); test("every check of a tier runs even after one fails, and each failure is reported by name", async () => { diff --git a/scripts/test-release-promotion.mjs b/scripts/test-release-promotion.mjs index 99b8cab8..fc60821f 100644 --- a/scripts/test-release-promotion.mjs +++ b/scripts/test-release-promotion.mjs @@ -97,15 +97,16 @@ test("windows-package keeps the console CLI, installed-CLI and portable zip step assert.match(w, /Verify silent per-user install runs the installed ARM64 CLI/); assert.match(w, /Build and verify ARM64 portable zip/); assert.match(w, /Verify installers carry the console CLI/); - // The one step that depends on the event is the Rust workspace test (#358): ci.yml's test-windows - // owns the x64 tests of a pull request, so this workflow skips them for x64 there. Releases come in - // through workflow_call from a push or a dispatch, never a pull request, and still run them. + // The steps that depend on the event are the Rust workspace test and the debug build that stands in + // for it (#358): ci.yml's test-windows owns the x64 tests of a pull request, so this workflow skips + // them for x64 there. Releases come in through workflow_call from a push or a dispatch, never a + // pull request, and still run them. const conditional = w .split("\n - name: ") .slice(1) .filter((step) => /if:[^\n]*github\.event_name/.test(step)) .map((step) => step.split("\n")[0]); - assert.deepEqual(conditional, ["Test and lint Rust workspace"]); + assert.deepEqual(conditional, ["Test and lint Rust workspace", "Build lean CLI (debug)"]); assert.match(w, /if: github\.event_name != 'pull_request' \|\| matrix\.architecture == 'arm64'\n/); }); From 3fdc07e85808578c2af0c288013ed21188a164e1 Mon Sep 17 00:00:00 2001 From: Magnus Gille Date: Wed, 14 Oct 2026 23:35:22 +0000 Subject: [PATCH 3/7] ci: key the Rust cache on the cargo gate list (#358) rust-cache saves only when its key is new, and the key follows Cargo.lock, so the artifacts of a changed gate list would never reach the cache and every run would recompile them. Appending the hash of scripts/ci/cargo-gates.mjs to the shared key makes a gate change a new key that the first run saves. Co-Authored-By: Claude Sonnet 5.5 --- .github/actions/setup/action.yml | 8 ++++++-- scripts/test-ci-parallel-gates.mjs | 5 +++++ 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/.github/actions/setup/action.yml b/.github/actions/setup/action.yml index a7c25ec5..a6dd3037 100644 --- a/.github/actions/setup/action.yml +++ b/.github/actions/setup/action.yml @@ -19,7 +19,11 @@ inputs: description: Extra toolchain components, comma separated (clippy, rustfmt). default: "" cache-key: - description: Swatinem/rust-cache shared-key. Lanes that compile alike share one; rust-cache keeps OS and target entries apart. + description: >- + Swatinem/rust-cache shared-key. Lanes that compile alike share one; rust-cache keeps OS and target + entries apart. The hash of scripts/ci/cargo-gates.mjs is appended: the cache holds what the gates + compile, and rust-cache saves only when its key is new, so a changed gate list needs a new key to + get its artifacts cached. required: true cache-workspaces: description: Cargo workspaces to cache, one per line. The standalone ORT engine host is its own workspace (#357). @@ -79,7 +83,7 @@ runs: uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 with: workspaces: ${{ inputs.cache-workspaces }} - shared-key: ${{ inputs.cache-key }} + shared-key: ${{ inputs.cache-key }}-${{ hashFiles('scripts/ci/cargo-gates.mjs') }} - if: ${{ inputs.models == 'true' }} name: Cache downloaded models diff --git a/scripts/test-ci-parallel-gates.mjs b/scripts/test-ci-parallel-gates.mjs index 4beb1827..692ae902 100644 --- a/scripts/test-ci-parallel-gates.mjs +++ b/scripts/test-ci-parallel-gates.mjs @@ -178,6 +178,11 @@ test("every lane that runs Python selects the supported runtime first", () => { assert.equal(actionInput(action, setupStep(workflow.job("msrv-linux")), "python"), "false"); }); +test("the Rust cache key follows the gate list: rust-cache saves only under a new key, so new gates need one to get cached", () => { + const cache = loadAction("setup").steps.find((step) => step.uses?.startsWith("Swatinem/rust-cache@")); + assert.equal(cache.with["shared-key"], "${{ inputs.cache-key }}-${{ hashFiles('scripts/ci/cargo-gates.mjs') }}"); +}); + test("only the native build lanes cache downloaded models", () => { const action = loadAction("setup"); for (const id of LANES) { From c08f53b05870e3f8ebea63890cda2118f2545b38 Mon Sep 17 00:00:00 2001 From: Magnus Gille Date: Thu, 15 Oct 2026 07:26:26 +0000 Subject: [PATCH 4/7] ci: let test-windows own the Windows tests on pull requests (#358) windows-package.yml no longer repeats cargo test --workspace and cargo clippy --workspace on a pull request, for either architecture. In that pwsh step only the last command's exit code counts, so a failing test or lint there never failed the job; test-windows runs the same workspace gates through scripts/ci/cargo-gates.mjs and does. Releases (workflow_call from prebuild-release.yml) and manual runs keep the step unchanged. A pull request builds the lean debug CLI in a step of its own, which the CPU policy checks read. Co-Authored-By: Claude Sonnet 5.5 --- .github/workflows/ci.yml | 8 ++++---- .github/workflows/windows-package.yml | 23 +++++++++++------------ docs/ARCHITECTURE.md | 8 ++++---- docs/windows-release.md | 4 ++-- scripts/test-ci-parallel-gates.mjs | 14 ++++++++------ scripts/test-release-promotion.mjs | 8 ++++---- 6 files changed, 33 insertions(+), 32 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a7b38839..68aab33a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -5,10 +5,10 @@ name: CI # scripts/ci/smoke-cli.mjs and scripts/ci/assert-lanes.mjs (the check-* aggregators). Branch # protection requires check-macos and check-windows by name: keep both, `needs:` every lane of their OS. # -# Windows tests have one owner per architecture: test-windows owns the x64 tests of every pull -# request (the required lane). windows-package.yml does not repeat `cargo test --workspace` and -# `cargo clippy --workspace` for x64 on pull requests; it keeps them for the native ARM64 runner -# (the only place tests execute on ARM64), for releases and for manual runs. +# Windows tests have one owner on a pull request: test-windows (x64), the required lane. +# windows-package.yml does not repeat `cargo test --workspace` and `cargo clippy --workspace` there +# (in its pwsh step a failure never failed the job); it keeps them for releases and manual runs, +# the only place ARM64 tests run. on: push: diff --git a/.github/workflows/windows-package.yml b/.github/workflows/windows-package.yml index 773b217b..fb4d97d7 100644 --- a/.github/workflows/windows-package.yml +++ b/.github/workflows/windows-package.yml @@ -251,15 +251,14 @@ jobs: npm run build npm run check - # Ownership of the Windows tests (#358): ci.yml's test-windows, a required check, owns the - # x64 tests of every pull request and runs these same workspace commands - # (scripts/ci/cargo-gates.mjs), so x64 does not repeat them on a pull request. They stay for - # the native ARM64 runner, the only place the tests execute on ARM64, and for releases - # (workflow_call from prebuild-release.yml) and manual runs, whose candidates are gated here. - # Note that in this pwsh step only the LAST command's exit code counts: a failing test or - # lint is reported by test-windows, not by this step. + # Ownership of the Windows tests (#358): ci.yml's test-windows, a required check, owns them on + # a pull request and runs these same workspace commands (scripts/ci/cargo-gates.mjs), so a pull + # request does not repeat them on either architecture. They stay for releases (workflow_call + # from prebuild-release.yml) and manual runs, whose candidates are gated here. In this pwsh + # step only the LAST command's exit code counts: a failing test or lint here never failed the + # job; test-windows is where it does (x64; ARM64 tests run only here, on releases and manual runs). - name: Test and lint Rust workspace - if: github.event_name != 'pull_request' || matrix.architecture == 'arm64' + if: github.event_name != 'pull_request' working-directory: src-tauri run: | # `cargo check --workspace` is omitted: clippy --all-targets below is a strict superset. @@ -267,9 +266,9 @@ jobs: cargo clippy --workspace --all-targets -- -D warnings cargo build -p sagascript-cli --no-default-features - # An x64 pull request skips the step above; the CPU policy check below still reads its debug tree. + # A pull request skips the step above; the CPU policy checks below still read its debug tree. - name: Build lean CLI (debug) - if: github.event_name == 'pull_request' && matrix.architecture == 'x64' + if: github.event_name == 'pull_request' working-directory: src-tauri run: cargo build -p sagascript-cli --no-default-features @@ -277,8 +276,8 @@ jobs: shell: pwsh run: | # The host triple was verified above. Reuse the workspace test binary - # instead of recompiling a duplicate explicit-target debug tree (on an x64 - # pull request there is none: this step then compiles the app's test binary). + # instead of recompiling a duplicate explicit-target debug tree (on a pull + # request there is none: this step then compiles the app's test binary). cargo test --manifest-path src-tauri/Cargo.toml -p sagascript native_runner_clipboard_transaction_smoke -- --ignored --test-threads=1 if ($LASTEXITCODE -ne 0) { throw "Native clipboard transaction gate failed" } diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index 3589e55e..25c75065 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -417,10 +417,10 @@ msrv-linux runs `cargo check --workspace --locked`, the lean CLI and the ORT hos toolchain (1.88), and check-linux requires it; test-macos runs `npm audit --audit-level=high`, the frontend suites (Prettier, ESLint, Vitest, Node), Svelte types, the Swift tests and the engine-host conformance subset; test-windows runs the same frontend suites except ESLint and the Windows build -identity gates. test-windows owns the x64 Windows tests of every pull request: `windows-package.yml` -(on PRs touching `src/`, `src-tauri/`, `scripts/`, package files, `.gitattributes` or the workflow -itself) repeats `cargo test --workspace` and clippy only on its native ARM64 runner on a pull request, -and on both architectures for releases (`prebuild-release.yml`) and manual runs; +identity gates. test-windows owns the Windows tests of every pull request: +`windows-package.yml` (on PRs touching `src/`, `src-tauri/`, `scripts/`, package files, +`.gitattributes` or the workflow itself) repeats `cargo test --workspace` and clippy only for +releases (`prebuild-release.yml`) and manual runs, on both architectures; `release-quality-gate.yml` runs them for a release. A PR that only touches `README.md`, `docs/release-readiness.md` or `docs/research/**/*.md` skips the native lanes (`scripts/ci-docs-scope.mjs`). Locally, `npm run verify` runs the test and lint suites above diff --git a/docs/windows-release.md b/docs/windows-release.md index a6a750c9..44fcb555 100644 --- a/docs/windows-release.md +++ b/docs/windows-release.md @@ -33,9 +33,9 @@ described as an official, signed, stable, or fully accepted Windows release. `.github/workflows/windows-package.yml` runs the release metadata, license, frontend, Rust, and real CPU-transcription gates on native `windows-latest` -(x64) and `windows-11-arm` (ARM64) runners. (On a pull request the x64 Rust +(x64) and `windows-11-arm` (ARM64) runners. (On a pull request the Rust tests and lints are `ci.yml`'s `test-windows`, a required check, so this -workflow repeats them only on ARM64; releases and manual runs run them on both.) Each runner then +workflow repeats them only for releases and manual runs, on both architectures.) Each runner then builds unsigned NSIS and MSI installers, a portable desktop executable, and a separate console CLI executable. It uploads them as one short-lived GitHub Actions artifact and never creates or modifies a GitHub Release. Its artifacts diff --git a/scripts/test-ci-parallel-gates.mjs b/scripts/test-ci-parallel-gates.mjs index 692ae902..865eb359 100644 --- a/scripts/test-ci-parallel-gates.mjs +++ b/scripts/test-ci-parallel-gates.mjs @@ -245,22 +245,24 @@ test("test-windows runs the build identity gates before npm ci and any Cargo wor } }); -test("windows-package.yml leaves the x64 Rust tests of a pull request to test-windows and keeps them for ARM64, releases and manual runs", () => { +test("windows-package.yml leaves the Rust tests of a pull request to test-windows and keeps them for releases and manual runs", () => { const windows = loadWorkflow("windows-package.yml").jobs[0]; const names = windows.steps.map((step) => step.name); const tests = windows.steps[names.indexOf("Test and lint Rust workspace")]; - assert.equal(tests.condition, "github.event_name != 'pull_request' || matrix.architecture == 'arm64'"); - // The debug lean CLI that the CPU policy checks read stays built on every architecture: inside the - // step above where it runs, in a step of its own exactly where that step is skipped. + assert.equal(tests.condition, "github.event_name != 'pull_request'"); + // The debug lean CLI that the CPU policy checks read stays built: inside the step above where it runs, + // in a step of its own exactly where that step is skipped. assert.ok(commandLines(tests).includes("cargo build -p sagascript-cli --no-default-features")); const build = windows.steps[names.indexOf("Build lean CLI (debug)")]; assert.ok(build, "the debug build has a step of its own for the skipped case"); - assert.equal(build.condition, "github.event_name == 'pull_request' && matrix.architecture == 'x64'"); + assert.equal(build.condition, "github.event_name == 'pull_request'"); assert.deepEqual(commandLines(build), ["cargo build -p sagascript-cli --no-default-features"]); assert.ok(names.indexOf(build.name) > names.indexOf(tests.name)); assert.ok(names.indexOf(build.name) < names.indexOf("Verify debug x64 inference CPU policy")); - // test-windows still runs the tests this one no longer repeats for x64. + assert.ok(names.indexOf(build.name) < names.indexOf("Verify debug ARM64 native CPU flags")); + // test-windows runs the tests this one no longer repeats on a pull request. assert.ok(gatesFor({ os: "windows" }).some((gate) => gate.args.join(" ") === "test --workspace")); + assert.ok(gatesFor({ os: "windows" }).some((gate) => gate.args.join(" ") === "clippy --workspace --all-targets -- -D warnings")); }); test("the workflow parses the same with CRLF line endings, as on a Windows checkout", () => { diff --git a/scripts/test-release-promotion.mjs b/scripts/test-release-promotion.mjs index fc60821f..c50ba398 100644 --- a/scripts/test-release-promotion.mjs +++ b/scripts/test-release-promotion.mjs @@ -98,16 +98,16 @@ test("windows-package keeps the console CLI, installed-CLI and portable zip step assert.match(w, /Build and verify ARM64 portable zip/); assert.match(w, /Verify installers carry the console CLI/); // The steps that depend on the event are the Rust workspace test and the debug build that stands in - // for it (#358): ci.yml's test-windows owns the x64 tests of a pull request, so this workflow skips - // them for x64 there. Releases come in through workflow_call from a push or a dispatch, never a - // pull request, and still run them. + // for it (#358): ci.yml's test-windows owns the tests of a pull request, so this workflow skips them + // there. Releases come in through workflow_call from a push or a dispatch, never a pull request, + // and still run them. const conditional = w .split("\n - name: ") .slice(1) .filter((step) => /if:[^\n]*github\.event_name/.test(step)) .map((step) => step.split("\n")[0]); assert.deepEqual(conditional, ["Test and lint Rust workspace", "Build lean CLI (debug)"]); - assert.match(w, /if: github\.event_name != 'pull_request' \|\| matrix\.architecture == 'arm64'\n/); + assert.match(w, /if: github\.event_name != 'pull_request'\n/); }); test("prebuild runs on main pushes only, never on pull requests, and never cancels", () => { From 1526c6a0d181774d6ebf3dca24510652be20579e Mon Sep 17 00:00:00 2001 From: Magnus Gille Date: Thu, 15 Oct 2026 08:29:04 +0000 Subject: [PATCH 5/7] build: declare yaml as a devDependency (#358) The CI structure tests parse the workflows with yaml, which was only vite's optional peer. yaml 2.9.1 is the version the lockfile already resolved, so the lockfile change is the root devDependencies entry and the dropped optional/peer flags of the existing node_modules/yaml entry; nothing new is downloaded. Co-Authored-By: Claude Sonnet 5.5 --- package-lock.json | 5 ++--- package.json | 3 ++- scripts/lib/ci-workflow.mjs | 7 +++---- scripts/node-test-groups.mjs | 3 +-- 4 files changed, 8 insertions(+), 10 deletions(-) diff --git a/package-lock.json b/package-lock.json index be5a4da7..b086d5da 100644 --- a/package-lock.json +++ b/package-lock.json @@ -35,7 +35,8 @@ "typescript": "^5.5.0", "typescript-eslint": "^8.71.0", "vite": "^6.0.0", - "vitest": "^5.0.2" + "vitest": "^5.0.2", + "yaml": "^2.9.1" }, "engines": { "node": ">=22" @@ -4146,8 +4147,6 @@ "integrity": "sha512-3NxN8+78OdzbT7C/WjGsyfPAtJaN3FNDsWxv7Y7mcDsT/oOmgW8BpyQQFFBnvZE3j9Y2Sdz1ULFLezL7Eb2yFw==", "dev": true, "license": "ISC", - "optional": true, - "peer": true, "bin": { "yaml": "bin.mjs" }, diff --git a/package.json b/package.json index 75c9dc9d..3c9ea8d3 100644 --- a/package.json +++ b/package.json @@ -52,7 +52,8 @@ "typescript": "^5.5.0", "typescript-eslint": "^8.71.0", "vite": "^6.0.0", - "vitest": "^5.0.2" + "vitest": "^5.0.2", + "yaml": "^2.9.1" }, "dependencies": { "@tauri-apps/api": "^2.0.0", diff --git a/scripts/lib/ci-workflow.mjs b/scripts/lib/ci-workflow.mjs index ad29be95..071ac37f 100644 --- a/scripts/lib/ci-workflow.mjs +++ b/scripts/lib/ci-workflow.mjs @@ -3,10 +3,9 @@ // which inputs, and keeps passing when the file is reformatted, re-commented or reordered // within what the assertion states. // -// The parser is the `yaml` package. It is not a dependency of this repository: it is vite's -// optional peer, which npm installs from package-lock.json. The ci test group lists it in -// GROUP_NPM_PACKAGES (scripts/node-test-groups.mjs), so a checkout where it is missing fails -// once with a clear message. If vite ever drops it, add `yaml` to devDependencies. +// The parser is the `yaml` package, a devDependency. The ci and release test groups list it in +// GROUP_NPM_PACKAGES (scripts/node-test-groups.mjs), so a checkout without `npm ci` fails once +// with a clear message instead of once per test file. // // Not for tests that run before `npm ci` (the Windows build identity gates in ci.yml's // test-windows): they must read text with node:fs only. diff --git a/scripts/node-test-groups.mjs b/scripts/node-test-groups.mjs index 88942f57..977e48da 100644 --- a/scripts/node-test-groups.mjs +++ b/scripts/node-test-groups.mjs @@ -26,8 +26,7 @@ export const GROUP_DESCRIPTIONS = { // failure per file. Groups not listed need nothing beyond Node itself. export const GROUP_NPM_PACKAGES = { ui: ["typescript"], - // `yaml` is vite's optional peer, which npm installs from the lockfile; scripts/lib/ci-workflow.mjs - // parses the workflows with it (#358). + // scripts/lib/ci-workflow.mjs parses the workflows with `yaml` (#358). ci: ["vite", "yaml"], release: ["yaml"], }; From 53bee3c9026cde45feca748902bdae347540e267 Mon Sep 17 00:00:00 2001 From: Magnus Gille Date: Thu, 15 Oct 2026 08:29:04 +0000 Subject: [PATCH 6/7] ci: key only gate lanes' Rust cache on the gate list; follow imports before npm ci (#358) - setup action: the native-build input replaces models. The native build lanes cache their models and keep the gate list out of their Rust cache key, so a gate change no longer evicts the caches of lanes that never run gates; the other lanes still carry the hash. A structural test checks both directions against the lanes that invoke the gate script. - scripts/lib/import-graph.mjs: the relative-import walker, shared by the test-group package check and by the check that the pre-npm-ci Windows identity tests reach no npm package even through a repository module. - ci.yml header: the check-* jobs run assert-lanes.mjs from the PR checkout, at the trust level of ci.yml itself. Co-Authored-By: Claude Sonnet 5.5 --- .github/actions/setup/action.yml | 17 ++++--- .github/workflows/ci.yml | 5 +- scripts/fixtures/import-graph/cycle-b.mjs | 1 + scripts/fixtures/import-graph/cycle-c.mjs | 1 + scripts/fixtures/import-graph/entry.mjs | 3 ++ scripts/fixtures/import-graph/lib/one.mjs | 1 + scripts/fixtures/import-graph/lib/three.mjs | 5 ++ scripts/fixtures/import-graph/lib/two.mjs | 1 + scripts/lib/import-graph.mjs | 38 +++++++++++++++ scripts/test-ci-parallel-gates.mjs | 52 +++++++++++++++------ scripts/test-frontend-test-groups.mjs | 32 ++++++------- 11 files changed, 113 insertions(+), 43 deletions(-) create mode 100644 scripts/fixtures/import-graph/cycle-b.mjs create mode 100644 scripts/fixtures/import-graph/cycle-c.mjs create mode 100644 scripts/fixtures/import-graph/entry.mjs create mode 100644 scripts/fixtures/import-graph/lib/one.mjs create mode 100644 scripts/fixtures/import-graph/lib/three.mjs create mode 100644 scripts/fixtures/import-graph/lib/two.mjs create mode 100644 scripts/lib/import-graph.mjs diff --git a/.github/actions/setup/action.yml b/.github/actions/setup/action.yml index a6dd3037..7675c045 100644 --- a/.github/actions/setup/action.yml +++ b/.github/actions/setup/action.yml @@ -21,15 +21,18 @@ inputs: cache-key: description: >- Swatinem/rust-cache shared-key. Lanes that compile alike share one; rust-cache keeps OS and target - entries apart. The hash of scripts/ci/cargo-gates.mjs is appended: the cache holds what the gates - compile, and rust-cache saves only when its key is new, so a changed gate list needs a new key to - get its artifacts cached. + entries apart. Unless `native-build` is set, the hash of scripts/ci/cargo-gates.mjs is appended: the + cache holds what the gates compile, and rust-cache saves only when its key is new, so a changed gate + list needs a new key to get its artifacts cached. required: true cache-workspaces: description: Cargo workspaces to cache, one per line. The standalone ORT engine host is its own workspace (#357). default: src-tauri - models: - description: Cache the models the CLI smoke tests download (macOS and Windows build lanes). + native-build: + description: >- + The lane builds the native release binary and runs the CLI smoke tests (the macOS and Windows build + lanes): it caches the models they download, and its Rust cache key leaves out the gate list, which + such a lane never runs, so a gate change does not evict its cache. default: "false" linux-deps: description: Install the Tauri system libraries and xdotool with apt (Linux lanes that compile the app crate). @@ -83,9 +86,9 @@ runs: uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 with: workspaces: ${{ inputs.cache-workspaces }} - shared-key: ${{ inputs.cache-key }}-${{ hashFiles('scripts/ci/cargo-gates.mjs') }} + shared-key: ${{ inputs.cache-key }}${{ inputs.native-build != 'true' && format('-{0}', hashFiles('scripts/ci/cargo-gates.mjs')) || '' }} - - if: ${{ inputs.models == 'true' }} + - if: ${{ inputs.native-build == 'true' }} name: Cache downloaded models uses: actions/cache@caa296126883cff596d87d8935842f9db880ef25 # v5.1.0 with: diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 68aab33a..77a9b48e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -4,6 +4,7 @@ name: CI # scripts/ci/cargo-gates.mjs (THE list of cargo gates; per-OS exclusions are documented there), # scripts/ci/smoke-cli.mjs and scripts/ci/assert-lanes.mjs (the check-* aggregators). Branch # protection requires check-macos and check-windows by name: keep both, `needs:` every lane of their OS. +# assert-lanes.mjs comes from the PR checkout, at the trust level of this file itself (contents: read, no secrets). # # Windows tests have one owner on a pull request: test-windows (x64), the required lane. # windows-package.yml does not repeat `cargo test --workspace` and `cargo clippy --workspace` there @@ -148,7 +149,7 @@ jobs: uses: ./.github/actions/setup with: cache-key: sagascript-ci-native - models: "true" + native-build: "true" - name: Build frontend run: npm run build # The smoke tests need the binary, not an installer (packaging: release.yml). @@ -311,7 +312,7 @@ jobs: uses: ./.github/actions/setup with: cache-key: sagascript-ci-native - models: "true" + native-build: "true" - name: Build frontend run: npm run build # The smoke tests need the binary, not an installer; packaging stays gated in release.yml. diff --git a/scripts/fixtures/import-graph/cycle-b.mjs b/scripts/fixtures/import-graph/cycle-b.mjs new file mode 100644 index 00000000..733af410 --- /dev/null +++ b/scripts/fixtures/import-graph/cycle-b.mjs @@ -0,0 +1 @@ +import "./cycle-c.mjs"; diff --git a/scripts/fixtures/import-graph/cycle-c.mjs b/scripts/fixtures/import-graph/cycle-c.mjs new file mode 100644 index 00000000..77f788a7 --- /dev/null +++ b/scripts/fixtures/import-graph/cycle-c.mjs @@ -0,0 +1 @@ +import "./cycle-b.mjs"; diff --git a/scripts/fixtures/import-graph/entry.mjs b/scripts/fixtures/import-graph/entry.mjs new file mode 100644 index 00000000..8360c0c0 --- /dev/null +++ b/scripts/fixtures/import-graph/entry.mjs @@ -0,0 +1,3 @@ +// Inputs of the import-graph test in scripts/test-frontend-test-groups.mjs. Never executed. +import { one } from "./lib/one.mjs"; +import "node:fs"; diff --git a/scripts/fixtures/import-graph/lib/one.mjs b/scripts/fixtures/import-graph/lib/one.mjs new file mode 100644 index 00000000..1ed52a9e --- /dev/null +++ b/scripts/fixtures/import-graph/lib/one.mjs @@ -0,0 +1 @@ +export { two as one } from "./two.mjs"; diff --git a/scripts/fixtures/import-graph/lib/three.mjs b/scripts/fixtures/import-graph/lib/three.mjs new file mode 100644 index 00000000..3acfa503 --- /dev/null +++ b/scripts/fixtures/import-graph/lib/three.mjs @@ -0,0 +1,5 @@ +// A comment that says import "commented-out" is not an import. +/* import "also-commented" */ +import "yaml"; +import "@scope/pkg/sub"; +import "./one.mjs"; diff --git a/scripts/fixtures/import-graph/lib/two.mjs b/scripts/fixtures/import-graph/lib/two.mjs new file mode 100644 index 00000000..d85a29bf --- /dev/null +++ b/scripts/fixtures/import-graph/lib/two.mjs @@ -0,0 +1 @@ +export const two = await import("./three.mjs"); diff --git a/scripts/lib/import-graph.mjs b/scripts/lib/import-graph.mjs new file mode 100644 index 00000000..d221332a --- /dev/null +++ b/scripts/lib/import-graph.mjs @@ -0,0 +1,38 @@ +// The npm packages a module needs, found by following its relative imports (#358). A test that runs +// before `npm ci` must not reach one even through a repository module it imports, and a test group's +// GROUP_NPM_PACKAGES (scripts/node-test-groups.mjs) must name every package its files reach. +// +// A static scan, not a resolver: it reads `from "x"`, `import "x"` and `import("x")` specifiers, after +// dropping block comments and full-line comments (a comment that mentions `import "name"` is no import). +import { readFileSync } from "node:fs"; +import { join, posix } from "node:path"; + +const BARE_IMPORT = /\bfrom\s*["']([@a-z][\w@./:-]*)["']|\bimport\s*\(?\s*["']([@a-z][\w@./:-]*)["']/g; +const RELATIVE_IMPORT = /\b(?:from|import)\s*\(?\s*["'](\.{1,2}\/[^"']+\.m?js)["']/g; + +/** The package a bare specifier names: `@scope/name` or `name`; null for a Node builtin (`node:fs`). */ +export function packageOf(specifier) { + if (specifier.startsWith("node:")) return null; + return specifier.startsWith("@") ? specifier.split("/").slice(0, 2).join("/") : specifier.split("/")[0]; +} + +/** + * `{ packages, files }` reached from `file` (repository relative, forward slashes) under `root`: + * the bare-import packages of every file on the way, and the files themselves, `file` included. + */ +export function importGraph(file, root) { + const packages = new Set(); + const files = new Set(); + const visit = (current) => { + if (files.has(current)) return; + files.add(current); + const source = readFileSync(join(root, current), "utf8").replace(/\/\*[\s\S]*?\*\/|^\s*\/\/.*$/gm, ""); + for (const match of source.matchAll(BARE_IMPORT)) { + const name = packageOf(match[1] ?? match[2]); + if (name !== null) packages.add(name); + } + for (const match of source.matchAll(RELATIVE_IMPORT)) visit(posix.normalize(posix.join(posix.dirname(current), match[1]))); + }; + visit(file); + return { packages, files }; +} diff --git a/scripts/test-ci-parallel-gates.mjs b/scripts/test-ci-parallel-gates.mjs index 865eb359..437c93bc 100644 --- a/scripts/test-ci-parallel-gates.mjs +++ b/scripts/test-ci-parallel-gates.mjs @@ -6,10 +6,12 @@ import { fileURLToPath } from "node:url"; import { actionInput, commandLines, gateInvocation, loadAction, loadWorkflow, parseWorkflow } from "./lib/ci-workflow.mjs"; import { gatesFor } from "./ci/cargo-gates.mjs"; import { laneProblems } from "./ci/assert-lanes.mjs"; +import { importGraph } from "./lib/import-graph.mjs"; // The parallel lanes of ci.yml (#358): what each one needs, how the shared setup action and the // aggregators are wired, and that the aggregators fail closed. The workflow is parsed, not matched. +const repoRoot = fileURLToPath(new URL("..", import.meta.url)); const workflow = loadWorkflow(); const ASSERT_LANES = fileURLToPath(new URL("./ci/assert-lanes.mjs", import.meta.url)); const SCOPE_CONDITION = "${{ needs.scope.result == 'success' && needs.scope.outputs.docs_only != 'true' }}"; @@ -178,22 +180,38 @@ test("every lane that runs Python selects the supported runtime first", () => { assert.equal(actionInput(action, setupStep(workflow.job("msrv-linux")), "python"), "false"); }); -test("the Rust cache key follows the gate list: rust-cache saves only under a new key, so new gates need one to get cached", () => { - const cache = loadAction("setup").steps.find((step) => step.uses?.startsWith("Swatinem/rust-cache@")); - assert.equal(cache.with["shared-key"], "${{ inputs.cache-key }}-${{ hashFiles('scripts/ci/cargo-gates.mjs') }}"); +test("the Rust cache key follows the gate list for the lanes that run gates, and for no other lane", () => { + // rust-cache saves only under a new key, so a changed gate list needs one to get its artifacts cached; a lane + // that never runs the gates (the native build lanes) must not lose its cache to a gate change. + const action = loadAction("setup"); + const cache = action.steps.find((step) => step.uses?.startsWith("Swatinem/rust-cache@")); + assert.equal( + cache.with["shared-key"], + "${{ inputs.cache-key }}${{ inputs.native-build != 'true' && format('-{0}', hashFiles('scripts/ci/cargo-gates.mjs')) || '' }}", + ); + for (const id of LANES) { + const job = workflow.job(id); + const runsGates = job.steps.some((step) => gateInvocation(step)); + const nativeBuild = actionInput(action, setupStep(job), "native-build") === "true"; + assert.equal( + nativeBuild, + !runsGates, + `${id}: ${runsGates ? "runs the cargo gates, so its cache key must carry their hash" : "runs no gate, so its cache key must not"}`, + ); + } }); test("only the native build lanes cache downloaded models", () => { const action = loadAction("setup"); for (const id of LANES) { assert.equal( - actionInput(action, setupStep(workflow.job(id)), "models"), + actionInput(action, setupStep(workflow.job(id)), "native-build"), id.startsWith("build-") ? "true" : "false", - `${id}: model cache`, + `${id}: native build`, ); } const step = action.steps.find((candidate) => candidate.name === "Cache downloaded models"); - assert.equal(step.condition, "${{ inputs.models == 'true' }}"); + assert.equal(step.condition, "${{ inputs.native-build == 'true' }}"); for (const source of ["settings/manager.rs", "transcription/model.rs", "diarization/model.rs"]) { assert.ok(step.with.key.includes(`sagascript-core/src/${source}`), `the model cache key hashes ${source}`); } @@ -233,15 +251,19 @@ test("test-windows runs the build identity gates before npm ci and any Cargo wor "the action must not run npm ci ahead of the identity gates", ); assert.ok(install > identity && gates > install, "npm ci follows the identity gates; Cargo work follows npm ci"); - // They run before node_modules exists, so they and what they import must not need it. - for (const file of [ - "test-ci-build-identity", - "test-windows-release-identity", - "test-windows-package-workflow", - "test-release-promotion", - ]) { - const source = readFileSync(new URL(`./${file}.mjs`, import.meta.url), "utf8"); - assert.ok(!/from "yaml"|ci-workflow\.mjs/.test(source), `${file}.mjs runs before npm ci and must not import the YAML reader`); + // They run before node_modules exists, so neither they nor anything they import, however indirectly, + // may need an npm package (the YAML reader of scripts/lib/ci-workflow.mjs is one). + const files = commandLines(steps[identity]) + .find((line) => line.startsWith("node --test ")) + .split(/\s+/) + .slice(2); + assert.ok( + files.length >= 4 && files.every((file) => /^scripts\/test-[\w-]+\.mjs$/.test(file)), + `the identity step runs test files: ${files}`, + ); + for (const file of files) { + const { packages } = importGraph(file, repoRoot); + assert.deepEqual([...packages], [], `${file} runs before npm ci but its imports reach npm packages`); } }); diff --git a/scripts/test-frontend-test-groups.mjs b/scripts/test-frontend-test-groups.mjs index 80305e9e..045283b0 100644 --- a/scripts/test-frontend-test-groups.mjs +++ b/scripts/test-frontend-test-groups.mjs @@ -2,9 +2,10 @@ import assert from "node:assert/strict"; import { spawn } from "node:child_process"; import { chmodSync, existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; -import { delimiter, dirname, join, posix } from "node:path"; +import { delimiter, dirname, join } from "node:path"; import { fileURLToPath } from "node:url"; import test from "node:test"; +import { importGraph } from "./lib/import-graph.mjs"; import { isProcessAlive } from "./lib/process-tree.mjs"; import { GROUP_NPM_PACKAGES, TEST_GROUPS } from "./node-test-groups.mjs"; import { classificationProblems, discoverTestFiles } from "./run-frontend-tests.mjs"; @@ -90,31 +91,24 @@ test("the real manifest classifies every Node test file exactly once", () => { }); test("each group declares the npm packages its tests import", () => { - // Static, side-effect and dynamic imports of a bare package specifier. - const bareImport = /\bfrom\s*["']([@a-z][\w@./:-]*)["']|\bimport\s*\(?\s*["']([@a-z][\w@./:-]*)["']/g; - const relativeImport = /\bfrom\s*["'](\.{1,2}\/[^"']+\.mjs)["']/g; // A test needs the packages of the repository modules it imports as well (scripts/lib/ci-workflow.mjs // reads the workflows with `yaml`, #358), so relative imports are followed. - function collect(file, imported, seen) { - if (seen.has(file)) return; - seen.add(file); - // Comments that mention imports (`import "name"`) are not imports. - const source = readFileSync(join(repoRoot, file), "utf8").replace(/\/\*[\s\S]*?\*\/|^\s*\/\/.*$/gm, ""); - for (const match of source.matchAll(bareImport)) { - const specifier = match[1] ?? match[2]; - if (specifier.startsWith("node:")) continue; - imported.add(specifier.startsWith("@") ? specifier.split("/").slice(0, 2).join("/") : specifier.split("/")[0]); - } - for (const match of source.matchAll(relativeImport)) - collect(posix.normalize(posix.join(posix.dirname(file), match[1])), imported, seen); - } for (const [group, files] of Object.entries(TEST_GROUPS)) { - const imported = new Set(); - for (const file of files) collect(file, imported, new Set()); + const imported = new Set(files.flatMap((file) => [...importGraph(file, repoRoot).packages])); assert.deepEqual([...imported].sort(), [...(GROUP_NPM_PACKAGES[group] ?? [])].sort(), `group ${group}`); } }); +test("the import graph follows relative imports through any depth, in every spelling, and ignores comments", () => { + // scripts/fixtures/import-graph: entry -> lib/one -> lib/two (dynamic import) -> lib/three (side-effect imports). + const root = join(repoRoot, "scripts/fixtures/import-graph"); + const { packages, files } = importGraph("entry.mjs", root); + assert.deepEqual([...packages].sort(), ["@scope/pkg", "yaml"]); + assert.deepEqual([...files].sort(), ["entry.mjs", "lib/one.mjs", "lib/three.mjs", "lib/two.mjs"]); + // A cycle terminates, and a file with no package imports reaches none. + assert.deepEqual([...importGraph("cycle-b.mjs", root).packages], []); +}); + test("npm run test:code-health runs exactly the codeHealth group from scripts/", () => { const script = JSON.parse(readFileSync(join(repoRoot, "package.json"), "utf8")).scripts["test:code-health"]; assert.deepEqual(script.match(/(? Date: Fri, 16 Oct 2026 06:57:31 +0000 Subject: [PATCH 7/7] build: list yaml in THIRD_PARTY_NOTICES.md (#358) Declaring yaml as a devDependency made it a non-optional lockfile package, so the notice generator now includes its ISC license. Co-Authored-By: Claude Opus 5.5 --- THIRD_PARTY_NOTICES.md | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/THIRD_PARTY_NOTICES.md b/THIRD_PARTY_NOTICES.md index ab102170..3f257aa4 100644 --- a/THIRD_PARTY_NOTICES.md +++ b/THIRD_PARTY_NOTICES.md @@ -21723,6 +21723,28 @@ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ~~~~ +### cf12d35c36ba + +Components: npm yaml@2.9.1 + +Source filenames: LICENSE + +~~~~text +Copyright Eemeli Aro + +Permission to use, copy, modify, and/or distribute this software for any purpose +with or without fee is hereby granted, provided that the above copyright notice +and this permission notice appear in all copies. + +THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH +REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND +FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, +INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS +OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER +TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF +THIS SOFTWARE. +~~~~ + ### cf798811e9a0 Components: Rust quick-error@2.0.1