diff --git a/.github/actions/setup/action.yml b/.github/actions/setup/action.yml new file mode 100644 index 00000000..7675c045 --- /dev/null +++ b/.github/actions/setup/action.yml @@ -0,0 +1,103 @@ +name: Set up the CI toolchain +description: >- + The preamble of every ci.yml lane (#358): Python, the Rust toolchain, Node.js, the Rust + dependency cache, the downloaded-model cache of the native build lanes and `npm ci`. The job + checks out the repository first, so this action always comes from the revision under test. + The toolchain and Node versions are pinned here once; scripts/test-rust-toolchain-pin.mjs + holds them to rust-toolchain.toml and .nvmrc. + +inputs: + python: + description: Install Python 3.12 (the dictation and diarization tests, the JSON stream verifier). + default: "true" + rust: + description: >- + "true" installs the pinned Rust toolchain. "msrv" installs the one `rust-version` declares + (minimal profile) and exports RUSTUP_TOOLCHAIN for every later step (#353). "false" installs none. + default: "true" + rust-components: + description: Extra toolchain components, comma separated (clippy, rustfmt). + default: "" + cache-key: + description: >- + Swatinem/rust-cache shared-key. Lanes that compile alike share one; rust-cache keeps OS and target + entries apart. Unless `native-build` is set, the hash of scripts/ci/cargo-gates.mjs is appended: the + cache holds what the gates compile, and rust-cache saves only when its key is new, so a changed gate + list needs a new key to get its artifacts cached. + required: true + cache-workspaces: + description: Cargo workspaces to cache, one per line. The standalone ORT engine host is its own workspace (#357). + default: src-tauri + native-build: + description: >- + The lane builds the native release binary and runs the CLI smoke tests (the macOS and Windows build + lanes): it caches the models they download, and its Rust cache key leaves out the gate list, which + such a lane never runs, so a gate change does not evict its cache. + default: "false" + linux-deps: + description: Install the Tauri system libraries and xdotool with apt (Linux lanes that compile the app crate). + default: "false" + npm-ci: + description: Run `npm ci`. test-windows turns it off to run its build identity gates first, then installs. + default: "true" + +runs: + using: composite + steps: + - if: ${{ inputs.linux-deps == 'true' }} + name: Install Tauri system dependencies + shell: bash + run: | + sudo apt-get update + sudo apt-get install -y libwebkit2gtk-4.1-dev libayatana-appindicator3-dev librsvg2-dev libasound2-dev build-essential file xdotool + + - if: ${{ inputs.python == 'true' }} + name: Select evaluation Python runtime + uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6 + with: + python-version: "3.12" + + - if: ${{ inputs.rust == 'true' }} + name: Install Rust + uses: dtolnay/rust-toolchain@e2a55d2ffb04f378e9626c28d38b36d230d1e12f # v1; toolchain must match rust-toolchain.toml + with: + toolchain: 1.99.0 + components: ${{ inputs.rust-components }} + + - if: ${{ inputs.rust == 'msrv' }} + name: Install the declared MSRV toolchain + shell: bash + run: | + msrv=$(sed -n 's/^rust-version = "\(.*\)"$/\1/p' src-tauri/Cargo.toml) + ort_msrv=$(sed -n 's/^rust-version = "\(.*\)"$/\1/p' src-tauri/engine-host/ort/Cargo.toml) + test -n "$msrv" + test "$msrv" = "$ort_msrv" + rustup toolchain install "$msrv" --profile minimal + echo "RUSTUP_TOOLCHAIN=$msrv" >> "$GITHUB_ENV" + echo "Checking against Rust $msrv" + + - name: Install Node.js + uses: actions/setup-node@249970729cb0ef3589644e2896645e5dc5ba9c38 # v6.5.0 + with: + node-version: 22 + cache: npm + + - name: Cache Rust dependencies + uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 + with: + workspaces: ${{ inputs.cache-workspaces }} + shared-key: ${{ inputs.cache-key }}${{ inputs.native-build != 'true' && format('-{0}', hashFiles('scripts/ci/cargo-gates.mjs')) || '' }} + + - if: ${{ inputs.native-build == 'true' }} + name: Cache downloaded models + uses: actions/cache@caa296126883cff596d87d8935842f9db880ef25 # v5.1.0 + with: + path: ${{ runner.os == 'Windows' && '~\AppData\Roaming\Sagascript\Models' || '~/Library/Application Support/Sagascript/Models' }} + key: ${{ runner.os == 'Windows' && 'windows' || 'macos' }}-models-${{ hashFiles('src-tauri/crates/sagascript-core/src/settings/manager.rs', 'src-tauri/crates/sagascript-core/src/transcription/model.rs', 'src-tauri/crates/sagascript-core/src/diarization/model.rs') }} + restore-keys: | + ${{ runner.os == 'Windows' && 'windows' || 'macos' }}-models- + + - if: ${{ inputs.npm-ci == 'true' }} + name: Install npm dependencies + shell: bash + run: npm ci diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 512521d8..dcde65ec 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -33,8 +33,11 @@ updates: patterns: ["*"] update-types: ["minor", "patch"] + # The local composite actions hold SHA pins too (#358): list each one, or its pins go stale. - package-ecosystem: github-actions - directory: "/" + directories: + - "/" + - "/.github/actions/setup" schedule: interval: monthly open-pull-requests-limit: 5 diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7a058b1a..77a9b48e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,40 +1,40 @@ name: CI +# Shared parts live once, outside this file (#358): .github/actions/setup (toolchain preamble), +# scripts/ci/cargo-gates.mjs (THE list of cargo gates; per-OS exclusions are documented there), +# scripts/ci/smoke-cli.mjs and scripts/ci/assert-lanes.mjs (the check-* aggregators). Branch +# protection requires check-macos and check-windows by name: keep both, `needs:` every lane of their OS. +# assert-lanes.mjs comes from the PR checkout, at the trust level of this file itself (contents: read, no secrets). +# +# Windows tests have one owner on a pull request: test-windows (x64), the required lane. +# windows-package.yml does not repeat `cargo test --workspace` and `cargo clippy --workspace` there +# (in its pwsh step a failure never failed the job); it keeps them for releases and manual runs, +# the only place ARM64 tests run. + on: push: branches: [main] pull_request: branches: [main] -# Least privilege (#354): no CI job writes to the repository, so the token is -# read-only. Every job also sets timeout-minutes (60 for native build/test -# lanes, 20 for the rest; at least twice the slowest observed run) instead of -# the 360-minute default. +# Least privilege (#354): read-only token; every job sets timeout-minutes. permissions: contents: read -# Cancel superseded PR runs (#178): pushes to a PR share one group, so a fix -# cancels the stale run instead of burning runner-minutes compiling it twice. -# Pushes (e.g. to `main`) get a unique group per run, so a newer push can -# neither cancel an in-progress run nor replace a pending one — with the -# default single-slot queue, same-group pending runs would otherwise be -# dropped. Never touches release.yml (tag builds must never cancel). +# Cancel superseded PR runs (#178); pushes (e.g. to `main`) get a unique group per run. concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.run_id }} cancel-in-progress: true -# Keep source/line backtraces for CI failures without full variable/type debug -# information in every dependency and PDB. Release profiles are unchanged. -# rust-cache includes CARGO_* environment variables in its compatibility key. +# Line-table backtraces without full debug information (rust-cache keys on CARGO_* variables). env: CARGO_PROFILE_DEV_DEBUG: line-tables-only CARGO_PROFILE_TEST_DEBUG: line-tables-only jobs: - # This job is deliberately dependency-free and checks out the PR base so a - # PR cannot change the allowlist used to decide whether native CI is needed. - # Pushes use --full because the workflow only runs on main and no comparison - # range is needed to keep that required path fail-closed. + # Checks out the PR base so a PR cannot change the allowlist that decides whether native CI is + # needed. Inline on purpose: the base may predate any script file (adoption), and a missing file + # would fail the required checks instead of falling back to full CI. Pushes use --full. scope: runs-on: ubuntu-24.04 timeout-minutes: 20 @@ -48,7 +48,6 @@ jobs: with: fetch-depth: 0 ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.base.sha || github.sha }} - - name: Classify changed paths id: classify env: @@ -66,9 +65,7 @@ jobs: } if [ "$EVENT_NAME" = pull_request ]; then - # The policy is read from the base checkout. During adoption the - # base may predate this script; a fork head may also be absent from - # the base repository. Both cases safely run the full CI lanes. + # An absent policy (adoption) or head commit (fork) safely runs the full CI lanes. if [ -f scripts/ci-docs-scope.mjs ] \ && [[ "$PR_BASE_SHA" =~ ^[0-9a-f]{40}$ ]] \ && [[ "$PR_HEAD_SHA" =~ ^[0-9a-f]{40}$ ]] \ @@ -87,77 +84,31 @@ jobs: if: ${{ needs.scope.result == 'success' && needs.scope.outputs.docs_only != 'true' }} runs-on: macos-14 timeout-minutes: 60 - steps: - - name: Checkout - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0 - - - name: Select evaluation Python runtime - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6 - with: - python-version: "3.12" - - - name: Test offline dictation evaluation tooling - run: python -m unittest discover -s scripts/dictation_eval -p 'test_*.py' - - - name: Test diarization reference contracts - run: python -m unittest discover -s scripts/diarization_sv -p 'test_reference_dataset.py' - - - name: Install Rust - uses: dtolnay/rust-toolchain@e2a55d2ffb04f378e9626c28d38b36d230d1e12f # v1; toolchain must match rust-toolchain.toml + - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0 + - name: Set up toolchain + uses: ./.github/actions/setup with: - toolchain: 1.99.0 - components: clippy - - - name: Install Node.js - uses: actions/setup-node@249970729cb0ef3589644e2896645e5dc5ba9c38 # v6.5.0 - with: - node-version: 22 - cache: npm - - - name: Cache Rust dependencies - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 - with: - workspaces: src-tauri - shared-key: sagascript-ci-checks - - - name: Install npm dependencies - run: npm ci - - # npm advisories (#355): fail on a high or critical finding anywhere in - # the root package-lock.json, dev dependencies included, since they - # build the shipped frontend bundle. site/ and tooling/code-health/ have - # their own lockfiles; neither ships in the app and neither is audited - # here. - - name: Audit npm dependencies - run: npm audit --audit-level=high - - # Prettier over src/, scripts/ and tests/, ESLint, and the Vitest unit - # and component tests (src/**/*.test.ts, #361). verify runs them as - # `format`, `lint` and `unit`. - - name: Check frontend formatting - run: npm run format:check - - - name: Lint frontend and tooling - run: npm run lint - - - name: Unit-test frontend modules - run: npm run test:unit - - - name: Test frontend lifecycle helpers - run: | + rust-components: clippy + cache-key: sagascript-ci-checks + - name: Test offline dictation evaluation tooling and diarization reference contracts + run: | + python -m unittest discover -s scripts/dictation_eval -p 'test_*.py' + python -m unittest discover -s scripts/diarization_sv -p 'test_reference_dataset.py' + - name: Frontend checks (npm audit, format, lint, unit tests, Node tests, types) + run: | + npm audit --audit-level=high + npm run format:check + npm run lint + npm run test:unit npm run test:frontend - node scripts/test-generate-updater-manifest.mjs - node --test scripts/diarization_sv/test_reference_review.mjs - - - name: Install browser regression runtime - run: npx playwright install chromium - + npx svelte-check --tsconfig ./tsconfig.json - name: Test transcription result tabs in browser env: QA_OUTPUT_DIR: ${{ runner.temp }}/transcription-browser - run: npm run test:browser - + run: | + npx playwright install chromium + npm run test:browser - name: Upload browser regression diagnostics if: failure() uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 @@ -165,70 +116,20 @@ jobs: name: transcription-browser-diagnostics path: ${{ runner.temp }}/transcription-browser if-no-files-found: ignore - - name: Build frontend run: npm run build - - - name: Cargo check - working-directory: src-tauri - run: cargo check - - - name: Cargo test - working-directory: src-tauri - run: cargo test - - - name: Cargo clippy - working-directory: src-tauri - run: cargo clippy --all-targets -- -D warnings - - # Separate invocations preserve the core no-features build; combining - # package flags would unify features with the CLI defaults. - - name: Cargo test (sagascript-core, no features) - working-directory: src-tauri - run: cargo test -p sagascript-core - - - name: Cargo test (sagascript-cli) - working-directory: src-tauri - run: cargo test -p sagascript-cli - - - name: Cargo test (sagascript-cli, no features) - working-directory: src-tauri - run: cargo test -p sagascript-cli --no-default-features - - - name: Cargo clippy (sagascript-core, no features) - working-directory: src-tauri - run: cargo clippy -p sagascript-core --all-targets -- -D warnings - - - name: Cargo clippy (sagascript-cli) - working-directory: src-tauri - run: cargo clippy -p sagascript-cli --all-targets -- -D warnings - - # Core defaults to no features. CLI defaults already include diarization - # and are covered above; core needs this explicit feature matrix. - - name: Cargo check (sagascript-core, diarization) - working-directory: src-tauri - run: cargo check -p sagascript-core --features diarization - - - name: Cargo test (sagascript-core, diarization) - working-directory: src-tauri - run: cargo test -p sagascript-core --features diarization - - - name: Cargo clippy (sagascript-core, diarization) - working-directory: src-tauri - run: cargo clippy -p sagascript-core --features diarization --all-targets -- -D warnings - - - name: Svelte check - run: npx svelte-check --tsconfig ./tsconfig.json - - - name: Build Core ML engine host and check its revision + - name: Cargo gates + run: node scripts/ci/cargo-gates.mjs --os macos + # Model-free subset of the engine-host conformance (#367); the full run needs a Pianissimo + # model and is in the release gate (release-build-macos.yml). + - name: Build the Core ML engine host, check its revision, run the conformance subset run: | ./scripts/stage-engine-host.sh ./scripts/check-engine-host-identity.sh build/engine-host/sagascript-engine-host "$GITHUB_SHA" - + python scripts/engine-host-conformance.py build/engine-host/sagascript-engine-host --no-model + # The tests use Swift Testing (Xcode 16+); the runner's default Xcode may be older. - name: Swift test (Core ML engine host) working-directory: src-tauri/engine-host/coreml - # The tests use Swift Testing (`import Testing`), which ships with Xcode 16+; - # the runner's default Xcode may be older, so run them with the newest one installed. run: | newest=$(find /Applications -maxdepth 1 -name "Xcode_*.app" | sort -V | tail -1) if [ -z "$newest" ]; then echo "No Xcode found in /Applications" >&2; exit 1; fi @@ -237,152 +138,29 @@ jobs: swift --version swift test - # Engine-host protocol conformance (#367), model-free subset, against the - # host staged above: hello, ping, status, the errors the client relies on - # (unsupported, model_missing, not_loaded), shutdown and EOF exit. The full - # run (load, windows, cancel) needs a Pianissimo model and is in the release - # gate (release-build-macos.yml, verify-release-draft.sh; docs/engine-host-protocol.md, - # Conformance). - - name: Engine-host conformance, model-free subset (Core ML host) - run: python scripts/engine-host-conformance.py build/engine-host/sagascript-engine-host --no-model - build-macos: needs: scope if: ${{ needs.scope.result == 'success' && needs.scope.outputs.docs_only != 'true' }} runs-on: macos-14 timeout-minutes: 60 - steps: - - name: Checkout - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0 - - - name: Select evaluation Python runtime - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6 - with: - python-version: "3.12" - - - name: Install Rust - uses: dtolnay/rust-toolchain@e2a55d2ffb04f378e9626c28d38b36d230d1e12f # v1; toolchain must match rust-toolchain.toml - with: - toolchain: 1.99.0 - - - name: Install Node.js - uses: actions/setup-node@249970729cb0ef3589644e2896645e5dc5ba9c38 # v6.5.0 - with: - node-version: 22 - cache: npm - - - name: Cache Rust dependencies - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 - with: - workspaces: src-tauri - shared-key: sagascript-ci-native - - - name: Cache downloaded models - uses: actions/cache@caa296126883cff596d87d8935842f9db880ef25 # v5.1.0 + - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0 + - name: Set up toolchain + uses: ./.github/actions/setup with: - path: ~/Library/Application Support/Sagascript/Models - key: macos-models-${{ hashFiles('src-tauri/crates/sagascript-core/src/settings/manager.rs', 'src-tauri/crates/sagascript-core/src/transcription/model.rs', 'src-tauri/crates/sagascript-core/src/diarization/model.rs') }} - restore-keys: | - macos-models- - - - name: Install npm dependencies - run: npm ci - + cache-key: sagascript-ci-native + native-build: "true" - name: Build frontend run: npm run build - - # PR validation needs the native release binary for the smoke tests, not - # an installer. Full packaging, signing and notarization remain gated in - # release.yml, so skipping bundling here removes duplicate work without - # weakening native macOS coverage. - + # The smoke tests need the binary, not an installer (packaging: release.yml). - name: Build native release binary (no installer) env: SAGASCRIPT_UPDATER_PUBKEY: ${{ vars.SAGASCRIPT_UPDATER_PUBKEY }} run: npx tauri build --config scripts/tauri-ci-prebuilt.json --no-bundle - - - name: Verify native release binary - run: | - BINARY=src-tauri/target/release/sagascript - [[ -x "$BINARY" ]] - file "$BINARY" - - # --- Tier 1: CLI smoke tests (no model needed) --- - - - name: Smoke test — help and version + - name: Smoke tests (no model; model download, transcription, JSON and JSONL streams) run: | - BINARY=src-tauri/target/release/sagascript - $BINARY --help - $BINARY transcribe --help - $BINARY record --help - - - name: Smoke test — config management - run: | - BINARY=src-tauri/target/release/sagascript - $BINARY config list - $BINARY config get language - $BINARY config set language sv - $BINARY config get language | grep -q sv - $BINARY config reset language - $BINARY config get language | grep -q en - $BINARY config path - - - name: Smoke test — model listing and formats - run: | - BINARY=src-tauri/target/release/sagascript - $BINARY list-models - $BINARY formats - - - name: Smoke test — shell completions - run: | - BINARY=src-tauri/target/release/sagascript - $BINARY completions zsh > /dev/null - $BINARY completions bash > /dev/null - - # --- Tier 2: Model download + transcription --- - - - name: Smoke test — download model and transcribe - run: | - BINARY=src-tauri/target/release/sagascript - $BINARY download-model nb-whisper-tiny - RESULT=$($BINARY transcribe --language no --model nb-whisper-tiny test-audio/norwegian-short-3s.mp3) - echo "Transcription: $RESULT" - echo "$RESULT" | grep -iq "stortinget" - - - name: Smoke test — transcribe with JSON output - run: | - BINARY=src-tauri/target/release/sagascript - STDOUT="$RUNNER_TEMP/transcription-smoke.json" - STDERR="$RUNNER_TEMP/transcription-smoke.stderr" - $BINARY transcribe --json --language no --model nb-whisper-tiny \ - test-audio/norwegian-short-3s.mp3 > "$STDOUT" 2> "$STDERR" - python3 scripts/verify-json-cli-streams.py "$STDOUT" "$STDERR" - - # The output contract of #364 with a real model: JSONL items, a reader - # that stops after the first item, and the error object of a failed --json run. - - name: Smoke test — JSONL, closed pipe and JSON error object - run: | - set -o pipefail - BINARY=src-tauri/target/release/sagascript - INPUTS="$RUNNER_TEMP/jsonl-inputs" - mkdir -p "$INPUTS" - cp test-audio/norwegian-short-3s.mp3 "$INPUTS/a.mp3" - cp test-audio/norwegian-medium-8s.mp3 "$INPUTS/b.mp3" - $BINARY transcribe --jsonl --language no --model nb-whisper-tiny "$INPUTS" \ - > "$RUNNER_TEMP/transcription-smoke.jsonl" 2> "$RUNNER_TEMP/transcription-smoke-jsonl.stderr" - python3 scripts/verify-json-cli-streams.py --jsonl \ - "$RUNNER_TEMP/transcription-smoke.jsonl" "$RUNNER_TEMP/transcription-smoke-jsonl.stderr" - # A smoke check with the real model (pipefail turns a panic or a failing exit into - # a red step). The deterministic check of the closed-pipe contract is - # `run_into_closed_pipe` in src-tauri/crates/sagascript-cli/tests/output_contract.rs. - $BINARY transcribe --jsonl --language no --model nb-whisper-tiny "$INPUTS" | head -1 > /dev/null - STATUS=0 - $BINARY transcribe --json --language no --model nb-whisper-tiny "$RUNNER_TEMP/missing.wav" \ - > "$RUNNER_TEMP/transcription-error.json" 2> "$RUNNER_TEMP/transcription-error.stderr" || STATUS=$? - test "$STATUS" -eq 1 - python3 scripts/verify-json-cli-streams.py --error \ - "$RUNNER_TEMP/transcription-error.json" "$RUNNER_TEMP/transcription-error.stderr" + node scripts/ci/smoke-cli.mjs basic + node scripts/ci/smoke-cli.mjs model check-macos: needs: [scope, test-macos, build-macos] @@ -390,172 +168,59 @@ jobs: runs-on: ubuntu-24.04 timeout-minutes: 20 env: - SCOPE_RESULT: ${{ needs.scope.result }} - SCOPE_TRUSTED: ${{ needs.scope.outputs.scope_trusted }} - DOCS_ONLY: ${{ needs.scope.outputs.docs_only }} - TEST_RESULT: ${{ needs.test-macos.result }} - BUILD_RESULT: ${{ needs.build-macos.result }} + NEEDS: ${{ toJSON(needs) }} steps: + - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0 - name: Assert macOS CI lanes passed - run: | - test "$SCOPE_RESULT" = success - test "$SCOPE_TRUSTED" = true - if [ "$DOCS_ONLY" = true ]; then - test "$TEST_RESULT" = skipped - test "$BUILD_RESULT" = skipped - else - test "$DOCS_ONLY" = false - test "$TEST_RESULT" = success - test "$BUILD_RESULT" = success - fi + run: node scripts/ci/assert-lanes.mjs test-linux: needs: scope if: ${{ needs.scope.result == 'success' && needs.scope.outputs.docs_only != 'true' }} runs-on: ubuntu-24.04 timeout-minutes: 60 - steps: - - name: Checkout - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0 - - - name: Select evaluation Python runtime - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6 - with: - python-version: "3.12" - - - name: Test offline dictation evaluation tooling - run: python -m unittest discover -s scripts/dictation_eval -p 'test_*.py' - - - name: Install Tauri system dependencies - run: | - sudo apt-get update - sudo apt-get install -y \ - libwebkit2gtk-4.1-dev \ - libayatana-appindicator3-dev \ - librsvg2-dev \ - libasound2-dev \ - build-essential \ - file \ - xdotool - - - name: Install Rust - uses: dtolnay/rust-toolchain@e2a55d2ffb04f378e9626c28d38b36d230d1e12f # v1; toolchain must match rust-toolchain.toml - with: - toolchain: 1.99.0 - components: clippy, rustfmt - - # Formatting gate (#352). Both Cargo workspaces must be rustfmt-clean: - # the main one and the standalone ORT engine host. Runs before any - # compilation, so an unformatted PR fails early. - - name: Check Rust formatting - working-directory: src-tauri - run: | - cargo fmt --all -- --check - cargo fmt --manifest-path engine-host/ort/Cargo.toml --all -- --check - - - name: Install Node.js - uses: actions/setup-node@249970729cb0ef3589644e2896645e5dc5ba9c38 # v6.5.0 + - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0 + - name: Set up toolchain + uses: ./.github/actions/setup with: - node-version: 22 - cache: npm - - # Same rust-cache scheme as the macOS job (#178); the shared key spans - # branches while rust-cache keeps OS/target entries separate. The - # standalone ORT engine host is its own Cargo workspace (#357). - - name: Cache Rust dependencies - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 - with: - workspaces: | + rust-components: clippy, rustfmt + linux-deps: "true" + cache-key: sagascript-ci + cache-workspaces: | src-tauri src-tauri/engine-host/ort - shared-key: sagascript-ci - - # Dependency policy (#355), before any compilation: RustSec advisories, - # licenses, banned or duplicated crates and allowed sources, as set in - # src-tauri/deny.toml, for both Cargo workspaces. The advisory database - # is fetched at run time. rust-cache keeps the installed binary. The ORT - # host run allows the policy entries only the main workspace uses. + - name: Test offline dictation evaluation tooling + run: python -m unittest discover -s scripts/dictation_eval -p 'test_*.py' + # The cargo-deny gates run first, before any compilation; rust-cache keeps the binary. - name: Install cargo-deny run: cargo install --locked cargo-deny --version 0.20.2 - - - name: Check Rust dependency policy (cargo-deny) - working-directory: src-tauri - run: | - cargo deny --locked --workspace check - cargo deny --locked --manifest-path engine-host/ort/Cargo.toml --workspace check -A license-not-encountered -A license-exception-not-encountered -A unmatched-source - - - name: Install npm dependencies - run: npm ci - + # tauri::generate_context! embeds ../dist, so the app crate needs it. - name: Build frontend run: npm run build - - # GUI build: compiles the cfg(target_os = "linux") Tauri path so it can't - # silently break on a future PR (the macOS/Windows runners never see it). - # The bare invocation covers the root (app) package, which pulls in - # sagascript-core with the `record` feature. - - name: Cargo check (GUI app) - working-directory: src-tauri - run: cargo check - - - name: Cargo clippy (GUI app) - working-directory: src-tauri - run: cargo clippy --all-targets -- -D warnings - - # Headless CLI (no Tauri) — the sagascript-cli member crate. Default - # features include `record` (cpal/ALSA); the --no-default-features runs - # cover the pure batch-transcribe build, which must need no ALSA. - - name: Cargo check (headless CLI, with record) - working-directory: src-tauri - run: cargo check -p sagascript-cli - - - name: Cargo check (headless CLI, batch-only) - working-directory: src-tauri - run: cargo check -p sagascript-cli --no-default-features - - - name: Cargo clippy (headless CLI, batch-only) - working-directory: src-tauri - run: cargo clippy -p sagascript-cli --all-targets --no-default-features -- -D warnings - - # Clippy policy lints (#356): unwrap/expect/panic/print/size counts in - # production code, per lint and crate, must not rise above - # src-tauri/clippy-policy-baseline.json. It re-lints only the workspace - # crates (workspace + batch-only CLI); dependencies come from the steps - # above. Measures Linux code only. See CONTRIBUTING.md. + - name: Cargo gates + run: node scripts/ci/cargo-gates.mjs --os linux + # Clippy policy lints (#356): counts must not rise above src-tauri/clippy-policy-baseline.json. + # Re-lints only the workspace crates, after the gates above compiled the dependencies. - name: Clippy policy lints (counts must not rise) run: node scripts/clippy-policy.mjs - - - name: Cargo test (headless CLI, batch-only) - working-directory: src-tauri - run: cargo test -p sagascript-cli --no-default-features - - - name: Build batch CLI for evaluation contracts - working-directory: src-tauri - run: cargo build -p sagascript-cli --no-default-features - - - name: Install offline diarization evaluation test dependencies - run: python -m pip install -r scripts/diarization_sv/requirements-evaluation.txt - - name: Test diarization reference and evidence contracts with native identity parity env: SAGASCRIPT_REFERENCE_CLI: ${{ github.workspace }}/src-tauri/target/debug/sagascript run: | + python -m pip install -r scripts/diarization_sv/requirements-evaluation.txt test -x "$SAGASCRIPT_REFERENCE_CLI" python -m unittest discover -s scripts/diarization_sv -p 'test_*.py' - - name: Cross-check native DER and JER against independent scorers run: | curl --fail --location --retry 2 \ https://raw.githubusercontent.com/nryant/dscore/e02f949ac6592279300a2c33d03daf9e0c12fd27/scorelib/metrics.py \ --output "$RUNNER_TEMP/dscore-metrics.py" - # The cross-check verifies the pinned source SHA-256 before loading - # only its numeric JER function. No Perl runner or model inference. + # Verifies the pinned source SHA-256, loads only its numeric JER function. python scripts/diarization_sv/crosscheck_metrics.py \ --binary "$GITHUB_WORKSPACE/src-tauri/target/debug/sagascript" \ --scratch "$RUNNER_TEMP/diarization-metric-crosscheck" \ --dscore-source "$RUNNER_TEMP/dscore-metrics.py" - - name: Upload diarization metric receipts if: always() uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 @@ -564,131 +229,30 @@ jobs: path: ${{ runner.temp }}/diarization-metric-crosscheck if-no-files-found: ignore - # Run core's own tests on Linux too — cargo does not run dependency - # crates' tests, so without these steps core is only compile-checked - # here (its tests would run on macOS/Windows only). - - name: Cargo test (sagascript-core, no features) - working-directory: src-tauri - run: cargo test -p sagascript-core - - - name: Cargo test (sagascript-core, with record) - working-directory: src-tauri - run: cargo test -p sagascript-core --features record - - # Regression guard for the no-ALSA property (#66): the batch-only CLI - # dependency tree must contain neither cpal nor alsa. (grep -c would - # exit 1 on zero matches and fail the step, hence the if/grep -q form.) - - name: Assert batch-only CLI has no ALSA/cpal dependency - working-directory: src-tauri - run: | - if cargo tree -p sagascript-cli --no-default-features -e normal | grep -qiE 'alsa|cpal'; then - echo "ERROR: cpal/ALSA leaked into the batch-only dependency tree:" - cargo tree -p sagascript-cli --no-default-features -e normal | grep -iE 'alsa|cpal' - exit 1 - fi - echo "OK: no cpal/ALSA in batch-only tree" - - # The Rust<->Swift engine-host wire format (#357). No CI lane runs - # `cargo test --workspace` and cargo never runs a dependency's tests, so - # these otherwise ran only in the release gate and windows-package.yml. - - name: Cargo test (sagascript-engine-protocol) - working-directory: src-tauri - run: cargo test -p sagascript-engine-protocol - - # The ONNX Runtime engine host is a separate Cargo workspace (#357), so - # the workspace steps above never reach it; windows-package.yml only - # builds it. Its tests include the context-biasing parity vectors shared - # with the Swift host. ort's `load-dynamic` means neither step needs an - # ONNX Runtime binary. rustfmt is gated in "Check Rust formatting". - - name: Cargo test (ORT engine host) - working-directory: src-tauri/engine-host/ort - run: cargo test --locked - - - name: Cargo clippy (ORT engine host) - working-directory: src-tauri/engine-host/ort - run: cargo clippy --all-targets --locked -- -D warnings - - # Release-profile guard (#357): `release_build_ignores_the_real_environment` - # proves a release build ignores SAGASCRIPT_ENGINE_HOST and - # SAGASCRIPT_PIANISSIMO_MODEL_DIR. It is compiled only without - # debug_assertions and the `dev-overrides` feature, so every debug - # `cargo test` above leaves it out. - - name: Cargo test (sagascript-core, release-profile dev-override guard) - working-directory: src-tauri - run: | - set -o pipefail - cargo test --release -p sagascript-core --lib dev_overrides 2>&1 | tee "$RUNNER_TEMP/release-guard.log" - # cargo test exits 0 when the filter matches nothing: prove the guard ran. - grep -q 'release_build_ignores_the_real_environment ... ok' "$RUNNER_TEMP/release-guard.log" - - # Minimum supported Rust version (#353). `rust-version` in - # src-tauri/Cargo.toml (and engine-host/ort) is a promise to people building - # from source; compile with exactly that toolchain so it stays true. Linux is - # the cheapest lane that builds every workspace member. RUSTUP_TOOLCHAIN - # overrides the newer development pin in rust-toolchain.toml. + # Minimum supported Rust version (#353): `rust-version` is a promise to people building from + # source, so compile with exactly that toolchain. The setup action installs it and exports + # RUSTUP_TOOLCHAIN (over the pin in rust-toolchain.toml); the msrv profile refuses to run without it. msrv-linux: needs: scope if: ${{ needs.scope.result == 'success' && needs.scope.outputs.docs_only != 'true' }} runs-on: ubuntu-24.04 timeout-minutes: 60 - steps: - - name: Checkout - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0 - - - name: Install Tauri system dependencies - run: | - sudo apt-get update - sudo apt-get install -y \ - libwebkit2gtk-4.1-dev \ - libayatana-appindicator3-dev \ - librsvg2-dev \ - libasound2-dev \ - build-essential \ - file - - - name: Install the declared MSRV toolchain - run: | - msrv=$(sed -n 's/^rust-version = "\(.*\)"$/\1/p' src-tauri/Cargo.toml) - ort_msrv=$(sed -n 's/^rust-version = "\(.*\)"$/\1/p' src-tauri/engine-host/ort/Cargo.toml) - test -n "$msrv" - test "$msrv" = "$ort_msrv" - rustup toolchain install "$msrv" --profile minimal - echo "RUSTUP_TOOLCHAIN=$msrv" >> "$GITHUB_ENV" - echo "Checking against Rust $msrv" - - - name: Install Node.js - uses: actions/setup-node@249970729cb0ef3589644e2896645e5dc5ba9c38 # v6.5.0 - with: - node-version: 22 - cache: npm - - - name: Cache Rust dependencies - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 + - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0 + - name: Set up toolchain + uses: ./.github/actions/setup with: - workspaces: | + python: "false" + rust: msrv + linux-deps: "true" + cache-key: sagascript-msrv + cache-workspaces: | src-tauri src-tauri/engine-host/ort - shared-key: sagascript-msrv - - - name: Install npm dependencies - run: npm ci - - # tauri::generate_context! embeds ../dist, so the app crate needs it. - name: Build frontend run: npm run build - - - name: Cargo check (workspace) on MSRV - working-directory: src-tauri - run: cargo check --workspace --locked - - - name: Cargo check (headless CLI, batch-only) on MSRV - working-directory: src-tauri - run: cargo check -p sagascript-cli --no-default-features --locked - - - name: Cargo check (ORT engine host) on MSRV - working-directory: src-tauri/engine-host/ort - run: cargo check --locked + - name: Cargo gates (MSRV profile) + run: node scripts/ci/cargo-gates.mjs --profile msrv check-linux: needs: [scope, test-linux, msrv-linux] @@ -696,259 +260,81 @@ jobs: runs-on: ubuntu-24.04 timeout-minutes: 20 env: - SCOPE_RESULT: ${{ needs.scope.result }} - SCOPE_TRUSTED: ${{ needs.scope.outputs.scope_trusted }} - DOCS_ONLY: ${{ needs.scope.outputs.docs_only }} - LINUX_RESULT: ${{ needs.test-linux.result }} - MSRV_RESULT: ${{ needs.msrv-linux.result }} + NEEDS: ${{ toJSON(needs) }} steps: + - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0 - name: Assert Linux CI lanes passed - run: | - test "$SCOPE_RESULT" = success - test "$SCOPE_TRUSTED" = true - if [ "$DOCS_ONLY" = true ]; then - test "$LINUX_RESULT" = skipped - test "$MSRV_RESULT" = skipped - else - test "$DOCS_ONLY" = false - test "$LINUX_RESULT" = success - test "$MSRV_RESULT" = success - fi + run: node scripts/ci/assert-lanes.mjs test-windows: needs: scope if: ${{ needs.scope.result == 'success' && needs.scope.outputs.docs_only != 'true' }} runs-on: windows-latest timeout-minutes: 60 - steps: - - name: Checkout - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0 - - - name: Select evaluation Python runtime - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6 + - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0 + - name: Set up toolchain + uses: ./.github/actions/setup with: - python-version: "3.12" - - - name: Test offline dictation evaluation tooling - run: python -m unittest discover -s scripts/dictation_eval -p 'test_*.py' - - - name: Install Rust - uses: dtolnay/rust-toolchain@e2a55d2ffb04f378e9626c28d38b36d230d1e12f # v1; toolchain must match rust-toolchain.toml - with: - toolchain: 1.99.0 - components: clippy - - - name: Install Node.js - uses: actions/setup-node@249970729cb0ef3589644e2896645e5dc5ba9c38 # v6.5.0 - with: - node-version: 22 - cache: npm - - # The standalone ORT engine host is its own Cargo workspace (#357); its - # tests run here too (last step below). - - name: Cache Rust dependencies - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 - with: - workspaces: | + rust-components: clippy + cache-key: sagascript-ci-checks + cache-workspaces: | src-tauri src-tauri/engine-host/ort - shared-key: sagascript-ci-checks - + npm-ci: "false" + # Before npm ci, so a build identity regression fails before any npm or Cargo work. - name: Test Windows build identity gates run: node --test scripts/test-ci-build-identity.mjs scripts/test-windows-release-identity.mjs scripts/test-windows-package-workflow.mjs scripts/test-release-promotion.mjs - - name: Install npm dependencies run: npm ci - - # The whole Node test suite on a Windows checkout (#357), not only the - # build identity gates above (which stay first so they fail before any - # npm or Cargo work). Platform-only cases skip themselves. - - name: Test frontend suite - run: npm run test:frontend - - - name: Unit-test frontend modules - run: npm run test:unit - - # The checkout has CRLF line endings (core.autocrlf); prettier.config.js - # accepts them on Windows, test-macos holds the repository to LF. - - name: Check frontend formatting - run: npm run format:check - + # The whole Node suite on a Windows checkout (#357; CRLF line endings, which prettier.config.js + # accepts here). bash, not pwsh: pwsh would report only the LAST native command's exit code. + - name: Python and frontend checks (dictation tooling, Node tests, unit tests, format) + shell: bash + run: | + python -m unittest discover -s scripts/dictation_eval -p 'test_*.py' + npm run test:frontend + npm run test:unit + npm run format:check - name: Build frontend run: npm run build - - - name: Cargo check - working-directory: src-tauri - run: cargo check - - - name: Cargo test - working-directory: src-tauri - run: cargo test - - - name: Cargo clippy - working-directory: src-tauri - run: cargo clippy --all-targets -- -D warnings - - # Separate invocations preserve the core no-features build; combining - # package flags would unify features with the CLI defaults. - - name: Cargo test (sagascript-core, no features) - working-directory: src-tauri - run: cargo test -p sagascript-core - - - name: Cargo test (sagascript-cli) - working-directory: src-tauri - run: cargo test -p sagascript-cli - - - name: Cargo clippy (sagascript-core, no features) - working-directory: src-tauri - run: cargo clippy -p sagascript-core --all-targets -- -D warnings - - - name: Cargo clippy (sagascript-cli) - working-directory: src-tauri - run: cargo clippy -p sagascript-cli --all-targets -- -D warnings - - # The ONNX Runtime engine host ships for Windows, and its golden protocol - # replay moves the POSIX vector paths (/vectors/...) under an absolute - # Windows directory (#367), which only a Windows run exercises. ort's - # `load-dynamic` means no ONNX Runtime binary is needed. - - name: Cargo test (ORT engine host) - working-directory: src-tauri/engine-host/ort - run: cargo test --locked + - name: Cargo gates + run: node scripts/ci/cargo-gates.mjs --os windows build-windows: needs: scope if: ${{ needs.scope.result == 'success' && needs.scope.outputs.docs_only != 'true' }} runs-on: windows-latest timeout-minutes: 60 - steps: - - name: Checkout - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0 - - - name: Install Rust - uses: dtolnay/rust-toolchain@e2a55d2ffb04f378e9626c28d38b36d230d1e12f # v1; toolchain must match rust-toolchain.toml - with: - toolchain: 1.99.0 - - - name: Install Node.js - uses: actions/setup-node@249970729cb0ef3589644e2896645e5dc5ba9c38 # v6.5.0 + - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0 + - name: Set up toolchain + uses: ./.github/actions/setup with: - node-version: 22 - cache: npm - - - name: Cache Rust dependencies - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 - with: - workspaces: src-tauri - shared-key: sagascript-ci-native - - - name: Cache downloaded models - uses: actions/cache@caa296126883cff596d87d8935842f9db880ef25 # v5.1.0 - with: - path: ~\AppData\Roaming\Sagascript\Models - key: windows-models-${{ hashFiles('src-tauri/crates/sagascript-core/src/settings/manager.rs', 'src-tauri/crates/sagascript-core/src/transcription/model.rs', 'src-tauri/crates/sagascript-core/src/diarization/model.rs') }} - restore-keys: | - windows-models- - - - name: Install npm dependencies - run: npm ci - + cache-key: sagascript-ci-native + native-build: "true" - name: Build frontend run: npm run build - - # PR validation needs the native release binary for the smoke tests, not - # an installer. Full packaging, signing and notarization remain gated in - # release.yml, so skipping bundling here removes duplicate work without - # weakening native Windows coverage. - + # The smoke tests need the binary, not an installer; packaging stays gated in release.yml. - name: Build native release binary (no installer) run: npx tauri build --config scripts/tauri-ci-prebuilt.json --no-bundle - - - name: Verify native release binary - run: | - $binary = "src-tauri\target\release\sagascript.exe" - if (-not (Test-Path -PathType Leaf $binary)) { - throw "Missing native release binary: $binary" - } - - # --- CLI smoke tests (Windows / PowerShell) --- - - - name: Smoke test — help and version - run: | - $env:BINARY = "src-tauri\target\release\sagascript.exe" - & $env:BINARY --help - & $env:BINARY transcribe --help - & $env:BINARY record --help - - - name: Smoke test — config management - run: | - $env:BINARY = "src-tauri\target\release\sagascript.exe" - & $env:BINARY config list - & $env:BINARY config get language - & $env:BINARY config set language sv - $result = & $env:BINARY config get language - if ($result -notmatch "sv") { throw "Expected 'sv'" } - & $env:BINARY config reset language - $result = & $env:BINARY config get language - if ($result -notmatch "en") { throw "Expected 'en'" } - & $env:BINARY config path - - - name: Smoke test — model listing and formats - run: | - $env:BINARY = "src-tauri\target\release\sagascript.exe" - & $env:BINARY list-models - & $env:BINARY formats - - - name: Smoke test — shell completions - run: | - $env:BINARY = "src-tauri\target\release\sagascript.exe" - & $env:BINARY completions powershell | Out-Null - - # The two model smoke steps stay non-blocking (#357), but no longer - # silently: a step's `conclusion` is reported AFTER continue-on-error, so - # it read "success" while the transcription actually died after - # "Encoding audio..." in about half of the runs since 2026-10-09 (masked - # exit 1; suspected non-portable whisper.cpp build in the shared native - # cache). The report step below turns every masked failure into a - # warning annotation and a job-summary line until the cause is fixed. - - name: Smoke test — download model and transcribe - id: smoke_transcribe - continue-on-error: true - run: | - $env:BINARY = "src-tauri\target\release\sagascript.exe" - & $env:BINARY download-model nb-whisper-tiny - $result = & $env:BINARY transcribe --language no --model nb-whisper-tiny test-audio/norwegian-short-3s.mp3 - Write-Host "Transcription: $result" - if ($result -notmatch "(?i)stortinget") { throw "Expected 'stortinget' in output" } - - - name: Smoke test — transcribe with JSON output - id: smoke_json + - name: Smoke test, tier 1 (no model) + run: node scripts/ci/smoke-cli.mjs basic + # Tier 2 is the only non-blocking step (#357): the transcription dies in about half of the + # runs. continue-on-error masks the step's `conclusion`, so the report step below turns its + # real `outcome` into a warning annotation and a job-summary line. + - name: Smoke test, tier 2 (model download, transcription, JSON and JSONL streams) + id: smoke_model continue-on-error: true - shell: cmd - run: | - set BINARY=src-tauri\target\release\sagascript.exe - %BINARY% transcribe --json --language no --model nb-whisper-tiny test-audio/norwegian-short-3s.mp3 > json_output.txt 2>nul - type json_output.txt - findstr /C:"text" json_output.txt - findstr /C:"duration_seconds" json_output.txt - findstr /C:"language" json_output.txt - + run: node scripts/ci/smoke-cli.mjs model - name: Report masked smoke test failures if: ${{ always() }} shell: bash env: - TRANSCRIBE_OUTCOME: ${{ steps.smoke_transcribe.outcome }} - JSON_OUTCOME: ${{ steps.smoke_json.outcome }} + MODEL_OUTCOME: ${{ steps.smoke_model.outcome }} run: | - for pair in "download model and transcribe=$TRANSCRIBE_OUTCOME" "transcribe with JSON output=$JSON_OUTCOME"; do - name=${pair%%=*} - outcome=${pair##*=} - echo "Windows smoke test — $name: $outcome" >> "$GITHUB_STEP_SUMMARY" - if [ "$outcome" != success ]; then - echo "::warning title=Windows smoke test failed (non-blocking)::Smoke test — $name: $outcome (masked by continue-on-error, see #357)" - fi - done + echo "Windows smoke test, tier 2 (model): $MODEL_OUTCOME" >> "$GITHUB_STEP_SUMMARY" + [ "$MODEL_OUTCOME" = success ] || echo "::warning title=Windows smoke test failed (non-blocking)::Tier 2 (model): $MODEL_OUTCOME, masked by continue-on-error (#357)" check-windows: needs: [scope, test-windows, build-windows] @@ -956,21 +342,8 @@ jobs: runs-on: ubuntu-24.04 timeout-minutes: 20 env: - SCOPE_RESULT: ${{ needs.scope.result }} - SCOPE_TRUSTED: ${{ needs.scope.outputs.scope_trusted }} - DOCS_ONLY: ${{ needs.scope.outputs.docs_only }} - TEST_RESULT: ${{ needs.test-windows.result }} - BUILD_RESULT: ${{ needs.build-windows.result }} + NEEDS: ${{ toJSON(needs) }} steps: + - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0 - name: Assert Windows CI lanes passed - run: | - test "$SCOPE_RESULT" = success - test "$SCOPE_TRUSTED" = true - if [ "$DOCS_ONLY" = true ]; then - test "$TEST_RESULT" = skipped - test "$BUILD_RESULT" = skipped - else - test "$DOCS_ONLY" = false - test "$TEST_RESULT" = success - test "$BUILD_RESULT" = success - fi + run: node scripts/ci/assert-lanes.mjs diff --git a/.github/workflows/windows-package.yml b/.github/workflows/windows-package.yml index 49159201..fb4d97d7 100644 --- a/.github/workflows/windows-package.yml +++ b/.github/workflows/windows-package.yml @@ -251,7 +251,14 @@ jobs: npm run build npm run check + # Ownership of the Windows tests (#358): ci.yml's test-windows, a required check, owns them on + # a pull request and runs these same workspace commands (scripts/ci/cargo-gates.mjs), so a pull + # request does not repeat them on either architecture. They stay for releases (workflow_call + # from prebuild-release.yml) and manual runs, whose candidates are gated here. In this pwsh + # step only the LAST command's exit code counts: a failing test or lint here never failed the + # job; test-windows is where it does (x64; ARM64 tests run only here, on releases and manual runs). - name: Test and lint Rust workspace + if: github.event_name != 'pull_request' working-directory: src-tauri run: | # `cargo check --workspace` is omitted: clippy --all-targets below is a strict superset. @@ -259,11 +266,18 @@ jobs: cargo clippy --workspace --all-targets -- -D warnings cargo build -p sagascript-cli --no-default-features + # A pull request skips the step above; the CPU policy checks below still read its debug tree. + - name: Build lean CLI (debug) + if: github.event_name == 'pull_request' + working-directory: src-tauri + run: cargo build -p sagascript-cli --no-default-features + - name: Gate native Windows clipboard transactions shell: pwsh run: | # The host triple was verified above. Reuse the workspace test binary - # instead of recompiling a duplicate explicit-target debug tree. + # instead of recompiling a duplicate explicit-target debug tree (on a pull + # request there is none: this step then compiles the app's test binary). cargo test --manifest-path src-tauri/Cargo.toml -p sagascript native_runner_clipboard_transaction_smoke -- --ignored --test-threads=1 if ($LASTEXITCODE -ne 0) { throw "Native clipboard transaction gate failed" } diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 1323849a..6a05a939 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -82,12 +82,12 @@ npm run verify -- --list # every check with its commands, budget | Area (`--only`) | Checks | What runs (working directory) | Needs | CI lane | |---|---|---|---|---| | `fmt` | `fmt`, `fmt-ort` | `cargo fmt --all -- --check` and `cargo fmt --manifest-path engine-host/ort/Cargo.toml --all -- --check` (`src-tauri`) | Rust | test-linux | -| `clippy` | `clippy-workspace`, `clippy-cli-lean`, `clippy-policy` (Linux); `clippy-app` (fast set) | `cargo clippy --workspace --all-targets -- -D warnings`; `cargo clippy -p sagascript-cli --all-targets --no-default-features -- -D warnings` (`src-tauri`); `node scripts/clippy-policy.mjs` (policy counts, see below) | Rust; Linux libraries; `dist/` (workspace, app, policy) | test-macos, test-linux, test-windows (per crate); policy: test-linux | +| `clippy` | `clippy-workspace`, `clippy-cli-lean`, `clippy-policy` (Linux); `clippy-app` (fast set) | `cargo clippy --workspace --all-targets -- -D warnings`; `cargo clippy -p sagascript-cli --all-targets --no-default-features -- -D warnings` (`src-tauri`); `node scripts/clippy-policy.mjs` (policy counts, see below) | Rust; Linux libraries; `dist/` (workspace, app, policy) | workspace and lean CLI: test-macos, test-linux, test-windows; policy: test-linux | | `rust-deps` | `cli-lean-deps` | `cargo tree -p sagascript-cli --no-default-features -e normal` contains no cpal/ALSA | Rust | test-linux | -| `rust-test` | `test-workspace`, `test-cli-lean` | `cargo test --workspace`; `cargo test -p sagascript-cli --no-default-features` | Rust; Linux libraries; `dist/` (workspace) | workspace: test-macos, test-windows (`cargo test` and per crate); lean CLI: test-macos, test-linux | -| `core-features` | `core-clippy`, `core-clippy-diarization`, `core-test`, `core-test-record`, `core-test-diarization` | `cargo clippy`/`cargo test -p sagascript-core` without features, with `record`, with `diarization` | Rust | test-macos, test-linux, test-windows | -| `engine-protocol` | `engine-protocol` | `cargo test -p sagascript-engine-protocol` | Rust | test-linux | -| `ort-host` | `ort-host-clippy`, `ort-host-test` | `cargo clippy --all-targets --locked -- -D warnings`, `cargo test --locked` (`src-tauri/engine-host/ort`) | Rust | test-linux; tests also test-windows | +| `rust-test` | `test-workspace`, `test-cli-lean` | `cargo test --workspace`; `cargo test -p sagascript-cli --no-default-features` | Rust; Linux libraries; `dist/` (workspace) | workspace and lean CLI: test-macos, test-linux, test-windows | +| `core-features` | `core-clippy`, `core-clippy-diarization`, `core-test`, `core-test-record`, `core-test-diarization` | `cargo clippy`/`cargo test -p sagascript-core` without features, with `record`, with `diarization` | Rust | without features: test-macos, test-linux, test-windows; with a feature: test-linux | +| `engine-protocol` | `engine-protocol` | `cargo test -p sagascript-engine-protocol` | Rust | every test lane, through `cargo test --workspace` | +| `ort-host` | `ort-host-clippy`, `ort-host-test` | `cargo clippy --all-targets --locked -- -D warnings`, `cargo test --locked` (`src-tauri/engine-host/ort`) | Rust | test-linux, test-windows | | `deps` | `deny`, `npm-audit` | `cargo deny --locked --workspace check` and the same for the ORT host (`src-tauri`; policy and what to do about a new advisory: `src-tauri/deny.toml`); `npm audit --audit-level=high` (root lockfile). Both fetch advisory data over the network | `cargo install --locked cargo-deny --version 0.20.2` | deny: test-linux; audit: test-macos | | `release-guard` | `release-guard` | `cargo test --release -p sagascript-core --lib dev_overrides` | Rust | test-linux | | `svelte-check` | `svelte-check` | `npx svelte-check --tsconfig ./tsconfig.json` | `npm ci` | test-macos | @@ -101,9 +101,10 @@ npm run verify -- --list # every check with its commands, budget | `msrv` | `msrv` (opt-in; run it for dependency bumps or newer language features) | `cargo check --workspace --locked`, the lean CLI and the ORT host on the `rust-version` toolchain | `rustup toolchain install 1.88 --profile minimal`; `dist/` | msrv-linux | | `browser` | `browser` (opt-in) | `npm run test:browser` (Playwright against the mocked IPC) | `npx playwright install chromium` | test-macos | -CI splits the cargo commands per crate and feature set; `--list` shows which CI commands each -check stands for, and `scripts/test-verify-ci-parity.mjs` fails when ci.yml's test lanes and -this list drift apart. Not part of verify: release builds and CLI smoke tests (build-macos, +CI keeps its cargo commands in one list, `scripts/ci/cargo-gates.mjs`, with a reason for every gate that +is not run on all three OSes; `--list` shows which CI commands each check stands for, and +`scripts/test-verify-ci-parity.mjs` fails when ci.yml's test lanes (gate list and setup action +included) and this list drift apart. Not part of verify: release builds and CLI smoke tests (build-macos, build-windows), engine-host staging, the DER/JER cross-check against a downloaded scorer, and the release checks `npm run release:check` and `npm run licenses:check`. Where every suite lives is in [docs/ARCHITECTURE.md](docs/ARCHITECTURE.md#5-where-tests-live-and-how-ci-runs-them). diff --git a/THIRD_PARTY_NOTICES.md b/THIRD_PARTY_NOTICES.md index ab102170..3f257aa4 100644 --- a/THIRD_PARTY_NOTICES.md +++ b/THIRD_PARTY_NOTICES.md @@ -21723,6 +21723,28 @@ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ~~~~ +### cf12d35c36ba + +Components: npm yaml@2.9.1 + +Source filenames: LICENSE + +~~~~text +Copyright Eemeli Aro + +Permission to use, copy, modify, and/or distribute this software for any purpose +with or without fee is hereby granted, provided that the above copyright notice +and this permission notice appear in all copies. + +THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH +REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND +FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, +INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS +OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER +TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF +THIS SOFTWARE. +~~~~ + ### cf798811e9a0 Components: Rust quick-error@2.0.1 diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index f534b52a..96cdd604 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -381,11 +381,11 @@ Shared fixtures: `sagascript_testsupport` (`TempDir`, `fake_settings`, `sha256_h | Layer | Location | Local command | CI | |---|---|---|---| -| Rust unit, app | sibling `foo_tests.rs` files, declared last in `foo.rs` as `#[cfg(test)] #[path = "foo_tests.rs"] mod tests;` (`commands_tests.rs`, `main_tests.rs`, `meeting_jobs_tests.rs`, `paste/windows_clipboard_tests.rs`); inline `#[cfg(test)]` modules in smaller files | `cargo test` (app only) | macOS, Windows | -| Rust unit, core/CLI/protocol | inline, or sibling `foo_tests.rs` as in the app: core `download_tests.rs`, `settings/manager_tests.rs`, `settings/store_tests.rs`, `transcription/whisper_backend_tests.rs`; CLI `transcribe_tests.rs`, `meeting_reprocessing_tests.rs`; `sagascript-engine-protocol/src/tests.rs` | `cargo test -p `; release-only guard: `cargo test --release -p sagascript-core --lib dev_overrides` | per-crate steps; protocol and the release guard in test-linux | +| Rust unit, app | sibling `foo_tests.rs` files, declared last in `foo.rs` as `#[cfg(test)] #[path = "foo_tests.rs"] mod tests;` (`commands_tests.rs`, `main_tests.rs`, `meeting_jobs_tests.rs`, `paste/windows_clipboard_tests.rs`); inline `#[cfg(test)]` modules in smaller files | `cargo test` (app only), `cargo test --workspace` (everything) | every OS lane (`cargo test --workspace`) | +| Rust unit, core/CLI/protocol | inline, or sibling `foo_tests.rs` as in the app: core `download_tests.rs`, `settings/manager_tests.rs`, `settings/store_tests.rs`, `transcription/whisper_backend_tests.rs`; CLI `transcribe_tests.rs`, `meeting_reprocessing_tests.rs`; `sagascript-engine-protocol/src/tests.rs` | `cargo test -p `; release-only guard: `cargo test --release -p sagascript-core --lib dev_overrides` | `cargo test --workspace` and the per-crate feature-subset gates on every OS; the release guard in test-linux | | Rust integration | `sagascript-core/tests/*.rs` (engine host against the fake host, chunk-merge golden, Pianissimo adapter); `sagascript-cli/tests/*.rs` (spawn `CARGO_BIN_EXE_sagascript`) | `cargo test -p sagascript-core`, `-p sagascript-cli` | macOS, Linux, Windows | -| ORT host | `src-tauri/engine-host/ort/src/**` | `cargo test` and `cargo clippy --all-targets -- -D warnings` in `engine-host/ort/` | test-linux (fmt, test, clippy); test-windows (test) | -| Engine-host protocol | golden transcripts `src-tauri/engine-host/test-vectors/protocol/*.jsonl` with the scripted `fake-engine.json` ([contract](engine-host-protocol.md#golden-transcripts)); replayed by `sagascript-engine-protocol/src/vector_tests.rs`, `engine-host/ort/src/server/golden_tests.rs` and `ProtocolServerTests.swift` | `cargo test -p sagascript-engine-protocol`; `cargo test --locked` in `engine-host/ort/`; `swift test` | test-linux (Rust, ORT host); test-windows (ORT host); test-macos (Swift) | +| ORT host | `src-tauri/engine-host/ort/src/**` | `cargo test` and `cargo clippy --all-targets -- -D warnings` in `engine-host/ort/` | test-linux (fmt, test, clippy); test-windows (test, clippy) | +| Engine-host protocol | golden transcripts `src-tauri/engine-host/test-vectors/protocol/*.jsonl` with the scripted `fake-engine.json` ([contract](engine-host-protocol.md#golden-transcripts)); replayed by `sagascript-engine-protocol/src/vector_tests.rs`, `engine-host/ort/src/server/golden_tests.rs` and `ProtocolServerTests.swift` | `cargo test -p sagascript-engine-protocol`; `cargo test --locked` in `engine-host/ort/`; `swift test` | `cargo test --workspace` in every OS lane (protocol crate); test-linux, test-windows (ORT host); test-macos (Swift) | | MSRV | whole workspace, lean CLI, ORT host | `RUSTUP_TOOLCHAIN=1.88 cargo check --workspace --locked` (see CONTRIBUTING.md) | msrv-linux | | Swift | `src-tauri/engine-host/coreml/Tests/EngineHostCoreTests/` | `scripts/build-engine-host.sh`, then `swift test` (`npm run verify -- --only swift`) | test-macos | | Frontend unit and component (Vitest) | `src/**/*.test.ts` next to the module or component, in happy-dom; typed IPC mock, setup and component backends in `src/test/` ([docs/testing/frontend.md](testing/frontend.md)) | `npm run test:unit` (`test:unit:coverage` for a report) | test-macos, test-windows; release-quality-gate, windows-package | @@ -397,29 +397,38 @@ Shared fixtures: `sagascript_testsupport` (`TempDir`, `fake_settings`, `sha256_h | Transport contract | Rust: `transport_contract*` tests in `events_tests.rs`, `commands_tests.rs`, `meeting_jobs_tests.rs` and `contract/` write or compare `contracts/transport/*.json` and `src/lib/ipc-contract.generated.ts`; TS: `src/test/transport.test.ts`, `src/lib/api.test.ts` ([contracts/README.md](../contracts/README.md)) | `cargo test contract` in `src-tauri/` (`SAGASCRIPT_BLESS_CONTRACTS=1` regenerates), `npm run test:unit`, svelte-check | with the app tests and the frontend suites | | Formatting | Prettier over `src/`, `scripts/`, `tests/` (not `.svelte` yet; `.prettierignore`) | `npm run format:check` (`npm run format` fixes) | test-macos, test-windows | | Lint | ESLint over `src/` (TypeScript, Svelte), `scripts/`, `tests/` (`eslint.config.js`, baseline in `eslint-suppressions.json`) | `npm run lint` | test-macos | -| CLI smoke | release binary in `ci.yml` build jobs: help, `config`, `list-models`, `formats`, completions, download + transcribe, `--json` streams (`scripts/verify-json-cli-streams.py`) | — | build-macos, build-windows | +| CLI smoke | release binary in `ci.yml` build jobs, `scripts/ci/smoke-cli.mjs` (one script for every OS; `basic` tier without a model: help, `config`, `list-models`, `formats`, completions of every shell; `model` tier: download + transcribe, `--json`, `--jsonl`, a closed pipe and the error object through `scripts/verify-json-cli-streams.py`) | `node scripts/ci/smoke-cli.mjs basic --binary ` | build-macos, build-windows (Windows tolerates the `model` tier, #357) | | Conformance | `scripts/engine-host-conformance.py --no-model` against a real host process; with `--model-dir ` the full run | model-free: `npm run verify -- --only engine-host-conformance` (macOS); full run by hand with a downloaded model | model-free subset: the staged Core ML host in test-macos and the signed host in `release-build-macos.yml`; full run (release gate, each time right after a Pianissimo smoke): `release-build-macos.yml` against the host in the installed signed app (prebuild and fallback paths), `scripts/verify-release-draft.sh` against the host in the downloaded draft (promote path in CI, fallback path by hand) and the manual `test-build.yml` ([spec](engine-host-protocol.md#conformance)) | -CI (`.github/workflows/ci.yml`) splits cargo per crate and feature set: test-linux runs both -`cargo fmt --check` gates, `cargo deny check` for both Cargo workspaces (`src-tauri/deny.toml`: -advisories, licenses, a single `reqwest`, allowed sources), app `check`/`clippy`, the CLI with -and without default features, core with no features and with `record`, the protocol crate, ORT -host test/clippy, the release-profile guard, the no-ALSA tree check and the clippy policy -counts (`scripts/clippy-policy.mjs`, which must not rise); msrv-linux runs `cargo -check --workspace --locked`, the lean CLI and the ORT host on the `rust-version` toolchain -(1.88), and check-linux requires it; test-macos runs `npm audit --audit-level=high`, app -`check`/`test`/`clippy`, core with no features and with `diarization`, the CLI both ways, Swift -tests and the frontend suites (Prettier, ESLint, Vitest, Node); test-windows runs the same -frontend suites except ESLint, app, core and CLI tests and clippy, and the ORT host tests. -`cargo test --workspace` (and the Vitest suite) runs in `windows-package.yml` (on PRs touching -`src/`, `src-tauri/`, `scripts/`, package files, `.gitattributes` or the workflow itself) and -`release-quality-gate.yml`. A PR that only touches +CI (`.github/workflows/ci.yml`) has a test lane per OS (test-macos, test-linux, test-windows), a native +build lane with the CLI smoke tests on macOS and Windows, an MSRV lane on Linux, and the `check-*` +aggregators that branch protection reads (`check-macos` and `check-windows` are required). The parts +every lane shares live once, outside the workflow (#358): `.github/actions/setup` (Python, the pinned +Rust toolchain, Node.js, the Rust cache, `npm ci`), `scripts/ci/cargo-gates.mjs` (the one list of cargo +commands, run by every lane with `--os`; the MSRV lane runs its `msrv` profile), `scripts/ci/smoke-cli.mjs` +and `scripts/ci/assert-lanes.mjs` (what an aggregator asserts about the lanes it needs). The same cargo +commands run on every OS except the gates marked `os:` in the list, each with its reason: formatting, +`cargo deny`, the core feature subsets (`diarization`, `record`), the lean-CLI build for the Python +suites, the ALSA tree check and the release-profile guard run on Linux only, the ORT host gates on +Linux and Windows. Everything else is `cargo clippy` and `cargo test` over the workspace, over core +and over the lean CLI (`--no-default-features`). Besides the cargo gates test-linux runs the clippy policy +counts (`scripts/clippy-policy.mjs`, which must not rise) and the offline diarization evaluation; +msrv-linux runs `cargo check --workspace --locked`, the lean CLI and the ORT host on the `rust-version` +toolchain (1.88), and check-linux requires it; test-macos runs `npm audit --audit-level=high`, the +frontend suites (Prettier, ESLint, Vitest, Node), Svelte types, the Swift tests and the engine-host +conformance subset; test-windows runs the same frontend suites except ESLint and the Windows build +identity gates. test-windows owns the Windows tests of every pull request: +`windows-package.yml` (on PRs touching `src/`, `src-tauri/`, `scripts/`, package files, +`.gitattributes` or the workflow itself) repeats `cargo test --workspace` and clippy only for +releases (`prebuild-release.yml`) and manual runs, on both architectures; +`release-quality-gate.yml` runs them for a release. A PR that only touches `README.md`, `docs/release-readiness.md` or `docs/research/**/*.md` skips the native lanes (`scripts/ci-docs-scope.mjs`). Locally, `npm run verify` runs the test and lint suites above (`scripts/verify.mjs`, documented in [CONTRIBUTING.md](../CONTRIBUTING.md#running-tests)); -`scripts/test-verify-ci-parity.mjs` keeps its check list in step with ci.yml's test lanes. -**Planned:** [#358](https://github.com/Magnus-Gille/sagascript/issues/358) lets CI call the -verify groups directly; [#361](https://github.com/Magnus-Gille/sagascript/issues/361) phase B +`scripts/test-verify-ci-parity.mjs` keeps its check list in step with the commands the test lanes run, +gate list and setup action included. The `scripts/test-ci-*.mjs` tests read the workflows as parsed +YAML (`scripts/lib/ci-workflow.mjs`), not as text. +**Planned:** [#361](https://github.com/Magnus-Gille/sagascript/issues/361) phase B replaces the remaining source-text tests in the `ui` group with component tests. ## 6. Known drift and planned changes diff --git a/docs/adr/0001-cli-first.md b/docs/adr/0001-cli-first.md index 7fdf2631..193e683b 100644 --- a/docs/adr/0001-cli-first.md +++ b/docs/adr/0001-cli-first.md @@ -22,7 +22,7 @@ The code follows it structurally. The desktop binary dispatches any subcommand t `sagascript_cli::run`; a headless build without Tauri exists for Linux and batch use (`cargo build -p sagascript-cli --no-default-features`); even GUI recovery has CLI entry points (`sagascript open`, `sagascript update`). CI exercises the release binary through the -CLI (`config`, `list-models`, `transcribe --json`, in `.github/workflows/ci.yml`), and +CLI (`config`, `list-models`, `transcribe --json`, in `scripts/ci/smoke-cli.mjs`, which `.github/workflows/ci.yml` runs), and `valid_keys_count_matches_settings_struct` in `crates/sagascript-cli/src/config.rs` fails when a `Settings` field has no `config` key. diff --git a/docs/recipes/add-cli-subcommand.md b/docs/recipes/add-cli-subcommand.md index d1ce7d09..f6a746f4 100644 --- a/docs/recipes/add-cli-subcommand.md +++ b/docs/recipes/add-cli-subcommand.md @@ -134,11 +134,13 @@ cargo run -q -p sagascript-cli -- example --json ## 6. CI smoke (model-free commands) -If the command is cheap and needs no model, add it to the "Tier 1: CLI smoke tests" steps -of `build-macos` (bash) and the matching steps of `build-windows` (PowerShell) in -`.github/workflows/ci.yml`. Some `scripts/test-ci-*.mjs` files assert on `ci.yml` text, so -run `npm run test:frontend` after editing it (tests in both the `ci` and `release` groups read -workflow files). +If the command is cheap and needs no model, add it to the `basic` tier of +`scripts/ci/smoke-cli.mjs`: one cross-platform script that `build-macos` and `build-windows` +in `.github/workflows/ci.yml` both run against the release binary (`node scripts/ci/smoke-cli.mjs +basic --binary ` runs it locally). `scripts/test-ci-smoke-cli.mjs` lists the commands each +tier runs, so extend it with the new one. The `scripts/test-ci-*.mjs` tests read the workflows as +parsed YAML; run `npm run test:frontend` after editing a workflow (tests in both the `ci` and +`release` groups read them). ## 7. Docs diff --git a/docs/windows-release.md b/docs/windows-release.md index c83c3b9c..44fcb555 100644 --- a/docs/windows-release.md +++ b/docs/windows-release.md @@ -33,7 +33,9 @@ described as an official, signed, stable, or fully accepted Windows release. `.github/workflows/windows-package.yml` runs the release metadata, license, frontend, Rust, and real CPU-transcription gates on native `windows-latest` -(x64) and `windows-11-arm` (ARM64) runners. Each runner then +(x64) and `windows-11-arm` (ARM64) runners. (On a pull request the Rust +tests and lints are `ci.yml`'s `test-windows`, a required check, so this +workflow repeats them only for releases and manual runs, on both architectures.) Each runner then builds unsigned NSIS and MSI installers, a portable desktop executable, and a separate console CLI executable. It uploads them as one short-lived GitHub Actions artifact and never creates or modifies a GitHub Release. Its artifacts diff --git a/eslint-suppressions.json b/eslint-suppressions.json index 48584107..3f2ed375 100644 --- a/eslint-suppressions.json +++ b/eslint-suppressions.json @@ -35,11 +35,6 @@ "count": 9 } }, - "scripts/test-ci-parallel-gates.mjs": { - "no-useless-escape": { - "count": 4 - } - }, "scripts/test-code-health-producer.mjs": { "no-control-regex": { "count": 1 diff --git a/package-lock.json b/package-lock.json index be5a4da7..b086d5da 100644 --- a/package-lock.json +++ b/package-lock.json @@ -35,7 +35,8 @@ "typescript": "^5.5.0", "typescript-eslint": "^8.71.0", "vite": "^6.0.0", - "vitest": "^5.0.2" + "vitest": "^5.0.2", + "yaml": "^2.9.1" }, "engines": { "node": ">=22" @@ -4146,8 +4147,6 @@ "integrity": "sha512-3NxN8+78OdzbT7C/WjGsyfPAtJaN3FNDsWxv7Y7mcDsT/oOmgW8BpyQQFFBnvZE3j9Y2Sdz1ULFLezL7Eb2yFw==", "dev": true, "license": "ISC", - "optional": true, - "peer": true, "bin": { "yaml": "bin.mjs" }, diff --git a/package.json b/package.json index 75c9dc9d..3c9ea8d3 100644 --- a/package.json +++ b/package.json @@ -52,7 +52,8 @@ "typescript": "^5.5.0", "typescript-eslint": "^8.71.0", "vite": "^6.0.0", - "vitest": "^5.0.2" + "vitest": "^5.0.2", + "yaml": "^2.9.1" }, "dependencies": { "@tauri-apps/api": "^2.0.0", diff --git a/scripts/ci/assert-lanes.mjs b/scripts/ci/assert-lanes.mjs new file mode 100644 index 00000000..0cbd98b9 --- /dev/null +++ b/scripts/ci/assert-lanes.mjs @@ -0,0 +1,52 @@ +#!/usr/bin/env node +// The body of the check-macos, check-windows and check-linux aggregators of +// .github/workflows/ci.yml (#358). Branch protection requires check-macos and +// check-windows by name; each job `needs:` the scope job and its OS's lanes and +// runs this script with `NEEDS: ${{ toJSON(needs) }}`. +// +// A lane that did not run is only acceptable when the scope job trusted its own +// classification and found nothing but documentation changed; in every other case +// every lane must have succeeded. Everything else fails closed: a missing or +// malformed input, an untrusted or unsuccessful scope, an unknown docs_only value, +// a lane that failed, was cancelled or was skipped on a full run, and an aggregator +// that waits for no lane at all. +import { pathToFileURL } from "node:url"; + +/** Returns the problems with a `needs` context; an empty list means the lanes are accepted. */ +export function laneProblems(needs) { + if (needs === null || typeof needs !== "object" || Array.isArray(needs)) return ["NEEDS must be the JSON object of the needs context"]; + const problems = []; + const scope = needs.scope; + if (scope?.result !== "success") problems.push(`scope: result is ${JSON.stringify(scope?.result)}, expected "success"`); + if (scope?.outputs?.scope_trusted !== "true") + problems.push(`scope: scope_trusted is ${JSON.stringify(scope?.outputs?.scope_trusted)}, expected "true"`); + const lanes = Object.entries(needs).filter(([id]) => id !== "scope"); + if (lanes.length === 0) problems.push("no lane job in needs: this aggregator would gate nothing"); + const docsOnly = scope?.outputs?.docs_only; + if (docsOnly !== "true" && docsOnly !== "false") { + problems.push(`scope: docs_only is ${JSON.stringify(docsOnly)}, expected "true" or "false"`); + } else { + const expected = docsOnly === "true" ? "skipped" : "success"; + for (const [id, lane] of lanes) { + if (lane?.result !== expected) + problems.push(`${id}: result is ${JSON.stringify(lane?.result)}, expected ${JSON.stringify(expected)} (docs_only=${docsOnly})`); + } + } + return problems; +} + +function main() { + let needs; + try { + needs = JSON.parse(process.env.NEEDS ?? ""); + } catch (error) { + console.error(`NEEDS is not valid JSON: ${error.message}`); + return 1; + } + const problems = laneProblems(needs); + for (const [id, entry] of Object.entries(needs ?? {})) console.log(`${id}: ${entry?.result}`); + for (const problem of problems) console.error(`::error title=CI lanes::${problem}`); + return problems.length === 0 ? 0 : 1; +} + +if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) process.exit(main()); diff --git a/scripts/ci/cargo-gates.mjs b/scripts/ci/cargo-gates.mjs new file mode 100644 index 00000000..5c2d9bfa --- /dev/null +++ b/scripts/ci/cargo-gates.mjs @@ -0,0 +1,257 @@ +#!/usr/bin/env node +// The one list of cargo gates of CI (#358). Every lane of .github/workflows/ci.yml +// runs `node scripts/ci/cargo-gates.mjs --os ` instead of +// writing the cargo commands out per OS, and the MSRV lane runs it with +// `--profile msrv`. scripts/test-ci-cargo-gates.mjs and +// scripts/test-verify-ci-parity.mjs read GATES from here, so the list cannot +// drift between the OS lanes, `npm run verify` (scripts/verify.mjs) and the tests. +// +// A gate runs on every OS unless it has an `os` list, and every `os` list says +// why in `why`: that is the ONE place the per-OS exclusions are documented. +// The rule: a gate whose subject is platform independent (source formatting, a +// Cargo feature subset, a dependency-graph assertion, a build-profile guard) +// runs once, on Linux, the cheapest runner. Everything that executes OS-specific +// code (the workspace tests and lints, the lean batch CLI, the ONNX host where +// it ships) runs on every OS it applies to. +// +// Why a plain `cargo clippy --workspace` / `cargo test --workspace` instead of a +// bare `cargo test` plus `-p sagascript-cli`: the workspace gate is a superset +// (app, CLI with its default features, protocol crate, test support) and, with +// resolver 2, builds core with exactly the features the app and CLI default to. +// The feature subsets below it (core alone, core with one feature, the lean CLI) +// stay separate invocations: combining package flags would unify their features. +// `cargo check` is not listed for the main profile because `cargo clippy +// --all-targets` compiles everything `cargo check` would, and more strictly. +// +// node scripts/ci/cargo-gates.mjs --os linux run, stop at the first failure +// node scripts/ci/cargo-gates.mjs --os linux --keep-going run all, fail at the end +// node scripts/ci/cargo-gates.mjs --os windows --list print the commands, run nothing +// node scripts/ci/cargo-gates.mjs --profile msrv needs RUSTUP_TOOLCHAIN (the lane sets it) +import { spawn } from "node:child_process"; +import { appendFileSync } from "node:fs"; +import { dirname, join } from "node:path"; +import { fileURLToPath, pathToFileURL } from "node:url"; + +export const REPO_ROOT = dirname(dirname(dirname(fileURLToPath(import.meta.url)))); +export const OSES = ["macos", "linux", "windows"]; +export const PROFILES = ["main", "msrv"]; +const OS_OF_PLATFORM = { darwin: "macos", linux: "linux", win32: "windows" }; + +const APP = "src-tauri"; +const ORT = "src-tauri/engine-host/ort"; +const LINUX = ["linux"]; +const gate = (id, cwd, args, extra = {}) => ({ id, cwd, args, profile: "main", ...extra }); + +// The release-profile guard exists only without debug assertions, and cargo +// test exits 0 when its name filter matches nothing: prove the test ran. +export const RELEASE_GUARD_PROOF = /release_build_ignores_the_real_environment \.\.\. ok/; + +/** + * Each gate: { id, cwd (repository relative), args (cargo arguments), profile, + * os? + why? (restriction and its reason), forbidOutput?/requireOutput? + * ({ pattern, message }), quiet? (show the output only on failure) }. + */ +export const GATES = [ + // Dependency policy (#355), before any compilation: RustSec advisories, licenses, banned or duplicated + // crates and allowed sources, as set in src-tauri/deny.toml, for both Cargo workspaces. The advisory + // database is fetched at run time. The lane installs cargo-deny first (ci.yml pins the version); the ORT + // host run allows only the policy entries the main workspace alone uses. + gate("deny", APP, ["deny", "--locked", "--workspace", "check"], { + os: LINUX, + why: "advisories, licenses and sources are properties of the lockfiles, not of the OS; deny.toml already judges every shipped target in one run", + }), + gate( + "deny-ort", + APP, + [ + ...["deny", "--locked", "--manifest-path", "engine-host/ort/Cargo.toml", "--workspace", "check"], + ...["-A", "license-not-encountered", "-A", "license-exception-not-encountered", "-A", "unmatched-source"], + ], + { os: LINUX, why: "same as deny, for the ORT host's own lockfile" }, + ), + // Formatting next, so an unformatted PR fails before anything compiles. + gate("fmt", APP, ["fmt", "--all", "--", "--check"], { + os: LINUX, + why: "rustfmt reads source text only; the result is identical on every OS (#352)", + }), + gate("fmt-ort", APP, ["fmt", "--manifest-path", "engine-host/ort/Cargo.toml", "--all", "--", "--check"], { + os: LINUX, + why: "rustfmt reads source text only; the standalone ORT host is its own Cargo workspace (#352)", + }), + + gate("clippy-workspace", APP, ["clippy", "--workspace", "--all-targets", "--", "-D", "warnings"]), + gate("test-workspace", APP, ["test", "--workspace"]), + + // Feature subsets. Separate invocations preserve the no-features builds. + gate("clippy-core", APP, ["clippy", "-p", "sagascript-core", "--all-targets", "--", "-D", "warnings"]), + gate("test-core", APP, ["test", "-p", "sagascript-core"]), + gate("clippy-cli-lean", APP, ["clippy", "-p", "sagascript-cli", "--all-targets", "--no-default-features", "--", "-D", "warnings"]), + gate("test-cli-lean", APP, ["test", "-p", "sagascript-cli", "--no-default-features"]), + gate( + "clippy-core-diarization", + APP, + ["clippy", "-p", "sagascript-core", "--features", "diarization", "--all-targets", "--", "-D", "warnings"], + { + os: LINUX, + why: "core's `diarization` and `record` features alone are a Cargo feature-graph property, not OS code; the workspace gates already build and test core with both on every OS", + }, + ), + gate("test-core-diarization", APP, ["test", "-p", "sagascript-core", "--features", "diarization"], { + os: LINUX, + why: "same as clippy-core-diarization: a Cargo feature subset, not OS code", + }), + gate("test-core-record", APP, ["test", "-p", "sagascript-core", "--features", "record"], { + os: LINUX, + why: "same as clippy-core-diarization (needs the ALSA headers the Linux lane installs)", + }), + + // The ONNX Runtime engine host is a separate Cargo workspace (#357). + gate("clippy-ort", ORT, ["clippy", "--all-targets", "--locked", "--", "-D", "warnings"], { + os: ["linux", "windows"], + why: "the ORT host ships for Windows and Linux; macOS ships the Core ML host (Swift tests, test-macos)", + }), + gate("test-ort", ORT, ["test", "--locked"], { + os: ["linux", "windows"], + why: "the ORT host ships for Windows and Linux; macOS ships the Core ML host (Swift tests, test-macos)", + }), + + // The debug lean CLI the offline diarization evaluation tests drive + // (SAGASCRIPT_REFERENCE_CLI=target/debug/sagascript). test-cli-lean built it already. + gate("build-cli-lean", APP, ["build", "-p", "sagascript-cli", "--no-default-features"], { + os: LINUX, + why: "only the Linux lane runs the Python diarization suite that drives this binary", + }), + // Regression guard for the no-ALSA property (#66): the lean CLI's dependency tree + // holds neither cpal nor alsa. + gate("no-alsa", APP, ["tree", "-p", "sagascript-cli", "--no-default-features", "-e", "normal"], { + os: LINUX, + why: "a dependency-graph assertion; cargo resolves the same graph for cpal on every OS, ALSA itself only on Linux", + quiet: true, + forbidOutput: { pattern: /alsa|cpal/i, message: "cpal/ALSA leaked into the batch-only dependency tree" }, + }), + // Release-profile guard (#357): a release build ignores SAGASCRIPT_ENGINE_HOST and + // SAGASCRIPT_PIANISSIMO_MODEL_DIR. Compiled only without debug_assertions and + // the `dev-overrides` feature, so every debug test run leaves it out. + gate("release-guard", APP, ["test", "--release", "-p", "sagascript-core", "--lib", "dev_overrides"], { + os: LINUX, + why: "a build-profile property of core, not OS code; one release compile is enough", + requireOutput: { pattern: RELEASE_GUARD_PROOF, message: "the release-profile guard test did not run" }, + }), + + // Minimum supported Rust version (#353): `rust-version` is a promise to people + // building from source. The lane exports RUSTUP_TOOLCHAIN=. + gate("msrv-workspace", APP, ["check", "--workspace", "--locked"], { profile: "msrv" }), + gate("msrv-cli-lean", APP, ["check", "-p", "sagascript-cli", "--no-default-features", "--locked"], { profile: "msrv" }), + gate("msrv-ort", ORT, ["check", "--locked"], { profile: "msrv" }), +]; + +export const commandOf = (item) => ["cargo", ...item.args].join(" "); + +/** The gates of one lane, in run order. */ +export function gatesFor({ os, profile = "main" }) { + if (!OSES.includes(os)) throw new Error(`unknown OS ${JSON.stringify(os)}; expected one of ${OSES.join(", ")}`); + if (!PROFILES.includes(profile)) throw new Error(`unknown profile ${JSON.stringify(profile)}; expected one of ${PROFILES.join(", ")}`); + return GATES.filter((item) => item.profile === profile && (item.os === undefined || item.os.includes(os))); +} + +/** + * Runs the gates through `execute(gate) -> { code, output }`, stopping at the + * first failure unless `keepGoing`. A gate also fails when the output it must + * not contain appears, or the output it must contain is absent. + */ +export async function runGates(gates, { execute, keepGoing = false, log = () => {} }) { + const results = []; + for (const item of gates) { + const started = Date.now(); + log(`::group::${commandOf(item)} (${item.cwd})`); + const { code, output } = await execute(item); + let failure = code === 0 ? null : `exited with status ${code}`; + if (failure === null && item.forbidOutput?.pattern.test(output)) failure = item.forbidOutput.message; + if (failure === null && item.requireOutput && !item.requireOutput.pattern.test(output)) failure = item.requireOutput.message; + if (failure !== null && item.quiet) log(output); + log("::endgroup::"); + if (failure !== null) log(`::error title=Cargo gate failed::${commandOf(item)} (${item.cwd}) ${failure}`); + results.push({ gate: item, failure, seconds: (Date.now() - started) / 1000 }); + if (failure !== null && !keepGoing) break; + } + return results; +} + +function executeWithCargo(item) { + return new Promise((resolve, reject) => { + const watched = item.forbidOutput !== undefined || item.requireOutput !== undefined; + const child = spawn("cargo", item.args, { cwd: join(REPO_ROOT, item.cwd), stdio: watched ? ["inherit", "pipe", "pipe"] : "inherit" }); + let output = ""; + for (const [stream, sink] of [ + [child.stdout, process.stdout], + [child.stderr, process.stderr], + ]) { + stream?.on("data", (chunk) => { + output += chunk; + if (!item.quiet) sink.write(chunk); + }); + } + child.on("error", reject); + child.on("close", (code, signal) => resolve({ code: code ?? (signal ? 128 : 1), output })); + }); +} + +function parseArguments(argv) { + const options = { os: OS_OF_PLATFORM[process.platform], profile: "main", list: false, keepGoing: false, explicitOs: false }; + for (let index = 0; index < argv.length; index += 1) { + const argument = argv[index]; + if (argument === "--os" || argument === "--profile") { + const value = argv[(index += 1)]; + if (value === undefined) throw new Error(`${argument} needs a value`); + if (argument === "--os") Object.assign(options, { os: value, explicitOs: true }); + else options.profile = value; + } else if (argument === "--list") options.list = true; + else if (argument === "--keep-going") options.keepGoing = true; + else throw new Error(`unknown argument ${argument}`); + } + return options; +} + +async function main(argv) { + const options = parseArguments(argv); + const gates = gatesFor(options); + if (options.list) { + for (const item of gates) console.log(`${item.cwd}: ${commandOf(item)}`); + return 0; + } + // A lane that names another OS would silently run the wrong list. + if (options.explicitOs && OS_OF_PLATFORM[process.platform] !== options.os) { + throw new Error(`refusing to run the ${options.os} gates on a ${OS_OF_PLATFORM[process.platform] ?? process.platform} runner`); + } + if (options.profile === "msrv" && !process.env.RUSTUP_TOOLCHAIN) { + throw new Error( + "the msrv profile needs RUSTUP_TOOLCHAIN set to the declared rust-version; without it cargo would use the development toolchain", + ); + } + const results = await runGates(gates, { execute: executeWithCargo, keepGoing: options.keepGoing, log: console.log }); + const skipped = gates.slice(results.length); + const lines = [ + `Cargo gates (${options.os}, ${options.profile}): ${results.length - results.filter((entry) => entry.failure).length} passed, ` + + `${results.filter((entry) => entry.failure).length} failed, ${skipped.length} not run`, + "", + "| Gate | Result | Seconds |", + "|---|---|---|", + ...results.map( + (entry) => `| \`${commandOf(entry.gate)}\` (${entry.gate.cwd}) | ${entry.failure ?? "passed"} | ${entry.seconds.toFixed(0)} |`, + ), + ...skipped.map((item) => `| \`${commandOf(item)}\` (${item.cwd}) | not run (an earlier gate failed) | |`), + ]; + console.log(lines.join("\n")); + if (process.env.GITHUB_STEP_SUMMARY) appendFileSync(process.env.GITHUB_STEP_SUMMARY, `${lines.join("\n")}\n`); + return results.some((entry) => entry.failure) ? 1 : 0; +} + +if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) { + main(process.argv.slice(2)).then( + (code) => process.exit(code), + (error) => { + console.error(error.message); + process.exit(2); + }, + ); +} diff --git a/scripts/ci/smoke-cli.mjs b/scripts/ci/smoke-cli.mjs new file mode 100644 index 00000000..1eef9105 --- /dev/null +++ b/scripts/ci/smoke-cli.mjs @@ -0,0 +1,244 @@ +#!/usr/bin/env node +// CLI smoke tests of the native release binary, one script for every OS (#358). +// ci.yml's build-macos and build-windows lanes run it after `tauri build --no-bundle` +// instead of carrying one inline bash and one inline PowerShell copy per check. +// +// node scripts/ci/smoke-cli.mjs basic tier 1, no model: the binary exists, help, config, +// list-models, formats, shell completions +// node scripts/ci/smoke-cli.mjs model tier 2: download nb-whisper-tiny, transcribe, and the +// output contract of #364 with a real model: --json, +// --jsonl, a reader that stops after the first item and +// the error object of a failed --json run +// --binary default src-tauri/target/release/sagascript[.exe] +// +// Every check of a tier runs even after a failure, and the script exits 1 when any failed, +// so one run names all of them. The model tier is the one the Windows lane tolerates +// (continue-on-error, #357); that decision lives in ci.yml, not here. +import { spawn, spawnSync } from "node:child_process"; +import { accessSync, constants, copyFileSync, existsSync, mkdirSync, mkdtempSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { dirname, join, resolve } from "node:path"; +import { fileURLToPath, pathToFileURL } from "node:url"; + +export const REPO_ROOT = dirname(dirname(dirname(fileURLToPath(import.meta.url)))); +const WINDOWS = process.platform === "win32"; +export const DEFAULT_BINARY = join("src-tauri", "target", "release", WINDOWS ? "sagascript.exe" : "sagascript"); +const MODEL = "nb-whisper-tiny"; +const SHORT_AUDIO = "test-audio/norwegian-short-3s.mp3"; +const MEDIUM_AUDIO = "test-audio/norwegian-medium-8s.mp3"; +const TRANSCRIBE = ["transcribe", "--language", "no", "--model", MODEL]; + +// Completions for every shell the binary supports on every OS, not one per OS. +export const COMPLETION_SHELLS = ["zsh", "bash", "powershell"]; + +const text = (value) => (typeof value === "string" ? value : ""); + +/** + * Each check: { name, run(harness) }. The harness is what a check may do: run the + * binary, write scratch files, run the stream verifier, and assert. + */ +export const TIERS = { + basic: [ + { + name: "the native release binary exists and is executable", + run(h) { + h.checkBinary(); + }, + }, + { + name: "help and version", + run(h) { + for (const args of [["--version"], ["--help"], ["transcribe", "--help"], ["record", "--help"]]) h.ok(args); + }, + }, + { + name: "config management", + run(h) { + h.ok(["config", "list"]); + h.ok(["config", "get", "language"]); + h.ok(["config", "set", "language", "sv"]); + h.assert(h.ok(["config", "get", "language"]).includes("sv"), "config get language should print sv after set"); + h.ok(["config", "reset", "language"]); + h.assert(h.ok(["config", "get", "language"]).includes("en"), "config get language should print en after reset"); + h.ok(["config", "path"]); + }, + }, + { + name: "model listing and formats", + run(h) { + h.ok(["list-models"]); + h.ok(["formats"]); + }, + }, + { + name: "shell completions", + run(h) { + for (const shell of COMPLETION_SHELLS) { + h.assert(h.ok(["completions", shell], { quiet: true }).trim() !== "", `completions ${shell} printed nothing`); + } + }, + }, + ], + model: [ + { + name: "download model and transcribe", + run(h) { + h.ok(["download-model", MODEL]); + const result = h.ok([...TRANSCRIBE, SHORT_AUDIO]); + console.log(`Transcription: ${result.trim()}`); + h.assert(/stortinget/i.test(result), "expected 'stortinget' in the transcription"); + }, + }, + { + name: "transcribe with JSON output", + run(h) { + const { stdout, stderr } = h.capture("transcription-smoke", [...TRANSCRIBE, "--json", SHORT_AUDIO]); + h.verifyStreams([], stdout, stderr); + }, + }, + { + name: "JSONL output", + run(h) { + const { stdout, stderr } = h.capture("transcription-smoke-jsonl", [...TRANSCRIBE, "--jsonl", h.audioDirectory()]); + h.verifyStreams(["--jsonl"], stdout, stderr); + }, + }, + { + // A smoke check with the real model: a panic or a failing exit on a reader that stops after the + // first item fails the check. The deterministic check of the closed-pipe contract is + // `run_into_closed_pipe` in src-tauri/crates/sagascript-cli/tests/output_contract.rs. + name: "closed pipe after the first JSONL item", + async run(h) { + const { code, sawLine } = await h.readFirstLineThenClose([...TRANSCRIBE, "--jsonl", h.audioDirectory()]); + h.assert(sawLine, "no JSONL item before the output ended"); + h.assert(code === 0, `exited with status ${code} after the reader closed the pipe`); + }, + }, + { + name: "JSON error object of a failed run", + run(h) { + const { code, stdout, stderr } = h.capture("transcription-error", [...TRANSCRIBE, "--json", join(h.scratch, "missing.wav")]); + h.assert(code === 1, `a failed --json run must exit 1, got ${code}`); + h.verifyStreams(["--error"], stdout, stderr); + }, + }, + ], +}; + +function createHarness({ binary, scratch }) { + const harness = { + binary, + scratch, + assert(condition, message) { + if (!condition) throw new Error(message); + }, + checkBinary() { + harness.assert(existsSync(binary), `missing native release binary: ${binary}`); + if (!WINDOWS) { + accessSync(binary, constants.X_OK); + harness.runProgram("file", [binary]); + } + }, + // `quiet` keeps a long stdout (a completion script) out of the log; stderr always shows. + runProgram(program, args, { quiet = false } = {}) { + console.log(`+ ${program} ${args.join(" ")}`); + const result = spawnSync(program, args, { cwd: REPO_ROOT, encoding: "utf8" }); + if (result.error) throw result.error; + if (!quiet) process.stdout.write(text(result.stdout)); + process.stderr.write(text(result.stderr)); + return { code: result.status, stdout: text(result.stdout), stderr: text(result.stderr) }; + }, + // Runs the binary; the exit status must be 0. Returns stdout. + ok(args, options) { + const result = harness.runProgram(binary, args, options); + harness.assert(result.code === 0, `${args.join(" ")} exited with status ${result.code}`); + return result.stdout; + }, + // Runs the binary with stdout and stderr in files (as a verifier needs them). Any exit status is returned. + capture(label, args) { + const result = harness.runProgram(binary, args); + const stdout = join(scratch, `${label}.stdout`); + const stderr = join(scratch, `${label}.stderr`); + writeFileSync(stdout, result.stdout); + writeFileSync(stderr, result.stderr); + return { code: result.code, stdout, stderr }; + }, + verifyStreams(mode, stdout, stderr) { + const python = WINDOWS ? "python" : "python3"; + const result = harness.runProgram(python, ["scripts/verify-json-cli-streams.py", ...mode, stdout, stderr]); + harness.assert(result.code === 0, `verify-json-cli-streams.py ${mode.join(" ")} exited with status ${result.code}`); + }, + // Two audio files in a directory of their own: `transcribe` takes a directory. + audioDirectory() { + const inputs = join(scratch, "jsonl-inputs"); + if (!existsSync(inputs)) { + mkdirSync(inputs, { recursive: true }); + copyFileSync(join(REPO_ROOT, SHORT_AUDIO), join(inputs, "a.mp3")); + copyFileSync(join(REPO_ROOT, MEDIUM_AUDIO), join(inputs, "b.mp3")); + } + return inputs; + }, + // `binary args | head -1` under pipefail: read one line, close the pipe, expect a clean exit. + readFirstLineThenClose(args) { + console.log(`+ ${binary} ${args.join(" ")} | head -1`); + return new Promise((resolve, reject) => { + const child = spawn(binary, args, { cwd: REPO_ROOT, stdio: ["ignore", "pipe", "inherit"] }); + let sawLine = false; + child.stdout.on("data", (chunk) => { + if (!sawLine && chunk.includes(10)) { + sawLine = true; + child.stdout.destroy(); + } + }); + child.on("error", reject); + child.on("close", (code) => resolve({ code, sawLine })); + }); + }, + }; + return harness; +} + +/** Runs every check of a tier, reporting each; resolves to the names of the failed ones. */ +export async function runTier(tier, harness, { log = console.log } = {}) { + const failed = []; + for (const check of TIERS[tier]) { + log(`::group::Smoke test: ${check.name}`); + try { + await check.run(harness); + log("::endgroup::"); + log(`ok: ${check.name}`); + } catch (error) { + log("::endgroup::"); + log(`::error title=Smoke test failed::${check.name}: ${error.message}`); + failed.push(check.name); + } + } + return failed; +} + +async function main(argv) { + const tier = argv[0]; + if (!Object.hasOwn(TIERS, tier)) throw new Error(`usage: smoke-cli.mjs <${Object.keys(TIERS).join("|")}> [--binary ]`); + const binaryIndex = argv.indexOf("--binary"); + const binary = resolve(REPO_ROOT, binaryIndex >= 0 ? argv[binaryIndex + 1] : DEFAULT_BINARY); + const scratch = mkdtempSync(join(process.env.RUNNER_TEMP || tmpdir(), "sagascript-smoke-")); + const failed = await runTier(tier, createHarness({ binary, scratch })); + if (failed.length > 0) { + console.error(`${failed.length} smoke check(s) failed in tier ${tier}: ${failed.join("; ")}`); + return 1; + } + return 0; +} + +if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) { + main(process.argv.slice(2)).then( + (code) => process.exit(code), + (error) => { + console.error(error.message); + process.exit(2); + }, + ); +} + +// Re-exported for the tests that run the script against a fake binary. +export { createHarness }; diff --git a/scripts/dictation_eval/test_evaluate_cli.py b/scripts/dictation_eval/test_evaluate_cli.py index 874c419a..1f9c0d38 100644 --- a/scripts/dictation_eval/test_evaluate_cli.py +++ b/scripts/dictation_eval/test_evaluate_cli.py @@ -2,7 +2,6 @@ import json from pathlib import Path -import re import subprocess import sys import tempfile @@ -15,20 +14,6 @@ class EvaluateCliTests(unittest.TestCase): - def test_ci_selects_supported_python_before_all_evaluator_suites(self): - workflow = SCRIPT.parents[2] / ".github" / "workflows" / "ci.yml" - source = workflow.read_text(encoding="utf-8").replace("\r\n", "\n") - for name in ("test-macos", "test-linux", "test-windows"): - marker = f" {name}:\n" - start = source.index(marker) + len(marker) - next_job = re.search(r"^ [A-Za-z0-9_-]+:\s*$", source[start:], re.MULTILINE) - end = start + next_job.start() if next_job else len(source) - job = source[start:end] - setup = job.index("actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1") - suite = job.index("Test offline dictation evaluation tooling") - self.assertLess(setup, suite) - self.assertIn('python-version: "3.12"', job[setup:suite]) - def run_cli(self, *args): return subprocess.run( [sys.executable, str(SCRIPT), *map(str, args)], diff --git a/scripts/fixtures/import-graph/cycle-b.mjs b/scripts/fixtures/import-graph/cycle-b.mjs new file mode 100644 index 00000000..733af410 --- /dev/null +++ b/scripts/fixtures/import-graph/cycle-b.mjs @@ -0,0 +1 @@ +import "./cycle-c.mjs"; diff --git a/scripts/fixtures/import-graph/cycle-c.mjs b/scripts/fixtures/import-graph/cycle-c.mjs new file mode 100644 index 00000000..77f788a7 --- /dev/null +++ b/scripts/fixtures/import-graph/cycle-c.mjs @@ -0,0 +1 @@ +import "./cycle-b.mjs"; diff --git a/scripts/fixtures/import-graph/entry.mjs b/scripts/fixtures/import-graph/entry.mjs new file mode 100644 index 00000000..8360c0c0 --- /dev/null +++ b/scripts/fixtures/import-graph/entry.mjs @@ -0,0 +1,3 @@ +// Inputs of the import-graph test in scripts/test-frontend-test-groups.mjs. Never executed. +import { one } from "./lib/one.mjs"; +import "node:fs"; diff --git a/scripts/fixtures/import-graph/lib/one.mjs b/scripts/fixtures/import-graph/lib/one.mjs new file mode 100644 index 00000000..1ed52a9e --- /dev/null +++ b/scripts/fixtures/import-graph/lib/one.mjs @@ -0,0 +1 @@ +export { two as one } from "./two.mjs"; diff --git a/scripts/fixtures/import-graph/lib/three.mjs b/scripts/fixtures/import-graph/lib/three.mjs new file mode 100644 index 00000000..3acfa503 --- /dev/null +++ b/scripts/fixtures/import-graph/lib/three.mjs @@ -0,0 +1,5 @@ +// A comment that says import "commented-out" is not an import. +/* import "also-commented" */ +import "yaml"; +import "@scope/pkg/sub"; +import "./one.mjs"; diff --git a/scripts/fixtures/import-graph/lib/two.mjs b/scripts/fixtures/import-graph/lib/two.mjs new file mode 100644 index 00000000..d85a29bf --- /dev/null +++ b/scripts/fixtures/import-graph/lib/two.mjs @@ -0,0 +1 @@ +export const two = await import("./three.mjs"); diff --git a/scripts/lib/ci-workflow.mjs b/scripts/lib/ci-workflow.mjs new file mode 100644 index 00000000..071ac37f --- /dev/null +++ b/scripts/lib/ci-workflow.mjs @@ -0,0 +1,127 @@ +// A structural reader for the workflow files under test (#358): the YAML is parsed, never matched +// with regular expressions, so a test asserts what a job `needs`, which step runs what and with +// which inputs, and keeps passing when the file is reformatted, re-commented or reordered +// within what the assertion states. +// +// The parser is the `yaml` package, a devDependency. The ci and release test groups list it in +// GROUP_NPM_PACKAGES (scripts/node-test-groups.mjs), so a checkout without `npm ci` fails once +// with a clear message instead of once per test file. +// +// Not for tests that run before `npm ci` (the Windows build identity gates in ci.yml's +// test-windows): they must read text with node:fs only. +import { readFileSync } from "node:fs"; +import { fileURLToPath } from "node:url"; +import { parse } from "yaml"; + +export const REPO_ROOT = fileURLToPath(new URL("../../", import.meta.url)); +export const readRepoFile = (path) => readFileSync(`${REPO_ROOT}${path}`, "utf8").replace(/\r\n?/g, "\n"); + +// A scalar `run` is a script; CRLF checkouts (core.autocrlf on Windows) must not leak into it. +const script = (value) => (typeof value === "string" ? value.replace(/\r\n?/g, "\n").trim() : ""); +const list = (value) => (value === undefined ? [] : Array.isArray(value) ? value : [value]); + +function normalizeStep(step, index) { + return { + index, + name: step.name, + id: step.id, + uses: step.uses, + run: script(step.run), + with: step.with ?? {}, + env: step.env ?? {}, + workingDirectory: step["working-directory"], + shell: step.shell, + condition: step.if === undefined ? undefined : String(step.if), + continueOnError: step["continue-on-error"], + }; +} + +function normalizeJob(id, job) { + return { + id, + needs: list(job.needs), + condition: job.if === undefined ? undefined : String(job.if), + runsOn: job["runs-on"], + timeoutMinutes: job["timeout-minutes"], + env: job.env ?? {}, + outputs: job.outputs ?? {}, + continueOnError: job["continue-on-error"], + steps: (job.steps ?? []).map(normalizeStep), + }; +} + +/** Parses workflow text into `{ doc, jobs, job(id) }`. */ +export function parseWorkflow(text) { + const doc = parse(text); + const jobs = Object.entries(doc.jobs ?? {}).map(([id, job]) => normalizeJob(id, job)); + return { + doc, + jobs, + jobIds: jobs.map((job) => job.id), + job(id) { + const found = jobs.find((job) => job.id === id); + if (!found) throw new Error(`workflow must define the job ${id}`); + return found; + }, + }; +} + +export const loadWorkflow = (file = "ci.yml") => parseWorkflow(readRepoFile(`.github/workflows/${file}`)); + +/** A local composite action: `{ inputs, steps }`. */ +export function loadAction(name) { + const doc = parse(readRepoFile(`.github/actions/${name}/action.yml`)); + return { doc, inputs: doc.inputs ?? {}, steps: (doc.runs?.steps ?? []).map(normalizeStep) }; +} + +/** The input a composite action step is called with, falling back to the action's default. */ +export function actionInput(action, step, name) { + return String(step.with[name] ?? action.inputs[name]?.default ?? ""); +} + +// Evaluates the only condition form the local actions use: `${{ inputs. == '' }}`. +function conditionHolds(condition, valueOf) { + if (condition === undefined) return true; + const match = /^\$\{\{\s*inputs\.([\w-]+) == '([^']*)'\s*\}\}$/.exec(condition.trim()); + if (!match) throw new Error(`unsupported condition in a local action: ${condition}`); + return valueOf(match[1]) === match[2]; +} + +/** + * The steps of a job with every `uses: ./.github/actions/` step replaced by the `run` steps of + * that action which its inputs enable (marked `viaAction`). Steps that `uses:` another action stay. + */ +export function expandedSteps(job) { + return job.steps.flatMap((step) => { + const local = /^\.\/\.github\/actions\/([\w-]+)$/.exec(step.uses ?? ""); + if (!local) return [step]; + const action = loadAction(local[1]); + return action.steps + .filter((inner) => inner.run !== "" && conditionHolds(inner.condition, (name) => actionInput(action, step, name))) + .map((inner) => ({ ...inner, viaAction: local[1] })); + }); +} + +/** The logical command lines of a step's script: continuations joined, blank and comment lines dropped. */ +export function commandLines(step) { + return step.run + .replace(/\\\n\s*/g, " ") + .split("\n") + .map((line) => line.trim()) + .filter((line) => line !== "" && !line.startsWith("#")); +} + +/** The `--os` and `--profile` a step passes to scripts/ci/cargo-gates.mjs, or null when it does not run it. */ +export function gateInvocation(step) { + const line = commandLines(step).find((candidate) => /^node scripts\/ci\/cargo-gates\.mjs(\s|$)/.test(candidate)); + if (line === undefined) return null; + const words = line.split(/\s+/); + const argument = (flag) => { + const index = words.indexOf(flag); + return index >= 0 ? words[index + 1] : undefined; + }; + return { os: argument("--os"), profile: argument("--profile") ?? "main", line }; +} + +/** Every step of every job, as `{ job, step }`. */ +export const allSteps = (workflow) => workflow.jobs.flatMap((job) => job.steps.map((step) => ({ job, step }))); diff --git a/scripts/lib/import-graph.mjs b/scripts/lib/import-graph.mjs new file mode 100644 index 00000000..d221332a --- /dev/null +++ b/scripts/lib/import-graph.mjs @@ -0,0 +1,38 @@ +// The npm packages a module needs, found by following its relative imports (#358). A test that runs +// before `npm ci` must not reach one even through a repository module it imports, and a test group's +// GROUP_NPM_PACKAGES (scripts/node-test-groups.mjs) must name every package its files reach. +// +// A static scan, not a resolver: it reads `from "x"`, `import "x"` and `import("x")` specifiers, after +// dropping block comments and full-line comments (a comment that mentions `import "name"` is no import). +import { readFileSync } from "node:fs"; +import { join, posix } from "node:path"; + +const BARE_IMPORT = /\bfrom\s*["']([@a-z][\w@./:-]*)["']|\bimport\s*\(?\s*["']([@a-z][\w@./:-]*)["']/g; +const RELATIVE_IMPORT = /\b(?:from|import)\s*\(?\s*["'](\.{1,2}\/[^"']+\.m?js)["']/g; + +/** The package a bare specifier names: `@scope/name` or `name`; null for a Node builtin (`node:fs`). */ +export function packageOf(specifier) { + if (specifier.startsWith("node:")) return null; + return specifier.startsWith("@") ? specifier.split("/").slice(0, 2).join("/") : specifier.split("/")[0]; +} + +/** + * `{ packages, files }` reached from `file` (repository relative, forward slashes) under `root`: + * the bare-import packages of every file on the way, and the files themselves, `file` included. + */ +export function importGraph(file, root) { + const packages = new Set(); + const files = new Set(); + const visit = (current) => { + if (files.has(current)) return; + files.add(current); + const source = readFileSync(join(root, current), "utf8").replace(/\/\*[\s\S]*?\*\/|^\s*\/\/.*$/gm, ""); + for (const match of source.matchAll(BARE_IMPORT)) { + const name = packageOf(match[1] ?? match[2]); + if (name !== null) packages.add(name); + } + for (const match of source.matchAll(RELATIVE_IMPORT)) visit(posix.normalize(posix.join(posix.dirname(current), match[1]))); + }; + visit(file); + return { packages, files }; +} diff --git a/scripts/node-test-groups.mjs b/scripts/node-test-groups.mjs index 19062fdf..3098b56a 100644 --- a/scripts/node-test-groups.mjs +++ b/scripts/node-test-groups.mjs @@ -26,7 +26,9 @@ export const GROUP_DESCRIPTIONS = { // failure per file. Groups not listed need nothing beyond Node itself. export const GROUP_NPM_PACKAGES = { ui: ["typescript"], - ci: ["vite"], + // scripts/lib/ci-workflow.mjs parses the workflows with `yaml` (#358). + ci: ["vite", "yaml"], + release: ["yaml"], }; export const TEST_GROUPS = { @@ -46,12 +48,14 @@ export const TEST_GROUPS = { "scripts/diarization_sv/test_reference_review.mjs", "scripts/test-browser-mock-commands.mjs", "scripts/test-ci-build-identity.mjs", + "scripts/test-ci-cargo-gates.mjs", "scripts/test-ci-cli-feature-coverage.mjs", "scripts/test-ci-cohort-timings.mjs", "scripts/test-ci-docs-scope.mjs", "scripts/test-ci-docs-workflow.mjs", "scripts/test-ci-parallel-gates.mjs", "scripts/test-ci-run-timings.mjs", + "scripts/test-ci-smoke-cli.mjs", "scripts/test-ci-unrun-suites.mjs", "scripts/test-clippy-policy.mjs", "scripts/test-dependency-policy.mjs", diff --git a/scripts/test-ci-cargo-gates.mjs b/scripts/test-ci-cargo-gates.mjs new file mode 100644 index 00000000..19f65d41 --- /dev/null +++ b/scripts/test-ci-cargo-gates.mjs @@ -0,0 +1,229 @@ +import assert from "node:assert/strict"; +import { spawnSync } from "node:child_process"; +import { existsSync } from "node:fs"; +import test from "node:test"; +import { join } from "node:path"; +import { fileURLToPath } from "node:url"; +import { allSteps, commandLines, gateInvocation, loadWorkflow } from "./lib/ci-workflow.mjs"; +import { GATES, OSES, PROFILES, RELEASE_GUARD_PROOF, REPO_ROOT, commandOf, gatesFor, runGates } from "./ci/cargo-gates.mjs"; + +// The list of cargo gates (scripts/ci/cargo-gates.mjs) is the one source of the cargo commands every +// lane of ci.yml runs (#358). These tests hold the list, its wiring into the lanes and its runner. + +const SCRIPT = fileURLToPath(new URL("./ci/cargo-gates.mjs", import.meta.url)); +const lines = (os, profile) => gatesFor({ os, profile }).map((gate) => `${gate.cwd}: ${commandOf(gate)}`); + +const COMMON = [ + "src-tauri: cargo clippy --workspace --all-targets -- -D warnings", + "src-tauri: cargo test --workspace", + "src-tauri: cargo clippy -p sagascript-core --all-targets -- -D warnings", + "src-tauri: cargo test -p sagascript-core", + "src-tauri: cargo clippy -p sagascript-cli --all-targets --no-default-features -- -D warnings", + "src-tauri: cargo test -p sagascript-cli --no-default-features", +]; +const ORT = [ + "src-tauri/engine-host/ort: cargo clippy --all-targets --locked -- -D warnings", + "src-tauri/engine-host/ort: cargo test --locked", +]; +const DENY = [ + "src-tauri: cargo deny --locked --workspace check", + "src-tauri: cargo deny --locked --manifest-path engine-host/ort/Cargo.toml --workspace check -A license-not-encountered -A license-exception-not-encountered -A unmatched-source", +]; + +// The commands each lane ran before the list existed, plus the ones the list added on purpose, +// spelled out so that dropping or adding a gate has to change this fixture too. +const EXPECTED = { + macos: COMMON, + windows: [...COMMON, ...ORT], + linux: [ + ...DENY, + "src-tauri: cargo fmt --all -- --check", + "src-tauri: cargo fmt --manifest-path engine-host/ort/Cargo.toml --all -- --check", + ...COMMON, + "src-tauri: cargo clippy -p sagascript-core --features diarization --all-targets -- -D warnings", + "src-tauri: cargo test -p sagascript-core --features diarization", + "src-tauri: cargo test -p sagascript-core --features record", + ...ORT, + "src-tauri: cargo build -p sagascript-cli --no-default-features", + "src-tauri: cargo tree -p sagascript-cli --no-default-features -e normal", + "src-tauri: cargo test --release -p sagascript-core --lib dev_overrides", + ], + msrv: [ + "src-tauri: cargo check --workspace --locked", + "src-tauri: cargo check -p sagascript-cli --no-default-features --locked", + "src-tauri/engine-host/ort: cargo check --locked", + ], +}; + +test("every gate is well formed, and every per-OS exclusion states its reason", () => { + assert.equal(new Set(GATES.map((gate) => gate.id)).size, GATES.length, "gate ids are unique"); + for (const gate of GATES) { + assert.ok(PROFILES.includes(gate.profile), `${gate.id}: profile`); + assert.ok(existsSync(join(REPO_ROOT, gate.cwd)), `${gate.id}: ${gate.cwd} exists`); + assert.ok(gate.args.length > 0 && gate.args.every((argument) => typeof argument === "string" && argument !== ""), `${gate.id}: args`); + if (gate.os === undefined) { + assert.equal(gate.why, undefined, `${gate.id} runs everywhere and needs no exclusion reason`); + } else { + assert.ok(gate.os.length > 0 && gate.os.every((os) => OSES.includes(os)), `${gate.id}: os`); + assert.ok(gate.os.length < OSES.length, `${gate.id} lists every OS; drop the restriction`); + assert.ok(typeof gate.why === "string" && gate.why.length >= 20, `${gate.id} excludes an OS and must say why`); + } + } +}); + +test("each lane runs exactly the expected cargo commands, in order", () => { + for (const os of OSES) assert.deepEqual(lines(os), EXPECTED[os], `${os} gates`); + assert.deepEqual(lines("linux", "msrv"), EXPECTED.msrv, "msrv gates"); +}); + +test("the core gates run on every OS; every other difference between the OS lists is an exclusion in the list", () => { + const everywhere = OSES.map((os) => new Set(lines(os))); + for (const command of COMMON) + assert.ok( + everywhere.every((set) => set.has(command)), + `${command} must run on every OS`, + ); + for (const gate of GATES.filter((entry) => entry.profile === "main")) { + const runsOn = OSES.filter((os) => lines(os).includes(`${gate.cwd}: ${commandOf(gate)}`)); + assert.deepEqual( + runsOn, + OSES.filter((os) => gate.os === undefined || gate.os.includes(os)), + `${gate.id} runs where it says`, + ); + } +}); + +test("the release guard proves its test ran and the dependency-tree gate forbids cpal and ALSA", () => { + const guard = GATES.find((gate) => gate.id === "release-guard"); + assert.ok(guard.requireOutput.pattern.test("test dev_overrides::tests::release_build_ignores_the_real_environment ... ok")); + assert.ok(!guard.requireOutput.pattern.test("running 0 tests")); + assert.equal(guard.requireOutput.pattern, RELEASE_GUARD_PROOF); + const tree = GATES.find((gate) => gate.id === "no-alsa"); + assert.ok(tree.forbidOutput.pattern.test("├── cpal v0.15.3") && tree.forbidOutput.pattern.test("alsa-sys")); + assert.ok(!tree.forbidOutput.pattern.test("sagascript-core v1.4.4")); +}); + +test("cargo-deny gates run first, before any gate that compiles", () => { + const linux = gatesFor({ os: "linux" }); + assert.deepEqual( + linux.slice(0, 2).map((gate) => gate.args[0]), + ["deny", "deny"], + ); + assert.ok(linux.slice(2).every((gate) => gate.args[0] !== "deny")); +}); + +test("every OS lane runs the gate script once with its own OS, and the MSRV lane its profile", () => { + const workflow = loadWorkflow(); + const osOf = (runsOn) => (/^macos/.test(runsOn) ? "macos" : /^windows/.test(runsOn) ? "windows" : "linux"); + const lanes = { "test-macos": "macos", "test-linux": "linux", "test-windows": "windows" }; + for (const [jobId, os] of Object.entries(lanes)) { + const job = workflow.job(jobId); + assert.equal(osOf(job.runsOn), os, `${jobId} runs on ${os}`); + const invocations = job.steps.map(gateInvocation).filter(Boolean); + assert.equal(invocations.length, 1, `${jobId} must run the gate script exactly once`); + assert.deepEqual([invocations[0].os, invocations[0].profile], [os, "main"]); + const step = job.steps.find((candidate) => gateInvocation(candidate)); + assert.equal(step.condition, undefined, `${jobId}: the gates are unconditional`); + assert.equal(step.continueOnError, undefined, `${jobId}: the gates are blocking`); + assert.equal(step.workingDirectory, undefined, "the script resolves each gate's directory itself"); + } + const msrv = workflow.job("msrv-linux"); + const [invocation] = msrv.steps.map(gateInvocation).filter(Boolean); + assert.deepEqual([invocation.os, invocation.profile], [undefined, "msrv"]); + for (const { job } of allSteps(workflow).filter(({ step }) => gateInvocation(step))) { + assert.ok([...Object.keys(lanes), "msrv-linux"].includes(job.id), `${job.id} must not run the cargo gates`); + } +}); + +test("ci.yml writes no cargo gate command of its own: the only cargo it runs is the cargo-deny install", () => { + const workflow = loadWorkflow(); + const cargo = allSteps(workflow).flatMap(({ job, step }) => + commandLines(step) + .filter((line) => /^cargo\s/.test(line)) + .map((line) => `${job.id}: ${line}`), + ); + assert.deepEqual(cargo, ["test-linux: cargo install --locked cargo-deny --version 0.20.2"]); +}); + +test("windows-package.yml keeps the workspace commands the gates run", () => { + const workflow = loadWorkflow("windows-package.yml"); + const step = workflow.jobs[0].steps.find((candidate) => candidate.name === "Test and lint Rust workspace"); + const commands = commandLines(step); + for (const id of ["test-workspace", "clippy-workspace"]) { + const gate = GATES.find((candidate) => candidate.id === id); + assert.ok(commands.includes(commandOf(gate)), `${commandOf(gate)} must stay in step with the gate ${id}`); + } +}); + +// -- the runner ---------------------------------------------------------------------------------- + +const fakeGate = (id, extra = {}) => ({ id, cwd: "src-tauri", args: ["test", id], profile: "main", ...extra }); +const executeWith = (outcomes) => async (gate) => outcomes[gate.id] ?? { code: 0, output: "" }; + +test("the runner stops at the first failing gate unless it keeps going", async () => { + const gates = [fakeGate("a"), fakeGate("b"), fakeGate("c")]; + const execute = executeWith({ b: { code: 101, output: "" } }); + const stopped = await runGates(gates, { execute }); + assert.deepEqual( + stopped.map((entry) => [entry.gate.id, entry.failure]), + [ + ["a", null], + ["b", "exited with status 101"], + ], + ); + const all = await runGates(gates, { execute, keepGoing: true }); + assert.deepEqual( + all.map((entry) => entry.gate.id), + ["a", "b", "c"], + ); + assert.equal(all.filter((entry) => entry.failure).length, 1); +}); + +test("a gate also fails on output it must not contain, or lacks the output it must contain", async () => { + const forbid = fakeGate("tree", { forbidOutput: { pattern: /alsa/, message: "alsa leaked" }, quiet: true }); + const require = fakeGate("guard", { requireOutput: { pattern: /ran ok/, message: "guard did not run" } }); + const logged = []; + const log = (line) => logged.push(line); + const [leak] = await runGates([forbid], { execute: executeWith({ tree: { code: 0, output: "alsa-sys" } }), log }); + assert.equal(leak.failure, "alsa leaked"); + assert.ok(logged.includes("alsa-sys"), "a quiet gate shows its output when it fails"); + logged.length = 0; + const [clean] = await runGates([forbid], { execute: executeWith({ tree: { code: 0, output: "cpal-free" } }), log }); + assert.equal(clean.failure, null); + assert.ok(!logged.includes("cpal-free"), "a quiet gate stays quiet when it passes"); + const [missing] = await runGates([require], { execute: executeWith({ guard: { code: 0, output: "running 0 tests" } }) }); + assert.equal(missing.failure, "guard did not run"); + const [proved] = await runGates([require], { execute: executeWith({ guard: { code: 0, output: "it ran ok" } }) }); + assert.equal(proved.failure, null); + const [failed] = await runGates([require], { execute: executeWith({ guard: { code: 1, output: "ran ok" } }) }); + assert.equal(failed.failure, "exited with status 1"); +}); + +// -- the command line ---------------------------------------------------------------------------- + +function cli(args, env = {}) { + const inherited = { ...process.env }; + delete inherited.RUSTUP_TOOLCHAIN; + return spawnSync(process.execPath, [SCRIPT, ...args], { encoding: "utf8", env: { ...inherited, ...env } }); +} + +test("--list prints a lane's commands and runs nothing", () => { + for (const os of OSES) { + const result = cli(["--os", os, "--list"]); + assert.equal(result.status, 0, result.stderr); + assert.deepEqual(result.stdout.trim().split("\n"), EXPECTED[os]); + } + assert.deepEqual(cli(["--profile", "msrv", "--list"]).stdout.trim().split("\n"), EXPECTED.msrv); +}); + +test("the script refuses to run another OS's gates, an unknown option and an MSRV run without a toolchain", () => { + const foreign = OSES.find((os) => os !== { darwin: "macos", linux: "linux", win32: "windows" }[process.platform]); + const wrongOs = cli(["--os", foreign]); + assert.equal(wrongOs.status, 2); + assert.match(wrongOs.stderr, /refusing to run the \w+ gates/); + assert.equal(cli(["--nonsense"]).status, 2); + assert.equal(cli(["--os", "plan9", "--list"]).status, 2); + const noToolchain = cli(["--profile", "msrv"]); + assert.equal(noToolchain.status, 2); + assert.match(noToolchain.stderr, /RUSTUP_TOOLCHAIN/); +}); diff --git a/scripts/test-ci-cli-feature-coverage.mjs b/scripts/test-ci-cli-feature-coverage.mjs index b21c8db7..fb45f685 100644 --- a/scripts/test-ci-cli-feature-coverage.mjs +++ b/scripts/test-ci-cli-feature-coverage.mjs @@ -1,39 +1,101 @@ import assert from "node:assert/strict"; -import { readFile } from "node:fs/promises"; import test from "node:test"; +import { gateInvocation, loadWorkflow, readRepoFile } from "./lib/ci-workflow.mjs"; +import { OSES, commandOf, gatesFor } from "./ci/cargo-gates.mjs"; -const [cliManifest, coreManifest, ciWorkflow] = await Promise.all([ - readFile(new URL("../src-tauri/crates/sagascript-cli/Cargo.toml", import.meta.url), "utf8"), - readFile(new URL("../src-tauri/crates/sagascript-core/Cargo.toml", import.meta.url), "utf8"), - readFile(new URL("../.github/workflows/ci.yml", import.meta.url), "utf8"), -]); +// How CI covers the Cargo features of the CLI and of core (#81, #358). Structural: the gate list +// that every lane runs, not the text of ci.yml. -function macosWorkflowFor(workflow) { - const normalized = workflow.replace(/\r\n/g, "\n"); - return normalized.slice(normalized.indexOf(" test-macos:"), normalized.indexOf(" build-macos:")); +const manifest = (crate) => readRepoFile(`src-tauri/crates/${crate}/Cargo.toml`); +const workflow = loadWorkflow(); +const lanes = { "test-macos": "macos", "test-linux": "linux", "test-windows": "windows" }; + +// The gates the lane really runs: found through the lane's own invocation of the gate script. +function gatesOfLane(jobId) { + const invocation = workflow.job(jobId).steps.map(gateInvocation).find(Boolean); + assert.ok(invocation, `${jobId} runs the gate script`); + return gatesFor({ os: invocation.os, profile: invocation.profile }); } -function assertMacosDiarizationCoverage(workflow) { - const macosWorkflow = macosWorkflowFor(workflow); - assert.match(cliManifest, /default\s*=\s*\[\s*"record"\s*,\s*"diarization"\s*\]/, "CLI default features must include diarization"); - assert.match(coreManifest, /default\s*=\s*\[\s*\]/, "core must retain its no-feature default"); +test("the CLI keeps diarization in its default features and core keeps none", () => { + assert.match( + manifest("sagascript-cli"), + /default\s*=\s*\[\s*"record"\s*,\s*"diarization"\s*\]/, + "CLI default features must include diarization", + ); + assert.match(manifest("sagascript-core"), /default\s*=\s*\[\s*\]/, "core must retain its no-feature default"); +}); - assert.match(macosWorkflow, /run: cargo test -p sagascript-cli\n/); - assert.match(macosWorkflow, /run: cargo clippy -p sagascript-cli --all-targets -- -D warnings\n/); - assert.doesNotMatch(macosWorkflow, /-p sagascript-cli --features diarization/); +test("every OS lint and tests the workspace, which builds the CLI with those defaults, with no package excluded", () => { + for (const [jobId, os] of Object.entries(lanes)) { + const gates = gatesOfLane(jobId); + for (const verb of ["clippy", "test"]) { + const gate = gates.find((candidate) => candidate.args[0] === verb && candidate.args.includes("--workspace")); + assert.ok(gate, `${os}: ${verb} --workspace`); + assert.ok( + !gate.args.some((argument) => ["--exclude", "--features", "--no-default-features", "-p"].includes(argument)), + `${os}: ${commandOf(gate)}`, + ); + } + assert.deepEqual( + gates.filter((gate) => gate.args.includes("--workspace") && ["clippy", "test"].includes(gate.args[0])).map((gate) => gate.args[0]), + ["clippy", "test"], + ); + assert.ok(OSES.includes(os)); + } +}); - assert.equal( - (macosWorkflow.match(/-p sagascript-core --features diarization/g) ?? []).length, - 3, - "core check, test, and all-target clippy gates must remain separate", - ); -} +test("feature sets stay separate invocations: no gate names two packages, and the CLI never gets `--features diarization`", () => { + for (const jobId of Object.keys(lanes)) { + for (const gate of gatesOfLane(jobId)) { + const packages = gate.args.filter((argument, index) => gate.args[index - 1] === "-p"); + assert.ok(packages.length <= 1, `${commandOf(gate)} combines packages, which would unify their features`); + assert.ok( + !(packages.includes("sagascript-cli") && gate.args.includes("--features")), + `${commandOf(gate)}: the CLI's features come from its defaults, not a flag`, + ); + } + } +}); -for (const [label, lineEnding] of [ - ["LF", "\n"], - ["CRLF", "\r\n"], -]) { - test(`macOS CI keeps CLI diarization in defaults and gates core explicitly (${label})`, () => { - assertMacosDiarizationCoverage(ciWorkflow.replace(/\r?\n/g, lineEnding)); - }); -} +test("core is gated explicitly with and without diarization, and the lean CLI without default features", () => { + const all = Object.keys(lanes).flatMap((jobId) => gatesOfLane(jobId)); + const matching = (predicate) => all.filter(predicate).map((gate) => gate.args.join(" ")); + const coreAlone = (gate) => + gate.args.includes("sagascript-core") && !gate.args.includes("--features") && !gate.args.includes("--release"); + const coreDiarization = (gate) => gate.args.includes("sagascript-core") && gate.args.join(" ").includes("--features diarization"); + // Core with no features, on every OS: clippy over all targets and the tests. + for (const jobId of Object.keys(lanes)) { + const gates = gatesOfLane(jobId); + assert.deepEqual( + gates.filter(coreAlone).map((gate) => gate.args[0]), + ["clippy", "test"], + `${jobId}: core without features`, + ); + } + // Core with diarization: a clippy over all targets and a test run, separate from the above. `cargo + // check` is not a third gate: clippy --all-targets compiles everything it would. + assert.deepEqual( + new Set(matching(coreDiarization)), + new Set([ + "clippy -p sagascript-core --features diarization --all-targets -- -D warnings", + "test -p sagascript-core --features diarization", + ]), + ); + assert.ok( + all.every((gate) => gate.args[0] !== "check" || gate.profile === "msrv"), + "no plain cargo check outside the MSRV profile", + ); + // The lean CLI, which strips diarization with record, is linted and tested on every OS. + for (const jobId of Object.keys(lanes)) { + const lean = gatesOfLane(jobId).filter( + (gate) => + ["clippy", "test"].includes(gate.args[0]) && gate.args.includes("sagascript-cli") && gate.args.includes("--no-default-features"), + ); + assert.deepEqual( + lean.map((gate) => gate.args[0]), + ["clippy", "test"], + `${jobId}: lean CLI`, + ); + } +}); diff --git a/scripts/test-ci-docs-workflow.mjs b/scripts/test-ci-docs-workflow.mjs index 95fc320f..31d25f24 100644 --- a/scripts/test-ci-docs-workflow.mjs +++ b/scripts/test-ci-docs-workflow.mjs @@ -4,83 +4,59 @@ import { copyFileSync, mkdtempSync, mkdirSync, readFileSync, rmSync, writeFileSy import { tmpdir } from "node:os"; import { join } from "node:path"; import test from "node:test"; +import { REPO_ROOT, loadWorkflow, parseWorkflow } from "./lib/ci-workflow.mjs"; -const workflow = readFileSync(new URL("../.github/workflows/ci.yml", import.meta.url), "utf8"); -function normalizeWorkflow(source) { - return source.replace(/\r\n?/g, "\n"); -} -function job(id, source = workflow) { - const normalized = normalizeWorkflow(source); - return normalized.split(`\n ${id}:\n`)[1].split(/\n [a-z][a-z0-9-]*:\n/)[0]; -} -function script(id, source = workflow) { - return job(id, source) - .split(" run: |\n")[1] - .split(/\n - /)[0] - .split("\n") - .map((line) => line.replace(/^ /, "")) - .join("\n"); +// The docs-only scope gate of ci.yml and what the aggregators make of it. The workflow is parsed, +// and the scripts under test are the ones its steps run. + +const workflow = loadWorkflow(); +const CHECKS = ["check-macos", "check-windows", "check-linux"]; + +function stepRun(source, jobId, predicate) { + const step = source.job(jobId).steps.find(predicate); + assert.ok(step, `${jobId} has the step`); + return step.run; } -function shell(command, env, cwd) { - return spawnSync("bash", ["-euo", "pipefail", "-c", command], { - cwd, - env: { ...process.env, ...env }, - encoding: "utf8", - }); +const scopeScript = (source = workflow) => stepRun(source, "scope", (step) => step.id === "classify"); +const shell = (command, env, cwd) => + spawnSync("bash", ["-euo", "pipefail", "-c", command], { cwd, env: { ...process.env, ...env }, encoding: "utf8" }); + +// A check job's own step, run as the workflow runs it, against a needs context built from the +// job's own `needs`. +function runCheck(jobId, { docsOnly, trusted = "true", scope = "success", results }) { + const job = workflow.job(jobId); + const needs = { scope: { result: scope, outputs: { docs_only: docsOnly, scope_trusted: trusted, scope_classification: "x" } } }; + const lanes = job.needs.filter((need) => need !== "scope"); + lanes.forEach((lane, index) => (needs[lane] = { result: results[index] ?? results[0], outputs: {} })); + const step = job.steps.find((candidate) => candidate.run.startsWith("node scripts/ci/assert-lanes.mjs")); + assert.equal(job.env.NEEDS, "${{ toJSON(needs) }}", "the step reads the job's needs context"); + return shell(step.run, { NEEDS: JSON.stringify(needs) }, REPO_ROOT); } test("required checks accept only successful full lanes or explicitly classified skipped docs lanes", () => { - const results = ["success", "failure", "cancelled", "skipped", ""]; - for (const id of ["check-macos", "check-windows", "check-linux"]) { - const run = script(id); - assert.match(job(id), /if: \$\{\{ always\(\) \}\}/); - for (const docsOnly of ["true", "false", ""]) { - for (const first of results) { - for (const second of results) { - const env = { - SCOPE_RESULT: "success", - SCOPE_TRUSTED: "true", - DOCS_ONLY: docsOnly, - TEST_RESULT: first, - BUILD_RESULT: second, - LINUX_RESULT: first, - MSRV_RESULT: second, - }; - const expected = - docsOnly === "true" - ? first === "skipped" && second === "skipped" - : docsOnly === "false" && first === "success" && second === "success"; - assert.equal(shell(run, env).status === 0, expected, `${id}: ${JSON.stringify(env)}`); + // One process per case: the matrix of every result is in test-ci-parallel-gates.mjs, in-process. + for (const id of CHECKS) { + for (const docsOnly of ["true", "false"]) { + for (const first of ["success", "failure", "cancelled", "skipped"]) { + for (const second of ["success", "skipped"]) { + const expected = docsOnly === "true" ? first === "skipped" && second === "skipped" : first === "success" && second === "success"; + assert.equal( + runCheck(id, { docsOnly, results: [first, second] }).status === 0, + expected, + `${id}: docs_only ${docsOnly}, ${first}/${second}`, + ); } } } - for (const result of results.filter((value) => value !== "success")) { - assert.notEqual( - shell(run, { - SCOPE_RESULT: result, - SCOPE_TRUSTED: "true", - DOCS_ONLY: "true", - TEST_RESULT: "skipped", - BUILD_RESULT: "skipped", - LINUX_RESULT: "skipped", - MSRV_RESULT: "skipped", - }).status, - 0, - `${id} rejects unsuccessful scope`, - ); + assert.notEqual(runCheck(id, { docsOnly: "", results: ["success"] }).status, 0, `${id} rejects an empty docs_only`); + for (const scope of ["failure", "cancelled", "skipped"]) { + assert.notEqual(runCheck(id, { docsOnly: "true", scope, results: ["skipped"] }).status, 0, `${id} rejects scope ${scope}`); } for (const trusted of ["false", ""]) { assert.notEqual( - shell(run, { - SCOPE_RESULT: "success", - SCOPE_TRUSTED: trusted, - DOCS_ONLY: "true", - TEST_RESULT: "skipped", - BUILD_RESULT: "skipped", - LINUX_RESULT: "skipped", - MSRV_RESULT: "skipped", - }).status, + runCheck(id, { docsOnly: "true", trusted, results: ["skipped"] }).status, 0, + `${id} rejects scope_trusted ${trusted}`, ); } } @@ -116,7 +92,7 @@ test("scope bootstrap and unavailable commits select full CI; known docs and pus const classify = (overrides = {}) => { writeFileSync(output, ""); const result = shell( - script("scope"), + scopeScript(), { ...env, EVENT_NAME: "pull_request", @@ -147,25 +123,28 @@ test("scope bootstrap and unavailable commits select full CI; known docs and pus assert.doesNotMatch(result.output, /docs_only=true/); }); -test("CRLF workflow text still parses and executes scope and aggregate scripts", () => { - const crlfWorkflow = normalizeWorkflow(workflow).replaceAll("\n", "\r\n"); - const scope = script("scope", crlfWorkflow); - const aggregateEnvironment = { - SCOPE_RESULT: "success", - SCOPE_TRUSTED: "true", - DOCS_ONLY: "false", - TEST_RESULT: "success", - BUILD_RESULT: "success", - LINUX_RESULT: "success", - MSRV_RESULT: "success", - }; +test("the scope job needs no script file of the revision it classifies", () => { + // It checks out the PR base, which may predate scripts/ci/ (adoption): its own script must be inline. + const scope = workflow.job("scope"); + assert.deepEqual(scope.needs, []); + const checkout = scope.steps[0]; + assert.equal(checkout.with.ref, "${{ github.event_name == 'pull_request' && github.event.pull_request.base.sha || github.sha }}"); + for (const step of scope.steps) assert.ok(!step.run.includes("scripts/ci/"), "scope must not depend on files added by the PR"); +}); - const scopeResult = shell(scope, { EVENT_NAME: "push", GITHUB_OUTPUT: "" }); +test("CRLF workflow text still parses and executes scope and aggregate scripts", () => { + const crlfWorkflow = parseWorkflow( + readFileSync(new URL("../.github/workflows/ci.yml", import.meta.url), "utf8") + .replace(/\r\n?/g, "\n") + .replaceAll("\n", "\r\n"), + ); + const scopeResult = shell(scopeScript(crlfWorkflow), { EVENT_NAME: "push", GITHUB_OUTPUT: "" }); assert.equal(scopeResult.status, 0, scopeResult.stderr); assert.match(scopeResult.stdout, /push-event-requires-full-ci/); - - for (const id of ["check-macos", "check-linux", "check-windows"]) { - const result = shell(script(id, crlfWorkflow), aggregateEnvironment); + for (const id of CHECKS) { + const result = runCheck(id, { docsOnly: "false", results: ["success"] }); assert.equal(result.status, 0, `${id}: ${result.stderr}`); + const step = crlfWorkflow.job(id).steps.find((candidate) => candidate.run.startsWith("node scripts/ci/assert-lanes.mjs")); + assert.equal(step.run, "node scripts/ci/assert-lanes.mjs"); } }); diff --git a/scripts/test-ci-parallel-gates.mjs b/scripts/test-ci-parallel-gates.mjs index 3318b6f6..437c93bc 100644 --- a/scripts/test-ci-parallel-gates.mjs +++ b/scripts/test-ci-parallel-gates.mjs @@ -1,369 +1,311 @@ import assert from "node:assert/strict"; -import { execFileSync } from "node:child_process"; -import { readFile } from "node:fs/promises"; +import { spawnSync } from "node:child_process"; +import { readFileSync } from "node:fs"; import test from "node:test"; +import { fileURLToPath } from "node:url"; +import { actionInput, commandLines, gateInvocation, loadAction, loadWorkflow, parseWorkflow } from "./lib/ci-workflow.mjs"; +import { gatesFor } from "./ci/cargo-gates.mjs"; +import { laneProblems } from "./ci/assert-lanes.mjs"; +import { importGraph } from "./lib/import-graph.mjs"; -const workflow = (await readFile(new URL("../.github/workflows/ci.yml", import.meta.url), "utf8")).replace(/\r\n/g, "\n"); +// The parallel lanes of ci.yml (#358): what each one needs, how the shared setup action and the +// aggregators are wired, and that the aggregators fail closed. The workflow is parsed, not matched. -function jobBlock(jobId) { - const start = workflow.indexOf(` ${jobId}:`); - assert.notEqual(start, -1, `workflow must define ${jobId}`); - const remainder = workflow.slice(start); - const body = remainder.slice(remainder.indexOf("\n") + 1); - const nextJob = body.search(/^ [a-z][a-z0-9-]*:/m); - return remainder.slice(0, nextJob === -1 ? remainder.length : remainder.indexOf("\n") + 1 + nextJob); -} - -function scopedAggregator(jobId, expectedNeeds, laneResultNames, stepName) { - const block = jobBlock(jobId); - const needs = /^ needs: \[([^\]]+)\]$/m - .exec(block)?.[1] - .split(",") - .map((value) => value.trim()); - assert.deepEqual(needs, expectedNeeds, `${jobId} must wait for scope and its applicable lane(s)`); - assert.match(block, /^ if: \$\{\{ always\(\) \}\}$/m); - assert.ok(block.includes("SCOPE_RESULT: ${{ needs.scope.result }}")); - assert.ok(block.includes("SCOPE_TRUSTED: ${{ needs.scope.outputs.scope_trusted }}")); - assert.ok(block.includes("DOCS_ONLY: ${{ needs.scope.outputs.docs_only }}")); - for (const [variable, lane] of Object.entries(laneResultNames)) { - assert.ok(block.includes(`${variable}: \${{ needs.${lane}.result }}`)); - } - const run = parseSteps(block).get(stepName); - assert.match(run, /test "\$SCOPE_RESULT" = success/); - assert.match(run, /test "\$SCOPE_TRUSTED" = true/); - assert.match(run, /if \[ "\$DOCS_ONLY" = true \]/); - return run; -} +const repoRoot = fileURLToPath(new URL("..", import.meta.url)); +const workflow = loadWorkflow(); +const ASSERT_LANES = fileURLToPath(new URL("./ci/assert-lanes.mjs", import.meta.url)); +const SCOPE_CONDITION = "${{ needs.scope.result == 'success' && needs.scope.outputs.docs_only != 'true' }}"; +const LANES = ["test-macos", "build-macos", "test-linux", "msrv-linux", "test-windows", "build-windows"]; +const CHECKS = { "check-macos": "macos", "check-linux": "linux", "check-windows": "windows" }; +const setupStep = (job) => job.steps.find((step) => step.uses === "./.github/actions/setup"); -function runScopedAggregator(run, values) { - execFileSync("bash", ["-euo", "pipefail", "-c", run], { - env: { PATH: process.env.PATH, ...values }, - stdio: "pipe", - }); -} +test("the job set is the scope job, the six lanes and the three aggregators", () => { + assert.deepEqual([...workflow.jobIds].sort(), ["scope", ...LANES, ...Object.keys(CHECKS)].sort()); +}); -for (const [jobId, needs, laneResultNames, stepName] of [ - [ - "check-macos", - ["scope", "test-macos", "build-macos"], - { TEST_RESULT: "test-macos", BUILD_RESULT: "build-macos" }, - "Assert macOS CI lanes passed", - ], - [ - "check-windows", - ["scope", "test-windows", "build-windows"], - { TEST_RESULT: "test-windows", BUILD_RESULT: "build-windows" }, - "Assert Windows CI lanes passed", - ], - [ - "check-linux", - ["scope", "test-linux", "msrv-linux"], - { LINUX_RESULT: "test-linux", MSRV_RESULT: "msrv-linux" }, - "Assert Linux CI lanes passed", - ], -]) { - test(`${jobId} accepts only trusted docs-only skips or successful full lanes`, () => { - const run = scopedAggregator(jobId, needs, laneResultNames, stepName); - const laneVariables = Object.keys(laneResultNames); - const correctDocsOnly = Object.fromEntries(laneVariables.map((variable) => [variable, "skipped"])); - const correctFull = Object.fromEntries(laneVariables.map((variable) => [variable, "success"])); +test("every lane waits only for the scope gate and is skipped together with every other lane on a trusted docs-only change", () => { + for (const id of LANES) { + const job = workflow.job(id); + assert.deepEqual(job.needs, ["scope"], `${id} must wait only for the scope gate`); + assert.equal(job.condition, SCOPE_CONDITION, `${id} must run only on a successful, non-docs-only scope`); + } +}); - assert.doesNotThrow(() => - runScopedAggregator(run, { - SCOPE_RESULT: "success", - SCOPE_TRUSTED: "true", - DOCS_ONLY: "true", - ...correctDocsOnly, - }), +test("check-macos and check-windows keep their names, need every lane of their OS and run whatever the lanes did", () => { + for (const [id, os] of Object.entries(CHECKS)) { + const job = workflow.job(id); + assert.equal(workflow.doc.jobs[id].name, undefined, `${id}: the check-run name is the job id; a name: would rename the required check`); + const lanes = LANES.filter((lane) => lane.endsWith(`-${os}`)); + assert.ok(lanes.length >= 2, `${os} has a test lane and a second lane`); + assert.deepEqual([...job.needs].sort(), ["scope", ...lanes].sort(), `${id} must wait for scope and every ${os} lane`); + assert.equal(job.condition, "${{ always() }}", `${id} must report even when a lane failed or was skipped`); + assert.equal(job.env.NEEDS, "${{ toJSON(needs) }}"); + assert.equal(job.timeoutMinutes, 20); + assert.equal(job.steps[0].uses?.split("@")[0], "actions/checkout"); + assert.deepEqual( + job.steps.slice(1).map((step) => step.run), + ["node scripts/ci/assert-lanes.mjs"], ); - assert.doesNotThrow(() => - runScopedAggregator(run, { - SCOPE_RESULT: "success", - SCOPE_TRUSTED: "true", - DOCS_ONLY: "false", - ...correctFull, - }), + assert.ok( + job.steps.every((step) => step.condition === undefined && step.continueOnError === undefined), + `${id}'s steps are unconditional`, ); + } +}); - for (const scopeResult of ["failure", "cancelled", "skipped"]) { +// The `needs` context a check job would see: its own needs, with the results given. +function needsContext(id, { scope = "success", trusted = "true", docsOnly = "false", lane = "success", overrides = {} }) { + const needs = { scope: { result: scope, outputs: { docs_only: docsOnly, scope_trusted: trusted, scope_classification: "x" } } }; + for (const name of workflow.job(id).needs.filter((need) => need !== "scope")) { + needs[name] = { result: overrides[name] ?? lane, outputs: {} }; + } + return needs; +} + +for (const id of Object.keys(CHECKS)) { + test(`${id} accepts only trusted docs-only skips or successful full lanes`, () => { + const accepts = (options) => laneProblems(needsContext(id, options)).length === 0; + assert.ok(accepts({ docsOnly: "true", lane: "skipped" })); + assert.ok(accepts({ docsOnly: "false", lane: "success" })); + for (const scope of ["failure", "cancelled", "skipped"]) { for (const docsOnly of ["true", "false"]) { - assert.throws(() => - runScopedAggregator(run, { - SCOPE_RESULT: scopeResult, - SCOPE_TRUSTED: "true", - DOCS_ONLY: docsOnly, - ...(docsOnly === "true" ? correctDocsOnly : correctFull), - }), + assert.ok( + !accepts({ scope, docsOnly, lane: docsOnly === "true" ? "skipped" : "success" }), + `scope ${scope}, docs_only ${docsOnly}`, ); } } for (const trusted of ["false", "", "untrusted"]) { - assert.throws(() => - runScopedAggregator(run, { - SCOPE_RESULT: "success", - SCOPE_TRUSTED: trusted, - DOCS_ONLY: "true", - ...correctDocsOnly, - }), - ); - } - for (const docsOnly of ["", "unknown"]) { - assert.throws(() => - runScopedAggregator(run, { - SCOPE_RESULT: "success", - SCOPE_TRUSTED: "true", - DOCS_ONLY: docsOnly, - ...correctFull, - }), - ); + assert.ok(!accepts({ trusted, docsOnly: "true", lane: "skipped" }), `scope_trusted ${trusted}`); } - for (const variable of laneVariables) { - for (const result of ["failure", "cancelled", "skipped"]) { - assert.throws(() => - runScopedAggregator(run, { - SCOPE_RESULT: "success", - SCOPE_TRUSTED: "true", - DOCS_ONLY: "false", - ...correctFull, - [variable]: result, - }), + for (const docsOnly of ["", "unknown"]) assert.ok(!accepts({ docsOnly }), `docs_only ${JSON.stringify(docsOnly)}`); + // Every lane the job waits for, one by one, with every result that is not the expected one. + for (const lane of workflow.job(id).needs.filter((need) => need !== "scope")) { + for (const result of ["failure", "cancelled", "skipped", ""]) { + assert.ok(!accepts({ overrides: { [lane]: result } }), `${lane} ${JSON.stringify(result)} on a full run`); + } + for (const result of ["success", "failure", "cancelled", ""]) { + assert.ok( + !accepts({ docsOnly: "true", lane: "skipped", overrides: { [lane]: result } }), + `${lane} ${JSON.stringify(result)} on a docs-only run`, ); } } }); } -function parseSteps(job) { - const lines = job.split("\n"); - const starts = lines.map((line, index) => (/^ - name: (.*)$/.exec(line) ? index : -1)).filter((index) => index >= 0); - const result = new Map(); +test("an aggregator that waits for no lane, or gets no usable needs context, fails", () => { + assert.notEqual(laneProblems({ scope: { result: "success", outputs: { docs_only: "false", scope_trusted: "true" } } }).length, 0); + for (const bad of [null, [], "success", 1]) assert.notEqual(laneProblems(bad).length, 0); + const run = (env) => spawnSync(process.execPath, [ASSERT_LANES], { encoding: "utf8", env: { PATH: process.env.PATH, ...env } }); + assert.equal(run({ NEEDS: JSON.stringify(needsContext("check-macos", {})) }).status, 0); + assert.equal(run({ NEEDS: JSON.stringify(needsContext("check-macos", { overrides: { "build-macos": "failure" } })) }).status, 1); + assert.equal(run({}).status, 1, "no NEEDS"); + assert.equal(run({ NEEDS: "not json" }).status, 1); + assert.match(run({ NEEDS: JSON.stringify(needsContext("check-macos", { lane: "failure" })) }).stderr, /test-macos: result is "failure"/); +}); - for (let i = 0; i < starts.length; i += 1) { - const start = starts[i]; - const end = starts[i + 1] ?? lines.length; - const segment = lines.slice(start, end); - const name = /^ - name: (.*)$/.exec(segment[0])[1]; - const runIndex = segment.findIndex((line) => /^ run:\s*/.test(line)); - if (runIndex < 0) continue; +test("the scope job publishes the outputs the aggregators read", () => { + const scope = workflow.job("scope"); + assert.deepEqual(Object.keys(scope.outputs).sort(), ["docs_only", "scope_classification", "scope_trusted"]); + for (const name of ["docs_only", "scope_trusted"]) assert.equal(scope.outputs[name], `\${{ steps.classify.outputs.${name} }}`); +}); - const runValue = segment[runIndex].replace(/^ run:\s*/, ""); - let command; - if (runValue === "|" || runValue === ">" || runValue === ">-") { - command = segment - .slice(runIndex + 1) - .filter((line) => line.length === 0 || line.startsWith(" ")) - .map((line) => line.replace(/^ /, "")) - .join("\n"); - } else { - command = runValue; +test("the setup action is the only toolchain preamble: every lane uses it with declared inputs and the cache its gates need", () => { + const action = loadAction("setup"); + const KEYS = { + "test-macos": "sagascript-ci-checks", + "test-windows": "sagascript-ci-checks", + "build-macos": "sagascript-ci-native", + "build-windows": "sagascript-ci-native", + "test-linux": "sagascript-ci", + "msrv-linux": "sagascript-msrv", + }; + for (const id of LANES) { + const job = workflow.job(id); + const setups = job.steps.filter((step) => step.uses === "./.github/actions/setup"); + assert.equal(setups.length, 1, `${id} sets up once, through the action`); + const [setup] = setups; + for (const name of Object.keys(setup.with)) assert.ok(Object.hasOwn(action.inputs, name), `${id} passes an undeclared input: ${name}`); + for (const [name, input] of Object.entries(action.inputs)) if (input.required) assert.ok(name in setup.with, `${id} must pass ${name}`); + assert.equal(setup.with["cache-key"], KEYS[id]); + assert.equal(job.steps[0].uses?.split("@")[0], "actions/checkout", `${id}: checkout stays in the job, ahead of the local action`); + assert.equal(job.steps.indexOf(setup), 1, `${id}: the setup follows the checkout directly`); + for (const step of job.steps) { + assert.ok( + !/^(actions\/setup-(python|node)|dtolnay\/rust-toolchain|Swatinem\/rust-cache)@/.test(step.uses ?? ""), + `${id} repeats a step of the setup action`, + ); } - result.set(name, command.trim()); - } - return result; -} - -function normalizeCommand(command) { - return command.replace(/\r\n/g, "\n").trim(); -} -// These are the Cargo/native/smoke run bodies in the pre-split check-macos and -// check-windows jobs at base 1ab1128. Keeping the fixture explicit makes a -// dropped invocation fail this test even if the replacement workflow remains valid YAML. -const BASELINE_GATES = { - macos: { - test: [ - ["Cargo check", "cargo check"], - ["Cargo test", "cargo test"], - ["Cargo clippy", "cargo clippy --all-targets -- -D warnings"], - ["Cargo test (sagascript-core, no features)", "cargo test -p sagascript-core"], - ["Cargo test (sagascript-cli)", "cargo test -p sagascript-cli"], - ["Cargo clippy (sagascript-core, no features)", "cargo clippy -p sagascript-core --all-targets -- -D warnings"], - ["Cargo clippy (sagascript-cli)", "cargo clippy -p sagascript-cli --all-targets -- -D warnings"], - ["Cargo check (sagascript-core, diarization)", "cargo check -p sagascript-core --features diarization"], - ["Cargo test (sagascript-core, diarization)", "cargo test -p sagascript-core --features diarization"], - [ - "Cargo clippy (sagascript-core, diarization)", - "cargo clippy -p sagascript-core --features diarization --all-targets -- -D warnings", - ], - ], - build: [ - ["Build native release binary (no installer)", "npx tauri build --config scripts/tauri-ci-prebuilt.json --no-bundle"], - ["Verify native release binary", 'BINARY=src-tauri/target/release/sagascript\n[[ -x "$BINARY" ]]\nfile "$BINARY"'], - [ - "Smoke test — help and version", - "BINARY=src-tauri/target/release/sagascript\n$BINARY --help\n$BINARY transcribe --help\n$BINARY record --help", - ], - [ - "Smoke test — config management", - "BINARY=src-tauri/target/release/sagascript\n$BINARY config list\n$BINARY config get language\n$BINARY config set language sv\n$BINARY config get language | grep -q sv\n$BINARY config reset language\n$BINARY config get language | grep -q en\n$BINARY config path", - ], - ["Smoke test — model listing and formats", "BINARY=src-tauri/target/release/sagascript\n$BINARY list-models\n$BINARY formats"], - [ - "Smoke test — shell completions", - "BINARY=src-tauri/target/release/sagascript\n$BINARY completions zsh > /dev/null\n$BINARY completions bash > /dev/null", - ], - [ - "Smoke test — download model and transcribe", - 'BINARY=src-tauri/target/release/sagascript\n$BINARY download-model nb-whisper-tiny\nRESULT=$($BINARY transcribe --language no --model nb-whisper-tiny test-audio/norwegian-short-3s.mp3)\necho "Transcription: $RESULT"\necho "$RESULT" | grep -iq "stortinget"', - ], - [ - "Smoke test — transcribe with JSON output", - 'BINARY=src-tauri/target/release/sagascript\nSTDOUT="$RUNNER_TEMP/transcription-smoke.json"\nSTDERR="$RUNNER_TEMP/transcription-smoke.stderr"\n$BINARY transcribe --json --language no --model nb-whisper-tiny \\\n test-audio/norwegian-short-3s.mp3 > "$STDOUT" 2> "$STDERR"\npython3 scripts/verify-json-cli-streams.py "$STDOUT" "$STDERR"', - ], - ], - }, - windows: { - test: [ - ["Cargo check", "cargo check"], - ["Cargo test", "cargo test"], - ["Cargo clippy", "cargo clippy --all-targets -- -D warnings"], - ["Cargo test (sagascript-core, no features)", "cargo test -p sagascript-core"], - ["Cargo test (sagascript-cli)", "cargo test -p sagascript-cli"], - ["Cargo clippy (sagascript-core, no features)", "cargo clippy -p sagascript-core --all-targets -- -D warnings"], - ["Cargo clippy (sagascript-cli)", "cargo clippy -p sagascript-cli --all-targets -- -D warnings"], - ], - build: [ - ["Build native release binary (no installer)", "npx tauri build --config scripts/tauri-ci-prebuilt.json --no-bundle"], - [ - "Verify native release binary", - '$binary = "src-tauri\\target\\release\\sagascript.exe"\nif (-not (Test-Path -PathType Leaf $binary)) {\n throw "Missing native release binary: $binary"\n}', - ], - [ - "Smoke test — help and version", - '$env:BINARY = "src-tauri\\target\\release\\sagascript.exe"\n& $env:BINARY --help\n& $env:BINARY transcribe --help\n& $env:BINARY record --help', - ], - [ - "Smoke test — config management", - '$env:BINARY = "src-tauri\\target\\release\\sagascript.exe"\n& $env:BINARY config list\n& $env:BINARY config get language\n& $env:BINARY config set language sv\n$result = & $env:BINARY config get language\nif ($result -notmatch "sv") { throw "Expected \'sv\'" }\n& $env:BINARY config reset language\n$result = & $env:BINARY config get language\nif ($result -notmatch "en") { throw "Expected \'en\'" }\n& $env:BINARY config path', - ], - [ - "Smoke test — model listing and formats", - '$env:BINARY = "src-tauri\\target\\release\\sagascript.exe"\n& $env:BINARY list-models\n& $env:BINARY formats', - ], - [ - "Smoke test — shell completions", - '$env:BINARY = "src-tauri\\target\\release\\sagascript.exe"\n& $env:BINARY completions powershell | Out-Null', - ], - [ - "Smoke test — download model and transcribe", - '$env:BINARY = "src-tauri\\target\\release\\sagascript.exe"\n& $env:BINARY download-model nb-whisper-tiny\n$result = & $env:BINARY transcribe --language no --model nb-whisper-tiny test-audio/norwegian-short-3s.mp3\nWrite-Host "Transcription: $result"\nif ($result -notmatch "(?i)stortinget") { throw "Expected \'stortinget\' in output" }', - ], - [ - "Smoke test — transcribe with JSON output", - 'set BINARY=src-tauri\\target\\release\\sagascript.exe\n%BINARY% transcribe --json --language no --model nb-whisper-tiny test-audio/norwegian-short-3s.mp3 > json_output.txt 2>nul\ntype json_output.txt\nfindstr /C:"text" json_output.txt\nfindstr /C:"duration_seconds" json_output.txt\nfindstr /C:"language" json_output.txt', - ], - ], - }, -}; + // Which cargo gates the lane runs decides which Rust components and workspaces it needs. + const invocation = job.steps.map(gateInvocation).find(Boolean); + const gates = invocation ? gatesFor({ os: invocation.os ?? "linux", profile: invocation.profile }) : []; + const components = actionInput(action, setup, "rust-components") + .split(",") + .map((part) => part.trim()); + if (gates.some((gate) => gate.args[0] === "clippy")) assert.ok(components.includes("clippy"), `${id} runs clippy`); + if (gates.some((gate) => gate.args[0] === "fmt")) assert.ok(components.includes("rustfmt"), `${id} runs rustfmt`); + const workspaces = actionInput(action, setup, "cache-workspaces") + .split("\n") + .map((line) => line.trim()); + assert.ok(workspaces.includes("src-tauri")); + if (gates.some((gate) => gate.cwd === "src-tauri/engine-host/ort")) { + assert.ok(workspaces.includes("src-tauri/engine-host/ort"), `${id} runs ORT host gates and must cache its workspace`); + } + } + const msrv = setupStep(workflow.job("msrv-linux")); + assert.equal(actionInput(action, msrv, "rust"), "msrv"); + assert.equal(actionInput(action, msrv, "python"), "false"); +}); -for (const [platform, jobIds] of Object.entries({ - macos: { test: "test-macos", build: "build-macos" }, - windows: { test: "test-windows", build: "build-windows" }, -})) { - for (const lane of ["test", "build"]) { - test(`${platform} ${lane} lane retains every baseline gate`, () => { - const block = jobBlock(jobIds[lane]); - assert.match(block, /^ needs: scope$/m, "each lane must wait only for the scope gate"); - assert.match(block, /^ if: \$\{\{ needs\.scope\.result == 'success' && needs\.scope\.outputs\.docs_only != 'true' \}\}$/m); - const commands = parseSteps(block); - for (const [name, expected] of BASELINE_GATES[platform][lane]) { - assert.equal(commands.get(name), normalizeCommand(expected), `${jobIds[lane]} must retain the baseline ${name} command`); - } - }); +test("every lane that runs Python selects the supported runtime first", () => { + // Moved here from scripts/dictation_eval/test_evaluate_cli.py: the workflow is parsed, not read as text. + const action = loadAction("setup"); + const python = action.steps.find((step) => step.uses?.startsWith("actions/setup-python@")); + assert.equal(python.with["python-version"], "3.12"); + assert.equal(python.condition, "${{ inputs.python == 'true' }}"); + assert.equal(action.inputs.python.default, "true"); + for (const id of ["test-macos", "test-linux", "test-windows"]) { + const steps = workflow.job(id).steps; + const setup = steps.findIndex((step) => step.uses === "./.github/actions/setup"); + const suites = steps.findIndex((step) => + commandLines(step).some((line) => line.startsWith("python -m unittest discover -s scripts/dictation_eval")), + ); + assert.equal(actionInput(action, steps[setup], "python"), "true", `${id} selects Python`); + assert.ok(setup >= 0 && suites > setup, `${id} selects Python before the evaluator suites`); } -} + // The native build lanes run the JSON stream verifier of the smoke tests with it; the MSRV lane runs no Python. + for (const id of ["build-macos", "build-windows"]) assert.equal(actionInput(action, setupStep(workflow.job(id)), "python"), "true"); + assert.equal(actionInput(action, setupStep(workflow.job("msrv-linux")), "python"), "false"); +}); -// The Windows model smoke steps are the only non-blocking exception (#357). -// continue-on-error masks their failures (a step's `conclusion` reads -// "success"), so a report step must surface every one from the real `outcome`. -test("only the two Windows smoke steps are non-blocking, and their failures are reported", () => { - const jobs = workflow.slice(workflow.indexOf("\njobs:\n")); - const jobIds = [...jobs.matchAll(/^ ([a-z][a-z0-9-]*):$/gm)].map((match) => match[1]); - for (const expected of ["test-macos", "build-macos", "test-linux", "msrv-linux", "test-windows", "build-windows"]) { - assert.ok(jobIds.includes(expected), `workflow must define ${expected}`); +test("the Rust cache key follows the gate list for the lanes that run gates, and for no other lane", () => { + // rust-cache saves only under a new key, so a changed gate list needs one to get its artifacts cached; a lane + // that never runs the gates (the native build lanes) must not lose its cache to a gate change. + const action = loadAction("setup"); + const cache = action.steps.find((step) => step.uses?.startsWith("Swatinem/rust-cache@")); + assert.equal( + cache.with["shared-key"], + "${{ inputs.cache-key }}${{ inputs.native-build != 'true' && format('-{0}', hashFiles('scripts/ci/cargo-gates.mjs')) || '' }}", + ); + for (const id of LANES) { + const job = workflow.job(id); + const runsGates = job.steps.some((step) => gateInvocation(step)); + const nativeBuild = actionInput(action, setupStep(job), "native-build") === "true"; + assert.equal( + nativeBuild, + !runsGates, + `${id}: ${runsGates ? "runs the cargo gates, so its cache key must carry their hash" : "runs no gate, so its cache key must not"}`, + ); } - for (const jobId of jobIds.filter((id) => id !== "build-windows")) { - assert.doesNotMatch(jobBlock(jobId), /continue-on-error:/, `${jobId} must not tolerate failures`); +}); + +test("only the native build lanes cache downloaded models", () => { + const action = loadAction("setup"); + for (const id of LANES) { + assert.equal( + actionInput(action, setupStep(workflow.job(id)), "native-build"), + id.startsWith("build-") ? "true" : "false", + `${id}: native build`, + ); } - const windows = jobBlock("build-windows"); - assert.doesNotMatch(windows, /^ continue-on-error:/m, "build-windows itself must stay blocking"); - assert.equal((windows.match(/^ continue-on-error: true$/gm) ?? []).length, 2); - const stepText = (name) => windows.slice(windows.indexOf(` - name: ${name}`)).split(/\n - name:/)[0]; - for (const [name, id] of [ - ["Smoke test — download model and transcribe", "smoke_transcribe"], - ["Smoke test — transcribe with JSON output", "smoke_json"], - ]) { - const step = stepText(name); - assert.match(step, /^ continue-on-error: true$/m, `${name} is a tolerated step`); - assert.match(step, new RegExp(`^ id: ${id}$`, "m"), `${name} needs id ${id} so its outcome can be reported`); + const step = action.steps.find((candidate) => candidate.name === "Cache downloaded models"); + assert.equal(step.condition, "${{ inputs.native-build == 'true' }}"); + for (const source of ["settings/manager.rs", "transcription/model.rs", "diarization/model.rs"]) { + assert.ok(step.with.key.includes(`sagascript-core/src/${source}`), `the model cache key hashes ${source}`); } - const reportName = "Report masked smoke test failures"; - const report = stepText(reportName); - assert.ok(windows.indexOf(reportName) > windows.indexOf("Smoke test — transcribe with JSON output")); - assert.match(report, /^ if: \$\{\{ always\(\) \}\}$/m); - assert.match(report, /TRANSCRIBE_OUTCOME: \$\{\{ steps\.smoke_transcribe\.outcome \}\}/); - assert.match(report, /JSON_OUTCOME: \$\{\{ steps\.smoke_json\.outcome \}\}/); - assert.match(report, /::warning /); - assert.match(report, /GITHUB_STEP_SUMMARY/); }); -test("model caches are isolated to native lanes", () => { - assert.doesNotMatch(jobBlock("test-macos"), /name: Cache downloaded models/); - assert.doesNotMatch(jobBlock("test-windows"), /name: Cache downloaded models/); - assert.match(jobBlock("build-macos"), /name: Cache downloaded models/); - assert.match(jobBlock("build-windows"), /name: Cache downloaded models/); - assert.match(jobBlock("test-macos"), /shared-key: sagascript-ci-checks/); - assert.match(jobBlock("test-windows"), /shared-key: sagascript-ci-checks/); - assert.match(jobBlock("build-macos"), /shared-key: sagascript-ci-native/); - assert.match(jobBlock("build-windows"), /shared-key: sagascript-ci-native/); +test("the setup action's steps run in the order the lanes depend on", () => { + const action = loadAction("setup"); + const order = action.steps.map((step) => step.name); + const before = (first, second) => + assert.ok(order.indexOf(first) >= 0 && order.indexOf(first) < order.indexOf(second), `${first} before ${second}`); + before("Install Tauri system dependencies", "Select evaluation Python runtime"); + before("Install Rust", "Cache Rust dependencies"); + before("Install the declared MSRV toolchain", "Cache Rust dependencies"); + before("Install Node.js", "Install npm dependencies"); + before("Cache Rust dependencies", "Install npm dependencies"); + // The MSRV toolchain is exported before anything later in the lane looks at rustc. + const lines = commandLines(action.steps.find((step) => step.name === "Install the declared MSRV toolchain")); + assert.ok(lines.includes('echo "RUSTUP_TOOLCHAIN=$msrv" >> "$GITHUB_ENV"')); + assert.ok( + lines.some((line) => line.includes("src-tauri/Cargo.toml")) && lines.some((line) => line.includes("engine-host/ort/Cargo.toml")), + ); }); -function aggregator(jobId, expectedNeeds, testResultName, buildResultName) { - const block = jobBlock(jobId); - const needs = /^ needs: \[([^\]]+)\]$/m - .exec(block)?.[1] - .split(",") - .map((value) => value.trim()); - assert.deepEqual(needs, expectedNeeds, `${jobId} must wait for scope and its applicable lane(s)`); - assert.match(block, /^ if: \$\{\{ always\(\) \}\}$/m); - assert.match(block, new RegExp(`TEST_RESULT: \\\$\\{\\{ needs\\.${testResultName}\\.result \\\}\\}`)); - assert.match(block, new RegExp(`BUILD_RESULT: \\\$\\{\\{ needs\\.${buildResultName}\\.result \\\}\\}`)); - const run = parseSteps(block).get(jobId === "check-macos" ? "Assert macOS CI lanes passed" : "Assert Windows CI lanes passed"); - assert.match(run, /test "\$TEST_RESULT" = success/); - assert.match(run, /test "\$BUILD_RESULT" = success/); - return run; -} +test("test-windows runs the build identity gates before npm ci and any Cargo work, from sources that need no npm packages", () => { + const steps = workflow.job("test-windows").steps; + const find = (predicate) => steps.findIndex((step) => commandLines(step).some(predicate)); + const identityCommand = + "node --test scripts/test-ci-build-identity.mjs scripts/test-windows-release-identity.mjs scripts/test-windows-package-workflow.mjs"; + const identity = find((line) => line.startsWith(identityCommand)); + const setup = steps.findIndex((step) => step.uses === "./.github/actions/setup"); + const install = find((line) => line === "npm ci"); + const gates = find((line) => line.startsWith("node scripts/ci/cargo-gates.mjs")); + assert.ok(setup >= 0 && identity > setup, "after Node is installed"); + assert.equal( + actionInput(loadAction("setup"), steps[setup], "npm-ci"), + "false", + "the action must not run npm ci ahead of the identity gates", + ); + assert.ok(install > identity && gates > install, "npm ci follows the identity gates; Cargo work follows npm ci"); + // They run before node_modules exists, so neither they nor anything they import, however indirectly, + // may need an npm package (the YAML reader of scripts/lib/ci-workflow.mjs is one). + const files = commandLines(steps[identity]) + .find((line) => line.startsWith("node --test ")) + .split(/\s+/) + .slice(2); + assert.ok( + files.length >= 4 && files.every((file) => /^scripts\/test-[\w-]+\.mjs$/.test(file)), + `the identity step runs test files: ${files}`, + ); + for (const file of files) { + const { packages } = importGraph(file, repoRoot); + assert.deepEqual([...packages], [], `${file} runs before npm ci but its imports reach npm packages`); + } +}); -function runAggregator(run, testResult, buildResult) { - execFileSync("bash", ["-euo", "pipefail", "-c", run], { - env: { - PATH: process.env.PATH, - SCOPE_RESULT: "success", - SCOPE_TRUSTED: "true", - DOCS_ONLY: "false", - TEST_RESULT: testResult, - BUILD_RESULT: buildResult, - }, - stdio: "pipe", - }); -} +test("windows-package.yml leaves the Rust tests of a pull request to test-windows and keeps them for releases and manual runs", () => { + const windows = loadWorkflow("windows-package.yml").jobs[0]; + const names = windows.steps.map((step) => step.name); + const tests = windows.steps[names.indexOf("Test and lint Rust workspace")]; + assert.equal(tests.condition, "github.event_name != 'pull_request'"); + // The debug lean CLI that the CPU policy checks read stays built: inside the step above where it runs, + // in a step of its own exactly where that step is skipped. + assert.ok(commandLines(tests).includes("cargo build -p sagascript-cli --no-default-features")); + const build = windows.steps[names.indexOf("Build lean CLI (debug)")]; + assert.ok(build, "the debug build has a step of its own for the skipped case"); + assert.equal(build.condition, "github.event_name == 'pull_request'"); + assert.deepEqual(commandLines(build), ["cargo build -p sagascript-cli --no-default-features"]); + assert.ok(names.indexOf(build.name) > names.indexOf(tests.name)); + assert.ok(names.indexOf(build.name) < names.indexOf("Verify debug x64 inference CPU policy")); + assert.ok(names.indexOf(build.name) < names.indexOf("Verify debug ARM64 native CPU flags")); + // test-windows runs the tests this one no longer repeats on a pull request. + assert.ok(gatesFor({ os: "windows" }).some((gate) => gate.args.join(" ") === "test --workspace")); + assert.ok(gatesFor({ os: "windows" }).some((gate) => gate.args.join(" ") === "clippy --workspace --all-targets -- -D warnings")); +}); -for (const [jobId, needs] of [ - ["check-macos", ["scope", "test-macos", "build-macos"]], - ["check-windows", ["scope", "test-windows", "build-windows"]], -]) { - test(`${jobId} fails closed for every non-success lane result`, () => { - const [, testLane, buildLane] = needs; - const run = aggregator(jobId, needs, testLane, buildLane); - for (const testResult of ["success", "failure", "cancelled", "skipped"]) { - for (const buildResult of ["success", "failure", "cancelled", "skipped"]) { - const invoke = () => runAggregator(run, testResult, buildResult); - if (testResult === "success" && buildResult === "success") { - assert.doesNotThrow(invoke); - } else { - assert.throws(invoke, `${jobId} must reject ${testResult}/${buildResult}`); - } - } +test("the workflow parses the same with CRLF line endings, as on a Windows checkout", () => { + const source = readFileSync(new URL("../.github/workflows/ci.yml", import.meta.url), "utf8").replace(/\r\n?/g, "\n"); + const lf = parseWorkflow(source); + const crlf = parseWorkflow(source.replaceAll("\n", "\r\n")); + assert.deepEqual(crlf.jobs, lf.jobs); + const classify = crlf.job("scope").steps.find((step) => step.id === "classify"); + assert.ok(!classify.run.includes("\r"), "scripts stay free of carriage returns"); +}); + +test("a multi-command step of a Windows lane runs under bash: pwsh reports only the last native command's exit code", () => { + // `npm run test:frontend` failing in front of a passing `npm run format:check` went green in a first draft of this file. + const windowsJobs = workflow.jobs.filter((job) => /^windows/.test(job.runsOn)); + assert.deepEqual(windowsJobs.map((job) => job.id).sort(), ["build-windows", "test-windows"]); + for (const job of windowsJobs) { + for (const step of job.steps.filter((candidate) => candidate.run !== "")) { + if (commandLines(step).length > 1) assert.equal(step.shell, "bash", `${job.id}: "${step.name}" runs several commands`); } - }); -} + } + for (const step of loadAction("setup").steps.filter((candidate) => candidate.run !== "")) { + assert.equal(step.shell, "bash", `the setup action's "${step.name}" must name its shell (and runs on Windows)`); + } +}); diff --git a/scripts/test-ci-smoke-cli.mjs b/scripts/test-ci-smoke-cli.mjs new file mode 100644 index 00000000..12ddca3d --- /dev/null +++ b/scripts/test-ci-smoke-cli.mjs @@ -0,0 +1,248 @@ +import assert from "node:assert/strict"; +import { spawnSync } from "node:child_process"; +import { chmodSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import test from "node:test"; +import { fileURLToPath } from "node:url"; +import { allSteps, commandLines, loadWorkflow } from "./lib/ci-workflow.mjs"; +import { COMPLETION_SHELLS, DEFAULT_BINARY, TIERS, runTier } from "./ci/smoke-cli.mjs"; + +// One smoke script for every OS (scripts/ci/smoke-cli.mjs, #358) replaces the inline bash and +// PowerShell blocks of the two native build lanes. These tests hold what it checks, how the +// lanes call it, and its behaviour against a fake binary. + +const SCRIPT = fileURLToPath(new URL("./ci/smoke-cli.mjs", import.meta.url)); + +// A harness that records the commands a tier runs instead of running anything. `language` plays +// the persisted setting `config set` / `config reset` change. +function recordingHarness() { + const calls = []; + let language = "en"; + return { + calls, + binary: "sagascript", + scratch: "/scratch", + assert(condition, message) { + if (!condition) throw new Error(message); + }, + checkBinary() { + calls.push("check binary"); + }, + runProgram(program, args) { + calls.push([program, ...args].join(" ")); + return { code: 0, stdout: "", stderr: "" }; + }, + ok(args) { + calls.push(args.join(" ")); + if (args[0] === "config" && args[1] === "set") language = args[3]; + if (args[0] === "config" && args[1] === "reset") language = "en"; + if (args[0] === "config" && args[1] === "get") return `${language}\n`; + if (args[0] === "transcribe") return "Stortinget har vedtatt\n"; + if (args[0] === "completions") return "# completion script\n"; + return ""; + }, + capture(label, args) { + calls.push(args.join(" ")); + return { code: args.at(-1).endsWith("missing.wav") ? 1 : 0, stdout: `${label}.stdout`, stderr: `${label}.stderr` }; + }, + verifyStreams(mode) { + calls.push(`verify ${mode.join(" ") || "--json"}`); + }, + audioDirectory: () => "/scratch/jsonl-inputs", + async readFirstLineThenClose(args) { + calls.push(`${args.join(" ")} | head -1`); + return { code: 0, sawLine: true }; + }, + }; +} + +const quiet = { log: () => {} }; + +test("the basic tier covers the binary, help, config, model listing, formats and completions of every shell", async () => { + const harness = recordingHarness(); + const failed = await runTier("basic", harness, quiet); + assert.deepEqual(failed, []); + assert.deepEqual(harness.calls, [ + "check binary", + "--version", + "--help", + "transcribe --help", + "record --help", + "config list", + "config get language", + "config set language sv", + "config get language", + "config reset language", + "config get language", + "config path", + "list-models", + "formats", + ...COMPLETION_SHELLS.map((shell) => `completions ${shell}`), + ]); + assert.deepEqual(COMPLETION_SHELLS, ["zsh", "bash", "powershell"], "completions for every shell on every OS"); +}); + +test("the model tier downloads, transcribes and verifies the JSON, JSONL and error streams of the output contract", async () => { + const harness = recordingHarness(); + const failed = await runTier("model", harness, quiet); + assert.deepEqual(failed, []); + const model = "--language no --model nb-whisper-tiny"; + // A Windows path joins with backslashes; the commands are compared in one spelling. + assert.deepEqual( + harness.calls.map((call) => call.replaceAll("\\", "/")), + [ + "download-model nb-whisper-tiny", + `transcribe ${model} test-audio/norwegian-short-3s.mp3`, + `transcribe ${model} --json test-audio/norwegian-short-3s.mp3`, + "verify --json", + `transcribe ${model} --jsonl /scratch/jsonl-inputs`, + "verify --jsonl", + `transcribe ${model} --jsonl /scratch/jsonl-inputs | head -1`, + `transcribe ${model} --json /scratch/missing.wav`, + "verify --error", + ], + ); +}); + +test("every check of a tier runs even after one fails, and each failure is reported by name", async () => { + const harness = recordingHarness(); + const original = harness.ok; + harness.ok = (args) => { + if (args[0] === "list-models") throw new Error("list-models exited with status 1"); + return original(args); + }; + const logged = []; + const failed = await runTier("basic", harness, { log: (line) => logged.push(line) }); + assert.deepEqual(failed, ["model listing and formats"]); + assert.ok(harness.calls.includes("completions powershell"), "later checks still ran"); + assert.ok(logged.some((line) => line.startsWith("::error title=Smoke test failed::model listing and formats"))); +}); + +test("the lanes run the smoke script after the native build, and only the Windows model tier is non-blocking", () => { + const workflow = loadWorkflow(); + for (const jobId of ["build-macos", "build-windows"]) { + const steps = workflow.job(jobId).steps; + const index = (predicate) => steps.findIndex((step) => commandLines(step).some(predicate)); + const build = index((line) => line.startsWith("npx tauri build")); + const basic = index((line) => line === "node scripts/ci/smoke-cli.mjs basic"); + const model = index((line) => line === "node scripts/ci/smoke-cli.mjs model"); + assert.ok(build >= 0 && basic > build && model >= basic, `${jobId}: build, then the basic tier, then the model tier`); + assert.equal(steps[basic].continueOnError, undefined, `${jobId}: the basic tier blocks`); + assert.equal(steps[model].continueOnError, jobId === "build-windows" ? true : undefined, `${jobId}: model tier tolerance`); + } + const tolerant = allSteps(workflow).filter(({ step }) => step.continueOnError !== undefined); + assert.deepEqual( + tolerant.map(({ job, step }) => `${job.id}:${step.id}`), + ["build-windows:smoke_model"], + "the Windows model tier is the only step that tolerates failure (#357)", + ); + assert.ok( + tolerant.every(({ job }) => job.continueOnError === undefined), + "no job tolerates failure", + ); +}); + +test("a masked Windows smoke failure is reported from the step's real outcome", () => { + const steps = loadWorkflow().job("build-windows").steps; + const model = steps.findIndex((step) => step.id === "smoke_model"); + const report = steps.findIndex((step) => step.name === "Report masked smoke test failures"); + assert.ok(report > model, "the report follows the tolerated step"); + assert.equal(steps[report].condition, "${{ always() }}"); + assert.equal(steps[report].env.MODEL_OUTCOME, "${{ steps.smoke_model.outcome }}"); + const lines = commandLines(steps[report]).join("\n"); + assert.ok(lines.includes("GITHUB_STEP_SUMMARY") && lines.includes("::warning "), "the summary and a warning annotation"); +}); + +// -- the script against a fake binary ---------------------------------------------------------- + +// A stand-in for `sagascript`: keeps `language` in a file, prints what the real CLI prints in the +// places the checks look. FAKE_FAIL names a behaviour to break. +const FAKE_BINARY = `#!/usr/bin/env node +const fs = require("node:fs"); +const path = require("node:path"); +const state = path.join(process.env.FAKE_STATE, "language"); +const args = process.argv.slice(2); +const fail = process.env.FAKE_FAIL; +const read = () => (fs.existsSync(state) ? fs.readFileSync(state, "utf8") : "en"); +const [command, sub] = args; +if (command === "config" && sub === "set") fs.writeFileSync(state, args[3]); +else if (command === "config" && sub === "reset") fs.rmSync(state, { force: true }); +else if (command === "config" && sub === "get") console.log(fail === "config" ? "xx" : read()); +else if (command === "completions") process.stdout.write(fail === "completions" ? "" : "# " + sub + " completion\\n"); +else if (command === "transcribe" && args.includes("--help")) console.log("transcribe help"); +else if (command === "record" && args.includes("--help")) console.log("record help"); +else if (command === "transcribe") { + const input = args.at(-1); + if (input.endsWith("missing.wav")) { + console.log(JSON.stringify({ status: "error", code: "file_decode_error", error: "no such file" })); + console.error("Error: Transcription failed"); + process.exit(1); + } else if (args.includes("--jsonl")) { + for (const name of ["a", "b"]) console.log(JSON.stringify({ status: "ok", source: name + ".mp3", result: { text: "Stortinget " + name } })); + } else if (args.includes("--json")) { + console.log(JSON.stringify({ text: "Stortinget har vedtatt", language: "no" })); + } else { + console.log("Stortinget har vedtatt"); + } +} else if (!["--version", "--help", "list-models", "formats", "config", "download-model"].includes(command)) { + process.exit(2); +} else if (command === "config" && !["list", "path"].includes(sub)) { + process.exit(2); +} +`; + +const python = spawnSync(process.platform === "win32" ? "python" : "python3", ["--version"]).status === 0; + +function withFakeBinary(callback) { + const directory = mkdtempSync(join(tmpdir(), "sagascript-smoke-test-")); + try { + const binary = join(directory, "sagascript"); + writeFileSync(binary, FAKE_BINARY); + chmodSync(binary, 0o755); + return callback(binary, directory); + } finally { + rmSync(directory, { recursive: true, force: true }); + } +} + +function runScript(tier, binary, directory, env = {}) { + return spawnSync(process.execPath, [SCRIPT, tier, "--binary", binary], { + encoding: "utf8", + env: { ...process.env, FAKE_STATE: directory, RUNNER_TEMP: directory, ...env }, + }); +} + +test("both tiers pass against a fake binary that keeps the contract", { skip: process.platform === "win32" || !python }, () => { + withFakeBinary((binary, directory) => { + for (const tier of Object.keys(TIERS)) { + const result = runScript(tier, binary, directory); + assert.equal(result.status, 0, `${tier}: ${result.stdout}${result.stderr}`); + } + }); +}); + +test( + "the script exits 1 and names the check when the binary breaks the contract", + { skip: process.platform === "win32" || !python }, + () => { + withFakeBinary((binary, directory) => { + for (const [fault, tier, check] of [ + ["config", "basic", "config management"], + ["completions", "basic", "shell completions"], + ]) { + const result = runScript(tier, binary, directory, { FAKE_FAIL: fault }); + assert.equal(result.status, 1, `${fault}: ${result.stdout}${result.stderr}`); + assert.match(result.stdout, new RegExp(`::error title=Smoke test failed::${check}`)); + } + const missing = runScript("basic", join(directory, "no-such-binary"), directory); + assert.equal(missing.status, 1); + assert.match(missing.stdout, /the native release binary exists and is executable/); + assert.equal(runScript("nonsense", binary, directory).status, 2); + }); + }, +); + +test("the default binary is the native release build the lanes just made", () => { + assert.equal(DEFAULT_BINARY.replaceAll("\\", "/").replace(/\.exe$/, ""), "src-tauri/target/release/sagascript"); +}); diff --git a/scripts/test-ci-unrun-suites.mjs b/scripts/test-ci-unrun-suites.mjs index 9febc4ba..a4e0f4bb 100644 --- a/scripts/test-ci-unrun-suites.mjs +++ b/scripts/test-ci-unrun-suites.mjs @@ -1,69 +1,86 @@ import assert from "node:assert/strict"; -import { readFile } from "node:fs/promises"; +import { readFileSync, readdirSync } from "node:fs"; import test from "node:test"; +import { fileURLToPath } from "node:url"; +import { commandOf, gatesFor } from "./ci/cargo-gates.mjs"; +import { actionInput, commandLines, loadAction, loadWorkflow, readRepoFile } from "./lib/ci-workflow.mjs"; -// #357: suites that used to run in no PR lane (or only in the release gate) -// are now part of PR CI. These assertions keep them there. +// #357: suites that used to run in no PR lane (or only in the release gate) are now part of PR CI. +// These assertions keep them there. The gate list (scripts/ci/cargo-gates.mjs) says which cargo +// commands each OS runs; the workflow says which lane runs the list, and when. -const read = async (path) => (await readFile(new URL(`../${path}`, import.meta.url), "utf8")).replace(/\r\n?/g, "\n"); +const workflow = loadWorkflow(); +const action = loadAction("setup"); +const devOverrides = readRepoFile("src-tauri/crates/sagascript-core/src/transcription/dev_overrides.rs"); +const workspaceManifest = readRepoFile("src-tauri/Cargo.toml"); +const ortManifest = readRepoFile("src-tauri/engine-host/ort/Cargo.toml"); -const [workflow, devOverrides, workspaceManifest, ortManifest] = await Promise.all([ - read(".github/workflows/ci.yml"), - read("src-tauri/crates/sagascript-core/src/transcription/dev_overrides.rs"), - read("src-tauri/Cargo.toml"), - read("src-tauri/engine-host/ort/Cargo.toml"), -]); - -function jobBlock(jobId) { - const start = workflow.indexOf(`\n ${jobId}:\n`); - assert.notEqual(start, -1, `workflow must define ${jobId}`); - const body = workflow.slice(start + 1); - const next = body.slice(1).search(/^ [a-z][a-z0-9-]*:$/m); - return next === -1 ? body : body.slice(0, next + 1); -} - -// Each step as { name, raw, run, workingDirectory }, in workflow order. -function steps(jobId) { - return jobBlock(jobId) - .split(/\n(?= - name: )/) - .slice(1) - .map((raw) => { - const name = /^ - name: (.*)$/m.exec(raw)[1]; - const inline = /^ run: (?![|>])(.*)$/m.exec(raw)?.[1]; - const block = /^ run: [|>]-?\n((?: .*\n?|\n)*)/m - .exec(raw)?.[1] - ?.split("\n") - .map((line) => line.replace(/^ /, "")) - .join("\n"); - const workingDirectory = /^ working-directory: (.*)$/m.exec(raw)?.[1]; - return { name, raw, run: (inline ?? block ?? "").trim(), workingDirectory }; - }); +const gateLines = (os) => gatesFor({ os }).map((gate) => `${gate.cwd}: ${commandOf(gate)}`); +const stepIndex = (jobId, predicate) => workflow.job(jobId).steps.findIndex(predicate); +const runs = (command) => (step) => commandLines(step).includes(command); +const gatesStep = (os) => (step) => commandLines(step).includes(`node scripts/ci/cargo-gates.mjs --os ${os}`); +// The step that runs `npm ci`: the setup action when it is asked to, otherwise an explicit step. +function npmInstall(jobId) { + return stepIndex( + jobId, + (step) => (step.uses === "./.github/actions/setup" && actionInput(action, step, "npm-ci") === "true") || runs("npm ci")(step), + ); } - -function stepRunning(jobId, command, workingDirectory) { - const matches = steps(jobId).filter((step) => step.run === command); - assert.equal(matches.length, 1, `${jobId} must run \`${command}\` exactly once as its own step`); - const [step] = matches; - assert.equal(step.workingDirectory, workingDirectory, `\`${command}\` must run in ${workingDirectory}`); - assert.doesNotMatch(step.raw, /^ (if|continue-on-error):/m, `\`${command}\` must be unconditional and blocking`); - return step; +function assertBlocking(jobId, index, what) { + const step = workflow.job(jobId).steps[index]; + assert.ok(step, `${jobId} must have ${what}`); + assert.equal(step.condition, undefined, `${what} must be unconditional`); + assert.equal(step.continueOnError, undefined, `${what} must be blocking`); } -test("test-linux runs the engine-protocol wire-format tests", () => { - stepRunning("test-linux", "cargo test -p sagascript-engine-protocol", "src-tauri"); +test("every lane runs the engine-protocol wire-format tests through the workspace tests", () => { + const members = /^members = \[(.*)\]$/m.exec(workspaceManifest)?.[1] ?? ""; + assert.ok(members.includes('"crates/*"'), "the workspace takes every crate under crates/"); + assert.ok( + readdirSync(fileURLToPath(new URL("../src-tauri/crates/", import.meta.url))).includes("sagascript-engine-protocol"), + "the protocol crate is one of them", + ); + for (const os of ["macos", "linux", "windows"]) { + assert.ok( + gateLines(os).includes("src-tauri: cargo test --workspace"), + `${os} runs cargo test --workspace, which reaches the protocol crate`, + ); + } + for (const [jobId, os] of [ + ["test-macos", "macos"], + ["test-linux", "linux"], + ["test-windows", "windows"], + ]) { + assertBlocking(jobId, stepIndex(jobId, gatesStep(os)), "the cargo gates"); + } }); -test("test-linux tests and lints the standalone ORT engine host workspace", () => { +test("test-linux and test-windows test and lint the standalone ORT engine host workspace", () => { assert.match(ortManifest, /^\[workspace\]$/m, "the ORT host is its own Cargo workspace"); - stepRunning("test-linux", "cargo test --locked", "src-tauri/engine-host/ort"); - stepRunning("test-linux", "cargo clippy --all-targets --locked -- -D warnings", "src-tauri/engine-host/ort"); - const fmt = steps("test-linux").find((step) => step.name === "Check Rust formatting"); - assert.ok(fmt, "test-linux must keep its rustfmt gate"); - assert.match(fmt.run, /^cargo fmt --manifest-path engine-host\/ort\/Cargo\.toml --all -- --check$/m); - - const cache = steps("test-linux").find((step) => step.name === "Cache Rust dependencies"); - assert.ok(cache, "test-linux must keep its Rust cache"); - assert.match(cache.raw, /^ workspaces: \|\n src-tauri\n src-tauri\/engine-host\/ort\n/m); + for (const os of ["linux", "windows"]) { + const lines = gateLines(os); + assert.ok(lines.includes("src-tauri/engine-host/ort: cargo test --locked"), `${os} tests the ORT host`); + assert.ok(lines.includes("src-tauri/engine-host/ort: cargo clippy --all-targets --locked -- -D warnings"), `${os} lints the ORT host`); + } + assert.ok( + !gateLines("macos").some((line) => line.startsWith("src-tauri/engine-host/ort")), + "macOS ships the Core ML host, not the ORT host", + ); + assert.ok( + gateLines("linux").includes("src-tauri: cargo fmt --manifest-path engine-host/ort/Cargo.toml --all -- --check"), + "test-linux must keep its rustfmt gate for the ORT host", + ); + for (const jobId of ["test-linux", "test-windows"]) { + const setup = workflow.job(jobId).steps.find((step) => step.uses === "./.github/actions/setup"); + assert.deepEqual( + actionInput(action, setup, "cache-workspaces") + .split("\n") + .map((line) => line.trim()) + .filter(Boolean), + ["src-tauri", "src-tauri/engine-host/ort"], + `${jobId} must cache both Cargo workspaces`, + ); + } }); test("test-linux runs the release-only dev-override guard in the release profile", () => { @@ -75,52 +92,78 @@ test("test-linux runs the release-only dev-override guard in the release profile // ...and its doc comment names the command CI must run. const command = "cargo test --release -p sagascript-core --lib dev_overrides"; assert.ok(devOverrides.includes(`(\`${command}\`)`), "the test's doc comment must name the CI command"); - const guard = steps("test-linux").filter((step) => step.run.split("\n").some((line) => line.startsWith(command))); - assert.equal(guard.length, 1, "test-linux must run the release guard exactly once"); - assert.equal(guard[0].workingDirectory, "src-tauri"); - assert.doesNotMatch(guard[0].raw, /^ (if|continue-on-error):/m, "the release guard must be unconditional and blocking"); - // cargo test exits 0 when the filter matches nothing; the step must prove - // the guard test itself ran, and must not lose cargo's exit code in a pipe. - assert.match(guard[0].run, /^set -o pipefail$/m); - assert.match(guard[0].run, /^grep -q 'release_build_ignores_the_real_environment \.\.\. ok' /m); + const guards = gatesFor({ os: "linux" }).filter((gate) => commandOf(gate) === command); + assert.equal(guards.length, 1, "test-linux must run the release guard exactly once"); + const [guard] = guards; + assert.equal(guard.cwd, "src-tauri"); + // cargo test exits 0 when the filter matches nothing; the gate must prove the guard test itself ran. + assert.ok( + guard.requireOutput?.pattern.test("release_build_ignores_the_real_environment ... ok"), + "the gate must require the guard's own pass line", + ); + assert.ok(!guard.requireOutput.pattern.test("running 0 tests")); + assert.ok( + ["macos", "windows"].every((os) => !gateLines(os).some((line) => line.includes("dev_overrides"))), + "the guard compiles core in the release profile once, on Linux", + ); // A release profile with debug assertions would compile the guard out and // leave the step green with zero tests. const release = /^\[profile\.release\]\n([\s\S]*?)(?=^\[|(?![\s\S]))/m.exec(workspaceManifest)?.[1]; assert.ok(release !== undefined, "src-tauri/Cargo.toml must define [profile.release]"); assert.doesNotMatch(release, /^debug-assertions\s*=\s*true/m); - assert.doesNotMatch(workflow, /CARGO_PROFILE_RELEASE_DEBUG_ASSERTIONS/); + const environments = [ + workflow.doc.env, + action.doc.runs.env, + ...workflow.jobs.flatMap((job) => [job.env, ...job.steps.map((step) => step.env)]), + ...action.steps.map((step) => step.env), + ]; + for (const environment of environments) { + assert.ok( + !Object.keys(environment ?? {}).includes("CARGO_PROFILE_RELEASE_DEBUG_ASSERTIONS"), + "no lane may switch debug assertions on in release", + ); + } + for (const name of readdirSync(fileURLToPath(new URL("../.github/workflows/", import.meta.url)))) { + assert.ok( + !readFileSync(new URL(`../.github/workflows/${name}`, import.meta.url), "utf8").includes("CARGO_PROFILE_RELEASE_DEBUG_ASSERTIONS"), + name, + ); + } }); -test("test-windows runs the full frontend suite after npm ci", () => { - const names = steps("test-windows").map((step) => step.name); - const suite = stepRunning("test-windows", "npm run test:frontend", undefined); - const index = names.indexOf(suite.name); - assert.ok(names.indexOf("Install npm dependencies") >= 0, "test-windows must install npm dependencies"); - assert.ok(index > names.indexOf("Install npm dependencies"), "the suite needs node_modules"); - assert.ok(index < names.indexOf("Cargo check"), "the suite must fail before Cargo work"); - const identity = names.indexOf("Test Windows build identity gates"); - assert.ok(identity >= 0 && identity < names.indexOf("Install npm dependencies"), "the early build identity gates stay ahead of npm ci"); +test("test-windows runs the full frontend suite after npm ci and before Cargo work", () => { + const suite = stepIndex("test-windows", runs("npm run test:frontend")); + const install = npmInstall("test-windows"); + assert.ok(install >= 0, "test-windows must install npm dependencies"); + assert.ok(suite > install, "the suite needs node_modules"); + assert.ok(suite < stepIndex("test-windows", gatesStep("windows")), "the suite must fail before Cargo work"); + assertBlocking("test-windows", suite, "`npm run test:frontend`"); + const identity = stepIndex("test-windows", (step) => + commandLines(step).some((line) => line.startsWith("node --test scripts/test-ci-build-identity.mjs")), + ); + assert.ok(identity >= 0 && identity < install, "the early build identity gates stay ahead of npm ci"); }); test("test-macos runs ESLint after npm ci and before Cargo work", () => { - const names = steps("test-macos").map((step) => step.name); - const step = stepRunning("test-macos", "npm run lint", undefined); - const index = names.indexOf(step.name); - assert.ok(index > names.indexOf("Install npm dependencies"), "`npm run lint` needs node_modules"); - assert.ok(index < names.indexOf("Cargo check"), "`npm run lint` must fail before Cargo work"); + const lint = stepIndex("test-macos", runs("npm run lint")); + assert.ok(lint > npmInstall("test-macos"), "`npm run lint` needs node_modules"); + assert.ok(lint < stepIndex("test-macos", gatesStep("macos")), "`npm run lint` must fail before Cargo work"); + assertBlocking("test-macos", lint, "`npm run lint`"); }); test("test-macos and test-windows run the Vitest unit tests and the Prettier check after npm ci", () => { - for (const job of ["test-macos", "test-windows"]) { - const names = steps(job).map((step) => step.name); - const install = names.indexOf("Install npm dependencies"); - assert.ok(install >= 0, `${job} must install npm dependencies`); + for (const [jobId, os] of [ + ["test-macos", "macos"], + ["test-windows", "windows"], + ]) { + const install = npmInstall(jobId); + assert.ok(install >= 0, `${jobId} must install npm dependencies`); for (const command of ["npm run test:unit", "npm run format:check"]) { - const step = stepRunning(job, command, undefined); - const index = names.indexOf(step.name); - assert.ok(index > install, `${job}: \`${command}\` needs node_modules`); - assert.ok(index < names.indexOf("Cargo check"), `${job}: \`${command}\` must fail before Cargo work`); + const index = stepIndex(jobId, runs(command)); + assert.ok(index > install, `${jobId}: \`${command}\` needs node_modules`); + assert.ok(index < stepIndex(jobId, gatesStep(os)), `${jobId}: \`${command}\` must fail before Cargo work`); + assertBlocking(jobId, index, `\`${command}\``); } } }); diff --git a/scripts/test-clippy-policy.mjs b/scripts/test-clippy-policy.mjs index 55a0c6db..c9417651 100644 --- a/scripts/test-clippy-policy.mjs +++ b/scripts/test-clippy-policy.mjs @@ -8,6 +8,8 @@ import { join } from "node:path"; import test from "node:test"; import { fileURLToPath } from "node:url"; +import { gatesFor } from "./ci/cargo-gates.mjs"; +import { commandLines, loadWorkflow } from "./lib/ci-workflow.mjs"; import { POLICY_LINTS, RUNS, @@ -409,17 +411,25 @@ test("clippy.toml exempts tests and sets the size thresholds", () => { }); test("test-linux runs the policy check after the clippy gates", () => { - const ci = read(".github/workflows/ci.yml"); - const start = ci.indexOf("\n test-linux:\n"); - const end = ci.indexOf("\n msrv-linux:\n"); - assert.ok(start >= 0 && end > start, "ci.yml must define test-linux before msrv-linux"); - const job = ci.slice(start, end); - const steps = job.split(/\n(?= - name: )/).slice(1); - const names = steps.map((step) => /^ - name: (.*)$/m.exec(step)[1]); - const index = steps.findIndex((step) => /^ run: node scripts\/clippy-policy\.mjs$/m.test(step)); - assert.ok(index >= 0, "test-linux must run `node scripts/clippy-policy.mjs` as its own step"); - assert.doesNotMatch(steps[index], /^ (if|continue-on-error|working-directory):/m); - assert.ok(index > names.indexOf("Install Node.js"), "the check needs Node.js"); - assert.ok(index > names.indexOf("Cargo clippy (GUI app)"), "the regular clippy gate runs first"); - assert.ok(index > names.indexOf("Cargo clippy (headless CLI, batch-only)"), "the regular clippy gate runs first"); + const steps = loadWorkflow().job("test-linux").steps; + const index = steps.findIndex((step) => commandLines(step).includes("node scripts/clippy-policy.mjs")); + assert.ok(index >= 0, "test-linux must run `node scripts/clippy-policy.mjs`"); + assert.deepEqual(commandLines(steps[index]), ["node scripts/clippy-policy.mjs"], "as its own step"); + assert.equal(steps[index].condition, undefined); + assert.equal(steps[index].continueOnError, undefined); + assert.equal(steps[index].workingDirectory, undefined); + const setup = steps.findIndex((step) => step.uses === "./.github/actions/setup"); + assert.ok(index > setup, "the check needs Node.js, which the setup action installs"); + // The regular clippy gates run first: they compile the dependencies the policy run reuses. + const gates = steps.findIndex((step) => commandLines(step).includes("node scripts/ci/cargo-gates.mjs --os linux")); + assert.ok(index > gates, "the regular clippy gates run first"); + const clippy = gatesFor({ os: "linux" }).filter((gate) => gate.args[0] === "clippy"); + assert.ok( + clippy.some((gate) => gate.args.includes("--workspace")), + "the workspace clippy gate", + ); + assert.ok( + clippy.some((gate) => gate.args.includes("--no-default-features") && gate.args.includes("sagascript-cli")), + "the lean CLI clippy gate", + ); }); diff --git a/scripts/test-dependency-policy.mjs b/scripts/test-dependency-policy.mjs index 80352628..c3406bad 100644 --- a/scripts/test-dependency-policy.mjs +++ b/scripts/test-dependency-policy.mjs @@ -6,11 +6,12 @@ import assert from "node:assert/strict"; import { readdirSync } from "node:fs"; import { readFile } from "node:fs/promises"; import test from "node:test"; +import { commandOf, gatesFor } from "./ci/cargo-gates.mjs"; +import { commandLines, loadWorkflow } from "./lib/ci-workflow.mjs"; const read = async (path) => (await readFile(new URL(`../${path}`, import.meta.url), "utf8")).replace(/\r\n?/g, "\n"); -const [workflow, deny, notices, ...manifests] = await Promise.all([ - read(".github/workflows/ci.yml"), +const [deny, notices, ...manifests] = await Promise.all([ read("src-tauri/deny.toml"), read("scripts/generate-third-party-notices.mjs"), ...[ @@ -20,40 +21,13 @@ const [workflow, deny, notices, ...manifests] = await Promise.all([ ].map(read), ]); -function jobBlock(jobId) { - const start = workflow.indexOf(`\n ${jobId}:\n`); - assert.notEqual(start, -1, `workflow must define ${jobId}`); - const body = workflow.slice(start + 1); - const next = body.slice(1).search(/^ [a-z][a-z0-9-]*:$/m); - return next === -1 ? body : body.slice(0, next + 1); -} - -// Each step as { name, raw, commands, workingDirectory }, in workflow order. -function steps(jobId) { - return jobBlock(jobId) - .split(/\n(?= - name: )/) - .slice(1) - .map((raw) => { - const name = /^ - name: (.*)$/m.exec(raw)[1]; - const inline = /^ run: (?![|>])(.*)$/m.exec(raw)?.[1]; - const block = /^ run: [|>]-?\n((?: .*\n?|\n)*)/m.exec(raw)?.[1]; - const commands = (inline ?? block ?? "") - .split("\n") - .map((line) => line.trim()) - .filter(Boolean); - const workingDirectory = /^ working-directory: (.*)$/m.exec(raw)?.[1]; - return { name, raw, commands, workingDirectory }; - }); -} - -function blockingStep(jobId, command, workingDirectory) { - const all = steps(jobId); - const matches = all.filter((step) => step.commands.includes(command)); - assert.equal(matches.length, 1, `${jobId} must run \`${command}\` exactly once`); - const [step] = matches; - assert.equal(step.workingDirectory, workingDirectory, `\`${command}\` must run in ${workingDirectory ?? "the repository root"}`); - assert.doesNotMatch(step.raw, /^ (if|continue-on-error):/m, `\`${command}\` must be unconditional and blocking`); - return all.indexOf(step); +const workflow = loadWorkflow(); +const stepIndex = (jobId, predicate) => workflow.job(jobId).steps.findIndex(predicate); +const runs = (command) => (step) => commandLines(step).includes(command); +// A step that gates unconditionally: no `if`, no `continue-on-error`. +function assertBlocking(step, what) { + assert.equal(step.condition, undefined, `${what} must be unconditional`); + assert.equal(step.continueOnError, undefined, `${what} must be blocking`); } // The string items of a TOML array `key = [ ... ]` inside `[section]`. @@ -66,37 +40,50 @@ function tomlStrings(text, section, key) { } test("test-linux gates both Cargo workspaces on cargo-deny before compiling", () => { - const names = steps("test-linux").map((step) => step.name); - const install = blockingStep("test-linux", "cargo install --locked cargo-deny --version 0.20.2", undefined); - const main = blockingStep("test-linux", "cargo deny --locked --workspace check", "src-tauri"); + const INSTALL = "cargo install --locked cargo-deny --version 0.20.2"; + const linux = gatesFor({ os: "linux" }); // The ORT host run may only silence the "policy entry not used here" notes, // not skip a check or allow a finding. assert.deepEqual( - steps("test-linux")[main].commands, + linux.slice(0, 2).map((gate) => [gate.cwd, commandOf(gate)]), [ - "cargo deny --locked --workspace check", - "cargo deny --locked --manifest-path engine-host/ort/Cargo.toml --workspace check -A license-not-encountered -A license-exception-not-encountered -A unmatched-source", + ["src-tauri", "cargo deny --locked --workspace check"], + [ + "src-tauri", + "cargo deny --locked --manifest-path engine-host/ort/Cargo.toml --workspace check -A license-not-encountered -A license-exception-not-encountered -A unmatched-source", + ], ], - "the cargo-deny step must check both workspaces, every check kind, against the committed lockfiles", + "the cargo-deny gates must check both workspaces, every check kind, against the committed lockfiles, ahead of every gate that compiles", + ); + assert.ok( + linux.slice(2).every((gate) => gate.args[0] !== "deny"), + "no other gate runs cargo-deny", ); - assert.ok(install < main, "cargo-deny must be installed before it runs"); - assert.ok(main > names.indexOf("Cache Rust dependencies"), "the cached cargo-deny binary must be restored first"); - assert.ok(main < names.indexOf("Cargo check (GUI app)"), "the policy must fail before any compilation"); + const steps = workflow.job("test-linux").steps; + const setup = stepIndex("test-linux", (step) => step.uses === "./.github/actions/setup"); + const install = stepIndex("test-linux", runs(INSTALL)); + const gates = stepIndex("test-linux", (step) => commandLines(step).includes("node scripts/ci/cargo-gates.mjs --os linux")); + assert.ok(setup >= 0 && install >= 0 && gates >= 0); + assertBlocking(steps[install], "the cargo-deny install"); + assertBlocking(steps[gates], "the cargo gates"); + assert.ok(setup < install, "the cached cargo-deny binary must be restored first"); + assert.ok(install < gates, "cargo-deny must be installed before it runs"); }); test("npm run verify runs the same cargo-deny release CI installs", async () => { const { CARGO_DENY_VERSION } = await import("./verify.mjs"); assert.ok( - steps("test-linux").some((step) => step.commands.includes(`cargo install --locked cargo-deny --version ${CARGO_DENY_VERSION}`)), + workflow.job("test-linux").steps.some(runs(`cargo install --locked cargo-deny --version ${CARGO_DENY_VERSION}`)), `verify's preflight asks for cargo-deny ${CARGO_DENY_VERSION}; test-linux must install the same release`, ); }); test("test-macos gates the npm lockfile on npm audit", () => { - const names = steps("test-macos").map((step) => step.name); - const audit = blockingStep("test-macos", "npm audit --audit-level=high", undefined); - assert.ok(audit > names.indexOf("Install npm dependencies"), "audit after npm ci"); - assert.ok(audit < names.indexOf("Build frontend"), "audit before the bundle is built"); + const audit = stepIndex("test-macos", runs("npm audit --audit-level=high")); + assert.ok(audit >= 0, "test-macos must run npm audit"); + assertBlocking(workflow.job("test-macos").steps[audit], "`npm audit --audit-level=high`"); + assert.ok(audit > stepIndex("test-macos", (step) => step.uses === "./.github/actions/setup"), "audit after npm ci (the setup action)"); + assert.ok(audit < stepIndex("test-macos", runs("npm run build")), "audit before the bundle is built"); }); // SPDX expression satisfiability: AND binds tighter than OR, the legacy "/" diff --git a/scripts/test-frontend-test-groups.mjs b/scripts/test-frontend-test-groups.mjs index 3957b807..045283b0 100644 --- a/scripts/test-frontend-test-groups.mjs +++ b/scripts/test-frontend-test-groups.mjs @@ -5,6 +5,7 @@ import { tmpdir } from "node:os"; import { delimiter, dirname, join } from "node:path"; import { fileURLToPath } from "node:url"; import test from "node:test"; +import { importGraph } from "./lib/import-graph.mjs"; import { isProcessAlive } from "./lib/process-tree.mjs"; import { GROUP_NPM_PACKAGES, TEST_GROUPS } from "./node-test-groups.mjs"; import { classificationProblems, discoverTestFiles } from "./run-frontend-tests.mjs"; @@ -90,22 +91,24 @@ test("the real manifest classifies every Node test file exactly once", () => { }); test("each group declares the npm packages its tests import", () => { - // Static, side-effect and dynamic imports of a bare package specifier. - const bareImport = /\bfrom\s*["']([@a-z][\w@./:-]*)["']|\bimport\s*\(?\s*["']([@a-z][\w@./:-]*)["']/g; + // A test needs the packages of the repository modules it imports as well (scripts/lib/ci-workflow.mjs + // reads the workflows with `yaml`, #358), so relative imports are followed. for (const [group, files] of Object.entries(TEST_GROUPS)) { - const imported = new Set(); - for (const file of files) { - const source = readFileSync(join(repoRoot, file), "utf8"); - for (const match of source.matchAll(bareImport)) { - const specifier = match[1] ?? match[2]; - if (specifier.startsWith("node:")) continue; - imported.add(specifier.startsWith("@") ? specifier.split("/").slice(0, 2).join("/") : specifier.split("/")[0]); - } - } + const imported = new Set(files.flatMap((file) => [...importGraph(file, repoRoot).packages])); assert.deepEqual([...imported].sort(), [...(GROUP_NPM_PACKAGES[group] ?? [])].sort(), `group ${group}`); } }); +test("the import graph follows relative imports through any depth, in every spelling, and ignores comments", () => { + // scripts/fixtures/import-graph: entry -> lib/one -> lib/two (dynamic import) -> lib/three (side-effect imports). + const root = join(repoRoot, "scripts/fixtures/import-graph"); + const { packages, files } = importGraph("entry.mjs", root); + assert.deepEqual([...packages].sort(), ["@scope/pkg", "yaml"]); + assert.deepEqual([...files].sort(), ["entry.mjs", "lib/one.mjs", "lib/three.mjs", "lib/two.mjs"]); + // A cycle terminates, and a file with no package imports reaches none. + assert.deepEqual([...importGraph("cycle-b.mjs", root).packages], []); +}); + test("npm run test:code-health runs exactly the codeHealth group from scripts/", () => { const script = JSON.parse(readFileSync(join(repoRoot, "package.json"), "utf8")).scripts["test:code-health"]; assert.deepEqual(script.match(/(? readFileSync(fileURLToPath(new URL(`../${path}`, import.meta.url)), "utf8"); const exists = (path) => existsSync(fileURLToPath(new URL(`../${path}`, import.meta.url))); @@ -74,10 +75,23 @@ test("release verifier checks the bundled host", () => { }); test("CI builds the host, runs swift test, and checks the revision", () => { - const content = read(".github/workflows/ci.yml"); - assert.match(content, /scripts\/stage-engine-host\.sh/); - assert.match(content, /swift test/); - assert.match(content, /check-engine-host-identity\.sh[^\n]*GITHUB_SHA/); + const steps = loadWorkflow().job("test-macos").steps; + const stepWith = (predicate) => steps.findIndex((step) => commandLines(step).some(predicate)); + const stage = stepWith((line) => line === "./scripts/stage-engine-host.sh"); + const identity = stepWith( + (line) => line === './scripts/check-engine-host-identity.sh build/engine-host/sagascript-engine-host "$GITHUB_SHA"', + ); + const swift = stepWith((line) => line === "swift test"); + for (const [what, index] of [ + ["stages the host", stage], + ["checks its revision against GITHUB_SHA", identity], + ["runs swift test", swift], + ]) { + assert.ok(index >= 0, `test-macos ${what}`); + assert.equal(steps[index].continueOnError, undefined, `test-macos ${what} and blocks on it`); + } + assert.ok(stage <= identity, "the host is staged before its revision is checked"); + assert.equal(steps[swift].workingDirectory, "src-tauri/engine-host/coreml"); }); test("Pianissimo smoke covers status, download, doctor and transcribe", () => { diff --git a/scripts/test-pipeline-cache.mjs b/scripts/test-pipeline-cache.mjs index 594f15ec..2b73c5ac 100644 --- a/scripts/test-pipeline-cache.mjs +++ b/scripts/test-pipeline-cache.mjs @@ -1,11 +1,11 @@ import assert from "node:assert/strict"; import { readFile } from "node:fs/promises"; import test from "node:test"; +import { allSteps, loadAction, loadWorkflow } from "./lib/ci-workflow.mjs"; const rustCacheRef = "Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6"; -const [ciWorkflow, testBuildWorkflow, windowsWorkflow] = await Promise.all([ - readFile(new URL("../.github/workflows/ci.yml", import.meta.url), "utf8"), +const [testBuildWorkflow, windowsWorkflow] = await Promise.all([ readFile(new URL("../.github/workflows/test-build.yml", import.meta.url), "utf8"), readFile(new URL("../.github/workflows/windows-package.yml", import.meta.url), "utf8"), ]); @@ -18,12 +18,15 @@ function section(content, startMarker, endMarker) { } test("CI rust caches use the reviewed immutable action revision", () => { - const refs = ciWorkflow.match(/uses: Swatinem\/rust-cache@[^\s]+/g) ?? []; - // test-macos, build-macos, test-linux, msrv-linux (#353), test-windows, build-windows - assert.equal(refs.length, 6); + const cacheUses = (steps) => steps.map((step) => step.uses).filter((uses) => uses?.startsWith("Swatinem/rust-cache@")); + // The cache is part of the setup action (#358), the one place that names the revision; ci.yml's + // six lanes (test-macos, build-macos, test-linux, msrv-linux #353, test-windows, build-windows) + // reach it through the action, which scripts/test-ci-parallel-gates.mjs checks lane by lane. + assert.deepEqual(cacheUses(loadAction("setup").steps), [rustCacheRef]); assert.deepEqual( - refs, - Array.from({ length: 6 }, () => `uses: ${rustCacheRef}`), + cacheUses(allSteps(loadWorkflow()).map(({ step }) => step)), + [], + "a ci.yml lane that names its own rust-cache would bypass the shared setup", ); }); diff --git a/scripts/test-release-promotion.mjs b/scripts/test-release-promotion.mjs index 330bf823..c50ba398 100644 --- a/scripts/test-release-promotion.mjs +++ b/scripts/test-release-promotion.mjs @@ -97,7 +97,17 @@ test("windows-package keeps the console CLI, installed-CLI and portable zip step assert.match(w, /Verify silent per-user install runs the installed ARM64 CLI/); assert.match(w, /Build and verify ARM64 portable zip/); assert.match(w, /Verify installers carry the console CLI/); - assert.doesNotMatch(w, /if:[^\n]*github\.event_name/); + // The steps that depend on the event are the Rust workspace test and the debug build that stands in + // for it (#358): ci.yml's test-windows owns the tests of a pull request, so this workflow skips them + // there. Releases come in through workflow_call from a push or a dispatch, never a pull request, + // and still run them. + const conditional = w + .split("\n - name: ") + .slice(1) + .filter((step) => /if:[^\n]*github\.event_name/.test(step)) + .map((step) => step.split("\n")[0]); + assert.deepEqual(conditional, ["Test and lint Rust workspace", "Build lean CLI (debug)"]); + assert.match(w, /if: github\.event_name != 'pull_request'\n/); }); test("prebuild runs on main pushes only, never on pull requests, and never cancels", () => { diff --git a/scripts/test-rust-toolchain-pin.mjs b/scripts/test-rust-toolchain-pin.mjs index e8a8ff88..f74434f5 100644 --- a/scripts/test-rust-toolchain-pin.mjs +++ b/scripts/test-rust-toolchain-pin.mjs @@ -5,17 +5,25 @@ import assert from "node:assert/strict"; import { readFileSync, readdirSync } from "node:fs"; import { fileURLToPath } from "node:url"; import test from "node:test"; +import { commandOf, gatesFor } from "./ci/cargo-gates.mjs"; +import { commandLines, loadAction, loadWorkflow } from "./lib/ci-workflow.mjs"; const read = (path) => readFileSync(fileURLToPath(new URL(`../${path}`, import.meta.url)), "utf8").replace(/\r\n?/g, "\n"); const toolchainFile = read("rust-toolchain.toml"); const channel = /^channel = "([^"]+)"\s*(?:#.*)?$/m.exec(toolchainFile)?.[1]; const nodeMajor = read(".nvmrc").trim(); -const workflows = Object.fromEntries( - readdirSync(fileURLToPath(new URL("../.github/workflows/", import.meta.url))) +// Every workflow, and the local composite actions (#358): ci.yml installs its toolchains through +// .github/actions/setup, so the pins live in the action file. +const workflows = Object.fromEntries([ + ...readdirSync(fileURLToPath(new URL("../.github/workflows/", import.meta.url))) .filter((name) => name.endsWith(".yml")) .map((name) => [name, read(`.github/workflows/${name}`)]), -); + ...readdirSync(fileURLToPath(new URL("../.github/actions/", import.meta.url))).map((name) => [ + `actions/${name}/action.yml`, + read(`.github/actions/${name}/action.yml`), + ]), +]); const rustVersion = (manifest) => /^rust-version = "([^"]+)"$/m.exec(read(manifest))?.[1]; const msrv = rustVersion("src-tauri/Cargo.toml"); @@ -49,7 +57,7 @@ test("every workflow Rust install is SHA-pinned and uses the rust-toolchain.toml assert.equal(withInputs(text, match.index).toolchain, channel, `${name}: toolchain must equal rust-toolchain.toml`); } } - assert.ok(installs >= 8, `expected the CI, release and packaging installs, found ${installs}`); + assert.ok(installs >= 6, `expected the setup action and the release, packaging and gate installs, found ${installs}`); }); test("a workflow that installs a different Rust release overrides the toolchain file explicitly", () => { @@ -80,12 +88,22 @@ test("the MSRV is declared once, inherited by every crate and checked in CI", () for (const crate of ["src-tauri", ...members]) { assert.match(read(`${crate}/Cargo.toml`), /^rust-version\.workspace = true$/m, `${crate} must inherit rust-version`); } - const ci = workflows["ci.yml"]; - const msrvJob = ci.slice(ci.indexOf("\n msrv-linux:\n"), ci.indexOf("\n check-linux:\n")); - assert.match(msrvJob, /rust-version = /, "msrv-linux must read the declared rust-version, not hardcode it"); - assert.match(msrvJob, /echo "RUSTUP_TOOLCHAIN=\$msrv" >> "\$GITHUB_ENV"/); - assert.match(msrvJob, /cargo check --workspace --locked/); - assert.match(ci, /^ check-linux:\n needs: \[[^\]]*msrv-linux[^\]]*\]$/m, "check-linux must gate on msrv-linux"); + // The MSRV lane installs the declared `rust-version` through the setup action, which exports + // RUSTUP_TOOLCHAIN, and compiles with the gates' msrv profile. + const workflow = loadWorkflow(); + const msrvLane = workflow.job("msrv-linux"); + const setup = msrvLane.steps.find((step) => step.uses === "./.github/actions/setup"); + assert.equal(setup?.with.rust, "msrv", "msrv-linux must install the declared MSRV, not a pinned release"); + const install = loadAction("setup").steps.find((step) => step.name === "Install the declared MSRV toolchain"); + const lines = commandLines(install); + assert.ok( + lines.some((line) => line.includes("rust-version = ")), + "the action must read the declared rust-version, not hardcode it", + ); + assert.ok(lines.includes('echo "RUSTUP_TOOLCHAIN=$msrv" >> "$GITHUB_ENV"')); + assert.ok(msrvLane.steps.some((step) => commandLines(step).includes("node scripts/ci/cargo-gates.mjs --profile msrv"))); + assert.ok(gatesFor({ os: "linux", profile: "msrv" }).some((gate) => commandOf(gate) === "cargo check --workspace --locked")); + assert.ok(workflow.job("check-linux").needs.includes("msrv-linux"), "check-linux must gate on msrv-linux"); }); test("Node.js is pinned to one major everywhere", () => { diff --git a/scripts/test-verify-ci-parity.mjs b/scripts/test-verify-ci-parity.mjs index 861c9a05..bb29da9d 100644 --- a/scripts/test-verify-ci-parity.mjs +++ b/scripts/test-verify-ci-parity.mjs @@ -4,28 +4,36 @@ import { fileURLToPath } from "node:url"; import test from "node:test"; import { TEST_GROUPS } from "./node-test-groups.mjs"; import { CHECKS } from "./verify.mjs"; +import { GATES, commandOf, gatesFor } from "./ci/cargo-gates.mjs"; +import { expandedSteps, gateInvocation, parseWorkflow } from "./lib/ci-workflow.mjs"; // Keeps scripts/verify.mjs (the local check list, #350) and the test lanes of -// .github/workflows/ci.yml from drifting apart until CI calls verify itself -// (#358): +// .github/workflows/ci.yml from drifting apart: // - every ci.yml job is either a test lane or a listed non-test job; -// - every command in a test lane's run scripts (named or nameless steps, any -// program, not only cargo/npm/node) is covered by a verify check, the -// Node test-group manifest, an allowlisted difference or a listed shell -// word that is not a check (each with its reason); +// - every command in a test lane (named or nameless steps, any program, not +// only cargo/npm/node) is covered by a verify check, the Node test-group +// manifest, an allowlisted difference or a listed shell word that is not a +// check (each with its reason); // - every CI command a verify check claims exists in a test lane. // +// What a lane runs is read from the parsed workflow (#358), through the two +// places the lanes keep commands outside their own `run:` scripts: the cargo +// gate list (a step running scripts/ci/cargo-gates.mjs stands for that OS's +// gates, in scripts/ci/cargo-gates.mjs) and the local setup action (a step +// using it stands for the action's `run` steps its inputs enable). +// // Limits: environment set elsewhere (job `env:`, $GITHUB_ENV, e.g. the MSRV // lane's RUSTUP_TOOLCHAIN or test-macos's DEVELOPER_DIR), `if:` conditions, -// `uses:` actions, commands inside $(…), PowerShell syntax and shell control -// flow beyond splitting at `;`, `&&`, `||` and `|` are not compared. +// third-party `uses:` actions, commands inside $(…), PowerShell syntax and +// shell control flow beyond splitting at `;`, `&&`, `||` and `|` are not +// compared. // // SAGASCRIPT_PARITY_CI_YML= checks another copy of ci.yml (used to show // the detections on mutated copies; never set in CI). const ciPath = process.env.SAGASCRIPT_PARITY_CI_YML ?? fileURLToPath(new URL("../.github/workflows/ci.yml", import.meta.url)); const read = (path) => readFileSync(path, "utf8").replace(/\r\n?/g, "\n"); -const workflow = read(ciPath); +const workflow = parseWorkflow(read(ciPath)); const packageJson = read(fileURLToPath(new URL("../package.json", import.meta.url))); const TEST_LANES = ["test-macos", "test-linux", "msrv-linux", "test-windows"]; @@ -92,49 +100,7 @@ const ALLOWLIST = [ ]; // --------------------------------------------------------------------------- -// A small reader for the subset of YAML and shell that ci.yml uses. - -function jobsSection(text) { - const start = text.indexOf("\njobs:\n"); - assert.notEqual(start, -1, "ci.yml has a jobs: section"); - return text.slice(start + "\njobs:\n".length - 1); -} - -function jobIds(text) { - return [...jobsSection(text).matchAll(/^ {2}([A-Za-z0-9_-]+):\s*$/gm)].map((match) => match[1]); -} - -function jobBlock(text, jobId) { - const jobs = jobsSection(text); - const start = jobs.indexOf(`\n ${jobId}:\n`); - assert.notEqual(start, -1, `ci.yml must define ${jobId}`); - const body = jobs.slice(start + 1); - const next = body.slice(1).search(/^ {2}[A-Za-z0-9_-]+:\s*$/m); - return next === -1 ? body : body.slice(0, next + 1); -} - -// Steps of a job as { name, run, cwd, env }, named or not. -function stepsOf(text, jobId) { - return jobBlock(text, jobId) - .split(/\n(?= {6}- )/) - .slice(1) - .map((chunk) => { - const raw = chunk.replace(/^ {6}- /, " "); - const inline = /^ {8}run: (?![|>])(.*)$/m.exec(raw)?.[1]; - const block = /^ {8}run: [|>]-?\n((?: {10}.*\n?|\n)*)/m - .exec(raw)?.[1] - ?.split("\n") - .map((line) => line.replace(/^ {10}/, "")) - .join("\n"); - const envBlock = /^ {8}env:\n((?: {10}.*\n?)*)/m.exec(raw)?.[1] ?? ""; - return { - name: /^ {8}name: (.*)$/m.exec(raw)?.[1], - run: inline ?? block ?? "", - cwd: /^ {8}working-directory: (.*)$/m.exec(raw)?.[1], - env: Object.fromEntries([...envBlock.matchAll(/^ {10}([A-Za-z_][A-Za-z0-9_]*): (.*)$/gm)].map((match) => [match[1], match[2]])), - }; - }); -} +// A small reader for the subset of shell that the lanes' run scripts use. // Split a script into simple commands, honouring quotes and $(…): each is a // list of words with leading VAR=value assignments and redirections removed. @@ -207,9 +173,21 @@ function commandsIn(script) { .map((words) => words.join(" ")); } -const ciCommands = TEST_LANES.flatMap((job) => - stepsOf(workflow, job).flatMap((step) => commandsIn(step.run).map((run) => ({ job, run, cwd: step.cwd ?? "." }))), -); +const OS_OF_RUNNER = (runsOn) => (/^macos/.test(runsOn) ? "macos" : /^windows/.test(runsOn) ? "windows" : "linux"); + +// The commands of a lane as { job, run, cwd, step }, gates and action steps expanded. +function laneCommands(jobId) { + const job = workflow.job(jobId); + return expandedSteps(job).flatMap((step) => { + const invocation = gateInvocation(step); + if (invocation) { + const gates = gatesFor({ os: invocation.os ?? OS_OF_RUNNER(job.runsOn), profile: invocation.profile }); + return gates.map((gate) => ({ job: jobId, run: commandOf(gate), cwd: gate.cwd, step })); + } + return commandsIn(step.run).map((run) => ({ job: jobId, run, cwd: step.workingDirectory ?? ".", step })); + }); +} +const ciCommands = TEST_LANES.flatMap(laneCommands); const key = ({ run, cwd }) => `${cwd ?? "."}: ${run}`; const classified = new Set(Object.values(TEST_GROUPS).flat()); @@ -260,10 +238,11 @@ jobs: steps: - run: echo hi `; - assert.deepEqual(jobIds(synthetic), ["lane", "other"]); - const steps = stepsOf(synthetic, "lane"); + const parsed = parseWorkflow(synthetic); + assert.deepEqual(parsed.jobIds, ["lane", "other"]); + const steps = parsed.job("lane").steps; assert.deepEqual( - steps.map((step) => [step.name, step.cwd]), + steps.map((step) => [step.name, step.workingDirectory]), [ [undefined, undefined], [undefined, undefined], @@ -282,7 +261,7 @@ jobs: }); test("every ci.yml job is a test lane or a listed non-test job", () => { - const jobs = jobIds(workflow); + const jobs = workflow.jobIds; const known = new Set([...TEST_LANES, ...Object.keys(NON_TEST_JOBS)]); assert.deepEqual( jobs.filter((job) => !known.has(job)), @@ -311,6 +290,21 @@ test("the parser finds the commands of the CI test lanes", () => { ciCommands.some((command) => command.job === job), `${job} has commands`, ); + // The commands the lanes take from the gate list and from the setup action are visible too. + for (const expected of ["src-tauri: cargo test --workspace", ".: npm ci"]) assert.ok(keys.has(expected), `parsed ${expected}`); + assert.ok( + ciCommands.some((command) => command.job === "test-linux" && command.run.startsWith("sudo apt-get")), + "the setup action's apt step", + ); + assert.ok( + ciCommands.some((command) => command.job === "msrv-linux" && command.run.startsWith("rustup toolchain install")), + "the MSRV install", + ); +}); + +test("every gate of the list is run by some test lane, so the list holds no command the parity test cannot see", () => { + const seen = new Set(ciCommands.map(key)); + for (const gate of GATES) assert.ok(seen.has(key({ run: commandOf(gate), cwd: gate.cwd })), `${gate.id} is run by no test lane`); }); test("every command in the CI test lanes has a verify check, a manifest entry or an allowlisted reason", () => { @@ -378,14 +372,23 @@ test("workflows outside ci.yml named in a check's lanes run its command", () => test("the release guard proves its test ran, locally as in CI", () => { // cargo test exits 0 when the name filter matches nothing. - const proof = /grep -q '(release_build_ignores_the_real_environment \.\.\. ok)'/.exec(workflow)?.[1]; - assert.ok(proof, "test-linux greps the release guard's output"); + const proof = GATES.find((gate) => gate.id === "release-guard").requireOutput?.pattern; + assert.ok(proof, "the gate list requires the release guard's output"); const [step] = CHECKS.find((check) => check.id === "release-guard").steps; - assert.ok(step.requireOutput?.pattern.test(proof), "verify's release-guard requires the same line"); + assert.ok( + step.requireOutput?.pattern.test("release_build_ignores_the_real_environment ... ok"), + "verify's release-guard requires the same line", + ); + assert.ok(proof.test("test tests::release_build_ignores_the_real_environment ... ok")); + assert.deepEqual( + [proof.source, step.requireOutput.pattern.source], + [proof.source, proof.source], + "verify and CI require the same output", + ); }); test("the browser check gives the QA scripts an output directory, as CI does", () => { - const ciStep = stepsOf(workflow, "test-macos").find((step) => commandsIn(step.run).includes("npm run test:browser")); + const ciStep = expandedSteps(workflow.job("test-macos")).find((step) => commandsIn(step.run).includes("npm run test:browser")); assert.ok(ciStep?.env.QA_OUTPUT_DIR, "test-macos sets QA_OUTPUT_DIR for npm run test:browser"); const [step] = CHECKS.find((check) => check.id === "browser").steps; assert.ok(step.env?.QA_OUTPUT_DIR, "verify's browser check sets QA_OUTPUT_DIR"); diff --git a/scripts/test-windows-package-workflow.mjs b/scripts/test-windows-package-workflow.mjs index f3f383b2..38676754 100644 --- a/scripts/test-windows-package-workflow.mjs +++ b/scripts/test-windows-package-workflow.mjs @@ -4,18 +4,8 @@ import { readFile } from "node:fs/promises"; import test from "node:test"; import { fileURLToPath } from "node:url"; -test("Windows PR CI runs build identity regression tests before compilation", async () => { - const ci = (await readFile(new URL("../.github/workflows/ci.yml", import.meta.url), "utf8")).replace(/\r\n?/g, "\n"); - const testWindowsStart = ci.indexOf(" test-windows:"); - const buildWindowsStart = ci.indexOf(" build-windows:", testWindowsStart); - assert.ok(testWindowsStart >= 0 && buildWindowsStart > testWindowsStart); - const windows = ci.slice(testWindowsStart, buildWindowsStart); - const command = - "node --test scripts/test-ci-build-identity.mjs scripts/test-windows-release-identity.mjs scripts/test-windows-package-workflow.mjs"; - const checks = windows.indexOf(command); - assert.ok(checks > windows.indexOf("name: Install Node.js")); - assert.ok(checks < windows.indexOf("name: Cargo check")); -}); +// The ordering of ci.yml's Windows build identity gates (they run before `npm ci`, so this file must +// stay free of npm packages) is asserted in scripts/test-ci-parallel-gates.mjs. test("Windows candidates pin and verify source identity around release compilation", async () => { const workflow = await readFile(new URL("../.github/workflows/windows-package.yml", import.meta.url), "utf8"); diff --git a/scripts/test-workflow-hardening.mjs b/scripts/test-workflow-hardening.mjs index 1f5da7be..2547439c 100644 --- a/scripts/test-workflow-hardening.mjs +++ b/scripts/test-workflow-hardening.mjs @@ -7,6 +7,7 @@ import assert from "node:assert/strict"; import { readFileSync, readdirSync } from "node:fs"; import test from "node:test"; import { fileURLToPath } from "node:url"; +import { parse } from "yaml"; const directory = fileURLToPath(new URL("../.github/workflows/", import.meta.url)); const workflows = Object.fromEntries( @@ -81,6 +82,60 @@ function writeScopes(scopes) { }); } +// Local composite actions (#358) hold steps a workflow used to hold: their `uses:` pins and tool +// installs are held to the same rules. +const actionsDirectory = fileURLToPath(new URL("../.github/actions/", import.meta.url)); +const localActions = Object.fromEntries( + readdirSync(actionsDirectory) + .sort() + .map((name) => [`actions/${name}/action.yml`, readFileSync(`${actionsDirectory}${name}/action.yml`, "utf8").replace(/\r\n?/g, "\n")]), +); + +function assertActionsPinned(name, text) { + let actions = 0; + for (const [line, ref, comment] of text.matchAll(/^\s*(?:-\s+)?uses:\s*(\S+)(.*)$/gm)) { + if (ref.startsWith("./")) continue; + actions += 1; + const pinned = /^([^@\s]+)@([0-9a-f]{40})$/.exec(ref); + assert.ok(pinned, `${name}: ${line.trim()} must be pinned to a full commit SHA`); + assert.match(comment, /^\s+# v\d+(?:\.\d+)*\b/, `${name}: ${line.trim()} must name the release it resolves to (# vX.Y.Z)`); + } + assert.ok(actions > 0, `${name} uses no actions; the pin check would be vacuous`); +} + +function assertToolsPinned(name, text) { + for (const [command] of text.matchAll(/^.*\bcargo install\b.*$/gm)) { + assert.match(command, /--locked\b/, `${name}: ${command.trim()} must use --locked`); + assert.match(command, /--version \d+\.\d+\.\d+\b/, `${name}: ${command.trim()} must pin --version`); + } + for (const [command, args] of text.matchAll(/^.*\bnpm (?:install|i) (?:-g|--global)\b(.*)$/gm)) { + const packages = args + .trim() + .split(/\s+/) + .filter((arg) => arg && !arg.startsWith("-")); + assert.ok(packages.length > 0, `${name}: ${command.trim()} installs nothing`); + for (const spec of packages) { + assert.match(spec, /^(?:@[^@/\s]+\/)?[^@\s]+@\d+\.\d+\.\d+$/, `${name}: ${spec} must be pinned to an exact version`); + } + } +} + +test("Dependabot keeps the pins of every local composite action current", () => { + const dependabot = parse(readFileSync(new URL("../.github/dependabot.yml", import.meta.url), "utf8")); + const actions = dependabot.updates.find((update) => update["package-ecosystem"] === "github-actions"); + assert.ok(actions, "Dependabot updates GitHub Actions"); + const directories = actions.directories ?? [actions.directory]; + assert.ok(directories.includes("/"), "the workflows"); + for (const name of Object.keys(localActions)) { + assert.ok(directories.includes(`/.github/${name.replace(/\/action\.yml$/, "")}`), `${name} is not watched by Dependabot`); + } +}); + +for (const [name, text] of Object.entries(localActions)) { + test(`${name} pins every action to a commit SHA with its release in a comment`, () => assertActionsPinned(name, text)); + test(`${name} installs command-line tools at exact versions`, () => assertToolsPinned(name, text)); +} + for (const [name, text] of Object.entries(workflows)) { test(`${name} declares a least-privilege top-level permissions block`, () => { const scopes = topLevelPermissions(text); @@ -110,17 +165,7 @@ for (const [name, text] of Object.entries(workflows)) { } }); - test(`${name} pins every action to a commit SHA with its release in a comment`, () => { - let actions = 0; - for (const [line, ref, comment] of text.matchAll(/^\s*(?:-\s+)?uses:\s*(\S+)(.*)$/gm)) { - if (ref.startsWith("./")) continue; - actions += 1; - const pinned = /^([^@\s]+)@([0-9a-f]{40})$/.exec(ref); - assert.ok(pinned, `${name}: ${line.trim()} must be pinned to a full commit SHA`); - assert.match(comment, /^\s+# v\d+(?:\.\d+)*\b/, `${name}: ${line.trim()} must name the release it resolves to (# vX.Y.Z)`); - } - assert.ok(actions > 0, `${name} uses no actions; the pin check would be vacuous`); - }); + test(`${name} pins every action to a commit SHA with its release in a comment`, () => assertActionsPinned(name, text)); test(`${name} grants write scopes only to reviewed publishing jobs`, () => { for (const job of jobs(text)) { @@ -131,22 +176,7 @@ for (const [name, text] of Object.entries(workflows)) { } }); - test(`${name} installs command-line tools at exact versions`, () => { - for (const [command] of text.matchAll(/^.*\bcargo install\b.*$/gm)) { - assert.match(command, /--locked\b/, `${name}: ${command.trim()} must use --locked`); - assert.match(command, /--version \d+\.\d+\.\d+\b/, `${name}: ${command.trim()} must pin --version`); - } - for (const [command, args] of text.matchAll(/^.*\bnpm (?:install|i) (?:-g|--global)\b(.*)$/gm)) { - const packages = args - .trim() - .split(/\s+/) - .filter((arg) => arg && !arg.startsWith("-")); - assert.ok(packages.length > 0, `${name}: ${command.trim()} installs nothing`); - for (const spec of packages) { - assert.match(spec, /^(?:@[^@/\s]+\/)?[^@\s]+@\d+\.\d+\.\d+$/, `${name}: ${spec} must be pinned to an exact version`); - } - } - }); + test(`${name} installs command-line tools at exact versions`, () => assertToolsPinned(name, text)); } test("every reviewed writer still exists and still needs its scope", () => { diff --git a/scripts/verify.mjs b/scripts/verify.mjs index e032482f..f00636d7 100644 --- a/scripts/verify.mjs +++ b/scripts/verify.mjs @@ -3,7 +3,8 @@ // CHECKS below is the source of truth for what "verify my change" means: // CONTRIBUTING.md documents it, `--list` prints it, and // scripts/test-verify-ci-parity.mjs keeps it in step with the test lanes of -// .github/workflows/ci.yml. Before anything runs, a preflight checks the +// .github/workflows/ci.yml, whose cargo gates are listed once, in +// scripts/ci/cargo-gates.mjs. Before anything runs, a preflight checks the // tools and packages the selected checks need and reports everything that is // missing in one error; it never installs anything. Every check has a time // budget; on timeout, Ctrl-C, SIGTERM or SIGHUP its whole process tree is @@ -218,12 +219,7 @@ export const CHECKS = [ platforms: ALL, requires: ["rust", "linux-libs", "frontend-dist"], timeoutMinutes: 45, - ci: [ - ci("cargo clippy --all-targets -- -D warnings", "src-tauri"), - ci("cargo clippy -p sagascript-cli --all-targets -- -D warnings", "src-tauri"), - ci("cargo check", "src-tauri"), - ci("cargo check -p sagascript-cli", "src-tauri"), - ], + ci: [ci("cargo clippy --workspace --all-targets -- -D warnings", "src-tauri")], lanes: ["test-macos", "test-linux", "test-windows"], }, { @@ -235,8 +231,9 @@ export const CHECKS = [ platforms: ALL, requires: ["rust", "linux-libs", "frontend-dist"], timeoutMinutes: 45, - ci: [ci("cargo clippy --all-targets -- -D warnings", "src-tauri")], - lanes: ["test-macos", "test-linux", "test-windows"], + // CI runs the workspace gate, which covers the app crate. + ci: [], + lanes: [], }, { id: "clippy-cli-lean", @@ -247,11 +244,8 @@ export const CHECKS = [ platforms: ALL, requires: ["rust"], timeoutMinutes: 30, - ci: [ - ci("cargo clippy -p sagascript-cli --all-targets --no-default-features -- -D warnings", "src-tauri"), - ci("cargo check -p sagascript-cli --no-default-features", "src-tauri"), - ], - lanes: ["test-linux"], + ci: [ci("cargo clippy -p sagascript-cli --all-targets --no-default-features -- -D warnings", "src-tauri")], + lanes: ["test-macos", "test-linux", "test-windows"], }, { id: "clippy-policy", @@ -317,7 +311,7 @@ export const CHECKS = [ requires: ["rust"], timeoutMinutes: 30, ci: [ci("cargo clippy -p sagascript-core --all-targets -- -D warnings", "src-tauri")], - lanes: ["test-macos", "test-windows"], + lanes: ["test-macos", "test-linux", "test-windows"], }, { id: "core-clippy-diarization", @@ -329,7 +323,7 @@ export const CHECKS = [ requires: ["rust"], timeoutMinutes: 30, ci: [ci("cargo clippy -p sagascript-core --features diarization --all-targets -- -D warnings", "src-tauri")], - lanes: ["test-macos"], + lanes: ["test-linux"], }, { id: "test-workspace", @@ -340,8 +334,8 @@ export const CHECKS = [ platforms: ALL, requires: ["rust", "linux-libs", "frontend-dist"], timeoutMinutes: 60, - ci: [ci("cargo test", "src-tauri"), ci("cargo test -p sagascript-cli", "src-tauri")], - lanes: ["test-macos", "test-windows"], + ci: [ci("cargo test --workspace", "src-tauri")], + lanes: ["test-macos", "test-linux", "test-windows"], }, { id: "test-cli-lean", @@ -353,7 +347,7 @@ export const CHECKS = [ requires: ["rust"], timeoutMinutes: 30, ci: [ci("cargo test -p sagascript-cli --no-default-features", "src-tauri")], - lanes: ["test-macos", "test-linux"], + lanes: ["test-macos", "test-linux", "test-windows"], }, { id: "engine-protocol", @@ -364,8 +358,9 @@ export const CHECKS = [ platforms: ALL, requires: ["rust"], timeoutMinutes: 15, - ci: [ci("cargo test -p sagascript-engine-protocol", "src-tauri")], - lanes: ["test-linux"], + // CI reaches the protocol crate through `cargo test --workspace` on every OS. + ci: [], + lanes: [], }, { id: "core-test", @@ -400,11 +395,8 @@ export const CHECKS = [ platforms: ALL, requires: ["rust"], timeoutMinutes: 30, - ci: [ - ci("cargo test -p sagascript-core --features diarization", "src-tauri"), - ci("cargo check -p sagascript-core --features diarization", "src-tauri"), - ], - lanes: ["test-macos"], + ci: [ci("cargo test -p sagascript-core --features diarization", "src-tauri")], + lanes: ["test-linux"], }, { id: "ort-host-clippy", @@ -416,7 +408,7 @@ export const CHECKS = [ requires: ["rust"], timeoutMinutes: 30, ci: [ci("cargo clippy --all-targets --locked -- -D warnings", "src-tauri/engine-host/ort")], - lanes: ["test-linux"], + lanes: ["test-linux", "test-windows"], }, { id: "ort-host-test",