From 78029d8253feddf86a205155b77dadb2285d9c01 Mon Sep 17 00:00:00 2001 From: Johnny D Date: Sun, 20 Sep 2026 15:51:21 -0400 Subject: [PATCH 01/21] ci: enforce one lean PR job and make native qualification opt-in Remove full macOS, Linux and Windows application builds, SDK qualification, and source archives from automatic PR checks. Keep the existing small production-code regressions and sanitizers in one standard Linux job. Enforce a shared 60-second suite budget, 20 seconds per command, and a 3-minute job ceiling. Add AGENTS.md and structural workflow policy tests that reject accidental fan-out, recursive checkout, automatic qualification, soft-fail gates and raised budgets. Preserve native packaging/launch steps behind explicit single-target manual dispatches. AI-assisted change. Policy/budget regression tests passed locally; full integration verification must use the exact-head fast GitHub Actions run. No expensive native builds or hardware checks were launched. --- .github/workflows/native-switch2kit.yml | 134 ++------------- .github/workflows/switch2kit-desktop.yml | 4 +- .github/workflows/switch2kit-linux.yml | 19 ++- .github/workflows/switch2kit-macos.yml | 120 +++++++++++++ .github/workflows/switch2kit-windows.yml | 4 +- AGENTS.md | 63 +++++++ Docs/Switch2Kit.md | 17 +- Tools/check_ci_policy.py | 129 ++++++++++++++ Tools/run_fast_tests.py | 79 +++++++++ Tools/test_ci_policy.py | 204 +++++++++++++++++++++++ 10 files changed, 644 insertions(+), 129 deletions(-) create mode 100644 .github/workflows/switch2kit-macos.yml create mode 100644 AGENTS.md create mode 100644 Tools/check_ci_policy.py create mode 100644 Tools/run_fast_tests.py create mode 100644 Tools/test_ci_policy.py diff --git a/.github/workflows/native-switch2kit.yml b/.github/workflows/native-switch2kit.yml index 469a5cb4bb61..8f12be717a61 100644 --- a/.github/workflows/native-switch2kit.yml +++ b/.github/workflows/native-switch2kit.yml @@ -1,3 +1,4 @@ +# This is the only automatic workflow. Budgets are enforced by Tools/check_ci_policy.py. name: Native Switch2Kit on: pull_request: @@ -9,126 +10,25 @@ concurrency: cancel-in-progress: true jobs: wiring: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 - with: - persist-credentials: false - - name: Check exact-width SDL adapter capacity - run: | - git submodule update --init Externals/Switch2Kit Externals/SDL/SDL - CXX=clang++ python3 Tools/test_switch2kit_capacity.py - CXX=g++ python3 Tools/test_switch2kit_capacity.py - - run: python3 Tools/test_switch2kit.py - - run: python3 Tools/test_switch2kit_autoconnect.py - - run: python3 Tools/test_switch2kit_mapping.py --sanitize - - name: Execute host lifecycle and identity regressions with sanitizers - run: python3 Tools/test_switch2kit_host.py --sanitize - sdk-regressions: - if: github.repository == 'jmonster/dolphin' - runs-on: macos-15 - timeout-minutes: 25 + runs-on: ubuntu-24.04 + timeout-minutes: 3 env: - DEVELOPER_DIR: /Applications/Xcode_26.3.app/Contents/Developer + PYTHONPATH: ${{ runner.temp }}/ci-policy steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 + timeout-minutes: 1 with: persist-credentials: false - - name: Initialize exactly the pinned SDK and Dolphin SDL - run: git submodule update --init Externals/Switch2Kit Externals/SDL/SDL - - name: Exercise the real SDK and in-process SDL adapter without hardware + fetch-depth: 1 + submodules: false + - name: Enforce the CI cost policy before fetching dependencies + timeout-minutes: 1 run: | - set -o pipefail - swift test --package-path Externals/Switch2Kit 2>&1 | tee switch2kit-sdk-tests.log - bash Externals/Switch2Kit/tests/rumble/run.sh 2>&1 | tee switch2kit-motor-tests.log - S2K_SDL_SOURCE="$PWD/Externals/SDL/SDL" \ - bash Externals/Switch2Kit/tests/sdl-inprocess/verify.sh \ - 2>&1 | tee switch2kit-sdl-tests.log - - name: SDK test diagnostics - if: always() - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 - with: - name: Switch2Kit-sdk-regressions - path: switch2kit-*-tests.log - macos: - if: github.repository == 'jmonster/dolphin' - strategy: - fail-fast: false - matrix: - include: - - os: macos-15 - arch: arm64 - - os: macos-15-intel - arch: x86_64 - runs-on: ${{ matrix.os }} - timeout-minutes: 90 - env: - DEVELOPER_DIR: /Applications/Xcode_26.3.app/Contents/Developer - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 - with: - submodules: recursive - persist-credentials: false - - name: Install build dependencies - run: brew install cmake ninja nasm automake libtool qt@6 - - name: Check source pin and integration - run: | - test "$(git -C Externals/Switch2Kit rev-parse HEAD)" = "$(git ls-tree HEAD Externals/Switch2Kit | awk '{print $3}')" - python3 Tools/test_switch2kit.py - python3 Tools/test_switch2kit_autoconnect.py - python3 Tools/test_switch2kit_host.py --sanitize - python3 Tools/test_switch2kit_mapping.py --sanitize - - name: Build the complete application - env: - ARCH: ${{ matrix.arch }} - run: | - set -o pipefail - test "$(uname -m)" = "$ARCH" - cmake -S . -B build-switch2kit -G Ninja \ - -DCMAKE_BUILD_TYPE=Release -DCMAKE_OSX_DEPLOYMENT_TARGET=15.0 \ - -DCMAKE_OSX_ARCHITECTURES="$ARCH" \ - -DENABLE_SWITCH2KIT=ON -DENABLE_QT=ON -DENABLE_SDL=ON \ - -DUSE_SYSTEM_SDL3=OFF \ - -DCMAKE_PREFIX_PATH="$(brew --prefix qt@6)" \ - -DENABLE_VULKAN=OFF -DENABLE_TESTS=OFF -DPOSTPROCESS_BUNDLE=ON \ - 2>&1 | tee switch2kit-configure.log - cmake --build build-switch2kit --target dolphin-emu --parallel 3 \ - 2>&1 | tee switch2kit-build.log - - name: Inspect and archive application - env: - ARCH: ${{ matrix.arch }} - run: | - app=build-switch2kit/Binaries/DolphinQt.app - test -d "$app" - test -f "$app/Contents/Frameworks/libSwitch2KitC.dylib" - /usr/libexec/PlistBuddy -c 'Print :NSBluetoothAlwaysUsageDescription' "$app/Contents/Info.plist" - lipo "$app/Contents/MacOS/DolphinQt" -verify_arch "$ARCH" - lipo "$app/Contents/Frameworks/libSwitch2KitC.dylib" -verify_arch "$ARCH" - test -f "$app/Contents/Resources/Sys/Profiles/GCPad/Switch2Kit GameCube.ini" - test -f "$app/Contents/Resources/Sys/Profiles/GCPad/Switch2Kit Pro Controller 2.ini" - ditto -c -k --keepParent "$app" "Dolphin-Switch2Kit-$ARCH.zip" - - name: Launch, normally quit and relaunch the exact ZIP with build roots denied - env: - ARCH: ${{ matrix.arch }} - run: | - python3 Tools/test_switch2kit_bundle.py "Dolphin-Switch2Kit-$ARCH.zip" \ - --architecture "$ARCH" --report switch2kit-validation.json - - name: Application artifact (not hardware-qualified or notarized) - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 - with: - name: Dolphin-Switch2Kit-${{ matrix.arch }} - path: | - Dolphin-Switch2Kit-${{ matrix.arch }}.zip - switch2kit-validation.json - if-no-files-found: error - - name: Build diagnostics - if: always() - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 - with: - name: Switch2Kit-${{ matrix.arch }}-diagnostics - path: | - switch2kit-*.log - switch2kit-*.sample - switch2kit-validation.json - build-switch2kit/CMakeCache.txt - build-switch2kit/CMakeFiles/CMakeConfigureLog.yaml + python3 -m pip install --disable-pip-version-check --no-deps --only-binary=:all: --retries 1 --timeout 10 --target "$RUNNER_TEMP/ci-policy" PyYAML==6.0.3 + python3 Tools/check_ci_policy.py + - name: Fetch only the pinned SDK and SDL, without their history + timeout-minutes: 1 + run: git submodule update --init --depth 1 Externals/Switch2Kit Externals/SDL/SDL + - name: Run the fast regressions (60 seconds total, 20 seconds per command) + timeout-minutes: 2 + run: python3 Tools/run_fast_tests.py diff --git a/.github/workflows/switch2kit-desktop.yml b/.github/workflows/switch2kit-desktop.yml index 1c3026902ad1..e791611e0861 100644 --- a/.github/workflows/switch2kit-desktop.yml +++ b/.github/workflows/switch2kit-desktop.yml @@ -1,6 +1,6 @@ +# Manual qualification/artifacts only. Never add automatic triggers; see AGENTS.md. name: Switch2Kit desktop platforms on: - pull_request: workflow_dispatch: permissions: contents: read @@ -10,7 +10,7 @@ concurrency: jobs: source: runs-on: ubuntu-24.04 - timeout-minutes: 10 + timeout-minutes: 3 steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 with: diff --git a/.github/workflows/switch2kit-linux.yml b/.github/workflows/switch2kit-linux.yml index e9fe1b9dc303..9df656b2c3b3 100644 --- a/.github/workflows/switch2kit-linux.yml +++ b/.github/workflows/switch2kit-linux.yml @@ -1,17 +1,27 @@ +# Manual qualification/artifacts only. Never add automatic triggers; see AGENTS.md. name: Switch2Kit Linux application on: - pull_request: workflow_dispatch: + inputs: + target: + description: 'Choose ONE complete application build (can take 25 minutes)' + required: true + type: choice + default: enabled + options: + - enabled + - disabled permissions: contents: read concurrency: - group: switch2kit-linux-${{ github.ref }} + group: switch2kit-linux-${{ github.ref }}-${{ inputs.target }} cancel-in-progress: true jobs: linux: + if: inputs.target == 'enabled' runs-on: ubuntu-24.04 container: swift:6.2.1-noble - timeout-minutes: 90 + timeout-minutes: 45 env: DEBIAN_FRONTEND: noninteractive steps: @@ -70,11 +80,12 @@ jobs: retention-days: 7 disabled: + if: inputs.target == 'disabled' # A complete ordinary application build in an image without Swift. This is # not a controller-enabled artifact and does not use the SDK build helper. runs-on: ubuntu-24.04 container: ubuntu:24.04 - timeout-minutes: 90 + timeout-minutes: 45 env: DEBIAN_FRONTEND: noninteractive steps: diff --git a/.github/workflows/switch2kit-macos.yml b/.github/workflows/switch2kit-macos.yml new file mode 100644 index 000000000000..66db931d929f --- /dev/null +++ b/.github/workflows/switch2kit-macos.yml @@ -0,0 +1,120 @@ +# Full native qualification is opt-in, never a pull-request test. See AGENTS.md. +name: Switch2Kit macOS qualification +on: + workflow_dispatch: + inputs: + target: + description: 'Explicit native qualification (application builds can take 45 minutes)' + required: true + type: choice + default: sdk + options: + - sdk + - arm64 + - x86_64 +permissions: + contents: read +concurrency: + group: switch2kit-macos-${{ github.ref }}-${{ inputs.target }} + cancel-in-progress: true +jobs: + sdk-regressions: + if: inputs.target == 'sdk' + runs-on: macos-15 + timeout-minutes: 5 + env: + DEVELOPER_DIR: /Applications/Xcode_26.3.app/Contents/Developer + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 + with: + persist-credentials: false + - name: Initialize exactly the pinned SDK and Dolphin SDL + run: git submodule update --init Externals/Switch2Kit Externals/SDL/SDL + - name: Exercise the real SDK and in-process SDL adapter without hardware + run: | + set -o pipefail + swift test --package-path Externals/Switch2Kit 2>&1 | tee switch2kit-sdk-tests.log + bash Externals/Switch2Kit/tests/rumble/run.sh 2>&1 | tee switch2kit-motor-tests.log + S2K_SDL_SOURCE="$PWD/Externals/SDL/SDL" \ + bash Externals/Switch2Kit/tests/sdl-inprocess/verify.sh \ + 2>&1 | tee switch2kit-sdl-tests.log + - name: SDK test diagnostics + if: always() + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 + with: + name: Switch2Kit-sdk-regressions + path: switch2kit-*-tests.log + macos: + if: inputs.target == 'arm64' || inputs.target == 'x86_64' + runs-on: ${{ inputs.target == 'arm64' && 'macos-15' || 'macos-15-intel' }} + timeout-minutes: 60 + env: + DEVELOPER_DIR: /Applications/Xcode_26.3.app/Contents/Developer + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 + with: + submodules: recursive + persist-credentials: false + - name: Install build dependencies + run: brew install cmake ninja nasm automake libtool qt@6 + - name: Check source pin and integration + run: | + test "$(git -C Externals/Switch2Kit rev-parse HEAD)" = "$(git ls-tree HEAD Externals/Switch2Kit | awk '{print $3}')" + python3 Tools/test_switch2kit.py + python3 Tools/test_switch2kit_autoconnect.py + python3 Tools/test_switch2kit_host.py --sanitize + python3 Tools/test_switch2kit_mapping.py --sanitize + - name: Build the complete application + env: + ARCH: ${{ inputs.target }} + run: | + set -o pipefail + test "$(uname -m)" = "$ARCH" + cmake -S . -B build-switch2kit -G Ninja \ + -DCMAKE_BUILD_TYPE=Release -DCMAKE_OSX_DEPLOYMENT_TARGET=15.0 \ + -DCMAKE_OSX_ARCHITECTURES="$ARCH" \ + -DENABLE_SWITCH2KIT=ON -DENABLE_QT=ON -DENABLE_SDL=ON \ + -DUSE_SYSTEM_SDL3=OFF \ + -DCMAKE_PREFIX_PATH="$(brew --prefix qt@6)" \ + -DENABLE_VULKAN=OFF -DENABLE_TESTS=OFF -DPOSTPROCESS_BUNDLE=ON \ + 2>&1 | tee switch2kit-configure.log + cmake --build build-switch2kit --target dolphin-emu --parallel 3 \ + 2>&1 | tee switch2kit-build.log + - name: Inspect and archive application + env: + ARCH: ${{ inputs.target }} + run: | + app=build-switch2kit/Binaries/DolphinQt.app + test -d "$app" + test -f "$app/Contents/Frameworks/libSwitch2KitC.dylib" + /usr/libexec/PlistBuddy -c 'Print :NSBluetoothAlwaysUsageDescription' "$app/Contents/Info.plist" + lipo "$app/Contents/MacOS/DolphinQt" -verify_arch "$ARCH" + lipo "$app/Contents/Frameworks/libSwitch2KitC.dylib" -verify_arch "$ARCH" + test -f "$app/Contents/Resources/Sys/Profiles/GCPad/Switch2Kit GameCube.ini" + test -f "$app/Contents/Resources/Sys/Profiles/GCPad/Switch2Kit Pro Controller 2.ini" + ditto -c -k --keepParent "$app" "Dolphin-Switch2Kit-$ARCH.zip" + - name: Launch, normally quit and relaunch the exact ZIP with build roots denied + env: + ARCH: ${{ inputs.target }} + run: | + python3 Tools/test_switch2kit_bundle.py "Dolphin-Switch2Kit-$ARCH.zip" \ + --architecture "$ARCH" --report switch2kit-validation.json + - name: Application artifact (not hardware-qualified or notarized) + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 + with: + name: Dolphin-Switch2Kit-${{ inputs.target }} + path: | + Dolphin-Switch2Kit-${{ inputs.target }}.zip + switch2kit-validation.json + if-no-files-found: error + - name: Build diagnostics + if: always() + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 + with: + name: Switch2Kit-${{ inputs.target }}-diagnostics + path: | + switch2kit-*.log + switch2kit-*.sample + switch2kit-validation.json + build-switch2kit/CMakeCache.txt + build-switch2kit/CMakeFiles/CMakeConfigureLog.yaml diff --git a/.github/workflows/switch2kit-windows.yml b/.github/workflows/switch2kit-windows.yml index 367ec7eb9ceb..a4199b2c3dcf 100644 --- a/.github/workflows/switch2kit-windows.yml +++ b/.github/workflows/switch2kit-windows.yml @@ -1,6 +1,6 @@ +# Manual qualification/artifacts only. Never add automatic triggers; see AGENTS.md. name: Switch2Kit Windows application on: - pull_request: workflow_dispatch: permissions: contents: read @@ -10,7 +10,7 @@ concurrency: jobs: windows: runs-on: windows-2025-vs2026 - timeout-minutes: 120 + timeout-minutes: 45 env: VCPKG_MAX_CONCURRENCY: 3 steps: diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 000000000000..8eeecf9faee3 --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,63 @@ +# Repository instructions + +These rules apply to the entire repository and to every agent/contributor. + +## CI cost and test speed are hard requirements + +Automatic tests must be lean and fast. A slow test must be reduced, replaced with +a focused regression, or removed from automatic CI. Do not solve a timeout by +raising the budget, adding runners, dropping assertions, skipping failures, or +turning off sanitizers. A warm cache is not evidence that a test is cheap. + +- Exactly ONE automatic GitHub Actions job: `Native Switch2Kit / wiring`, on + `ubuntu-24.04`, for pull requests. No automatic macOS/Windows jobs, matrices, + scheduled builds, `workflow_run` chains, reusable-workflow escape hatches, or + duplicate branch-push jobs. Keep the gate unfiltered so changes to workflows, + this file, and test infrastructure are always checked. +- Hard budgets: **60 seconds for the entire regression suite**, **20 seconds per + test command**, and **3 minutes for the whole job**, including setup. The job + timeout is a safety ceiling, not a target. Report cold-run timing when changing + tests; fail closed when a budget is exceeded. +- Test production behavior at the smallest boundary. Small C/C++ harnesses, + sanitizer checks, source checks and tiny CMake fixtures are appropriate. Do not + compile Dolphin, Qt, SDL, the Swift SDK, or all dependencies to test a small + policy/disabled-feature branch. Do not build an application just to inspect it. +- Only shallow checkout and the explicitly needed pinned Switch2Kit/SDL submodules + belong in the PR lane. No recursive submodule fetch, platform toolchain install, + source/application archive, or artifact upload. The one pinned PyYAML wheel is + installed into a temporary directory for structural workflow validation. +- Add focused tests to `Tools/run_fast_tests.py`, not separate workflow jobs. + Preserve both compiler capacity probes, ASan/UBSan host/mapping coverage, consent, + and feature-disabled checks. Keep fixtures tiny; do not claim they qualify a + native platform, packaged application, Bluetooth hardware or gameplay. + +## Full application qualification is separate and explicitly requested + +The macOS, Linux, Windows and source-archive workflows are **workflow_dispatch +only**. They are optional application/SDK qualification tools, not PR tests. +Do not dispatch them without a human explicitly requesting that expensive work. +Select one architecture/configuration at a time; do not recreate an automatic +build matrix. Preserve real build, packaging, dependency and launch assertions +when changing those tools. A passing fast PR check is not native qualification. + +## Required verification for CI/test edits + +Install `PyYAML==6.0.3` in your development environment, then run: + +```sh +python3 Tools/check_ci_policy.py +python3 Tools/test_ci_policy.py +git submodule update --init --depth 1 Externals/Switch2Kit Externals/SDL/SDL +python3 Tools/run_fast_tests.py # Linux; enforces the shared deadline +``` + +`check_ci_policy.py` scans every `.yml`/`.yaml` workflow, rejects additional +automatic triggers/jobs and allowlists the fast lane's steps. Its tests must catch +attempts to restore expensive CI. Do not weaken the checker or runner to make a +change pass. Budget/policy exceptions need explicit repository-owner approval. +Repository-side checks are not a security boundary: keep `wiring` required in +branch protection and review changes to this policy and its enforcement together. + +Make targeted changes, preserve controller behavior and user-data safeguards, and +disclose AI-assisted changes and validation limitations in pull requests. Never +claim a test, native build, or hardware check was performed unless it actually ran. diff --git a/Docs/Switch2Kit.md b/Docs/Switch2Kit.md index 49a754a8811e..140515a4a04e 100644 --- a/Docs/Switch2Kit.md +++ b/Docs/Switch2Kit.md @@ -76,12 +76,20 @@ An automated callback test is not a physical rumble test. ## Applications and prerequisites -For development artifacts, open this repository's **Actions** tab and choose a successful -run for the revision being tested. The workflows and application artifacts are: +Pull requests run only the fast Linux regression gate; they do not build applications or +upload artifacts. Its suite is limited to 60 seconds, with a 3-minute job ceiling including +setup. See [AGENTS.md](../AGENTS.md) for the enforced CI policy and local test command. + +Application qualification is expensive and **manual only**. For a development artifact, +open this repository's **Actions** tab, choose the platform workflow below, select **Run +workflow** and the revision to qualify. On macOS, select `arm64` or `x86_64` for an app; +`sdk` runs only SDK checks. On Linux, select `enabled` for the controller app or `disabled` +for the separate no-Swift qualification. Each dispatch selects one configuration, not a +build matrix. Download only from a successful run for the exact revision being tested. | Platform | Workflow | Artifact | | --- | --- | --- | -| macOS | [Native Switch2Kit](../.github/workflows/native-switch2kit.yml) | `Dolphin-Switch2Kit-arm64` or `Dolphin-Switch2Kit-x86_64` | +| macOS | [Switch2Kit macOS qualification](../.github/workflows/switch2kit-macos.yml) | `Dolphin-Switch2Kit-arm64` or `Dolphin-Switch2Kit-x86_64` | | Linux | [Switch2Kit Linux application](../.github/workflows/switch2kit-linux.yml) | `Dolphin-Switch2Kit-linux-x86_64` | | Windows | [Switch2Kit Windows application](../.github/workflows/switch2kit-windows.yml) | `Dolphin-Switch2Kit-windows-x86_64` | @@ -198,7 +206,8 @@ controller absent after a search, check adapter power/access, Sync mode, competi connections and the displayed status, then retry Find. Installing a dashboard or replacing system SDL is not a remedy. -The workflows check builds and extracted-package launch using private test settings. +The manually requested qualification workflows check builds and extracted-package +launch using private test settings. A fast PR check is not application qualification. They do not establish pristine first-use dialogs, downloaded-app approval, Bluetooth hardware or gameplay. Mapping and host regressions cover cancellation, backup/rollback, identity, saved consent, explicit stop and shutdown ordering; keep those checks when diff --git a/Tools/check_ci_policy.py b/Tools/check_ci_policy.py new file mode 100644 index 000000000000..813c566438ab --- /dev/null +++ b/Tools/check_ci_policy.py @@ -0,0 +1,129 @@ +#!/usr/bin/env python3 +# Copyright 2026 Dolphin Emulator Project +# SPDX-License-Identifier: GPL-2.0-or-later +"""Fail closed on accidental CI cost regressions. Requires PyYAML 6.0.3.""" +from pathlib import Path +import sys + +import yaml + +ROOT = Path(__file__).resolve().parents[1] +AUTOMATIC_WORKFLOW = "native-switch2kit.yml" +CHECKOUT = "actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1" +BOOTSTRAP = ('python3 -m pip install --disable-pip-version-check --no-deps ' + '--only-binary=:all: --retries 1 --timeout 10 ' + '--target "$RUNNER_TEMP/ci-policy" PyYAML==6.0.3\n' + 'python3 Tools/check_ci_policy.py') +SUBMODULES = "git submodule update --init --depth 1 Externals/Switch2Kit Externals/SDL/SDL" +FAST_TESTS = "python3 Tools/run_fast_tests.py" + + +class PolicyError(ValueError): + pass + + +class UniqueLoader(yaml.BaseLoader): + # BaseLoader preserves 'on' as a string (YAML 1.1 otherwise makes it True). + def construct_mapping(self, node, deep=False): + keys = [self.construct_object(key, deep=deep) for key, _ in node.value] + if any(not isinstance(key, str) for key in keys) or len(keys) != len(set(keys)): + raise PolicyError("Workflow mapping keys must be unique strings") + return super().construct_mapping(node, deep=deep) + + +def load_workflow(text): + # No aliases/merge keys hiding extra triggers, jobs or overridden budgets. + if any(isinstance(token, (yaml.tokens.AnchorToken, yaml.tokens.AliasToken)) + for token in yaml.scan(text)): + raise PolicyError("Workflow anchors and aliases are not allowed") + workflow = yaml.load(text, Loader=UniqueLoader) + if not isinstance(workflow, dict): + raise PolicyError("Workflow must be a mapping") + return workflow + + +def require(condition, message): + if not condition: + raise PolicyError(message) + + +def event_names(events): + if isinstance(events, str): + return {events} + if isinstance(events, (dict, list)): + return set(events) + raise PolicyError("Invalid workflow triggers") + + +def validate_workflow(name, workflow): + events = event_names(workflow.get("on")) + jobs = workflow.get("jobs") + require(isinstance(jobs, dict) and jobs, "Workflow must contain jobs") + if name != AUTOMATIC_WORKFLOW: + require(events == {"workflow_dispatch"}, + "Only native-switch2kit.yml may run automatically; use workflow_dispatch") + for job in jobs.values(): + require(isinstance(job, dict), "Invalid manual job") + limit = job.get("timeout-minutes", "") + require(str(limit).isdigit() and 0 < int(limit) <= 60, + "Manual qualification jobs need an explicit timeout <= 60 minutes") + require("strategy" not in job, "Select one manual target; no build matrices") + return + + require(set(workflow) == {"name", "on", "permissions", "concurrency", "jobs"}, + "Automatic workflow has unreviewed top-level settings") + require(workflow["on"] == {"pull_request": "", "workflow_dispatch": ""}, + "Keep the gate on every PR, without path filters, duplicate pushes or schedules") + require(workflow["name"] == "Native Switch2Kit", "Keep the required check name stable") + require(workflow["permissions"] == {"contents": "read"}, "CI must be read-only") + require(workflow["concurrency"] == { + "group": "native-switch2kit-${{ github.ref }}", "cancel-in-progress": "true"}, + "Cancel superseded runs for the same ref") + require(set(jobs) == {"wiring"}, "Automatic CI has exactly one job: wiring") + job = jobs["wiring"] + require(isinstance(job, dict), "Invalid automatic job") + require(set(job) == {"runs-on", "timeout-minutes", "env", "steps"}, + "No matrices, services, containers, reusable jobs or conditional/soft-fail gates") + require(job["runs-on"] == "ubuntu-24.04", "Only one standard Linux runner is permitted") + require(job["timeout-minutes"] == "3", "Automatic job hard limit is 3 minutes") + require(job["env"] == {"PYTHONPATH": "${{ runner.temp }}/ci-policy"}, + "Only the isolated YAML parser path belongs in the automatic job environment") + expected = [ + {"uses": CHECKOUT, "timeout-minutes": "1", "with": { + "persist-credentials": "false", "fetch-depth": "1", "submodules": "false"}}, + {"run": BOOTSTRAP, "timeout-minutes": "1"}, + {"run": SUBMODULES, "timeout-minutes": "1"}, + {"run": FAST_TESTS, "timeout-minutes": "2"}, + ] + steps = job["steps"] + require(isinstance(steps, list) and all(isinstance(step, dict) for step in steps), + "Invalid automatic steps") + normalized = [] + for step in steps: + step = {key: value for key, value in step.items() if key != "name"} + if isinstance(step.get("run"), str): + step["run"] = step["run"].strip() + normalized.append(step) + require(normalized == expected, + "Automatic steps are allowlisted: shallow checkout, policy, two submodules, " + "bounded tests. No full builds, installers, artifacts or bypasses") + + +def check_repository(root=ROOT): + require((root / "AGENTS.md").is_file(), "Root AGENTS.md CI policy is required") + files = sorted(path for path in (root / ".github/workflows").iterdir() + if path.suffix in (".yml", ".yaml")) + require(any(path.name == AUTOMATIC_WORKFLOW for path in files), "Fast PR gate is missing") + for path in files: + try: + validate_workflow(path.name, load_workflow(path.read_text(encoding="utf-8"))) + except (ValueError, TypeError, yaml.YAMLError) as error: + raise PolicyError(f"{path.name}: {error}") from error + print(f"CI policy passed: {len(files)} workflows; one automatic 3-minute Linux job.") + + +if __name__ == "__main__": + try: + check_repository() + except (PolicyError, OSError) as error: + sys.exit(f"CI POLICY FAILURE: {error}\nSee AGENTS.md; do not raise the budget to pass.") diff --git a/Tools/run_fast_tests.py b/Tools/run_fast_tests.py new file mode 100644 index 000000000000..0e139764bf79 --- /dev/null +++ b/Tools/run_fast_tests.py @@ -0,0 +1,79 @@ +#!/usr/bin/env python3 +# Copyright 2026 Dolphin Emulator Project +# SPDX-License-Identifier: GPL-2.0-or-later +"""Run the small production-code regressions, never a full Dolphin/SDK build. + +The suite has a shared wall-clock deadline, not N independent timeout allowances. +A failure or timeout is fatal; kill the entire compiler/test process group. +""" +import os +from pathlib import Path +import signal +import subprocess +import sys +import time + +ROOT = Path(__file__).resolve().parents[1] +SUITE_SECONDS = 60 +TEST_SECONDS = 20 +# Keep both compiler capacity probes and the existing sanitizer coverage. +TESTS = ( + ("CI policy regressions", ("test_ci_policy.py",), {}), + ("SDL capacity (Clang)", ("test_switch2kit_capacity.py",), {"CXX": "clang++"}), + ("SDL capacity (GCC)", ("test_switch2kit_capacity.py",), {"CXX": "g++"}), + ("Integration and disabled CMake", ("test_switch2kit.py",), {}), + ("Connection consent", ("test_switch2kit_autoconnect.py",), {}), + ("Mapping ASan/UBSan", ("test_switch2kit_mapping.py", "--sanitize"), {}), + ("Host lifecycle ASan/UBSan", ("test_switch2kit_host.py", "--sanitize"), {}), + ("Tiny deployment fixtures", ("test_switch2kit_runtime.py",), {}), +) + + +class BudgetExceeded(RuntimeError): + pass + + +def run_command(command, *, seconds, env, cwd): + if seconds <= 0: + raise BudgetExceeded("Shared suite deadline exhausted before starting the next test") + # Automatic CI is Linux. Do not leave grandchildren (e.g. cc1plus) running. + process = subprocess.Popen(command, env=env, cwd=cwd, start_new_session=True) + try: + result = process.wait(timeout=seconds) + except subprocess.TimeoutExpired as error: + raise BudgetExceeded(f"Test exceeded its {seconds:.2f}s remaining allowance") from error + finally: + try: + os.killpg(process.pid, signal.SIGKILL) + except ProcessLookupError: + pass + process.wait() + if result: + raise subprocess.CalledProcessError(result, command) + + +def run_suite(tests=TESTS, *, suite_seconds=SUITE_SECONDS, test_seconds=TEST_SECONDS, + root=ROOT): + started = time.monotonic() + deadline = started + suite_seconds + for name, arguments, extra_env in tests: + print(f"\n=== {name} ===", flush=True) + before = time.monotonic() + run_command([sys.executable, str(root / "Tools" / arguments[0]), *arguments[1:]], + seconds=min(test_seconds, deadline - before), + env={**os.environ, **extra_env}, cwd=root) + print(f"PASS {name}: {time.monotonic() - before:.2f}s", flush=True) + elapsed = time.monotonic() - started + if elapsed > suite_seconds: + raise BudgetExceeded(f"Suite exceeded {suite_seconds}s") + print(f"\nAll {len(tests)} checks passed in {elapsed:.2f}s (budget {suite_seconds}s).", + flush=True) + + +if __name__ == "__main__": + if sys.platform != "linux": + sys.exit("The budgeted PR suite requires Linux; run individual portable tests locally.") + try: + run_suite() + except (BudgetExceeded, subprocess.CalledProcessError, OSError) as error: + sys.exit(f"FAST CI FAILED: {error}\nFix or reduce the test; do not increase the budget.") diff --git a/Tools/test_ci_policy.py b/Tools/test_ci_policy.py new file mode 100644 index 000000000000..c56fb71f21f5 --- /dev/null +++ b/Tools/test_ci_policy.py @@ -0,0 +1,204 @@ +#!/usr/bin/env python3 +# Copyright 2026 Dolphin Emulator Project +# SPDX-License-Identifier: GPL-2.0-or-later +"""Small regression tests for CI fan-out, YAML parsing and runtime limits.""" +import contextlib +import copy +import io +import os +from pathlib import Path +import signal +import subprocess +import sys +import tempfile +import unittest +from unittest.mock import patch + +import yaml +from check_ci_policy import (AUTOMATIC_WORKFLOW, ROOT, PolicyError, check_repository, + load_workflow, validate_workflow) +from run_fast_tests import (BudgetExceeded, SUITE_SECONDS, TEST_SECONDS, TESTS, + run_command, run_suite) + + +class WorkflowPolicyTests(unittest.TestCase): + def setUp(self): + self.workflow = load_workflow( + (ROOT / '.github/workflows' / AUTOMATIC_WORKFLOW).read_text()) + + def reject(self, workflow, name=AUTOMATIC_WORKFLOW): + with self.assertRaises(PolicyError): + validate_workflow(name, workflow) + + def test_repository_policy(self): + check_repository() + + def test_only_one_automatic_job_and_runner(self): + for field, value in (('runs-on', 'macos-15'), ('runs-on', 'windows-2025-vs2026'), + ('runs-on', ['self-hosted']), ('timeout-minutes', '90'), + ('strategy', {'matrix': {'os': ['ubuntu-24.04', 'macos-15']}}), + ('container', 'swift:6.2.1-noble'), ('services', {}), + ('uses', './.github/workflows/build.yml'), + ('continue-on-error', 'true'), ('if', 'false')): + with self.subTest(field=field, value=value): + workflow = copy.deepcopy(self.workflow) + workflow['jobs']['wiring'][field] = value + self.reject(workflow) + self.workflow['jobs']['build'] = copy.deepcopy(self.workflow['jobs']['wiring']) + self.reject(self.workflow) + + def test_cannot_remove_gates_or_add_full_builds(self): + for mutation in ('extra', 'removed', 'recursive', 'soft-fail', 'skip', 'replace'): + with self.subTest(mutation=mutation): + workflow = copy.deepcopy(self.workflow) + steps = workflow['jobs']['wiring']['steps'] + if mutation == 'extra': + steps.append({'run': 'cmake --build build --target dolphin-emu'}) + elif mutation == 'removed': + steps.pop(1) + elif mutation == 'recursive': + steps[0]['with']['submodules'] = 'recursive' + elif mutation == 'soft-fail': + steps[-1]['continue-on-error'] = 'true' + elif mutation == 'skip': + steps[-1]['if'] = 'false' + else: + steps[-1]['run'] = 'python3 Tools/run_fast_tests.py || true' + self.reject(workflow) + + def test_unfiltered_pr_gate_and_no_duplicate_events(self): + for events in ({'pull_request': {'paths': ['Source/**']}, 'workflow_dispatch': ''}, + {'push': '', 'pull_request': '', 'workflow_dispatch': ''}, + {'workflow_dispatch': ''}): + self.workflow['on'] = events + self.reject(self.workflow) + + def test_manual_workflows_reject_all_automatic_trigger_syntaxes(self): + for trigger in ('pull_request', 'push', 'schedule', 'workflow_run', + 'workflow_call', 'pull_request_target', 'issue_comment'): + for events in (trigger, [trigger, 'workflow_dispatch'], + {trigger: '', 'workflow_dispatch': ''}): + with self.subTest(events=events): + workflow = {'on': events, 'jobs': {'build': {'timeout-minutes': '45'}}} + self.reject(workflow, 'new-expensive.yaml') + + def test_manual_dispatch_forms_remain_usable(self): + for events in ('workflow_dispatch', ['workflow_dispatch'], {'workflow_dispatch': ''}): + validate_workflow('manual.yml', { + 'on': events, 'jobs': {'build': {'timeout-minutes': '45'}}}) + + def test_manual_timeouts_and_matrices(self): + for job in ({}, {'timeout-minutes': '120'}, {'timeout-minutes': '${{ inputs.limit }}'}, + {'timeout-minutes': '0'}, {'timeout-minutes': '45', 'strategy': {}}): + self.reject({'on': 'workflow_dispatch', 'jobs': {'build': job}}, 'manual.yml') + + def test_real_yaml_parser_preserves_on_and_rejects_ambiguous_yaml(self): + self.assertEqual(load_workflow('on: [pull_request, workflow_dispatch]')['on'], + ['pull_request', 'workflow_dispatch']) + for text in ('on: workflow_dispatch\non: push\n', + 'on: workflow_dispatch\njobs:\n build:\n runs-on: linux\n runs-on: macos\n', + 'on: &events [push]\njobs: *events\n', 'on: [unterminated', '- not-a-workflow'): + with self.subTest(text=text), self.assertRaises((PolicyError, yaml.YAMLError)): + load_workflow(text) + + def test_new_yaml_file_is_scanned_and_missing_gate_fails(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + workflows = root / '.github/workflows' + workflows.mkdir(parents=True) + (root / 'AGENTS.md').write_text('policy') + with self.assertRaises(PolicyError): + check_repository(root) + (workflows / AUTOMATIC_WORKFLOW).write_text( + (ROOT / '.github/workflows' / AUTOMATIC_WORKFLOW).read_text()) + (workflows / 'surprise.yaml').write_text('on: push\njobs:\n big:\n timeout-minutes: 90\n') + with self.assertRaises(PolicyError): + check_repository(root) + (workflows / 'surprise.yaml').unlink() + (root / 'AGENTS.md').unlink() + with self.assertRaises(PolicyError): + check_repository(root) + + +class RuntimeBudgetTests(unittest.TestCase): + def test_budgets_and_existing_behavior_checks_are_preserved(self): + self.assertEqual((SUITE_SECONDS, TEST_SECONDS), (60, 20)) + commands = [(arguments, env) for _, arguments, env in TESTS] + for arguments, env in ( + (('test_ci_policy.py',), {}), + (('test_switch2kit_capacity.py',), {'CXX': 'clang++'}), + (('test_switch2kit_capacity.py',), {'CXX': 'g++'}), + (('test_switch2kit_mapping.py', '--sanitize'), {}), + (('test_switch2kit_host.py', '--sanitize'), {}), + (('test_switch2kit.py',), {}), + (('test_switch2kit_autoconnect.py',), {}), + (('test_switch2kit_runtime.py',), {})): + self.assertIn((arguments, env), commands) + + def test_nonzero_exit_is_not_a_pass(self): + with self.assertRaises(subprocess.CalledProcessError): + run_command([sys.executable, '-c', 'raise SystemExit(7)'], seconds=2, + env=os.environ, cwd=ROOT) + + def test_expired_deadline_never_starts_another_process(self): + with patch('run_fast_tests.subprocess.Popen') as spawn: + with self.assertRaises(BudgetExceeded): + run_command(['unused'], seconds=0, env={}, cwd=ROOT) + spawn.assert_not_called() + + def test_shared_deadline_not_a_fresh_allowance_per_test(self): + ticks = iter((0, 0, 0.25, 0.8, 0.9, 0.9)) + tests = (('one', ('one.py',), {}), ('two', ('two.py',), {})) + with patch('run_fast_tests.time.monotonic', side_effect=lambda: next(ticks)), \ + patch('run_fast_tests.run_command') as command, \ + contextlib.redirect_stdout(io.StringIO()): + run_suite(tests, suite_seconds=1, test_seconds=0.75) + self.assertAlmostEqual(command.call_args_list[0].kwargs['seconds'], 0.75) + self.assertAlmostEqual(command.call_args_list[1].kwargs['seconds'], 0.2) + + def test_failure_stops_the_suite_immediately(self): + tests = (('one', ('one.py',), {}), ('two', ('two.py',), {})) + with patch('run_fast_tests.run_command', + side_effect=subprocess.CalledProcessError(1, 'one')) as command, \ + contextlib.redirect_stdout(io.StringIO()): + with self.assertRaises(subprocess.CalledProcessError): + run_suite(tests) + self.assertEqual(command.call_count, 1) + + @unittest.skipUnless(sys.platform == 'linux', 'The budgeted PR lane is Linux') + def test_timeout_kills_the_entire_process_group(self): + # Start a grandchild that ignores SIGTERM. Simulate only the wait timeout, + # so this test is deterministic and does not sleep for an actual budget. + process = subprocess.Popen( + [sys.executable, '-u', '-c', + 'import os,subprocess,sys,time; ' + 'child=subprocess.Popen([sys.executable, "-u", "-c", ' + '"import signal,time; signal.signal(signal.SIGTERM,signal.SIG_IGN); ' + 'print(123,flush=True); time.sleep(30)"],stdout=subprocess.PIPE); ' + 'child.stdout.readline(); print(os.getpgrp(),flush=True); time.sleep(30)'], + stdout=subprocess.PIPE, text=True, start_new_session=True) + try: + self.assertEqual(process.stdout.readline().strip(), str(process.pid)) + actual_wait = process.wait + def wait(timeout=None): + if timeout is not None: + raise subprocess.TimeoutExpired(process.args, timeout) + return actual_wait(timeout=2) + with patch('run_fast_tests.subprocess.Popen', return_value=process), \ + patch.object(process, 'wait', side_effect=wait), \ + patch('run_fast_tests.os.killpg', wraps=os.killpg) as kill: + with self.assertRaises(BudgetExceeded): + run_command(['unused'], seconds=1, env={}, cwd=ROOT) + kill.assert_called_once_with(process.pid, signal.SIGKILL) + self.assertEqual(process.returncode, -signal.SIGKILL) + finally: + try: + os.killpg(process.pid, signal.SIGKILL) + except ProcessLookupError: + pass + process.wait(timeout=2) + process.stdout.close() + + +if __name__ == '__main__': + unittest.main(verbosity=2) From 169117a03b659f06e9ac52d8515eba1ae181f33a Mon Sep 17 00:00:00 2001 From: Johnny D Date: Sun, 20 Sep 2026 15:56:10 -0400 Subject: [PATCH 02/21] ci: resolve runner-temporary parser path in step context GitHub does not allow the runner context in job-level env. Scope the isolated parser path to its two consuming steps instead, and cover the invalid job-level configuration in the policy mutation tests. Local policy and all 15 enforcement tests pass. Keep all runtime budgets, production regression commands and manual qualification gates unchanged. --- .github/workflows/native-switch2kit.yml | 6 ++++-- Tools/check_ci_policy.py | 11 ++++++----- Tools/test_ci_policy.py | 1 + 3 files changed, 11 insertions(+), 7 deletions(-) diff --git a/.github/workflows/native-switch2kit.yml b/.github/workflows/native-switch2kit.yml index 8f12be717a61..81bde228bb5f 100644 --- a/.github/workflows/native-switch2kit.yml +++ b/.github/workflows/native-switch2kit.yml @@ -12,8 +12,6 @@ jobs: wiring: runs-on: ubuntu-24.04 timeout-minutes: 3 - env: - PYTHONPATH: ${{ runner.temp }}/ci-policy steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 timeout-minutes: 1 @@ -23,6 +21,8 @@ jobs: submodules: false - name: Enforce the CI cost policy before fetching dependencies timeout-minutes: 1 + env: + PYTHONPATH: ${{ runner.temp }}/ci-policy run: | python3 -m pip install --disable-pip-version-check --no-deps --only-binary=:all: --retries 1 --timeout 10 --target "$RUNNER_TEMP/ci-policy" PyYAML==6.0.3 python3 Tools/check_ci_policy.py @@ -31,4 +31,6 @@ jobs: run: git submodule update --init --depth 1 Externals/Switch2Kit Externals/SDL/SDL - name: Run the fast regressions (60 seconds total, 20 seconds per command) timeout-minutes: 2 + env: + PYTHONPATH: ${{ runner.temp }}/ci-policy run: python3 Tools/run_fast_tests.py diff --git a/Tools/check_ci_policy.py b/Tools/check_ci_policy.py index 813c566438ab..fe0a643f4876 100644 --- a/Tools/check_ci_policy.py +++ b/Tools/check_ci_policy.py @@ -82,18 +82,19 @@ def validate_workflow(name, workflow): require(set(jobs) == {"wiring"}, "Automatic CI has exactly one job: wiring") job = jobs["wiring"] require(isinstance(job, dict), "Invalid automatic job") - require(set(job) == {"runs-on", "timeout-minutes", "env", "steps"}, + require(set(job) == {"runs-on", "timeout-minutes", "steps"}, "No matrices, services, containers, reusable jobs or conditional/soft-fail gates") require(job["runs-on"] == "ubuntu-24.04", "Only one standard Linux runner is permitted") require(job["timeout-minutes"] == "3", "Automatic job hard limit is 3 minutes") - require(job["env"] == {"PYTHONPATH": "${{ runner.temp }}/ci-policy"}, - "Only the isolated YAML parser path belongs in the automatic job environment") + # runner.temp is available only after a runner is assigned (step env), + # not in jobs..env. Keep the parser isolated to its two consumers. + parser_env = {"PYTHONPATH": "${{ runner.temp }}/ci-policy"} expected = [ {"uses": CHECKOUT, "timeout-minutes": "1", "with": { "persist-credentials": "false", "fetch-depth": "1", "submodules": "false"}}, - {"run": BOOTSTRAP, "timeout-minutes": "1"}, + {"run": BOOTSTRAP, "timeout-minutes": "1", "env": parser_env}, {"run": SUBMODULES, "timeout-minutes": "1"}, - {"run": FAST_TESTS, "timeout-minutes": "2"}, + {"run": FAST_TESTS, "timeout-minutes": "2", "env": parser_env}, ] steps = job["steps"] require(isinstance(steps, list) and all(isinstance(step, dict) for step in steps), diff --git a/Tools/test_ci_policy.py b/Tools/test_ci_policy.py index c56fb71f21f5..98bfea534286 100644 --- a/Tools/test_ci_policy.py +++ b/Tools/test_ci_policy.py @@ -38,6 +38,7 @@ def test_only_one_automatic_job_and_runner(self): ('runs-on', ['self-hosted']), ('timeout-minutes', '90'), ('strategy', {'matrix': {'os': ['ubuntu-24.04', 'macos-15']}}), ('container', 'swift:6.2.1-noble'), ('services', {}), + ('env', {'PYTHONPATH': '${{ runner.temp }}/ci-policy'}), ('uses', './.github/workflows/build.yml'), ('continue-on-error', 'true'), ('if', 'false')): with self.subTest(field=field, value=value): From 53c6845c0edcf5623d4ef95f88413491e27f81da Mon Sep 17 00:00:00 2001 From: Johnny D Date: Sun, 20 Sep 2026 15:58:01 -0400 Subject: [PATCH 03/21] test: follow the bounded runner when checking retained CI coverage Keep every consent/startup assertion intact. The CI-retention check now checks the actual runner argument tuples, including sanitizer flags, instead of requiring individual commands inline in the workflow YAML. The focused retention test and all 15 policy tests pass locally. --- Tools/test_switch2kit_autoconnect.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/Tools/test_switch2kit_autoconnect.py b/Tools/test_switch2kit_autoconnect.py index 42c147c69b98..fac8b28d08ef 100755 --- a/Tools/test_switch2kit_autoconnect.py +++ b/Tools/test_switch2kit_autoconnect.py @@ -63,9 +63,15 @@ def test_polling_and_initialization_never_start_bluetooth(self): def test_ci_retains_regressions(self): workflow = self.read(".github/workflows/native-switch2kit.yml") - for script in ("test_switch2kit.py", "test_switch2kit_host.py --sanitize", - "test_switch2kit_mapping.py --sanitize", "test_switch2kit_autoconnect.py"): - self.assertIn(script, workflow) + self.assertIn("python3 Tools/run_fast_tests.py", workflow) + # The budgeted runner owns these commands now. Check actual arguments, + # not script names that could appear only in workflow comments. + from run_fast_tests import TESTS + commands = {arguments for _, arguments, _ in TESTS} + for command in (("test_switch2kit.py",), ("test_switch2kit_host.py", "--sanitize"), + ("test_switch2kit_mapping.py", "--sanitize"), + ("test_switch2kit_autoconnect.py",)): + self.assertIn(command, commands) if __name__ == "__main__": From e433603bac39fd5d5a1457033dacb481112242f8 Mon Sep 17 00:00:00 2001 From: Johnny D Date: Sun, 20 Sep 2026 16:53:06 -0400 Subject: [PATCH 04/21] ci: follow upstream CTest and restore input-scoped native validation Replace the fork-wide YAML policy and bespoke Python deadline runner with ordinary CTest registration for the existing focused regression harnesses. Preserve upstream tests/unittests and run its Google Test suite in the real Swift-free Linux application build tree. Restore automatic real SDK/rumble/SDL tests for changed SDK inputs and all native build, architecture, dependency and exact-package launch checks for changed production/build inputs. Scope platform-only changes to their own jobs; test selection against renames, deletions and Git failures. Aggregate selected job results so the small wiring check cannot hide missing native qualification. Cancel obsolete runs and avoid duplicate PR triggers. Use standard CMake sccache launchers and a pinned Swift test-build cache; never cache a passing test result. Keep application archives for launch validation but upload successful distributions only on explicit dispatch. Rewrite AGENTS.md around upstream compatibility, efficient coverage and honest reporting instead of a blanket ban on native tests. AI-assisted change. Nine change-selection regressions, CTest enumeration, the CI-retention regression, Python compilation and YAML parsing passed locally in a partial checkout. Full exact-head CI must pass; no native performance improvement or native validation result is assumed. --- .github/workflows/native-switch2kit.yml | 115 ++++++++++-- .github/workflows/switch2kit-linux.yml | 43 +++-- .github/workflows/switch2kit-macos.yml | 74 +++----- .github/workflows/switch2kit-windows.yml | 13 +- AGENTS.md | 110 ++++++----- Docs/Switch2Kit.md | 21 +-- Docs/Switch2KitCI.md | 77 ++++++++ Source/UnitTests/CMakeLists.txt | 6 + Source/UnitTests/Switch2Kit/CMakeLists.txt | 31 ++++ Tools/check_ci_policy.py | 130 ------------- Tools/run_fast_tests.py | 79 -------- Tools/switch2kit_ci.py | 81 ++++++++ Tools/test_ci_policy.py | 205 --------------------- Tools/test_switch2kit_autoconnect.py | 28 ++- Tools/test_switch2kit_ci.py | 92 +++++++++ 15 files changed, 530 insertions(+), 575 deletions(-) create mode 100644 Docs/Switch2KitCI.md create mode 100644 Source/UnitTests/Switch2Kit/CMakeLists.txt delete mode 100644 Tools/check_ci_policy.py delete mode 100644 Tools/run_fast_tests.py create mode 100644 Tools/switch2kit_ci.py delete mode 100644 Tools/test_ci_policy.py mode change 100755 => 100644 Tools/test_switch2kit_autoconnect.py create mode 100644 Tools/test_switch2kit_ci.py diff --git a/.github/workflows/native-switch2kit.yml b/.github/workflows/native-switch2kit.yml index 81bde228bb5f..f09e3a8a1d47 100644 --- a/.github/workflows/native-switch2kit.yml +++ b/.github/workflows/native-switch2kit.yml @@ -1,8 +1,14 @@ -# This is the only automatic workflow. Budgets are enforced by Tools/check_ci_policy.py. +# Upstream CMake/CTest plus scoped Switch2Kit qualification. See AGENTS.md. name: Native Switch2Kit on: pull_request: workflow_dispatch: + inputs: + target: + description: 'Tests only, or explicit native qualification/cache priming' + type: choice + default: tests + options: [tests, sdk, macos, linux, windows, all] permissions: contents: read concurrency: @@ -12,25 +18,106 @@ jobs: wiring: runs-on: ubuntu-24.04 timeout-minutes: 3 + outputs: + sdk: ${{ steps.scope.outputs.sdk }} + macos: ${{ steps.scope.outputs.macos }} + linux: ${{ steps.scope.outputs.linux }} + windows: ${{ steps.scope.outputs.windows }} steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 - timeout-minutes: 1 with: persist-credentials: false fetch-depth: 1 - submodules: false - - name: Enforce the CI cost policy before fetching dependencies - timeout-minutes: 1 + - name: Select validation from the actual changed inputs + id: scope env: - PYTHONPATH: ${{ runner.temp }}/ci-policy + BASE_SHA: ${{ github.event.pull_request.base.sha }} + TARGET: ${{ inputs.target || 'tests' }} run: | - python3 -m pip install --disable-pip-version-check --no-deps --only-binary=:all: --retries 1 --timeout 10 --target "$RUNNER_TEMP/ci-policy" PyYAML==6.0.3 - python3 Tools/check_ci_policy.py - - name: Fetch only the pinned SDK and SDL, without their history - timeout-minutes: 1 + if [ -n "$BASE_SHA" ]; then + git fetch --no-tags --depth 1 origin "$BASE_SHA" + python3 Tools/switch2kit_ci.py --base "$BASE_SHA" + else + python3 Tools/switch2kit_ci.py --target "$TARGET" + fi + - name: Fetch only the two dependencies used by the focused tests run: git submodule update --init --depth 1 Externals/Switch2Kit Externals/SDL/SDL - - name: Run the fast regressions (60 seconds total, 20 seconds per command) - timeout-minutes: 2 + - name: Run the additions through CTest without building the emulator + run: | + cmake -S Source/UnitTests/Switch2Kit -B build-switch2kit-tests + ctest --test-dir build-switch2kit-tests --output-on-failure --no-tests=error + sdk-regressions: + needs: wiring + if: needs.wiring.outputs.sdk == 'true' + runs-on: macos-15 + timeout-minutes: 5 + env: + DEVELOPER_DIR: /Applications/Xcode_26.3.app/Contents/Developer + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 + with: + persist-credentials: false + - name: Initialize exactly the pinned SDK and Dolphin SDL + run: git submodule update --init --depth 1 Externals/Switch2Kit Externals/SDL/SDL + - name: Reuse the pinned Swift test build, not a previous test result + uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 + with: + path: Externals/Switch2Kit/.build + key: s2k-sdk-tests-${{ runner.os }}-${{ runner.arch }}-xcode26.3-${{ hashFiles('Externals/Switch2Kit/Package.swift', 'Externals/Switch2Kit/Package.resolved', 'Externals/Switch2Kit/Sources/**', 'Externals/Switch2Kit/Tests/**') }} + - name: Exercise the real SDK and in-process SDL adapter without hardware + run: | + set -o pipefail + swift test --package-path Externals/Switch2Kit 2>&1 | tee switch2kit-sdk-tests.log + bash Externals/Switch2Kit/tests/rumble/run.sh 2>&1 | tee switch2kit-motor-tests.log + S2K_SDL_SOURCE="$PWD/Externals/SDL/SDL" \ + bash Externals/Switch2Kit/tests/sdl-inprocess/verify.sh \ + 2>&1 | tee switch2kit-sdl-tests.log + - name: SDK test diagnostics + if: failure() || github.event_name == 'workflow_dispatch' + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 + with: + name: Switch2Kit-sdk-regressions + path: switch2kit-*-tests.log + macos: + needs: [wiring, sdk-regressions] + if: >- + !cancelled() && needs.wiring.result == 'success' && + (needs.sdk-regressions.result == 'success' || needs.sdk-regressions.result == 'skipped') && + needs.wiring.outputs.macos == 'true' + uses: ./.github/workflows/switch2kit-macos.yml + linux: + needs: [wiring, sdk-regressions] + if: >- + !cancelled() && needs.wiring.result == 'success' && + (needs.sdk-regressions.result == 'success' || needs.sdk-regressions.result == 'skipped') && + needs.wiring.outputs.linux == 'true' + uses: ./.github/workflows/switch2kit-linux.yml + windows: + needs: [wiring, sdk-regressions] + if: >- + !cancelled() && needs.wiring.result == 'success' && + (needs.sdk-regressions.result == 'success' || needs.sdk-regressions.result == 'skipped') && + needs.wiring.outputs.windows == 'true' + uses: ./.github/workflows/switch2kit-windows.yml + checks: + name: Switch2Kit checks + needs: [wiring, sdk-regressions, macos, linux, windows] + if: always() + runs-on: ubuntu-24.04 + timeout-minutes: 1 + steps: + - name: Require every selected check, including native qualification env: - PYTHONPATH: ${{ runner.temp }}/ci-policy - run: python3 Tools/run_fast_tests.py + RESULTS: ${{ toJSON(needs) }} + run: | + python3 - <<'PYTHON' + import json, os + jobs = json.loads(os.environ['RESULTS']) + assert jobs['wiring']['result'] == 'success', 'Focused tests or change selection failed' + selected = jobs['wiring']['outputs'] + for target, job in [('sdk', 'sdk-regressions'), ('macos', 'macos'), + ('linux', 'linux'), ('windows', 'windows')]: + assert selected.get(target) in ('true', 'false'), f'Missing selection for {target}' + expected = 'success' if selected[target] == 'true' else 'skipped' + assert jobs[job]['result'] == expected, f'{job}: {jobs[job]["result"]}, expected {expected}' + PYTHON diff --git a/.github/workflows/switch2kit-linux.yml b/.github/workflows/switch2kit-linux.yml index 9df656b2c3b3..05fc48166e0b 100644 --- a/.github/workflows/switch2kit-linux.yml +++ b/.github/workflows/switch2kit-linux.yml @@ -1,29 +1,23 @@ -# Manual qualification/artifacts only. Never add automatic triggers; see AGENTS.md. +# Called automatically for affected inputs; never a duplicate PR trigger. name: Switch2Kit Linux application on: + workflow_call: workflow_dispatch: - inputs: - target: - description: 'Choose ONE complete application build (can take 25 minutes)' - required: true - type: choice - default: enabled - options: - - enabled - - disabled permissions: contents: read concurrency: - group: switch2kit-linux-${{ github.ref }}-${{ inputs.target }} + group: switch2kit-linux-${{ github.ref }} cancel-in-progress: true jobs: linux: - if: inputs.target == 'enabled' runs-on: ubuntu-24.04 container: swift:6.2.1-noble timeout-minutes: 45 env: DEBIAN_FRONTEND: noninteractive + SCCACHE_GHA_ENABLED: 'true' + CMAKE_C_COMPILER_LAUNCHER: sccache + CMAKE_CXX_COMPILER_LAUNCHER: sccache steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 with: @@ -34,6 +28,10 @@ jobs: apt-get update apt-get install -y --no-install-recommends build-essential cmake ninja-build python3 pkg-config qt6-base-dev qt6-base-private-dev qt6-svg-dev libbluetooth-dev libevdev-dev libudev-dev libusb-1.0-0-dev libasound2-dev libpulse-dev libx11-dev libxi-dev libxrandr-dev libegl1-mesa-dev libgl1-mesa-dev libsystemd0 dbus xvfb xauth openbox wmctrl x11-utils git config --global --add safe.directory "$PWD" + - name: Cache native compiler results (tests still run) + uses: Mozilla-Actions/sccache-action@7d986dd989559c6ecdb630a3fd2557667be217ad + with: + version: v0.10.0 - name: Build and install the controller-enabled application shell: bash run: | @@ -63,6 +61,7 @@ jobs: --forbidden-root "$PWD" --forbidden-root "$RUNNER_TEMP/disabled-build-tree" \ --forbidden-root "$RUNNER_TEMP/unavailable-staged-application" - name: Controller-enabled development build + if: github.event_name == 'workflow_dispatch' uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 with: name: Dolphin-Switch2Kit-linux-x86_64 @@ -70,7 +69,7 @@ jobs: retention-days: 14 if-no-files-found: error - name: Native diagnostics - if: always() + if: failure() || github.event_name == 'workflow_dispatch' uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 with: name: Dolphin-Switch2Kit-linux-diagnostics @@ -80,7 +79,7 @@ jobs: retention-days: 7 disabled: - if: inputs.target == 'disabled' + needs: linux # A complete ordinary application build in an image without Swift. This is # not a controller-enabled artifact and does not use the SDK build helper. runs-on: ubuntu-24.04 @@ -88,6 +87,9 @@ jobs: timeout-minutes: 45 env: DEBIAN_FRONTEND: noninteractive + SCCACHE_GHA_ENABLED: 'true' + CMAKE_C_COMPILER_LAUNCHER: sccache + CMAKE_CXX_COMPILER_LAUNCHER: sccache steps: - name: Install ordinary native dependencies without Swift run: | @@ -101,6 +103,10 @@ jobs: with: persist-credentials: false submodules: recursive + - name: Cache native compiler results (tests still run) + uses: Mozilla-Actions/sccache-action@7d986dd989559c6ecdb630a3fd2557667be217ad + with: + version: v0.10.0 - name: Build the complete application with Switch2Kit disabled shell: bash run: | @@ -108,9 +114,10 @@ jobs: git config --global --add safe.directory "$PWD" cmake -S . -B build-disabled -G Ninja -DCMAKE_BUILD_TYPE=Release \ -DENABLE_SWITCH2KIT=OFF -DENABLE_SDL=ON -DENABLE_QT=ON \ - -DUSE_SYSTEM_SDL3=OFF -DENABLE_TESTS=OFF -DENABLE_VULKAN=OFF \ + -DUSE_SYSTEM_SDL3=OFF -DENABLE_TESTS=ON -DENABLE_VULKAN=OFF \ -DENABLE_AUTOUPDATE=OFF 2>&1 | tee disabled-configure.log cmake --build build-disabled --target dolphin-emu --parallel 3 2>&1 | tee disabled-build.log + cmake --build build-disabled --target unittests --parallel 3 2>&1 | tee upstream-unittests.log ldd build-disabled/Binaries/dolphin-emu | tee disabled-dependencies.log ! grep -q 'not found' disabled-dependencies.log ! grep -Ei 'Switch2Kit|libswift|libFoundation|libdispatch|libBlocksRuntime' disabled-dependencies.log @@ -121,9 +128,11 @@ jobs: HOME="$profile" XDG_CONFIG_HOME="$profile/config" XDG_CACHE_HOME="$profile/cache" \ QT_QPA_PLATFORM=offscreen build-disabled/Binaries/dolphin-emu --user "$profile/user" --version - name: Backend-disabled build diagnostics - if: always() + if: failure() || github.event_name == 'workflow_dispatch' uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 with: name: Dolphin-without-Switch2Kit-linux-diagnostics - path: disabled-*.log + path: | + disabled-*.log + upstream-unittests.log retention-days: 7 diff --git a/.github/workflows/switch2kit-macos.yml b/.github/workflows/switch2kit-macos.yml index 66db931d929f..beaac0b94b55 100644 --- a/.github/workflows/switch2kit-macos.yml +++ b/.github/workflows/switch2kit-macos.yml @@ -1,60 +1,39 @@ -# Full native qualification is opt-in, never a pull-request test. See AGENTS.md. +# Called automatically for affected inputs; never a duplicate PR trigger. name: Switch2Kit macOS qualification on: + workflow_call: workflow_dispatch: - inputs: - target: - description: 'Explicit native qualification (application builds can take 45 minutes)' - required: true - type: choice - default: sdk - options: - - sdk - - arm64 - - x86_64 permissions: contents: read concurrency: - group: switch2kit-macos-${{ github.ref }}-${{ inputs.target }} + group: switch2kit-macos-${{ github.ref }} cancel-in-progress: true jobs: - sdk-regressions: - if: inputs.target == 'sdk' - runs-on: macos-15 - timeout-minutes: 5 - env: - DEVELOPER_DIR: /Applications/Xcode_26.3.app/Contents/Developer - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 - with: - persist-credentials: false - - name: Initialize exactly the pinned SDK and Dolphin SDL - run: git submodule update --init Externals/Switch2Kit Externals/SDL/SDL - - name: Exercise the real SDK and in-process SDL adapter without hardware - run: | - set -o pipefail - swift test --package-path Externals/Switch2Kit 2>&1 | tee switch2kit-sdk-tests.log - bash Externals/Switch2Kit/tests/rumble/run.sh 2>&1 | tee switch2kit-motor-tests.log - S2K_SDL_SOURCE="$PWD/Externals/SDL/SDL" \ - bash Externals/Switch2Kit/tests/sdl-inprocess/verify.sh \ - 2>&1 | tee switch2kit-sdl-tests.log - - name: SDK test diagnostics - if: always() - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 - with: - name: Switch2Kit-sdk-regressions - path: switch2kit-*-tests.log macos: - if: inputs.target == 'arm64' || inputs.target == 'x86_64' - runs-on: ${{ inputs.target == 'arm64' && 'macos-15' || 'macos-15-intel' }} + strategy: + fail-fast: true + matrix: + include: + - os: macos-15 + arch: arm64 + - os: macos-15-intel + arch: x86_64 + runs-on: ${{ matrix.os }} timeout-minutes: 60 env: DEVELOPER_DIR: /Applications/Xcode_26.3.app/Contents/Developer + SCCACHE_GHA_ENABLED: 'true' + CMAKE_C_COMPILER_LAUNCHER: sccache + CMAKE_CXX_COMPILER_LAUNCHER: sccache steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 with: submodules: recursive persist-credentials: false + - name: Cache native compiler results (tests still run) + uses: Mozilla-Actions/sccache-action@7d986dd989559c6ecdb630a3fd2557667be217ad + with: + version: v0.10.0 - name: Install build dependencies run: brew install cmake ninja nasm automake libtool qt@6 - name: Check source pin and integration @@ -66,7 +45,7 @@ jobs: python3 Tools/test_switch2kit_mapping.py --sanitize - name: Build the complete application env: - ARCH: ${{ inputs.target }} + ARCH: ${{ matrix.arch }} run: | set -o pipefail test "$(uname -m)" = "$ARCH" @@ -82,7 +61,7 @@ jobs: 2>&1 | tee switch2kit-build.log - name: Inspect and archive application env: - ARCH: ${{ inputs.target }} + ARCH: ${{ matrix.arch }} run: | app=build-switch2kit/Binaries/DolphinQt.app test -d "$app" @@ -95,23 +74,24 @@ jobs: ditto -c -k --keepParent "$app" "Dolphin-Switch2Kit-$ARCH.zip" - name: Launch, normally quit and relaunch the exact ZIP with build roots denied env: - ARCH: ${{ inputs.target }} + ARCH: ${{ matrix.arch }} run: | python3 Tools/test_switch2kit_bundle.py "Dolphin-Switch2Kit-$ARCH.zip" \ --architecture "$ARCH" --report switch2kit-validation.json - name: Application artifact (not hardware-qualified or notarized) + if: github.event_name == 'workflow_dispatch' uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 with: - name: Dolphin-Switch2Kit-${{ inputs.target }} + name: Dolphin-Switch2Kit-${{ matrix.arch }} path: | - Dolphin-Switch2Kit-${{ inputs.target }}.zip + Dolphin-Switch2Kit-${{ matrix.arch }}.zip switch2kit-validation.json if-no-files-found: error - name: Build diagnostics - if: always() + if: failure() || github.event_name == 'workflow_dispatch' uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 with: - name: Switch2Kit-${{ inputs.target }}-diagnostics + name: Switch2Kit-${{ matrix.arch }}-diagnostics path: | switch2kit-*.log switch2kit-*.sample diff --git a/.github/workflows/switch2kit-windows.yml b/.github/workflows/switch2kit-windows.yml index a4199b2c3dcf..835f1f70d13c 100644 --- a/.github/workflows/switch2kit-windows.yml +++ b/.github/workflows/switch2kit-windows.yml @@ -1,6 +1,7 @@ -# Manual qualification/artifacts only. Never add automatic triggers; see AGENTS.md. +# Called automatically for affected inputs; never a duplicate PR trigger. name: Switch2Kit Windows application on: + workflow_call: workflow_dispatch: permissions: contents: read @@ -13,11 +14,18 @@ jobs: timeout-minutes: 45 env: VCPKG_MAX_CONCURRENCY: 3 + SCCACHE_GHA_ENABLED: 'true' + CMAKE_C_COMPILER_LAUNCHER: sccache + CMAKE_CXX_COMPILER_LAUNCHER: sccache steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 with: persist-credentials: false submodules: recursive + - name: Cache native compiler results (tests still run) + uses: Mozilla-Actions/sccache-action@7d986dd989559c6ecdb630a3fd2557667be217ad + with: + version: v0.10.0 - uses: compnerd/gha-setup-swift@397094e75494a93fa8d81db0268dbc8f5d6cf7c6 with: # Swift 6.2 ships Clang 19, which the VS 2026 STL rejects. @@ -49,6 +57,7 @@ jobs: & ./Tools/test_switch2kit_windows_launch.ps1 -Archive 'Dolphin-Switch2Kit-windows-x86_64.zip' -ForbiddenRoot "$PWD" if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } - name: Controller-enabled development application + if: github.event_name == 'workflow_dispatch' uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 with: name: Dolphin-Switch2Kit-windows-x86_64 @@ -56,7 +65,7 @@ jobs: retention-days: 14 if-no-files-found: error - name: Native diagnostics - if: always() + if: failure() || github.event_name == 'workflow_dispatch' uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 with: name: Dolphin-Switch2Kit-windows-diagnostics diff --git a/AGENTS.md b/AGENTS.md index 8eeecf9faee3..1ba904c23f5b 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -1,63 +1,57 @@ -# Repository instructions - -These rules apply to the entire repository and to every agent/contributor. - -## CI cost and test speed are hard requirements - -Automatic tests must be lean and fast. A slow test must be reduced, replaced with -a focused regression, or removed from automatic CI. Do not solve a timeout by -raising the budget, adding runners, dropping assertions, skipping failures, or -turning off sanitizers. A warm cache is not evidence that a test is cheap. - -- Exactly ONE automatic GitHub Actions job: `Native Switch2Kit / wiring`, on - `ubuntu-24.04`, for pull requests. No automatic macOS/Windows jobs, matrices, - scheduled builds, `workflow_run` chains, reusable-workflow escape hatches, or - duplicate branch-push jobs. Keep the gate unfiltered so changes to workflows, - this file, and test infrastructure are always checked. -- Hard budgets: **60 seconds for the entire regression suite**, **20 seconds per - test command**, and **3 minutes for the whole job**, including setup. The job - timeout is a safety ceiling, not a target. Report cold-run timing when changing - tests; fail closed when a budget is exceeded. -- Test production behavior at the smallest boundary. Small C/C++ harnesses, - sanitizer checks, source checks and tiny CMake fixtures are appropriate. Do not - compile Dolphin, Qt, SDL, the Swift SDK, or all dependencies to test a small - policy/disabled-feature branch. Do not build an application just to inspect it. -- Only shallow checkout and the explicitly needed pinned Switch2Kit/SDL submodules - belong in the PR lane. No recursive submodule fetch, platform toolchain install, - source/application archive, or artifact upload. The one pinned PyYAML wheel is - installed into a temporary directory for structural workflow validation. -- Add focused tests to `Tools/run_fast_tests.py`, not separate workflow jobs. - Preserve both compiler capacity probes, ASan/UBSan host/mapping coverage, consent, - and feature-disabled checks. Keep fixtures tiny; do not claim they qualify a - native platform, packaged application, Bluetooth hardware or gameplay. - -## Full application qualification is separate and explicitly requested - -The macOS, Linux, Windows and source-archive workflows are **workflow_dispatch -only**. They are optional application/SDK qualification tools, not PR tests. -Do not dispatch them without a human explicitly requesting that expensive work. -Select one architecture/configuration at a time; do not recreate an automatic -build matrix. Preserve real build, packaging, dependency and launch assertions -when changing those tools. A passing fast PR check is not native qualification. - -## Required verification for CI/test edits - -Install `PyYAML==6.0.3` in your development environment, then run: +# Working on this Dolphin fork + +## Upstream first + +Keep Dolphin's upstream build system, Google Test suite, coding conventions and +project documentation intact. Make the smallest additive change for Switch2Kit. +Do not introduce a fork-wide test framework, custom YAML policy language, or a +blanket rule forbidding native validation. Upstream's external Buildbot service +is not automatically inherited by this fork; do not claim otherwise. + +The existing `unittests` target remains the baseline. Register our focused tests +with CTest in `Source/UnitTests/Switch2Kit`, not a parallel Python test runner. +The same registration can be configured alone for fast iteration: ```sh -python3 Tools/check_ci_policy.py -python3 Tools/test_ci_policy.py git submodule update --init --depth 1 Externals/Switch2Kit Externals/SDL/SDL -python3 Tools/run_fast_tests.py # Linux; enforces the shared deadline +cmake -S Source/UnitTests/Switch2Kit -B build-switch2kit-tests +ctest --test-dir build-switch2kit-tests --output-on-failure --no-tests=error ``` -`check_ci_policy.py` scans every `.yml`/`.yaml` workflow, rejects additional -automatic triggers/jobs and allowlists the fast lane's steps. Its tests must catch -attempts to restore expensive CI. Do not weaken the checker or runner to make a -change pass. Budget/policy exceptions need explicit repository-owner approval. -Repository-side checks are not a security boundary: keep `wiring` required in -branch protection and review changes to this policy and its enforcement together. - -Make targeted changes, preserve controller behavior and user-data safeguards, and -disclose AI-assisted changes and validation limitations in pull requests. Never -claim a test, native build, or hardware check was performed unless it actually ran. +## Efficiency without loss of validation + +Fast tests must exercise the smallest relevant production boundary. Keep ASan / +UBSan, both Linux compiler-capacity probes, real CMake feature guards, and mapping, +consent, identity, lifecycle and deployment regressions. The local CTest suite +should remain under one minute; each command has a 20-second timeout and the CI +job a 3-minute ceiling. Fix slow tests rather than silently dropping assertions, +disabling sanitizers, increasing timeouts, or claiming skipped tests passed. + +Use changed inputs to avoid unrelated work. Documentation must not rebuild five +applications. SDK/SDL pin changes require the real SDK, rumble and SDL integration +suites plus native builds. Shared production/build/resource changes require native +build/link/package/launch validation on all supported desktop architectures. +Platform-specific scripts require their affected platform. Test the selector for +renames, deletions, shared inputs and failure cases; unknown inputs fail toward +more validation, not less. Never use labels or manual dispatch as a substitute +for necessary automatic validation. + +Use compiler caches and the ordinary CMake build graph. Reuse upstream core +objects rather than duplicating standalone builds. Run the unchanged upstream +unit suite in the Linux application build tree. Do not duplicate it per platform +without a demonstrated need. Cache hits must never skip test execution, relinking +or package checks. Report cold/warm timings separately; caches can miss or expire. +Cancel obsolete runs, fail early, and upload application artifacts only when +requested. Do not add nightly builds, extra matrices or broader triggers without +an explicit need and a measured runtime/cost impact. + +## Claims and review + +A source check is not a native build; a fixture is not the actual SDK; an SDK test +is not Bluetooth hardware acceptance. Preserve native architecture, dependency, +relocation and exact-archive launch checks. Never trade away required coverage to +advertise a seconds-only result. CI changes must state what still runs, when it +runs, what is not covered, and which exact revision actually passed. Do not merge +based on an earlier revision's green result. Disclose AI assistance and leave +controller behavior, dependency pins and user-data safeguards unchanged unless +the task explicitly requires changing them. diff --git a/Docs/Switch2Kit.md b/Docs/Switch2Kit.md index 140515a4a04e..e105f24812d8 100644 --- a/Docs/Switch2Kit.md +++ b/Docs/Switch2Kit.md @@ -76,20 +76,14 @@ An automated callback test is not a physical rumble test. ## Applications and prerequisites -Pull requests run only the fast Linux regression gate; they do not build applications or -upload artifacts. Its suite is limited to 60 seconds, with a 3-minute job ceiling including -setup. See [AGENTS.md](../AGENTS.md) for the enforced CI policy and local test command. - -Application qualification is expensive and **manual only**. For a development artifact, -open this repository's **Actions** tab, choose the platform workflow below, select **Run -workflow** and the revision to qualify. On macOS, select `arm64` or `x86_64` for an app; -`sdk` runs only SDK checks. On Linux, select `enabled` for the controller app or `disabled` -for the separate no-Swift qualification. Each dispatch selects one configuration, not a -build matrix. Download only from a successful run for the exact revision being tested. +For development artifacts, open **Actions > Native Switch2Kit > Run workflow**, choose +the revision and the `macos`, `linux` or `windows` target, then use the successful run's +artifacts. PRs automatically qualify affected platforms but upload application archives +only when explicitly requested. The workflows and application artifacts are: | Platform | Workflow | Artifact | | --- | --- | --- | -| macOS | [Switch2Kit macOS qualification](../.github/workflows/switch2kit-macos.yml) | `Dolphin-Switch2Kit-arm64` or `Dolphin-Switch2Kit-x86_64` | +| macOS | [Native Switch2Kit](../.github/workflows/native-switch2kit.yml) | `Dolphin-Switch2Kit-arm64` or `Dolphin-Switch2Kit-x86_64` | | Linux | [Switch2Kit Linux application](../.github/workflows/switch2kit-linux.yml) | `Dolphin-Switch2Kit-linux-x86_64` | | Windows | [Switch2Kit Windows application](../.github/workflows/switch2kit-windows.yml) | `Dolphin-Switch2Kit-windows-x86_64` | @@ -206,8 +200,9 @@ controller absent after a search, check adapter power/access, Sync mode, competi connections and the displayed status, then retry Find. Installing a dashboard or replacing system SDL is not a remedy. -The manually requested qualification workflows check builds and extracted-package -launch using private test settings. A fast PR check is not application qualification. +See [testing and CI](Switch2KitCI.md) for the upstream unit-test baseline, focused CTest +commands, automatic change selection and compiler caching. Selected native workflows +check full builds and extracted-package launch using private test settings. They do not establish pristine first-use dialogs, downloaded-app approval, Bluetooth hardware or gameplay. Mapping and host regressions cover cancellation, backup/rollback, identity, saved consent, explicit stop and shutdown ordering; keep those checks when diff --git a/Docs/Switch2KitCI.md b/Docs/Switch2KitCI.md new file mode 100644 index 000000000000..9a7de8aa7082 --- /dev/null +++ b/Docs/Switch2KitCI.md @@ -0,0 +1,77 @@ +# Testing the Switch2Kit additions + +## Upstream baseline + +Dolphin's checked-in unit-test system is Google Test plus CMake/CTest, under +`Source/UnitTests`. Its external Buildbot infrastructure is not a GitHub Actions +workflow that a fork inherits. This change preserves the existing upstream +`tests` / `unittests` targets and all their tests. The only extension to the parent +unit-test CMake file is the optional Switch2Kit subdirectory. + +The full Linux backend-disabled build now uses `ENABLE_TESTS=ON` and runs the +ordinary upstream `unittests` target in that same build tree. This checks the +unmodified emulator baseline without a second standalone core build. The image +still contains no Swift, and its existing no-SDK/runtime-dependency assertions +remain mandatory. + +## Fast iteration uses the same CTest registration + +```sh +git submodule update --init --depth 1 Externals/Switch2Kit Externals/SDL/SDL +cmake -S Source/UnitTests/Switch2Kit -B build-switch2kit-tests +ctest --test-dir build-switch2kit-tests --output-on-failure --no-tests=error +``` + +This executes the existing mapping and host C++ harnesses with ASan/UBSan, real +CMake guard and deployment fixtures, connection-consent checks, and the real +adapter source's type/capacity probes under both Linux compilers. The same tests +are registered in ordinary POSIX builds with both ENABLE_TESTS and +ENABLE_SWITCH2KIT enabled. There is no separate Python suite runner or PyYAML +workflow-policy dependency. A local fixture pass does not qualify the actual SDK, +Qt UI, complete application, native packaging or Bluetooth hardware. + +## Automatic selection + +`Native Switch2Kit` always runs the small CTest suite and computes changes against +the PR event's base SHA, using the actual merge checkout. It does not use GitHub's +limited path-filter file list. Git failures fail the job. Renames include both +the removed and added path; unknown inputs conservatively request native checks. + +| Changed inputs | Additional automatic validation | +| --- | --- | +| Documentation only, or portable fixture tests only | No application builds; the CTest suite still runs | +| SDK/SDL pin or sources, or `.gitmodules` | Real Swift SDK, rumble and in-process SDL suites, then all native builds | +| Shared production code, CMake, dependencies or packaged resources | macOS arm64 + x86_64, Linux enabled + genuinely Swift-free disabled, Windows x64 | +| Platform-specific build/launch script or workflow | That platform's real build and package checks | +| Upstream unit-test sources | Linux application build and ordinary upstream unit suite | +| Change selector or parent workflow | SDK and every native job; validate the actual orchestration | + +The three platform workflows are reusable workflows called from the parent, +not independent PR triggers. A selected check must succeed: the aggregate +**Switch2Kit checks** job rejects a failed, cancelled or unexpectedly skipped job. +Make that aggregate check required in repository protection; `wiring` alone is +not sufficient. This PR does not change branch-protection settings. + +## Reuse work, not test results + +Native C/C++ builds use the standard CMake compiler-launcher mechanism with +sccache. Unchanged objects can be reused, but the real build graph, linking, +architecture/dependency checks and exact-archive launch tests still execute. +Linux disabled follows enabled so unchanged upstream objects can be reused +across the two configurations where compiler/cache keys match. SDK tests cache +the pinned Swift build, but always execute `swift test`, rumble and real SDL tests. + +Caches can miss or expire. GitHub cache scope also matters: a PR cache is not a +shared default-branch cache for future PRs. A manual run on `master` can prime +base-branch caches; it is optional and expensive, not run automatically by this +change. Cold native qualification remains a full build and is **not** a +seconds-only operation. Measure cold/warm native timings before claiming savings. +Job timeouts are safety ceilings, not benchmark results. + +Pull requests still construct and launch the exact application archives. They +do not upload successful application archives merely to test them. Failure +diagnostics are uploaded; successful distributables are uploaded only for an +explicit Actions **Run workflow** request. Select `macos`, `linux`, `windows`, +`sdk` or `all` in the parent workflow; `tests` is the cheap manual default. +The source-archive workflow remains manual because archiving adds no behavioral +coverage. No hardware qualification is inferred from any CI result. diff --git a/Source/UnitTests/CMakeLists.txt b/Source/UnitTests/CMakeLists.txt index 659d9e5b28fa..ebe023650d78 100644 --- a/Source/UnitTests/CMakeLists.txt +++ b/Source/UnitTests/CMakeLists.txt @@ -23,3 +23,9 @@ endmacro() add_subdirectory(Common) add_subdirectory(Core) add_subdirectory(VideoCommon) + +# These existing fixture harnesses use a POSIX compiler driver. Native Windows +# compilation and package launch are checked separately, without these stubs. +if(ENABLE_SWITCH2KIT AND NOT WIN32) + add_subdirectory(Switch2Kit) +endif() diff --git a/Source/UnitTests/Switch2Kit/CMakeLists.txt b/Source/UnitTests/Switch2Kit/CMakeLists.txt new file mode 100644 index 000000000000..e25dbb7a7c0f --- /dev/null +++ b/Source/UnitTests/Switch2Kit/CMakeLists.txt @@ -0,0 +1,31 @@ +# Also usable alone: configure this directory to test the additions without +# building the unrelated emulator. The normal ENABLE_TESTS build includes it. +cmake_minimum_required(VERSION 3.25) +if(CMAKE_SOURCE_DIR STREQUAL CMAKE_CURRENT_SOURCE_DIR) + project(DolphinSwitch2KitTests LANGUAGES NONE) + enable_testing() +endif() + +find_package(Python3 REQUIRED COMPONENTS Interpreter) +get_filename_component(DOLPHIN_ROOT "${CMAKE_CURRENT_LIST_DIR}/../../.." ABSOLUTE) + +function(add_switch2kit_test name script) + add_test(NAME Switch2Kit.${name} + COMMAND "${Python3_EXECUTABLE}" "${DOLPHIN_ROOT}/Tools/${script}" ${ARGN}) + set_tests_properties(Switch2Kit.${name} PROPERTIES TIMEOUT 20 LABELS Switch2Kit) +endfunction() + +add_switch2kit_test(ChangeSelection test_switch2kit_ci.py) +add_switch2kit_test(Integration test_switch2kit.py) +add_switch2kit_test(AutoConnect test_switch2kit_autoconnect.py) +add_switch2kit_test(Mapping test_switch2kit_mapping.py --sanitize) +add_switch2kit_test(Host test_switch2kit_host.py --sanitize) +find_program(SWITCH2KIT_CLANGXX clang++ REQUIRED) +add_switch2kit_test(CapacityClang test_switch2kit_capacity.py) +set_tests_properties(Switch2Kit.CapacityClang PROPERTIES ENVIRONMENT "CXX=${SWITCH2KIT_CLANGXX}") +if(CMAKE_HOST_SYSTEM_NAME STREQUAL "Linux") + find_program(SWITCH2KIT_GXX g++ REQUIRED) + add_switch2kit_test(CapacityGCC test_switch2kit_capacity.py) + set_tests_properties(Switch2Kit.CapacityGCC PROPERTIES ENVIRONMENT "CXX=${SWITCH2KIT_GXX}") + add_switch2kit_test(Runtime test_switch2kit_runtime.py) +endif() diff --git a/Tools/check_ci_policy.py b/Tools/check_ci_policy.py deleted file mode 100644 index fe0a643f4876..000000000000 --- a/Tools/check_ci_policy.py +++ /dev/null @@ -1,130 +0,0 @@ -#!/usr/bin/env python3 -# Copyright 2026 Dolphin Emulator Project -# SPDX-License-Identifier: GPL-2.0-or-later -"""Fail closed on accidental CI cost regressions. Requires PyYAML 6.0.3.""" -from pathlib import Path -import sys - -import yaml - -ROOT = Path(__file__).resolve().parents[1] -AUTOMATIC_WORKFLOW = "native-switch2kit.yml" -CHECKOUT = "actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1" -BOOTSTRAP = ('python3 -m pip install --disable-pip-version-check --no-deps ' - '--only-binary=:all: --retries 1 --timeout 10 ' - '--target "$RUNNER_TEMP/ci-policy" PyYAML==6.0.3\n' - 'python3 Tools/check_ci_policy.py') -SUBMODULES = "git submodule update --init --depth 1 Externals/Switch2Kit Externals/SDL/SDL" -FAST_TESTS = "python3 Tools/run_fast_tests.py" - - -class PolicyError(ValueError): - pass - - -class UniqueLoader(yaml.BaseLoader): - # BaseLoader preserves 'on' as a string (YAML 1.1 otherwise makes it True). - def construct_mapping(self, node, deep=False): - keys = [self.construct_object(key, deep=deep) for key, _ in node.value] - if any(not isinstance(key, str) for key in keys) or len(keys) != len(set(keys)): - raise PolicyError("Workflow mapping keys must be unique strings") - return super().construct_mapping(node, deep=deep) - - -def load_workflow(text): - # No aliases/merge keys hiding extra triggers, jobs or overridden budgets. - if any(isinstance(token, (yaml.tokens.AnchorToken, yaml.tokens.AliasToken)) - for token in yaml.scan(text)): - raise PolicyError("Workflow anchors and aliases are not allowed") - workflow = yaml.load(text, Loader=UniqueLoader) - if not isinstance(workflow, dict): - raise PolicyError("Workflow must be a mapping") - return workflow - - -def require(condition, message): - if not condition: - raise PolicyError(message) - - -def event_names(events): - if isinstance(events, str): - return {events} - if isinstance(events, (dict, list)): - return set(events) - raise PolicyError("Invalid workflow triggers") - - -def validate_workflow(name, workflow): - events = event_names(workflow.get("on")) - jobs = workflow.get("jobs") - require(isinstance(jobs, dict) and jobs, "Workflow must contain jobs") - if name != AUTOMATIC_WORKFLOW: - require(events == {"workflow_dispatch"}, - "Only native-switch2kit.yml may run automatically; use workflow_dispatch") - for job in jobs.values(): - require(isinstance(job, dict), "Invalid manual job") - limit = job.get("timeout-minutes", "") - require(str(limit).isdigit() and 0 < int(limit) <= 60, - "Manual qualification jobs need an explicit timeout <= 60 minutes") - require("strategy" not in job, "Select one manual target; no build matrices") - return - - require(set(workflow) == {"name", "on", "permissions", "concurrency", "jobs"}, - "Automatic workflow has unreviewed top-level settings") - require(workflow["on"] == {"pull_request": "", "workflow_dispatch": ""}, - "Keep the gate on every PR, without path filters, duplicate pushes or schedules") - require(workflow["name"] == "Native Switch2Kit", "Keep the required check name stable") - require(workflow["permissions"] == {"contents": "read"}, "CI must be read-only") - require(workflow["concurrency"] == { - "group": "native-switch2kit-${{ github.ref }}", "cancel-in-progress": "true"}, - "Cancel superseded runs for the same ref") - require(set(jobs) == {"wiring"}, "Automatic CI has exactly one job: wiring") - job = jobs["wiring"] - require(isinstance(job, dict), "Invalid automatic job") - require(set(job) == {"runs-on", "timeout-minutes", "steps"}, - "No matrices, services, containers, reusable jobs or conditional/soft-fail gates") - require(job["runs-on"] == "ubuntu-24.04", "Only one standard Linux runner is permitted") - require(job["timeout-minutes"] == "3", "Automatic job hard limit is 3 minutes") - # runner.temp is available only after a runner is assigned (step env), - # not in jobs..env. Keep the parser isolated to its two consumers. - parser_env = {"PYTHONPATH": "${{ runner.temp }}/ci-policy"} - expected = [ - {"uses": CHECKOUT, "timeout-minutes": "1", "with": { - "persist-credentials": "false", "fetch-depth": "1", "submodules": "false"}}, - {"run": BOOTSTRAP, "timeout-minutes": "1", "env": parser_env}, - {"run": SUBMODULES, "timeout-minutes": "1"}, - {"run": FAST_TESTS, "timeout-minutes": "2", "env": parser_env}, - ] - steps = job["steps"] - require(isinstance(steps, list) and all(isinstance(step, dict) for step in steps), - "Invalid automatic steps") - normalized = [] - for step in steps: - step = {key: value for key, value in step.items() if key != "name"} - if isinstance(step.get("run"), str): - step["run"] = step["run"].strip() - normalized.append(step) - require(normalized == expected, - "Automatic steps are allowlisted: shallow checkout, policy, two submodules, " - "bounded tests. No full builds, installers, artifacts or bypasses") - - -def check_repository(root=ROOT): - require((root / "AGENTS.md").is_file(), "Root AGENTS.md CI policy is required") - files = sorted(path for path in (root / ".github/workflows").iterdir() - if path.suffix in (".yml", ".yaml")) - require(any(path.name == AUTOMATIC_WORKFLOW for path in files), "Fast PR gate is missing") - for path in files: - try: - validate_workflow(path.name, load_workflow(path.read_text(encoding="utf-8"))) - except (ValueError, TypeError, yaml.YAMLError) as error: - raise PolicyError(f"{path.name}: {error}") from error - print(f"CI policy passed: {len(files)} workflows; one automatic 3-minute Linux job.") - - -if __name__ == "__main__": - try: - check_repository() - except (PolicyError, OSError) as error: - sys.exit(f"CI POLICY FAILURE: {error}\nSee AGENTS.md; do not raise the budget to pass.") diff --git a/Tools/run_fast_tests.py b/Tools/run_fast_tests.py deleted file mode 100644 index 0e139764bf79..000000000000 --- a/Tools/run_fast_tests.py +++ /dev/null @@ -1,79 +0,0 @@ -#!/usr/bin/env python3 -# Copyright 2026 Dolphin Emulator Project -# SPDX-License-Identifier: GPL-2.0-or-later -"""Run the small production-code regressions, never a full Dolphin/SDK build. - -The suite has a shared wall-clock deadline, not N independent timeout allowances. -A failure or timeout is fatal; kill the entire compiler/test process group. -""" -import os -from pathlib import Path -import signal -import subprocess -import sys -import time - -ROOT = Path(__file__).resolve().parents[1] -SUITE_SECONDS = 60 -TEST_SECONDS = 20 -# Keep both compiler capacity probes and the existing sanitizer coverage. -TESTS = ( - ("CI policy regressions", ("test_ci_policy.py",), {}), - ("SDL capacity (Clang)", ("test_switch2kit_capacity.py",), {"CXX": "clang++"}), - ("SDL capacity (GCC)", ("test_switch2kit_capacity.py",), {"CXX": "g++"}), - ("Integration and disabled CMake", ("test_switch2kit.py",), {}), - ("Connection consent", ("test_switch2kit_autoconnect.py",), {}), - ("Mapping ASan/UBSan", ("test_switch2kit_mapping.py", "--sanitize"), {}), - ("Host lifecycle ASan/UBSan", ("test_switch2kit_host.py", "--sanitize"), {}), - ("Tiny deployment fixtures", ("test_switch2kit_runtime.py",), {}), -) - - -class BudgetExceeded(RuntimeError): - pass - - -def run_command(command, *, seconds, env, cwd): - if seconds <= 0: - raise BudgetExceeded("Shared suite deadline exhausted before starting the next test") - # Automatic CI is Linux. Do not leave grandchildren (e.g. cc1plus) running. - process = subprocess.Popen(command, env=env, cwd=cwd, start_new_session=True) - try: - result = process.wait(timeout=seconds) - except subprocess.TimeoutExpired as error: - raise BudgetExceeded(f"Test exceeded its {seconds:.2f}s remaining allowance") from error - finally: - try: - os.killpg(process.pid, signal.SIGKILL) - except ProcessLookupError: - pass - process.wait() - if result: - raise subprocess.CalledProcessError(result, command) - - -def run_suite(tests=TESTS, *, suite_seconds=SUITE_SECONDS, test_seconds=TEST_SECONDS, - root=ROOT): - started = time.monotonic() - deadline = started + suite_seconds - for name, arguments, extra_env in tests: - print(f"\n=== {name} ===", flush=True) - before = time.monotonic() - run_command([sys.executable, str(root / "Tools" / arguments[0]), *arguments[1:]], - seconds=min(test_seconds, deadline - before), - env={**os.environ, **extra_env}, cwd=root) - print(f"PASS {name}: {time.monotonic() - before:.2f}s", flush=True) - elapsed = time.monotonic() - started - if elapsed > suite_seconds: - raise BudgetExceeded(f"Suite exceeded {suite_seconds}s") - print(f"\nAll {len(tests)} checks passed in {elapsed:.2f}s (budget {suite_seconds}s).", - flush=True) - - -if __name__ == "__main__": - if sys.platform != "linux": - sys.exit("The budgeted PR suite requires Linux; run individual portable tests locally.") - try: - run_suite() - except (BudgetExceeded, subprocess.CalledProcessError, OSError) as error: - sys.exit(f"FAST CI FAILED: {error}\nFix or reduce the test; do not increase the budget.") diff --git a/Tools/switch2kit_ci.py b/Tools/switch2kit_ci.py new file mode 100644 index 000000000000..aa3a819f7586 --- /dev/null +++ b/Tools/switch2kit_ci.py @@ -0,0 +1,81 @@ +#!/usr/bin/env python3 +# Copyright 2026 Dolphin Emulator Project +# SPDX-License-Identifier: GPL-2.0-or-later +"""Select validation from changed inputs, not from labels or guessed coverage. + +PRs compare their merge result with the event's base SHA. Renames are expanded +into deletion/addition pairs so moving a production file to Docs cannot hide it. +Unknown files conservatively select all native builds. Git errors fail the job. +""" +import argparse +import os +from pathlib import Path +import subprocess + +TARGETS = frozenset(('sdk', 'macos', 'linux', 'windows')) +PLATFORM_TOOLS = { + 'Tools/build-switch2kit-linux.sh': 'linux', + 'Tools/build-switch2kit-windows.ps1': 'windows', + 'Tools/test_switch2kit_windows_launch.ps1': 'windows', + 'Tools/test_switch2kit_bundle.py': 'macos', + 'Tools/mac-codesign.sh': 'macos', +} + + +def select_checks(paths): + selected = set() + for path in paths: + if path == '.github/workflows/switch2kit-desktop.yml': + continue # Source archiving has no compilation or behavioral coverage. + if path in PLATFORM_TOOLS: + selected.add(PLATFORM_TOOLS[path]) + elif path in ('.github/workflows/switch2kit-macos.yml', + '.github/workflows/switch2kit-linux.yml', + '.github/workflows/switch2kit-windows.yml'): + selected.add(Path(path).stem.removeprefix('switch2kit-')) + elif path in ('Tools/switch2kit_ci.py', 'Tools/test_switch2kit_ci.py', + '.github/workflows/native-switch2kit.yml', '.gitmodules'): + selected.update(TARGETS) + elif path == 'Externals/Switch2Kit' or path.startswith(('Externals/Switch2Kit/', + 'Externals/SDL/')): + selected.update(TARGETS) + elif path.startswith(('Source/UnitTests/', 'Tools/switch2kit/')): + # The normal upstream test target is executed in the Linux native job. + if path.startswith('Source/UnitTests/') and not path.startswith('Source/UnitTests/Switch2Kit/'): + selected.add('linux') + elif path.startswith('Tools/test_switch2kit') and path.endswith('.py'): + continue # These portable tests always execute in the CTest job. + elif path.startswith(('Docs/', '.tx/')) or path in ( + 'Readme.md', 'Contributing.md', 'CODE_OF_CONDUCT.md', 'COPYING', + 'AGENTS.md', '.mailmap', '.git-blame-ignore-revs', '.editorconfig'): + continue + else: + # Includes Source/Core, CMake, resources, other Externals and unknown + # build inputs. Do not assume a platform-independent edit is harmless. + selected.update(('macos', 'linux', 'windows')) + return selected + + +def changed_paths(base, head='HEAD'): + result = subprocess.run(['git', 'diff', '--no-renames', '--name-only', '-z', base, head], + check=True, stdout=subprocess.PIPE, timeout=15) + return [os.fsdecode(path) for path in result.stdout.split(b'\0') if path] + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--base') + parser.add_argument('--target', choices=('tests', 'all', *sorted(TARGETS)), default='tests') + args = parser.parse_args() + selected = select_checks(changed_paths(args.base)) if args.base else ( + set(TARGETS) if args.target == 'all' else set() if args.target == 'tests' else {args.target}) + # Outputs are derived only from constant target names, never from filenames. + output = ''.join(f'{target}={str(target in selected).lower()}\n' for target in sorted(TARGETS)) + print(output, end='') + if os.environ.get('GITHUB_OUTPUT'): + with open(os.environ['GITHUB_OUTPUT'], 'a', encoding='utf-8') as stream: + stream.write(output) + + +if __name__ == '__main__': + main() diff --git a/Tools/test_ci_policy.py b/Tools/test_ci_policy.py deleted file mode 100644 index 98bfea534286..000000000000 --- a/Tools/test_ci_policy.py +++ /dev/null @@ -1,205 +0,0 @@ -#!/usr/bin/env python3 -# Copyright 2026 Dolphin Emulator Project -# SPDX-License-Identifier: GPL-2.0-or-later -"""Small regression tests for CI fan-out, YAML parsing and runtime limits.""" -import contextlib -import copy -import io -import os -from pathlib import Path -import signal -import subprocess -import sys -import tempfile -import unittest -from unittest.mock import patch - -import yaml -from check_ci_policy import (AUTOMATIC_WORKFLOW, ROOT, PolicyError, check_repository, - load_workflow, validate_workflow) -from run_fast_tests import (BudgetExceeded, SUITE_SECONDS, TEST_SECONDS, TESTS, - run_command, run_suite) - - -class WorkflowPolicyTests(unittest.TestCase): - def setUp(self): - self.workflow = load_workflow( - (ROOT / '.github/workflows' / AUTOMATIC_WORKFLOW).read_text()) - - def reject(self, workflow, name=AUTOMATIC_WORKFLOW): - with self.assertRaises(PolicyError): - validate_workflow(name, workflow) - - def test_repository_policy(self): - check_repository() - - def test_only_one_automatic_job_and_runner(self): - for field, value in (('runs-on', 'macos-15'), ('runs-on', 'windows-2025-vs2026'), - ('runs-on', ['self-hosted']), ('timeout-minutes', '90'), - ('strategy', {'matrix': {'os': ['ubuntu-24.04', 'macos-15']}}), - ('container', 'swift:6.2.1-noble'), ('services', {}), - ('env', {'PYTHONPATH': '${{ runner.temp }}/ci-policy'}), - ('uses', './.github/workflows/build.yml'), - ('continue-on-error', 'true'), ('if', 'false')): - with self.subTest(field=field, value=value): - workflow = copy.deepcopy(self.workflow) - workflow['jobs']['wiring'][field] = value - self.reject(workflow) - self.workflow['jobs']['build'] = copy.deepcopy(self.workflow['jobs']['wiring']) - self.reject(self.workflow) - - def test_cannot_remove_gates_or_add_full_builds(self): - for mutation in ('extra', 'removed', 'recursive', 'soft-fail', 'skip', 'replace'): - with self.subTest(mutation=mutation): - workflow = copy.deepcopy(self.workflow) - steps = workflow['jobs']['wiring']['steps'] - if mutation == 'extra': - steps.append({'run': 'cmake --build build --target dolphin-emu'}) - elif mutation == 'removed': - steps.pop(1) - elif mutation == 'recursive': - steps[0]['with']['submodules'] = 'recursive' - elif mutation == 'soft-fail': - steps[-1]['continue-on-error'] = 'true' - elif mutation == 'skip': - steps[-1]['if'] = 'false' - else: - steps[-1]['run'] = 'python3 Tools/run_fast_tests.py || true' - self.reject(workflow) - - def test_unfiltered_pr_gate_and_no_duplicate_events(self): - for events in ({'pull_request': {'paths': ['Source/**']}, 'workflow_dispatch': ''}, - {'push': '', 'pull_request': '', 'workflow_dispatch': ''}, - {'workflow_dispatch': ''}): - self.workflow['on'] = events - self.reject(self.workflow) - - def test_manual_workflows_reject_all_automatic_trigger_syntaxes(self): - for trigger in ('pull_request', 'push', 'schedule', 'workflow_run', - 'workflow_call', 'pull_request_target', 'issue_comment'): - for events in (trigger, [trigger, 'workflow_dispatch'], - {trigger: '', 'workflow_dispatch': ''}): - with self.subTest(events=events): - workflow = {'on': events, 'jobs': {'build': {'timeout-minutes': '45'}}} - self.reject(workflow, 'new-expensive.yaml') - - def test_manual_dispatch_forms_remain_usable(self): - for events in ('workflow_dispatch', ['workflow_dispatch'], {'workflow_dispatch': ''}): - validate_workflow('manual.yml', { - 'on': events, 'jobs': {'build': {'timeout-minutes': '45'}}}) - - def test_manual_timeouts_and_matrices(self): - for job in ({}, {'timeout-minutes': '120'}, {'timeout-minutes': '${{ inputs.limit }}'}, - {'timeout-minutes': '0'}, {'timeout-minutes': '45', 'strategy': {}}): - self.reject({'on': 'workflow_dispatch', 'jobs': {'build': job}}, 'manual.yml') - - def test_real_yaml_parser_preserves_on_and_rejects_ambiguous_yaml(self): - self.assertEqual(load_workflow('on: [pull_request, workflow_dispatch]')['on'], - ['pull_request', 'workflow_dispatch']) - for text in ('on: workflow_dispatch\non: push\n', - 'on: workflow_dispatch\njobs:\n build:\n runs-on: linux\n runs-on: macos\n', - 'on: &events [push]\njobs: *events\n', 'on: [unterminated', '- not-a-workflow'): - with self.subTest(text=text), self.assertRaises((PolicyError, yaml.YAMLError)): - load_workflow(text) - - def test_new_yaml_file_is_scanned_and_missing_gate_fails(self): - with tempfile.TemporaryDirectory() as directory: - root = Path(directory) - workflows = root / '.github/workflows' - workflows.mkdir(parents=True) - (root / 'AGENTS.md').write_text('policy') - with self.assertRaises(PolicyError): - check_repository(root) - (workflows / AUTOMATIC_WORKFLOW).write_text( - (ROOT / '.github/workflows' / AUTOMATIC_WORKFLOW).read_text()) - (workflows / 'surprise.yaml').write_text('on: push\njobs:\n big:\n timeout-minutes: 90\n') - with self.assertRaises(PolicyError): - check_repository(root) - (workflows / 'surprise.yaml').unlink() - (root / 'AGENTS.md').unlink() - with self.assertRaises(PolicyError): - check_repository(root) - - -class RuntimeBudgetTests(unittest.TestCase): - def test_budgets_and_existing_behavior_checks_are_preserved(self): - self.assertEqual((SUITE_SECONDS, TEST_SECONDS), (60, 20)) - commands = [(arguments, env) for _, arguments, env in TESTS] - for arguments, env in ( - (('test_ci_policy.py',), {}), - (('test_switch2kit_capacity.py',), {'CXX': 'clang++'}), - (('test_switch2kit_capacity.py',), {'CXX': 'g++'}), - (('test_switch2kit_mapping.py', '--sanitize'), {}), - (('test_switch2kit_host.py', '--sanitize'), {}), - (('test_switch2kit.py',), {}), - (('test_switch2kit_autoconnect.py',), {}), - (('test_switch2kit_runtime.py',), {})): - self.assertIn((arguments, env), commands) - - def test_nonzero_exit_is_not_a_pass(self): - with self.assertRaises(subprocess.CalledProcessError): - run_command([sys.executable, '-c', 'raise SystemExit(7)'], seconds=2, - env=os.environ, cwd=ROOT) - - def test_expired_deadline_never_starts_another_process(self): - with patch('run_fast_tests.subprocess.Popen') as spawn: - with self.assertRaises(BudgetExceeded): - run_command(['unused'], seconds=0, env={}, cwd=ROOT) - spawn.assert_not_called() - - def test_shared_deadline_not_a_fresh_allowance_per_test(self): - ticks = iter((0, 0, 0.25, 0.8, 0.9, 0.9)) - tests = (('one', ('one.py',), {}), ('two', ('two.py',), {})) - with patch('run_fast_tests.time.monotonic', side_effect=lambda: next(ticks)), \ - patch('run_fast_tests.run_command') as command, \ - contextlib.redirect_stdout(io.StringIO()): - run_suite(tests, suite_seconds=1, test_seconds=0.75) - self.assertAlmostEqual(command.call_args_list[0].kwargs['seconds'], 0.75) - self.assertAlmostEqual(command.call_args_list[1].kwargs['seconds'], 0.2) - - def test_failure_stops_the_suite_immediately(self): - tests = (('one', ('one.py',), {}), ('two', ('two.py',), {})) - with patch('run_fast_tests.run_command', - side_effect=subprocess.CalledProcessError(1, 'one')) as command, \ - contextlib.redirect_stdout(io.StringIO()): - with self.assertRaises(subprocess.CalledProcessError): - run_suite(tests) - self.assertEqual(command.call_count, 1) - - @unittest.skipUnless(sys.platform == 'linux', 'The budgeted PR lane is Linux') - def test_timeout_kills_the_entire_process_group(self): - # Start a grandchild that ignores SIGTERM. Simulate only the wait timeout, - # so this test is deterministic and does not sleep for an actual budget. - process = subprocess.Popen( - [sys.executable, '-u', '-c', - 'import os,subprocess,sys,time; ' - 'child=subprocess.Popen([sys.executable, "-u", "-c", ' - '"import signal,time; signal.signal(signal.SIGTERM,signal.SIG_IGN); ' - 'print(123,flush=True); time.sleep(30)"],stdout=subprocess.PIPE); ' - 'child.stdout.readline(); print(os.getpgrp(),flush=True); time.sleep(30)'], - stdout=subprocess.PIPE, text=True, start_new_session=True) - try: - self.assertEqual(process.stdout.readline().strip(), str(process.pid)) - actual_wait = process.wait - def wait(timeout=None): - if timeout is not None: - raise subprocess.TimeoutExpired(process.args, timeout) - return actual_wait(timeout=2) - with patch('run_fast_tests.subprocess.Popen', return_value=process), \ - patch.object(process, 'wait', side_effect=wait), \ - patch('run_fast_tests.os.killpg', wraps=os.killpg) as kill: - with self.assertRaises(BudgetExceeded): - run_command(['unused'], seconds=1, env={}, cwd=ROOT) - kill.assert_called_once_with(process.pid, signal.SIGKILL) - self.assertEqual(process.returncode, -signal.SIGKILL) - finally: - try: - os.killpg(process.pid, signal.SIGKILL) - except ProcessLookupError: - pass - process.wait(timeout=2) - process.stdout.close() - - -if __name__ == '__main__': - unittest.main(verbosity=2) diff --git a/Tools/test_switch2kit_autoconnect.py b/Tools/test_switch2kit_autoconnect.py old mode 100755 new mode 100644 index fac8b28d08ef..0c6b11a437f8 --- a/Tools/test_switch2kit_autoconnect.py +++ b/Tools/test_switch2kit_autoconnect.py @@ -2,7 +2,10 @@ # Copyright 2026 Dolphin Emulator Project # SPDX-License-Identifier: GPL-2.0-or-later """Static UI/startup guards; execute test_switch2kit_host.py for host behavior.""" +import json from pathlib import Path +import subprocess +import tempfile import unittest ROOT = Path(__file__).resolve().parents[1] @@ -62,16 +65,21 @@ def test_polling_and_initialization_never_start_bluetooth(self): self.assertIn("s_started = false;", stop) def test_ci_retains_regressions(self): - workflow = self.read(".github/workflows/native-switch2kit.yml") - self.assertIn("python3 Tools/run_fast_tests.py", workflow) - # The budgeted runner owns these commands now. Check actual arguments, - # not script names that could appear only in workflow comments. - from run_fast_tests import TESTS - commands = {arguments for _, arguments, _ in TESTS} - for command in (("test_switch2kit.py",), ("test_switch2kit_host.py", "--sanitize"), - ("test_switch2kit_mapping.py", "--sanitize"), - ("test_switch2kit_autoconnect.py",)): - self.assertIn(command, commands) + # Check CTest's actual commands, not copies of command names in comments. + with tempfile.TemporaryDirectory() as build: + subprocess.run(['cmake', '-S', str(ROOT / 'Source/UnitTests/Switch2Kit'), + '-B', build], check=True, capture_output=True, timeout=15) + result = subprocess.run(['ctest', '--test-dir', build, '--show-only=json-v1'], + check=True, capture_output=True, text=True, timeout=5) + commands = {tuple(test['command'][1:]) for test in json.loads(result.stdout)['tests']} + for script, flags in (('test_switch2kit.py', ()), + ('test_switch2kit_host.py', ('--sanitize',)), + ('test_switch2kit_mapping.py', ('--sanitize',)), + ('test_switch2kit_autoconnect.py', ())): + self.assertIn((str(ROOT / 'Tools' / script), *flags), commands) + workflow = self.read('.github/workflows/native-switch2kit.yml') + self.assertIn('ctest --test-dir build-switch2kit-tests --output-on-failure --no-tests=error', + workflow) if __name__ == "__main__": diff --git a/Tools/test_switch2kit_ci.py b/Tools/test_switch2kit_ci.py new file mode 100644 index 000000000000..c3b2e7abaf44 --- /dev/null +++ b/Tools/test_switch2kit_ci.py @@ -0,0 +1,92 @@ +#!/usr/bin/env python3 +# Copyright 2026 Dolphin Emulator Project +# SPDX-License-Identifier: GPL-2.0-or-later +"""Small regressions for change selection; no bespoke workflow parser or runner.""" +import os +from pathlib import Path +import subprocess +import tempfile +import unittest +from unittest.mock import patch + +from switch2kit_ci import TARGETS, changed_paths, select_checks + +NATIVE = {'macos', 'linux', 'windows'} + + +class ChangeSelectionTests(unittest.TestCase): + def test_documentation_does_not_build_applications(self): + self.assertEqual(select_checks(['Readme.md', 'Docs/Switch2Kit.md', 'AGENTS.md']), set()) + + def test_every_shared_production_input_rebuilds_native_applications(self): + for path in ('Source/Core/DolphinQt/Config/ControllersPane.cpp', + 'Source/Core/Common/Config/Config.h', 'CMakeLists.txt', + 'CMake/DolphinSwitch2Kit.cmake', 'Externals/fmt/fmt', + 'Data/Sys/Profiles/GCPad/Switch2Kit GameCube.ini', 'new-build-input'): + with self.subTest(path=path): + self.assertEqual(select_checks([path]), NATIVE) + + def test_sdk_and_sdl_pins_restore_real_sdk_and_all_native_checks(self): + for path in ('Externals/Switch2Kit', 'Externals/SDL/SDL', '.gitmodules', + 'Externals/Switch2Kit/Package.swift'): + self.assertEqual(select_checks([path]), TARGETS) + + def test_platform_specific_tools_do_not_rebuild_unaffected_platforms(self): + for target in NATIVE: + self.assertEqual(select_checks([f'.github/workflows/switch2kit-{target}.yml']), {target}) + self.assertEqual(select_checks(['Tools/build-switch2kit-linux.sh']), {'linux'}) + self.assertEqual(select_checks(['Tools/test_switch2kit_bundle.py']), {'macos'}) + self.assertEqual(select_checks(['Tools/test_switch2kit_windows_launch.ps1']), {'windows'}) + + def test_orchestration_changes_validate_the_entire_graph(self): + for path in ('Tools/switch2kit_ci.py', 'Tools/test_switch2kit_ci.py', + '.github/workflows/native-switch2kit.yml'): + self.assertEqual(select_checks([path]), TARGETS) + + def test_fast_fixture_changes_stay_in_the_always_run_ctest_suite(self): + self.assertEqual(select_checks(['Tools/test_switch2kit_host.py', + 'Tools/switch2kit/HostTest.cpp', + 'Source/UnitTests/Switch2Kit/CMakeLists.txt']), set()) + self.assertEqual(select_checks(['Source/UnitTests/CMakeLists.txt']), {'linux'}) + + def test_multiple_changes_union_the_required_checks(self): + self.assertEqual(select_checks(['Docs/Switch2Kit.md', + 'Tools/build-switch2kit-linux.sh', + 'Tools/test_switch2kit_bundle.py']), {'linux', 'macos'}) + + def test_git_failure_cannot_be_reported_as_no_changes(self): + with patch('switch2kit_ci.subprocess.run', side_effect=subprocess.CalledProcessError(1, 'git')): + with self.assertRaises(subprocess.CalledProcessError): + changed_paths('missing-base') + + def test_deleted_renamed_and_whitespace_paths_are_not_lost(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + def git(*args): + return subprocess.run(['git', '-C', str(root), *args], check=True, + stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) + git('init', '-q') + git('config', 'user.name', 'Test') + git('config', 'user.email', 'test@example.invalid') + source = root / 'Source/Core/file with space.cpp' + source.parent.mkdir(parents=True) + source.write_text('int value;\n') + git('add', '.') + git('commit', '-qm', 'base') + (root / 'Docs').mkdir() + source.rename(root / 'Docs/moved.cpp') + git('add', '-A') + git('commit', '-qm', 'rename') + previous = os.getcwd() + try: + os.chdir(root) + paths = changed_paths('HEAD~1') + finally: + os.chdir(previous) + self.assertIn('Source/Core/file with space.cpp', paths) + self.assertIn('Docs/moved.cpp', paths) + self.assertEqual(select_checks(paths), NATIVE) + + +if __name__ == '__main__': + unittest.main(verbosity=2) From ed3493a585da506f64c98bf51005140e5b45d4fe Mon Sep 17 00:00:00 2001 From: Johnny D Date: Sun, 20 Sep 2026 17:33:51 -0400 Subject: [PATCH 05/21] test: include Switch2Kit regressions in the ordinary unit suite Remove ENABLE_SWITCH2KIT from test registration. The existing ENABLE_TESTS path now includes the POSIX fixture regressions even when the application backend is disabled; preserve the existing Windows harness limitation. No new test option, target, workflow or backend default is introduced. Extend the existing retention tests to compare CTest's actual commands in the standalone and normal parent registrations with the backend OFF/ON. The regression fails with the old gate and passes with the new wiring. Supply Clang in the Swift-free Linux test image for the retained compiler capacity probe. Keep upstream tests, sanitizers and native qualification. Update AGENTS.md and the CI documentation to prohibit a separate opt-in. AI-assisted change. Registration regression, Python compilation, YAML comparison and whitespace checks passed locally in a partial snapshot. Full exact-head CI still must pass; no native build result is inferred. --- .github/workflows/switch2kit-linux.yml | 2 +- AGENTS.md | 3 +++ Docs/Switch2KitCI.md | 16 ++++++----- Source/UnitTests/CMakeLists.txt | 6 ++--- Tools/test_switch2kit_autoconnect.py | 37 ++++++++++++++++++++++++++ 5 files changed, 53 insertions(+), 11 deletions(-) diff --git a/.github/workflows/switch2kit-linux.yml b/.github/workflows/switch2kit-linux.yml index 05fc48166e0b..761f1cbd85a1 100644 --- a/.github/workflows/switch2kit-linux.yml +++ b/.github/workflows/switch2kit-linux.yml @@ -94,7 +94,7 @@ jobs: - name: Install ordinary native dependencies without Swift run: | apt-get update - apt-get install -y --no-install-recommends git ca-certificates build-essential cmake ninja-build python3 pkg-config qt6-base-dev qt6-base-private-dev qt6-svg-dev libbluetooth-dev libevdev-dev libudev-dev libusb-1.0-0-dev libasound2-dev libpulse-dev libx11-dev libxi-dev libxrandr-dev libegl1-mesa-dev libgl1-mesa-dev libsystemd0 dbus + apt-get install -y --no-install-recommends git ca-certificates build-essential clang cmake ninja-build python3 pkg-config qt6-base-dev qt6-base-private-dev qt6-svg-dev libbluetooth-dev libevdev-dev libudev-dev libusb-1.0-0-dev libasound2-dev libpulse-dev libx11-dev libxi-dev libxrandr-dev libegl1-mesa-dev libgl1-mesa-dev libsystemd0 dbus if command -v swift || command -v swiftc; then echo 'The backend-disabled qualification image must not contain Swift.' >&2 exit 1 diff --git a/AGENTS.md b/AGENTS.md index 1ba904c23f5b..240505ee86b4 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -10,6 +10,9 @@ is not automatically inherited by this fork; do not claim otherwise. The existing `unittests` target remains the baseline. Register our focused tests with CTest in `Source/UnitTests/Switch2Kit`, not a parallel Python test runner. +On supported POSIX hosts they belong to the ordinary `ENABLE_TESTS` suite even +with `ENABLE_SWITCH2KIT=OFF`. Never add a separate test opt-in or gate the fixture +regressions on the application backend; they do not need Swift or Bluetooth. The same registration can be configured alone for fast iteration: ```sh diff --git a/Docs/Switch2KitCI.md b/Docs/Switch2KitCI.md index 9a7de8aa7082..8f5c45567d26 100644 --- a/Docs/Switch2KitCI.md +++ b/Docs/Switch2KitCI.md @@ -6,13 +6,14 @@ Dolphin's checked-in unit-test system is Google Test plus CMake/CTest, under `Source/UnitTests`. Its external Buildbot infrastructure is not a GitHub Actions workflow that a fork inherits. This change preserves the existing upstream `tests` / `unittests` targets and all their tests. The only extension to the parent -unit-test CMake file is the optional Switch2Kit subdirectory. +unit-test CMake file registers the Switch2Kit subdirectory on POSIX hosts. +There is no Switch2Kit-specific test opt-in. The full Linux backend-disabled build now uses `ENABLE_TESTS=ON` and runs the -ordinary upstream `unittests` target in that same build tree. This checks the -unmodified emulator baseline without a second standalone core build. The image -still contains no Swift, and its existing no-SDK/runtime-dependency assertions -remain mandatory. +ordinary `unittests` target, including the Switch2Kit regressions, in that same +build tree. This checks the upstream baseline and our additions together without +a second standalone core build. The image still contains no Swift, and its +existing no-SDK/runtime-dependency assertions remain mandatory. ## Fast iteration uses the same CTest registration @@ -25,8 +26,9 @@ ctest --test-dir build-switch2kit-tests --output-on-failure --no-tests=error This executes the existing mapping and host C++ harnesses with ASan/UBSan, real CMake guard and deployment fixtures, connection-consent checks, and the real adapter source's type/capacity probes under both Linux compilers. The same tests -are registered in ordinary POSIX builds with both ENABLE_TESTS and -ENABLE_SWITCH2KIT enabled. There is no separate Python suite runner or PyYAML +run in ordinary POSIX test builds under `ENABLE_TESTS`, regardless of +`ENABLE_SWITCH2KIT`. The existing Windows exclusion reflects the POSIX compiler +harnesses, not a user-selectable test option. There is no separate runner or PyYAML workflow-policy dependency. A local fixture pass does not qualify the actual SDK, Qt UI, complete application, native packaging or Bluetooth hardware. diff --git a/Source/UnitTests/CMakeLists.txt b/Source/UnitTests/CMakeLists.txt index ebe023650d78..48c9c5d41cd5 100644 --- a/Source/UnitTests/CMakeLists.txt +++ b/Source/UnitTests/CMakeLists.txt @@ -24,8 +24,8 @@ add_subdirectory(Common) add_subdirectory(Core) add_subdirectory(VideoCommon) -# These existing fixture harnesses use a POSIX compiler driver. Native Windows -# compilation and package launch are checked separately, without these stubs. -if(ENABLE_SWITCH2KIT AND NOT WIN32) +# Run these source-level regressions even with the backend disabled. The +# harnesses require a POSIX compiler driver; native Windows is checked separately. +if(NOT WIN32) add_subdirectory(Switch2Kit) endif() diff --git a/Tools/test_switch2kit_autoconnect.py b/Tools/test_switch2kit_autoconnect.py index 0c6b11a437f8..9a9b48792b62 100644 --- a/Tools/test_switch2kit_autoconnect.py +++ b/Tools/test_switch2kit_autoconnect.py @@ -64,6 +64,43 @@ def test_polling_and_initialization_never_start_bluetooth(self): self.assertIn("s_auto_start_pending = false;", stop) self.assertIn("s_started = false;", stop) + def test_normal_test_build_does_not_require_the_backend(self): + # Evaluate the real parent registration with empty upstream targets; + # this checks CTest membership, not an emulator build or native behavior. + def catalogue(source, build, *flags): + subprocess.run(['cmake', '-S', str(source), '-B', str(build), *flags], + check=True, capture_output=True, timeout=15) + result = subprocess.run(['ctest', '--test-dir', str(build), '--show-only=json-v1'], + check=True, capture_output=True, text=True, timeout=5) + return {test['name']: test.get('command') + for test in json.loads(result.stdout)['tests']} + + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + expected = catalogue(ROOT / 'Source/UnitTests/Switch2Kit', root / 'standalone') + self.assertTrue(expected, 'The focused test catalogue must not be empty') + for name in ('Common', 'Core', 'VideoCommon', 'Switch2Kit'): + (root / name).mkdir() + (root / name / 'CMakeLists.txt').write_text('') + (root / 'Switch2Kit/CMakeLists.txt').write_text( + f'include("{(ROOT / "Source/UnitTests/Switch2Kit/CMakeLists.txt").as_posix()}")\n') + for name in ('UnitTestsMain.cpp', 'StubHost.cpp'): + (root / name).write_text('// Configure-only registration fixture.\n') + (root / 'CMakeLists.txt').write_text( + 'cmake_minimum_required(VERSION 3.25)\n' + 'project(TestRegistration LANGUAGES CXX)\n' + 'set(CMAKE_RUNTIME_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/Binaries")\n' + 'foreach(dependency fmt::fmt gtest::gtest core uicommon)\n' + ' add_library(${dependency} INTERFACE IMPORTED)\n' + 'endforeach()\n' + f'include("{(ROOT / "Source/UnitTests/CMakeLists.txt").as_posix()}")\n') + for enabled in ('OFF', 'ON'): + with self.subTest(ENABLE_SWITCH2KIT=enabled): + registered = catalogue(root, root / 'normal', f'-DENABLE_SWITCH2KIT={enabled}') + self.assertIn('tests', registered, 'Keep the upstream test registration') + self.assertEqual(expected, {name: command for name, command in registered.items() + if name.startswith('Switch2Kit.')}) + def test_ci_retains_regressions(self): # Check CTest's actual commands, not copies of command names in comments. with tempfile.TemporaryDirectory() as build: From b23e1ccd69c4855f400fc6936d4b9db11cb6cce0 Mon Sep 17 00:00:00 2001 From: Johnny D Date: Sun, 20 Sep 2026 19:06:43 -0400 Subject: [PATCH 06/21] ci: provision shared memory for upstream Linux unit tests The disabled application builds, but upstream PageTableHostMappingTest crashes with SIGBUS during memory.Init(). The container's default 64 MiB /dev/shm is smaller than Dolphin's default shared-memory allocation. All eight Switch2Kit CTest groups passed in the same failed run. Give only the Swift-free unit-test container 512 MiB of shared-memory capacity. Check for at least 128 MiB free before installing dependencies or compiling so this configuration regression fails immediately. Keep every build, upstream/addition test, sanitizer, dependency/launch assertion, timeout, and aggregate gate unchanged. No privileged container, new job, test filter, or application behavior change. AI-assisted change. Original blob hash verified; YAML and shell syntax checked; exact preflight accepts adequate capacity and rejects 64 MiB, below-threshold, malformed/missing capacity and df failure locally. The actual upstream unit suite still requires exact-head CI verification. --- .github/workflows/switch2kit-linux.yml | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/.github/workflows/switch2kit-linux.yml b/.github/workflows/switch2kit-linux.yml index 761f1cbd85a1..cb97d94213b9 100644 --- a/.github/workflows/switch2kit-linux.yml +++ b/.github/workflows/switch2kit-linux.yml @@ -83,7 +83,10 @@ jobs: # A complete ordinary application build in an image without Swift. This is # not a controller-enabled artifact and does not use the SDK build helper. runs-on: ubuntu-24.04 - container: ubuntu:24.04 + container: + image: ubuntu:24.04 + # Upstream memory-mapping tests exceed Docker's default 64 MiB /dev/shm. + options: --shm-size=512m timeout-minutes: 45 env: DEBIAN_FRONTEND: noninteractive @@ -91,6 +94,16 @@ jobs: CMAKE_C_COMPILER_LAUNCHER: sccache CMAKE_CXX_COMPILER_LAUNCHER: sccache steps: + - name: Check shared memory before installing dependencies or compiling + shell: bash + run: | + set -euo pipefail + df -h /dev/shm + available=$(df -B1 --output=avail /dev/shm | tail -n 1) + if ! [ "$available" -ge "$((128 * 1024 * 1024))" ]; then + echo '::error::Upstream memory tests need at least 128 MiB free in /dev/shm. Keep --shm-size=512m on this container.' >&2 + exit 1 + fi - name: Install ordinary native dependencies without Swift run: | apt-get update From 109472bd17005e29d0ae362441ea1601f6dcd25d Mon Sep 17 00:00:00 2001 From: Johnny D Date: Sun, 20 Sep 2026 21:50:32 -0400 Subject: [PATCH 07/21] ci: bound native smoke builds and stop optimizing unused throughput Keep complete native application builds and all existing SDK, sanitizer, upstream unit, architecture, dependency, relocation and exact-archive launch assertions. Do not replace native checks with fixtures or manual labels. Automatic native jobs now have a 10-minute ceiling including setup. A timeout remains an aggregate failure, not a performance success. For automatic macOS/Windows smoke builds, keep Release defines and CRT/ABI but disable expensive C/C++ optimization; these jobs do not benchmark the emulator. Linux optimized builds and Swift SDK configurations are unchanged. Explicit artifact builds and local defaults retain optimized Release flags. This does not claim optimized macOS/Windows code-generation qualification. Cache the pinned Windows Swift installer; avoid upgrading installed Homebrew build tools; use the standard runner's existing logical CPUs; expose compiler cache statistics. No new jobs, runners, matrices, triggers or test opt-ins. AGENTS.md now requires measured exact-head jobs within budget, including cold cache runs, and forbids deadline inflation or reporting timed-out work as fast. AI-assisted. Original Git blobs verified. YAML and POSIX shell syntax, retained validation-step equality, automatic/manual argument construction, a real CMake compiler-flag fixture and diff whitespace checks passed locally. Native performance and PowerShell execution still require exact-head CI. --- .github/workflows/switch2kit-linux.yml | 4 +-- .github/workflows/switch2kit-macos.yml | 32 ++++++++++++++++++++---- .github/workflows/switch2kit-windows.yml | 17 +++++++++++-- AGENTS.md | 24 ++++++++++++++++++ Tools/build-switch2kit-windows.ps1 | 8 +++--- 5 files changed, 72 insertions(+), 13 deletions(-) diff --git a/.github/workflows/switch2kit-linux.yml b/.github/workflows/switch2kit-linux.yml index cb97d94213b9..40ff2368f5f1 100644 --- a/.github/workflows/switch2kit-linux.yml +++ b/.github/workflows/switch2kit-linux.yml @@ -12,7 +12,7 @@ jobs: linux: runs-on: ubuntu-24.04 container: swift:6.2.1-noble - timeout-minutes: 45 + timeout-minutes: ${{ github.event_name == 'workflow_dispatch' && 45 || 10 }} env: DEBIAN_FRONTEND: noninteractive SCCACHE_GHA_ENABLED: 'true' @@ -87,7 +87,7 @@ jobs: image: ubuntu:24.04 # Upstream memory-mapping tests exceed Docker's default 64 MiB /dev/shm. options: --shm-size=512m - timeout-minutes: 45 + timeout-minutes: ${{ github.event_name == 'workflow_dispatch' && 45 || 10 }} env: DEBIAN_FRONTEND: noninteractive SCCACHE_GHA_ENABLED: 'true' diff --git a/.github/workflows/switch2kit-macos.yml b/.github/workflows/switch2kit-macos.yml index beaac0b94b55..a878f21ae603 100644 --- a/.github/workflows/switch2kit-macos.yml +++ b/.github/workflows/switch2kit-macos.yml @@ -19,8 +19,10 @@ jobs: - os: macos-15-intel arch: x86_64 runs-on: ${{ matrix.os }} - timeout-minutes: 60 + timeout-minutes: ${{ github.event_name == 'workflow_dispatch' && 60 || 10 }} env: + HOMEBREW_NO_AUTO_UPDATE: 1 + HOMEBREW_NO_INSTALL_CLEANUP: 1 DEVELOPER_DIR: /Applications/Xcode_26.3.app/Contents/Developer SCCACHE_GHA_ENABLED: 'true' CMAKE_C_COMPILER_LAUNCHER: sccache @@ -35,7 +37,15 @@ jobs: with: version: v0.10.0 - name: Install build dependencies - run: brew install cmake ninja nasm automake libtool qt@6 + run: | + set -euo pipefail + missing=() + for formula in cmake ninja nasm automake libtool qt@6; do + brew list --versions "$formula" >/dev/null 2>&1 || missing+=("$formula") + done + if [ "${#missing[@]}" -ne 0 ]; then + brew install "${missing[@]}" + fi - name: Check source pin and integration run: | test "$(git -C Externals/Switch2Kit rev-parse HEAD)" = "$(git ls-tree HEAD Externals/Switch2Kit | awk '{print $3}')" @@ -47,18 +57,30 @@ jobs: env: ARCH: ${{ matrix.arch }} run: | - set -o pipefail + set -euo pipefail + # CI validates a complete native application, not emulator throughput. + # Keep Release defines/ABI and every launch check; avoid optimizing + # thousands of unrelated C/C++ files. Explicit artifact builds retain + # upstream's ordinary optimized Release flags. + flags=() + if [ "$GITHUB_EVENT_NAME" != workflow_dispatch ]; then + flags=("-DCMAKE_C_FLAGS_RELEASE=-O0 -DNDEBUG" + "-DCMAKE_CXX_FLAGS_RELEASE=-O0 -DNDEBUG") + fi test "$(uname -m)" = "$ARCH" cmake -S . -B build-switch2kit -G Ninja \ - -DCMAKE_BUILD_TYPE=Release -DCMAKE_OSX_DEPLOYMENT_TARGET=15.0 \ + -DCMAKE_BUILD_TYPE=Release "${flags[@]}" -DCMAKE_OSX_DEPLOYMENT_TARGET=15.0 \ -DCMAKE_OSX_ARCHITECTURES="$ARCH" \ -DENABLE_SWITCH2KIT=ON -DENABLE_QT=ON -DENABLE_SDL=ON \ -DUSE_SYSTEM_SDL3=OFF \ -DCMAKE_PREFIX_PATH="$(brew --prefix qt@6)" \ -DENABLE_VULKAN=OFF -DENABLE_TESTS=OFF -DPOSTPROCESS_BUNDLE=ON \ 2>&1 | tee switch2kit-configure.log - cmake --build build-switch2kit --target dolphin-emu --parallel 3 \ + cmake --build build-switch2kit --target dolphin-emu --parallel "$(sysctl -n hw.logicalcpu)" \ 2>&1 | tee switch2kit-build.log + - name: Compiler cache statistics + if: always() + run: sccache --show-stats - name: Inspect and archive application env: ARCH: ${{ matrix.arch }} diff --git a/.github/workflows/switch2kit-windows.yml b/.github/workflows/switch2kit-windows.yml index 835f1f70d13c..8b3c5b97fb80 100644 --- a/.github/workflows/switch2kit-windows.yml +++ b/.github/workflows/switch2kit-windows.yml @@ -11,7 +11,7 @@ concurrency: jobs: windows: runs-on: windows-2025-vs2026 - timeout-minutes: 45 + timeout-minutes: ${{ github.event_name == 'workflow_dispatch' && 45 || 10 }} env: VCPKG_MAX_CONCURRENCY: 3 SCCACHE_GHA_ENABLED: 'true' @@ -31,11 +31,24 @@ jobs: # Swift 6.2 ships Clang 19, which the VS 2026 STL rejects. swift-version: swift-6.3.3-release swift-build: 6.3.3-RELEASE + cache: true - name: Build the complete controller-enabled application shell: pwsh run: | - & ./Tools/build-switch2kit-windows.ps1 2>&1 | Tee-Object windows-build.log + # Match the Release CRT/defines without spending CI time optimizing + # the emulator. The SDK build and all native launch checks stay real. + # Explicit artifact builds keep the upstream optimized Release flags. + $cmakeArgs = @() + if ($env:GITHUB_EVENT_NAME -ne 'workflow_dispatch') { + $cmakeArgs = @('-DCMAKE_C_FLAGS_RELEASE=/MD /Od /Ob0 /DNDEBUG', + '-DCMAKE_CXX_FLAGS_RELEASE=/MD /Od /Ob0 /DNDEBUG') + } + & ./Tools/build-switch2kit-windows.ps1 -CMakeArgs $cmakeArgs 2>&1 | Tee-Object windows-build.log if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } + - name: Compiler cache statistics + if: always() + shell: pwsh + run: sccache --show-stats - name: Stage and launch the application away from the build tree shell: pwsh run: | diff --git a/AGENTS.md b/AGENTS.md index 240505ee86b4..28a6e01750f6 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -48,6 +48,30 @@ Cancel obsolete runs, fail early, and upload application artifacts only when requested. Do not add nightly builds, extra matrices or broader triggers without an explicit need and a measured runtime/cost impact. +## Automatic-job runtime budget + +Every automatic job, including native setup/build/package/launch work, has a +10-minute maximum. The focused suite keeps its stricter existing limits. A timed +out or cancelled native job is a failure, not a successful fast run. Do not raise +this budget, shard the same work into more billed jobs, switch to larger paid +runners, or delete checks to make the dashboard look fast. A green run above the +budget is not merge-ready. Verify actual job durations on the exact PR head. + +Native macOS/Windows smoke builds keep Release defines/runtime ABI but compile +C/C++ without expensive optimization. They still compile/link the complete real +application and execute every architecture/dependency/relocation/archive check. +Linux's optimized builds/upstream unit suite, real Swift SDK tests and sanitizer +regressions remain unchanged. These smoke builds do not establish optimized +macOS/Windows code-generation or gameplay performance correctness. The existing +explicit artifact build uses normal optimized Release flags; local developer +build defaults are unchanged. Do not describe unoptimized smoke builds as release +qualification or distribute them as optimized releases. + +Cache the pinned Windows installer, do not upgrade already installed Homebrew +build dependencies, and use the CPUs already assigned to each standard runner. +Compiler-cache statistics must be visible; measure rather than assume a hit. +Cold-cache failures must be fixed, not hidden by reporting only a warm rerun. + ## Claims and review A source check is not a native build; a fixture is not the actual SDK; an SDK test diff --git a/Tools/build-switch2kit-windows.ps1 b/Tools/build-switch2kit-windows.ps1 index 46eaf724b1d4..63c2f123e04c 100644 --- a/Tools/build-switch2kit-windows.ps1 +++ b/Tools/build-switch2kit-windows.ps1 @@ -1,4 +1,4 @@ -param([switch]$Run) +param([switch]$Run, [string[]]$CMakeArgs = @()) $ErrorActionPreference = 'Stop' Set-StrictMode -Version Latest Set-Location (Join-Path $PSScriptRoot '..') @@ -44,15 +44,15 @@ if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } cmake -S . -B build-switch2kit-windows -G Ninja -DCMAKE_BUILD_TYPE=Release ` -DENABLE_SWITCH2KIT=ON -DENABLE_SDL=ON -DENABLE_QT=ON -DUSE_SYSTEM_SDL3=OFF ` -DENABLE_TESTS=OFF -DENABLE_CLI_TOOL=OFF -DENABLE_AUTOUPDATE=OFF ` - "-DSWITCH2KIT_SWIFT=$swift" "-DSWITCH2KIT_SWIFTC=$swiftc" + "-DSWITCH2KIT_SWIFT=$swift" "-DSWITCH2KIT_SWIFTC=$swiftc" @CMakeArgs if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } # Compile the real Swift/C++ WinRT library first. Fail on toolchain/SDK # incompatibility before spending time on Dolphin's other dependencies; do not # bypass the Microsoft STL or Dolphin compiler guards. This target is reused by # the application build, with the same SwiftPM configuration and scratch path. -cmake --build build-switch2kit-windows --target Switch2KitCBuild --parallel 3 +cmake --build build-switch2kit-windows --target Switch2KitCBuild --parallel ([Environment]::ProcessorCount) if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } -cmake --build build-switch2kit-windows --target dolphin-emu --parallel 3 +cmake --build build-switch2kit-windows --target dolphin-emu --parallel ([Environment]::ProcessorCount) if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } $app = Join-Path $PWD 'build-switch2kit-windows/Binaries/Dolphin.exe' if (-not (Test-Path $app) -or -not (Test-Path (Join-Path (Split-Path $app) 'Switch2KitC.dll'))) { From e8dfb13e1057d07e6d900b09d828b6eb179b69e1 Mon Sep 17 00:00:00 2001 From: Johnny D Date: Sun, 20 Sep 2026 22:43:07 -0400 Subject: [PATCH 08/21] ci: eliminate repeated header parsing and per-object remote cache traffic Apply smoke flags after upstream MSVC initialization through CMake's project include hook; keep upstream build sources and local defaults unchanged. Reuse the existing upstream PCH header per POSIX target and preserve Windows' existing shared PCH. Keep every translation unit, native platform, sanitizer, SDK test, upstream unit test and exact-archive launch assertion. Restore bounded local compiler caches once per job instead of issuing remote cache requests for each compilation. Use the same distro Clang 18, build path and install prefix for the isolated optimized Linux ON/OFF configurations so compatible objects can be shared. Do not restore built applications or tests. Fetch pinned submodules in parallel and install only required Qt components. Extend the existing CTest-registered CI regressions to catch MSVC flag shadowing and validate real mixed C/C++ per-target precompiled-header builds. Keep the 10-minute automatic job budgets and all native validation steps. AI-assisted. Verified original blob hashes, YAML/shell syntax, unchanged native job topology/budgets and validation-step bodies. All 12 CI setup and selection tests passed locally; a shadowed-MSVC-flags mutation failed as expected. Native performance remains subject to the exact-head Actions run. --- .github/workflows/switch2kit-linux.yml | 56 ++++++++++++--- .github/workflows/switch2kit-macos.yml | 31 +++++---- .github/workflows/switch2kit-windows.yml | 25 ++++--- AGENTS.md | 16 ++++- Tools/build-switch2kit-linux.sh | 7 +- Tools/ci-native.cmake | 41 +++++++++++ Tools/test_switch2kit_ci.py | 86 ++++++++++++++++++++++++ 7 files changed, 227 insertions(+), 35 deletions(-) create mode 100644 Tools/ci-native.cmake diff --git a/.github/workflows/switch2kit-linux.yml b/.github/workflows/switch2kit-linux.yml index 40ff2368f5f1..e49eb8f34922 100644 --- a/.github/workflows/switch2kit-linux.yml +++ b/.github/workflows/switch2kit-linux.yml @@ -15,7 +15,12 @@ jobs: timeout-minutes: ${{ github.event_name == 'workflow_dispatch' && 45 || 10 }} env: DEBIAN_FRONTEND: noninteractive - SCCACHE_GHA_ENABLED: 'true' + # One cache archive per job, not network requests for every object. + SCCACHE_DIR: ${{ github.workspace }}/.ci-cache/sccache + SCCACHE_CACHE_SIZE: 2G + GIT_CONFIG_COUNT: 1 + GIT_CONFIG_KEY_0: submodule.fetchJobs + GIT_CONFIG_VALUE_0: 8 CMAKE_C_COMPILER_LAUNCHER: sccache CMAKE_CXX_COMPILER_LAUNCHER: sccache steps: @@ -26,8 +31,16 @@ jobs: - name: Install native dependencies run: | apt-get update - apt-get install -y --no-install-recommends build-essential cmake ninja-build python3 pkg-config qt6-base-dev qt6-base-private-dev qt6-svg-dev libbluetooth-dev libevdev-dev libudev-dev libusb-1.0-0-dev libasound2-dev libpulse-dev libx11-dev libxi-dev libxrandr-dev libegl1-mesa-dev libgl1-mesa-dev libsystemd0 dbus xvfb xauth openbox wmctrl x11-utils + apt-get install -y --no-install-recommends build-essential clang cmake ninja-build python3 pkg-config qt6-base-dev qt6-base-private-dev qt6-svg-dev libbluetooth-dev libevdev-dev libudev-dev libusb-1.0-0-dev libasound2-dev libpulse-dev libx11-dev libxi-dev libxrandr-dev libegl1-mesa-dev libgl1-mesa-dev libsystemd0 dbus xvfb xauth openbox wmctrl x11-utils git config --global --add safe.directory "$PWD" + - name: Restore compiler objects (never test results or applications) + uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 + with: + path: ${{ env.SCCACHE_DIR }} + key: s2k-objects-v2-linux-clang18-${{ github.event_name == 'workflow_dispatch' && 'release' || 'smoke' }}-${{ github.sha }}-${{ github.job }} + restore-keys: | + s2k-objects-v2-linux-clang18-${{ github.event_name == 'workflow_dispatch' && 'release' || 'smoke' }}-${{ github.sha }}- + s2k-objects-v2-linux-clang18-${{ github.event_name == 'workflow_dispatch' && 'release' || 'smoke' }}- - name: Cache native compiler results (tests still run) uses: Mozilla-Actions/sccache-action@7d986dd989559c6ecdb630a3fd2557667be217ad with: @@ -36,7 +49,13 @@ jobs: shell: bash run: | set -o pipefail - bash Tools/build-switch2kit-linux.sh 2>&1 | tee linux-build.log + S2K_BUILD_JOBS="$(nproc)" bash Tools/build-switch2kit-linux.sh \ + -DCMAKE_C_COMPILER=/usr/bin/clang-18 -DCMAKE_CXX_COMPILER=/usr/bin/clang++-18 \ + "-DCMAKE_PROJECT_dolphin-emu_INCLUDE=$PWD/Tools/ci-native.cmake" \ + 2>&1 | tee linux-build.log + - name: Compiler cache statistics + if: always() + run: sccache --show-stats - name: Execute controller policies run: | python3 Tools/test_switch2kit_mapping.py --sanitize @@ -90,7 +109,12 @@ jobs: timeout-minutes: ${{ github.event_name == 'workflow_dispatch' && 45 || 10 }} env: DEBIAN_FRONTEND: noninteractive - SCCACHE_GHA_ENABLED: 'true' + # One cache archive per job, not network requests for every object. + SCCACHE_DIR: ${{ github.workspace }}/.ci-cache/sccache + SCCACHE_CACHE_SIZE: 2G + GIT_CONFIG_COUNT: 1 + GIT_CONFIG_KEY_0: submodule.fetchJobs + GIT_CONFIG_VALUE_0: 8 CMAKE_C_COMPILER_LAUNCHER: sccache CMAKE_CXX_COMPILER_LAUNCHER: sccache steps: @@ -116,6 +140,14 @@ jobs: with: persist-credentials: false submodules: recursive + - name: Restore compiler objects (never test results or applications) + uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 + with: + path: ${{ env.SCCACHE_DIR }} + key: s2k-objects-v2-linux-clang18-${{ github.event_name == 'workflow_dispatch' && 'release' || 'smoke' }}-${{ github.sha }}-${{ github.job }} + restore-keys: | + s2k-objects-v2-linux-clang18-${{ github.event_name == 'workflow_dispatch' && 'release' || 'smoke' }}-${{ github.sha }}- + s2k-objects-v2-linux-clang18-${{ github.event_name == 'workflow_dispatch' && 'release' || 'smoke' }}- - name: Cache native compiler results (tests still run) uses: Mozilla-Actions/sccache-action@7d986dd989559c6ecdb630a3fd2557667be217ad with: @@ -125,13 +157,16 @@ jobs: run: | set -euo pipefail git config --global --add safe.directory "$PWD" - cmake -S . -B build-disabled -G Ninja -DCMAKE_BUILD_TYPE=Release \ + cmake -S . -B build-switch2kit -G Ninja -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_C_COMPILER=/usr/bin/clang-18 -DCMAKE_CXX_COMPILER=/usr/bin/clang++-18 \ + "-DCMAKE_PROJECT_dolphin-emu_INCLUDE=$PWD/Tools/ci-native.cmake" \ + -DCMAKE_INSTALL_PREFIX="$PWD/build-switch2kit/install" \ -DENABLE_SWITCH2KIT=OFF -DENABLE_SDL=ON -DENABLE_QT=ON \ -DUSE_SYSTEM_SDL3=OFF -DENABLE_TESTS=ON -DENABLE_VULKAN=OFF \ -DENABLE_AUTOUPDATE=OFF 2>&1 | tee disabled-configure.log - cmake --build build-disabled --target dolphin-emu --parallel 3 2>&1 | tee disabled-build.log - cmake --build build-disabled --target unittests --parallel 3 2>&1 | tee upstream-unittests.log - ldd build-disabled/Binaries/dolphin-emu | tee disabled-dependencies.log + cmake --build build-switch2kit --target dolphin-emu --parallel "$(nproc)" 2>&1 | tee disabled-build.log + cmake --build build-switch2kit --target unittests --parallel "$(nproc)" 2>&1 | tee upstream-unittests.log + ldd build-switch2kit/Binaries/dolphin-emu | tee disabled-dependencies.log ! grep -q 'not found' disabled-dependencies.log ! grep -Ei 'Switch2Kit|libswift|libFoundation|libdispatch|libBlocksRuntime' disabled-dependencies.log # Qt initializes for --version too. Keep this disabled-backend smoke @@ -139,7 +174,10 @@ jobs: profile=$(mktemp -d) trap 'rm -rf "$profile"' EXIT HOME="$profile" XDG_CONFIG_HOME="$profile/config" XDG_CACHE_HOME="$profile/cache" \ - QT_QPA_PLATFORM=offscreen build-disabled/Binaries/dolphin-emu --user "$profile/user" --version + QT_QPA_PLATFORM=offscreen build-switch2kit/Binaries/dolphin-emu --user "$profile/user" --version + - name: Compiler cache statistics + if: always() + run: sccache --show-stats - name: Backend-disabled build diagnostics if: failure() || github.event_name == 'workflow_dispatch' uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 diff --git a/.github/workflows/switch2kit-macos.yml b/.github/workflows/switch2kit-macos.yml index a878f21ae603..f89b6ff06af7 100644 --- a/.github/workflows/switch2kit-macos.yml +++ b/.github/workflows/switch2kit-macos.yml @@ -24,7 +24,12 @@ jobs: HOMEBREW_NO_AUTO_UPDATE: 1 HOMEBREW_NO_INSTALL_CLEANUP: 1 DEVELOPER_DIR: /Applications/Xcode_26.3.app/Contents/Developer - SCCACHE_GHA_ENABLED: 'true' + # One cache archive per job, not network requests for every object. + SCCACHE_DIR: ${{ github.workspace }}/.ci-cache/sccache + SCCACHE_CACHE_SIZE: 2G + GIT_CONFIG_COUNT: 1 + GIT_CONFIG_KEY_0: submodule.fetchJobs + GIT_CONFIG_VALUE_0: 8 CMAKE_C_COMPILER_LAUNCHER: sccache CMAKE_CXX_COMPILER_LAUNCHER: sccache steps: @@ -32,6 +37,14 @@ jobs: with: submodules: recursive persist-credentials: false + - name: Restore compiler objects (never test results or applications) + uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 + with: + path: ${{ env.SCCACHE_DIR }} + key: s2k-objects-v2-${{ matrix.os }}-${{ matrix.arch }}-xcode26.3-${{ github.event_name == 'workflow_dispatch' && 'release' || 'smoke' }}-${{ github.sha }}-${{ github.job }} + restore-keys: | + s2k-objects-v2-${{ matrix.os }}-${{ matrix.arch }}-xcode26.3-${{ github.event_name == 'workflow_dispatch' && 'release' || 'smoke' }}-${{ github.sha }}- + s2k-objects-v2-${{ matrix.os }}-${{ matrix.arch }}-xcode26.3-${{ github.event_name == 'workflow_dispatch' && 'release' || 'smoke' }}- - name: Cache native compiler results (tests still run) uses: Mozilla-Actions/sccache-action@7d986dd989559c6ecdb630a3fd2557667be217ad with: @@ -40,7 +53,7 @@ jobs: run: | set -euo pipefail missing=() - for formula in cmake ninja nasm automake libtool qt@6; do + for formula in cmake ninja nasm automake libtool qtbase qtsvg; do brew list --versions "$formula" >/dev/null 2>&1 || missing+=("$formula") done if [ "${#missing[@]}" -ne 0 ]; then @@ -58,22 +71,14 @@ jobs: ARCH: ${{ matrix.arch }} run: | set -euo pipefail - # CI validates a complete native application, not emulator throughput. - # Keep Release defines/ABI and every launch check; avoid optimizing - # thousands of unrelated C/C++ files. Explicit artifact builds retain - # upstream's ordinary optimized Release flags. - flags=() - if [ "$GITHUB_EVENT_NAME" != workflow_dispatch ]; then - flags=("-DCMAKE_C_FLAGS_RELEASE=-O0 -DNDEBUG" - "-DCMAKE_CXX_FLAGS_RELEASE=-O0 -DNDEBUG") - fi test "$(uname -m)" = "$ARCH" cmake -S . -B build-switch2kit -G Ninja \ - -DCMAKE_BUILD_TYPE=Release "${flags[@]}" -DCMAKE_OSX_DEPLOYMENT_TARGET=15.0 \ + -DCMAKE_BUILD_TYPE=Release -DCMAKE_OSX_DEPLOYMENT_TARGET=15.0 \ + "-DCMAKE_PROJECT_dolphin-emu_INCLUDE=$PWD/Tools/ci-native.cmake" \ -DCMAKE_OSX_ARCHITECTURES="$ARCH" \ -DENABLE_SWITCH2KIT=ON -DENABLE_QT=ON -DENABLE_SDL=ON \ -DUSE_SYSTEM_SDL3=OFF \ - -DCMAKE_PREFIX_PATH="$(brew --prefix qt@6)" \ + -DCMAKE_PREFIX_PATH="$(brew --prefix qtbase);$(brew --prefix qtsvg)" \ -DENABLE_VULKAN=OFF -DENABLE_TESTS=OFF -DPOSTPROCESS_BUNDLE=ON \ 2>&1 | tee switch2kit-configure.log cmake --build build-switch2kit --target dolphin-emu --parallel "$(sysctl -n hw.logicalcpu)" \ diff --git a/.github/workflows/switch2kit-windows.yml b/.github/workflows/switch2kit-windows.yml index 8b3c5b97fb80..65c1a63c7dd2 100644 --- a/.github/workflows/switch2kit-windows.yml +++ b/.github/workflows/switch2kit-windows.yml @@ -14,7 +14,12 @@ jobs: timeout-minutes: ${{ github.event_name == 'workflow_dispatch' && 45 || 10 }} env: VCPKG_MAX_CONCURRENCY: 3 - SCCACHE_GHA_ENABLED: 'true' + # One cache archive per job, not network requests for every object. + SCCACHE_DIR: ${{ github.workspace }}/.ci-cache/sccache + SCCACHE_CACHE_SIZE: 2G + GIT_CONFIG_COUNT: 1 + GIT_CONFIG_KEY_0: submodule.fetchJobs + GIT_CONFIG_VALUE_0: 8 CMAKE_C_COMPILER_LAUNCHER: sccache CMAKE_CXX_COMPILER_LAUNCHER: sccache steps: @@ -22,6 +27,14 @@ jobs: with: persist-credentials: false submodules: recursive + - name: Restore compiler objects (never test results or applications) + uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 + with: + path: ${{ env.SCCACHE_DIR }} + key: s2k-objects-v2-windows-vs2026-${{ github.event_name == 'workflow_dispatch' && 'release' || 'smoke' }}-${{ github.sha }}-${{ github.job }} + restore-keys: | + s2k-objects-v2-windows-vs2026-${{ github.event_name == 'workflow_dispatch' && 'release' || 'smoke' }}-${{ github.sha }}- + s2k-objects-v2-windows-vs2026-${{ github.event_name == 'workflow_dispatch' && 'release' || 'smoke' }}- - name: Cache native compiler results (tests still run) uses: Mozilla-Actions/sccache-action@7d986dd989559c6ecdb630a3fd2557667be217ad with: @@ -35,14 +48,8 @@ jobs: - name: Build the complete controller-enabled application shell: pwsh run: | - # Match the Release CRT/defines without spending CI time optimizing - # the emulator. The SDK build and all native launch checks stay real. - # Explicit artifact builds keep the upstream optimized Release flags. - $cmakeArgs = @() - if ($env:GITHUB_EVENT_NAME -ne 'workflow_dispatch') { - $cmakeArgs = @('-DCMAKE_C_FLAGS_RELEASE=/MD /Od /Ob0 /DNDEBUG', - '-DCMAKE_CXX_FLAGS_RELEASE=/MD /Od /Ob0 /DNDEBUG') - } + # Load after upstream's MSVC flag initialization, not as shadowed -D flags. + $cmakeArgs = @("-DCMAKE_PROJECT_dolphin-emu_INCLUDE=$PWD/Tools/ci-native.cmake") & ./Tools/build-switch2kit-windows.ps1 -CMakeArgs $cmakeArgs 2>&1 | Tee-Object windows-build.log if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } - name: Compiler cache statistics diff --git a/AGENTS.md b/AGENTS.md index 28a6e01750f6..ee1e3c60daf7 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -60,8 +60,8 @@ budget is not merge-ready. Verify actual job durations on the exact PR head. Native macOS/Windows smoke builds keep Release defines/runtime ABI but compile C/C++ without expensive optimization. They still compile/link the complete real application and execute every architecture/dependency/relocation/archive check. -Linux's optimized builds/upstream unit suite, real Swift SDK tests and sanitizer -regressions remain unchanged. These smoke builds do not establish optimized +Linux keeps optimized builds and the complete upstream unit suite; real Swift +SDK tests and sanitizer regressions remain unchanged. These smoke builds do not establish optimized macOS/Windows code-generation or gameplay performance correctness. The existing explicit artifact build uses normal optimized Release flags; local developer build defaults are unchanged. Do not describe unoptimized smoke builds as release @@ -72,6 +72,18 @@ build dependencies, and use the CPUs already assigned to each standard runner. Compiler-cache statistics must be visible; measure rather than assume a hit. Cold-cache failures must be fixed, not hidden by reporting only a warm rerun. +Keep build acceleration CI-local through `Tools/ci-native.cmake`; do not rewrite +upstream compiler defaults. Verify effective MSVC flags after upstream's +`FlagsOverride.cmake`, not only the command-line arguments. Reuse upstream's PCH +header on POSIX and its existing PCH implementation on Windows. Retain every +translation unit; do not use unity builds to mask missing includes or collisions. +Install the Qt components Dolphin actually requires, not the all-modules Qt +metapackage. Parallelize pinned submodule fetching. Keep compiler objects in a +bounded local cache restored once per job, rather than paying a remote-cache +round trip per translation unit. Cache only compiler outputs, never successful +test results or a prebuilt application. Both Linux configurations must continue +to configure/build/link separately, with the disabled one in a Swift-free image. + ## Claims and review A source check is not a native build; a fixture is not the actual SDK; an SDK test diff --git a/Tools/build-switch2kit-linux.sh b/Tools/build-switch2kit-linux.sh index 3ff8871dbc00..808f785c1d15 100755 --- a/Tools/build-switch2kit-linux.sh +++ b/Tools/build-switch2kit-linux.sh @@ -6,13 +6,16 @@ if [[ $(uname -s) != Linux ]]; then echo 'This helper requires Linux.' >&2; exit for tool in cmake ninja git swift python3 pkg-config; do command -v "$tool" >/dev/null || { echo "Missing build tool: $tool" >&2; exit 1; } done +# Remaining arguments are ordinary CMake configure arguments, as in the Windows helper. +run=false +if [[ ${1:-} == --run ]]; then run=true; shift; fi git submodule update --init --recursive cmake -S . -B build-switch2kit -G Ninja \ -DCMAKE_BUILD_TYPE=Release -DENABLE_SWITCH2KIT=ON \ -DENABLE_SDL=ON -DENABLE_QT=ON -DUSE_SYSTEM_SDL3=OFF \ -DENABLE_TESTS=OFF -DENABLE_VULKAN=OFF -DENABLE_AUTOUPDATE=OFF \ - -DCMAKE_INSTALL_PREFIX="$PWD/build-switch2kit/install" + -DCMAKE_INSTALL_PREFIX="$PWD/build-switch2kit/install" "$@" cmake --build build-switch2kit --parallel "${S2K_BUILD_JOBS:-3}" cmake --install build-switch2kit printf 'Built: %s/build-switch2kit/install/bin/dolphin-emu\n' "$PWD" -if [[ ${1:-} == --run ]]; then exec "$PWD/build-switch2kit/install/bin/dolphin-emu"; fi +if "$run"; then exec "$PWD/build-switch2kit/install/bin/dolphin-emu"; fi diff --git a/Tools/ci-native.cmake b/Tools/ci-native.cmake new file mode 100644 index 000000000000..cb215aaf1749 --- /dev/null +++ b/Tools/ci-native.cmake @@ -0,0 +1,41 @@ +# CI-only build acceleration, loaded through CMAKE_PROJECT_dolphin-emu_INCLUDE. +# Keep upstream source/build defaults and the complete native build graph intact. +if(NOT "$ENV{GITHUB_ACTIONS}" STREQUAL "true") + message(FATAL_ERROR "Tools/ci-native.cmake is only for GitHub Actions builds") +endif() + +# project() has now loaded upstream FlagsOverride.cmake. Command-line -D flags +# alone are shadowed by that file's normal variables on MSVC. +if(NOT "$ENV{GITHUB_EVENT_NAME}" STREQUAL "workflow_dispatch") + if(MSVC) + set(CMAKE_C_FLAGS_RELEASE "/Od /Ob0 /DNDEBUG /Z7") + set(CMAKE_CXX_FLAGS_RELEASE "/Od /Ob0 /DNDEBUG /Z7") + elseif(APPLE) + set(CMAKE_C_FLAGS_RELEASE "-O0 -DNDEBUG") + set(CMAKE_CXX_FLAGS_RELEASE "-O0 -DNDEBUG") + endif() +endif() +message(STATUS "Native CI C Release flags: ${CMAKE_C_FLAGS_RELEASE}") +message(STATUS "Native CI C++ Release flags: ${CMAKE_CXX_FLAGS_RELEASE}") + +# Windows already shares upstream's PCH. On POSIX, compile the same upstream +# header once per target instead of parsing it for every translation unit. +# Per-target PCHs preserve each target's own defines, include paths and flags. +# Do not use unity builds (which change translation-unit boundaries), touch the +# production sources, replace libraries, or exclude a source from compilation. +function(switch2kit_ci_precompile_headers) + if(MSVC) + return() + endif() + foreach(target common audiocommon inputcommon videocommon discio core dolphin-emu) + if(TARGET ${target}) + target_precompile_headers(${target} PRIVATE + "$<$:${PROJECT_SOURCE_DIR}/Source/PCH/pch.h>") + endif() + endforeach() + if(TARGET dolphin-emu AND ENABLE_QT) + target_precompile_headers(dolphin-emu PRIVATE + "$<$:>") + endif() +endfunction() +cmake_language(DEFER CALL switch2kit_ci_precompile_headers) diff --git a/Tools/test_switch2kit_ci.py b/Tools/test_switch2kit_ci.py index c3b2e7abaf44..5be65bbd2562 100644 --- a/Tools/test_switch2kit_ci.py +++ b/Tools/test_switch2kit_ci.py @@ -5,6 +5,7 @@ import os from pathlib import Path import subprocess +import sys import tempfile import unittest from unittest.mock import patch @@ -87,6 +88,91 @@ def git(*args): self.assertIn('Docs/moved.cpp', paths) self.assertEqual(select_checks(paths), NATIVE) +class NativeBuildSetupTests(unittest.TestCase): + root = Path(__file__).resolve().parents[1] + + def configure(self, root, event='pull_request', **extra_env): + env = dict(os.environ, GITHUB_ACTIONS='true', GITHUB_EVENT_NAME=event, **extra_env) + # Fixtures exercise CMake, not the surrounding native job's cache server. + for name in ('CMAKE_C_COMPILER_LAUNCHER', 'CMAKE_CXX_COMPILER_LAUNCHER'): + env.pop(name, None) + result = subprocess.run(['cmake', '-S', str(root), '-B', str(root / 'build'), + '-G', 'Ninja', '-DCMAKE_BUILD_TYPE=Release', + f'-DCMAKE_PROJECT_dolphin-emu_INCLUDE={self.root}/Tools/ci-native.cmake'], + env=env, capture_output=True, text=True, timeout=10) + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + + def test_release_flags_after_upstream_msvc_override(self): + # This is a CMake scope regression, not a simulated Windows compiler. + # Load the real upstream flag file and then use the actual project hook. + for event, expected in [('pull_request', '/Od /Ob0 /DNDEBUG /Z7'), + ('workflow_dispatch', '/O2 /DNDEBUG /Z7')]: + with self.subTest(event=event), tempfile.TemporaryDirectory() as directory: + root = Path(directory) + (root / 'CMakeLists.txt').write_text(f''' +cmake_minimum_required(VERSION 3.25) +set(MSVC TRUE) +set(CMAKE_CXX_COMPILER_ID MSVC) +set(CMAKE_CXX_FLAGS_RELEASE "/Od /DNDEBUG" CACHE STRING "") +include("{self.root}/CMake/FlagsOverride.cmake") +project(dolphin-emu LANGUAGES NONE) +file(WRITE "${{CMAKE_BINARY_DIR}}/flags.txt" "${{CMAKE_CXX_FLAGS_RELEASE}}") +''') + self.configure(root, event) + self.assertEqual((root / 'build/flags.txt').read_text(), expected) + + def test_pch_keeps_per_target_flags_and_linux_optimization(self): + import json + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + (root / 'Source/PCH').mkdir(parents=True) + (root / 'Source/PCH/pch.h').write_text(''' +#include +#include +#ifndef FIXTURE_VALUE +#error Per-target definitions must reach the PCH. +#endif +#ifndef NDEBUG +#error Release definitions must be retained. +#endif +''') + (root / 'QtWidgets').write_text('#pragma once\n') + (root / 'value.cpp').write_text('int value() { return FIXTURE_VALUE; }\n') + (root / 'plain.c').write_text('int c_value(void) { return 1; }\n') + (root / 'main.cpp').write_text(''' +#include +extern int value(); +int main() { std::vector values{value()}; return values[0] == 7 ? 0 : 1; } +''') + (root / 'CMakeLists.txt').write_text(''' +cmake_minimum_required(VERSION 3.25) +project(dolphin-emu LANGUAGES C CXX) +set(CMAKE_CXX_STANDARD 23) +set(CMAKE_EXPORT_COMPILE_COMMANDS ON) +set(ENABLE_QT ON) +add_library(common value.cpp plain.c) +target_compile_definitions(common PRIVATE FIXTURE_VALUE=7) +add_executable(dolphin-emu main.cpp) +target_compile_definitions(dolphin-emu PRIVATE FIXTURE_VALUE=9) +target_include_directories(dolphin-emu PRIVATE "${CMAKE_SOURCE_DIR}") +target_link_libraries(dolphin-emu PRIVATE common) +''') + self.configure(root) + result = subprocess.run(['cmake', '--build', str(root / 'build'), '--parallel', '2'], + capture_output=True, text=True, timeout=10) + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + subprocess.run([str(root / 'build/dolphin-emu')], check=True, timeout=2) + commands = json.loads((root / 'build/compile_commands.json').read_text()) + plain = next(entry['command'] for entry in commands if entry['file'].endswith('plain.c')) + self.assertNotIn('cmake_pch.hxx', plain) + cpp = next(entry['command'] for entry in commands if entry['file'].endswith('value.cpp')) + self.assertIn('cmake_pch.hxx', cpp) + self.assertIn('-O0' if sys.platform == 'darwin' else '-O3', cpp) + self.assertIn('-DNDEBUG', cpp) + + def test_ci_build_acceleration_changes_select_all_native_platforms(self): + self.assertEqual(select_checks(['Tools/ci-native.cmake']), NATIVE) + if __name__ == '__main__': unittest.main(verbosity=2) From 01e1feaf997535aede1b0cd257e19e7388b397ca Mon Sep 17 00:00:00 2001 From: Johnny D Date: Sun, 20 Sep 2026 22:52:13 -0400 Subject: [PATCH 09/21] ci: preserve per-source flags when precompiling upstream headers Compile files with source-specific flags normally instead of forcing them to consume a target PCH compiled with different ISA/language/define options. In particular, upstream ARM crypto files retain their architecture flags. No source, test, sanitizer or native check is excluded from compilation. Extend the existing Clang fixture with a subdirectory-scoped C++20 source beside a C++23 target PCH. The fixture fails without this correction and passes with it, while checking the special file still compiles with its own options and the ordinary files use the PCH. All 12 existing and new CI setup/selection tests pass locally. Native timings still require CI. AI-assisted. --- Tools/ci-native.cmake | 20 ++++++++++++++++++++ Tools/test_switch2kit_ci.py | 15 ++++++++++++--- 2 files changed, 32 insertions(+), 3 deletions(-) diff --git a/Tools/ci-native.cmake b/Tools/ci-native.cmake index cb215aaf1749..b3aefcfedeb2 100644 --- a/Tools/ci-native.cmake +++ b/Tools/ci-native.cmake @@ -29,6 +29,26 @@ function(switch2kit_ci_precompile_headers) endif() foreach(target common audiocommon inputcommon videocommon discio core dolphin-emu) if(TARGET ${target}) + # Upstream gives some files different flags (e.g. ARM crypto ISA flags). + # They still compile normally; a target-wide PCH cannot represent those + # per-source options. Resolve properties in the target's source directory. + get_target_property(source_dir ${target} SOURCE_DIR) + get_target_property(sources ${target} SOURCES) + foreach(source IN LISTS sources) + if(source MATCHES "\\$<") + continue() + endif() + if(NOT IS_ABSOLUTE "${source}") + set(source "${source_dir}/${source}") + endif() + foreach(property COMPILE_FLAGS COMPILE_OPTIONS COMPILE_DEFINITIONS) + get_source_file_property(value "${source}" DIRECTORY "${source_dir}" ${property}) + if(value) + set_source_files_properties("${source}" DIRECTORY "${source_dir}" + PROPERTIES SKIP_PRECOMPILE_HEADERS ON) + endif() + endforeach() + endforeach() target_precompile_headers(${target} PRIVATE "$<$:${PROJECT_SOURCE_DIR}/Source/PCH/pch.h>") endif() diff --git a/Tools/test_switch2kit_ci.py b/Tools/test_switch2kit_ci.py index 5be65bbd2562..b1e859109d46 100644 --- a/Tools/test_switch2kit_ci.py +++ b/Tools/test_switch2kit_ci.py @@ -138,7 +138,14 @@ def test_pch_keeps_per_target_flags_and_linux_optimization(self): ''') (root / 'QtWidgets').write_text('#pragma once\n') (root / 'value.cpp').write_text('int value() { return FIXTURE_VALUE; }\n') + (root / 'special.cpp').write_text('int special() { return 2; }\n') (root / 'plain.c').write_text('int c_value(void) { return 1; }\n') + (root / 'component').mkdir() + (root / 'component/CMakeLists.txt').write_text(''' +add_library(common ../value.cpp ../plain.c ../special.cpp) +set_source_files_properties(../special.cpp PROPERTIES COMPILE_OPTIONS -std=c++20) +target_compile_definitions(common PRIVATE FIXTURE_VALUE=7) +''') (root / 'main.cpp').write_text(''' #include extern int value(); @@ -150,14 +157,13 @@ def test_pch_keeps_per_target_flags_and_linux_optimization(self): set(CMAKE_CXX_STANDARD 23) set(CMAKE_EXPORT_COMPILE_COMMANDS ON) set(ENABLE_QT ON) -add_library(common value.cpp plain.c) -target_compile_definitions(common PRIVATE FIXTURE_VALUE=7) +add_subdirectory(component) add_executable(dolphin-emu main.cpp) target_compile_definitions(dolphin-emu PRIVATE FIXTURE_VALUE=9) target_include_directories(dolphin-emu PRIVATE "${CMAKE_SOURCE_DIR}") target_link_libraries(dolphin-emu PRIVATE common) ''') - self.configure(root) + self.configure(root, CC='clang', CXX='clang++') result = subprocess.run(['cmake', '--build', str(root / 'build'), '--parallel', '2'], capture_output=True, text=True, timeout=10) self.assertEqual(result.returncode, 0, result.stdout + result.stderr) @@ -165,6 +171,9 @@ def test_pch_keeps_per_target_flags_and_linux_optimization(self): commands = json.loads((root / 'build/compile_commands.json').read_text()) plain = next(entry['command'] for entry in commands if entry['file'].endswith('plain.c')) self.assertNotIn('cmake_pch.hxx', plain) + special = next(entry['command'] for entry in commands if entry['file'].endswith('special.cpp')) + self.assertNotIn('cmake_pch.hxx', special) + self.assertIn('-std=c++20', special) cpp = next(entry['command'] for entry in commands if entry['file'].endswith('value.cpp')) self.assertIn('cmake_pch.hxx', cpp) self.assertIn('-O0' if sys.platform == 'darwin' else '-O3', cpp) From e511b0d5e1bd6699b93f45c59476b4a2017a9033 Mon Sep 17 00:00:00 2001 From: Johnny D Date: Sun, 20 Sep 2026 22:55:23 -0400 Subject: [PATCH 10/21] ci: resolve split Homebrew Qt modules through their shared prefix Homebrew intentionally installs split Qt formulae with HOMEBREW_PREFIX as the Qt installation root. Pointing Qt6_DIR at the qtbase keg made Qt's component lookup miss the separately installed qtsvg keg. Use the common linked prefix and verify both package configs before configuring Dolphin. Retain the minimal qtbase/qtsvg dependency set, every native check and the ten-minute automatic job limit. AI-assisted. --- .github/workflows/switch2kit-macos.yml | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/.github/workflows/switch2kit-macos.yml b/.github/workflows/switch2kit-macos.yml index f89b6ff06af7..d6dc805ad613 100644 --- a/.github/workflows/switch2kit-macos.yml +++ b/.github/workflows/switch2kit-macos.yml @@ -59,6 +59,15 @@ jobs: if [ "${#missing[@]}" -ne 0 ]; then brew install "${missing[@]}" fi + # Split Qt formulae share Homebrew's linked installation prefix. + # A qtbase keg is not a complete Qt prefix: Svg lives in qtsvg. + qt_prefix="$(brew --prefix)" + for component in Qt6 Qt6Svg; do + test -f "$qt_prefix/lib/cmake/$component/${component}Config.cmake" || { + echo "::error::Missing linked Qt package: $component in $qt_prefix" >&2 + exit 1 + } + done - name: Check source pin and integration run: | test "$(git -C Externals/Switch2Kit rev-parse HEAD)" = "$(git ls-tree HEAD Externals/Switch2Kit | awk '{print $3}')" @@ -78,7 +87,8 @@ jobs: -DCMAKE_OSX_ARCHITECTURES="$ARCH" \ -DENABLE_SWITCH2KIT=ON -DENABLE_QT=ON -DENABLE_SDL=ON \ -DUSE_SYSTEM_SDL3=OFF \ - -DCMAKE_PREFIX_PATH="$(brew --prefix qtbase);$(brew --prefix qtsvg)" \ + -DCMAKE_PREFIX_PATH="$(brew --prefix)" \ + -DQt6_DIR="$(brew --prefix)/lib/cmake/Qt6" \ -DENABLE_VULKAN=OFF -DENABLE_TESTS=OFF -DPOSTPROCESS_BUNDLE=ON \ 2>&1 | tee switch2kit-configure.log cmake --build build-switch2kit --target dolphin-emu --parallel "$(sysctl -n hw.logicalcpu)" \ From cb88683b887070681af8298861dbbebf0569a68a Mon Sep 17 00:00:00 2001 From: Johnny D Date: Sun, 20 Sep 2026 23:08:12 -0400 Subject: [PATCH 11/21] ci: use a Linux compiler that supports upstream std::expected Clang 18 with Noble's libstdc++ does not expose std::expected, so the native build failed in DiscIO after compiling hundreds of objects. Use Ubuntu's packaged Clang 19 for both application configurations, with a tiny compiler/library probe before any application build. Keep the ordinary Clang/GCC regression probes, optimized builds, Swift-free disabled image, full upstream tests, native launch checks and budgets. AI-assisted. --- .github/workflows/switch2kit-linux.yml | 32 ++++++++++++++++++-------- 1 file changed, 22 insertions(+), 10 deletions(-) diff --git a/.github/workflows/switch2kit-linux.yml b/.github/workflows/switch2kit-linux.yml index e49eb8f34922..ab7a0fa6dc17 100644 --- a/.github/workflows/switch2kit-linux.yml +++ b/.github/workflows/switch2kit-linux.yml @@ -31,16 +31,22 @@ jobs: - name: Install native dependencies run: | apt-get update - apt-get install -y --no-install-recommends build-essential clang cmake ninja-build python3 pkg-config qt6-base-dev qt6-base-private-dev qt6-svg-dev libbluetooth-dev libevdev-dev libudev-dev libusb-1.0-0-dev libasound2-dev libpulse-dev libx11-dev libxi-dev libxrandr-dev libegl1-mesa-dev libgl1-mesa-dev libsystemd0 dbus xvfb xauth openbox wmctrl x11-utils + apt-get install -y --no-install-recommends build-essential clang clang-19 cmake ninja-build python3 pkg-config qt6-base-dev qt6-base-private-dev qt6-svg-dev libbluetooth-dev libevdev-dev libudev-dev libusb-1.0-0-dev libasound2-dev libpulse-dev libx11-dev libxi-dev libxrandr-dev libegl1-mesa-dev libgl1-mesa-dev libsystemd0 dbus xvfb xauth openbox wmctrl x11-utils git config --global --add safe.directory "$PWD" + - name: Verify upstream C++23 library support before compiling Dolphin + run: | + # Clang 18 cannot use libstdc++'s std::expected. Do not spoof feature macros. + printf '%s\n' '#include ' \ + 'static_assert(std::expected(42).value() == 42);' | \ + /usr/bin/clang++-19 -std=c++23 -x c++ -fsyntax-only - - name: Restore compiler objects (never test results or applications) uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 with: path: ${{ env.SCCACHE_DIR }} - key: s2k-objects-v2-linux-clang18-${{ github.event_name == 'workflow_dispatch' && 'release' || 'smoke' }}-${{ github.sha }}-${{ github.job }} + key: s2k-objects-v2-linux-clang19-${{ github.event_name == 'workflow_dispatch' && 'release' || 'smoke' }}-${{ github.sha }}-${{ github.job }} restore-keys: | - s2k-objects-v2-linux-clang18-${{ github.event_name == 'workflow_dispatch' && 'release' || 'smoke' }}-${{ github.sha }}- - s2k-objects-v2-linux-clang18-${{ github.event_name == 'workflow_dispatch' && 'release' || 'smoke' }}- + s2k-objects-v2-linux-clang19-${{ github.event_name == 'workflow_dispatch' && 'release' || 'smoke' }}-${{ github.sha }}- + s2k-objects-v2-linux-clang19-${{ github.event_name == 'workflow_dispatch' && 'release' || 'smoke' }}- - name: Cache native compiler results (tests still run) uses: Mozilla-Actions/sccache-action@7d986dd989559c6ecdb630a3fd2557667be217ad with: @@ -50,7 +56,7 @@ jobs: run: | set -o pipefail S2K_BUILD_JOBS="$(nproc)" bash Tools/build-switch2kit-linux.sh \ - -DCMAKE_C_COMPILER=/usr/bin/clang-18 -DCMAKE_CXX_COMPILER=/usr/bin/clang++-18 \ + -DCMAKE_C_COMPILER=/usr/bin/clang-19 -DCMAKE_CXX_COMPILER=/usr/bin/clang++-19 \ "-DCMAKE_PROJECT_dolphin-emu_INCLUDE=$PWD/Tools/ci-native.cmake" \ 2>&1 | tee linux-build.log - name: Compiler cache statistics @@ -131,11 +137,17 @@ jobs: - name: Install ordinary native dependencies without Swift run: | apt-get update - apt-get install -y --no-install-recommends git ca-certificates build-essential clang cmake ninja-build python3 pkg-config qt6-base-dev qt6-base-private-dev qt6-svg-dev libbluetooth-dev libevdev-dev libudev-dev libusb-1.0-0-dev libasound2-dev libpulse-dev libx11-dev libxi-dev libxrandr-dev libegl1-mesa-dev libgl1-mesa-dev libsystemd0 dbus + apt-get install -y --no-install-recommends git ca-certificates build-essential clang clang-19 cmake ninja-build python3 pkg-config qt6-base-dev qt6-base-private-dev qt6-svg-dev libbluetooth-dev libevdev-dev libudev-dev libusb-1.0-0-dev libasound2-dev libpulse-dev libx11-dev libxi-dev libxrandr-dev libegl1-mesa-dev libgl1-mesa-dev libsystemd0 dbus if command -v swift || command -v swiftc; then echo 'The backend-disabled qualification image must not contain Swift.' >&2 exit 1 fi + - name: Verify upstream C++23 library support before compiling Dolphin + run: | + # Use the same real compiler/library pair as the enabled build. + printf '%s\n' '#include ' \ + 'static_assert(std::expected(42).value() == 42);' | \ + /usr/bin/clang++-19 -std=c++23 -x c++ -fsyntax-only - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 with: persist-credentials: false @@ -144,10 +156,10 @@ jobs: uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 with: path: ${{ env.SCCACHE_DIR }} - key: s2k-objects-v2-linux-clang18-${{ github.event_name == 'workflow_dispatch' && 'release' || 'smoke' }}-${{ github.sha }}-${{ github.job }} + key: s2k-objects-v2-linux-clang19-${{ github.event_name == 'workflow_dispatch' && 'release' || 'smoke' }}-${{ github.sha }}-${{ github.job }} restore-keys: | - s2k-objects-v2-linux-clang18-${{ github.event_name == 'workflow_dispatch' && 'release' || 'smoke' }}-${{ github.sha }}- - s2k-objects-v2-linux-clang18-${{ github.event_name == 'workflow_dispatch' && 'release' || 'smoke' }}- + s2k-objects-v2-linux-clang19-${{ github.event_name == 'workflow_dispatch' && 'release' || 'smoke' }}-${{ github.sha }}- + s2k-objects-v2-linux-clang19-${{ github.event_name == 'workflow_dispatch' && 'release' || 'smoke' }}- - name: Cache native compiler results (tests still run) uses: Mozilla-Actions/sccache-action@7d986dd989559c6ecdb630a3fd2557667be217ad with: @@ -158,7 +170,7 @@ jobs: set -euo pipefail git config --global --add safe.directory "$PWD" cmake -S . -B build-switch2kit -G Ninja -DCMAKE_BUILD_TYPE=Release \ - -DCMAKE_C_COMPILER=/usr/bin/clang-18 -DCMAKE_CXX_COMPILER=/usr/bin/clang++-18 \ + -DCMAKE_C_COMPILER=/usr/bin/clang-19 -DCMAKE_CXX_COMPILER=/usr/bin/clang++-19 \ "-DCMAKE_PROJECT_dolphin-emu_INCLUDE=$PWD/Tools/ci-native.cmake" \ -DCMAKE_INSTALL_PREFIX="$PWD/build-switch2kit/install" \ -DENABLE_SWITCH2KIT=OFF -DENABLE_SDL=ON -DENABLE_QT=ON \ From de1bfc5f4512d98399bbbe8c8a45e0d917a0c612 Mon Sep 17 00:00:00 2001 From: Johnny D Date: Mon, 21 Sep 2026 00:26:44 -0400 Subject: [PATCH 12/21] ci: remove serial gitlink fetches and redundant compiler preprocessing Fetch complete pinned submodule updates concurrently, verify every selected HEAD against the committed gitlink, and exclude only foreign Windows binaries and Android-only dependencies. Keep the actual native build graph and tests. Do not wrap configure probes or native PCH consumers in sccache. Retain the bounded compiler-object cache for ordinary targets. Apply the MSVC smoke options as inherited directory options so child project flag initialization cannot silently restore /O2. Upstream and explicit release defaults stay intact. Extend the existing CTest CI group with real local-git pin regressions and CMake cache/PCH/child-scope checks. No raised budgets, skipped assertions, manual-only replacement, runner upgrades or build shards. AI-assisted. --- .github/workflows/switch2kit-linux.yml | 16 +-- .github/workflows/switch2kit-macos.yml | 7 +- .github/workflows/switch2kit-windows.yml | 7 +- Tools/checkout_native.py | 80 ++++++++++++++ Tools/ci-native.cmake | 38 +++++++ Tools/test_native_ci_setup.py | 131 +++++++++++++++++++++++ Tools/test_switch2kit_ci.py | 1 + 7 files changed, 264 insertions(+), 16 deletions(-) create mode 100644 Tools/checkout_native.py create mode 100644 Tools/test_native_ci_setup.py diff --git a/.github/workflows/switch2kit-linux.yml b/.github/workflows/switch2kit-linux.yml index ab7a0fa6dc17..9209b949c036 100644 --- a/.github/workflows/switch2kit-linux.yml +++ b/.github/workflows/switch2kit-linux.yml @@ -18,21 +18,20 @@ jobs: # One cache archive per job, not network requests for every object. SCCACHE_DIR: ${{ github.workspace }}/.ci-cache/sccache SCCACHE_CACHE_SIZE: 2G - GIT_CONFIG_COUNT: 1 - GIT_CONFIG_KEY_0: submodule.fetchJobs - GIT_CONFIG_VALUE_0: 8 CMAKE_C_COMPILER_LAUNCHER: sccache CMAKE_CXX_COMPILER_LAUNCHER: sccache steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 with: persist-credentials: false - submodules: recursive + submodules: false - name: Install native dependencies run: | apt-get update apt-get install -y --no-install-recommends build-essential clang clang-19 cmake ninja-build python3 pkg-config qt6-base-dev qt6-base-private-dev qt6-svg-dev libbluetooth-dev libevdev-dev libudev-dev libusb-1.0-0-dev libasound2-dev libpulse-dev libx11-dev libxi-dev libxrandr-dev libegl1-mesa-dev libgl1-mesa-dev libsystemd0 dbus xvfb xauth openbox wmctrl x11-utils git config --global --add safe.directory "$PWD" + - name: Fetch and verify native gitlinks in parallel + run: python3 Tools/checkout_native.py - name: Verify upstream C++23 library support before compiling Dolphin run: | # Clang 18 cannot use libstdc++'s std::expected. Do not spoof feature macros. @@ -118,9 +117,6 @@ jobs: # One cache archive per job, not network requests for every object. SCCACHE_DIR: ${{ github.workspace }}/.ci-cache/sccache SCCACHE_CACHE_SIZE: 2G - GIT_CONFIG_COUNT: 1 - GIT_CONFIG_KEY_0: submodule.fetchJobs - GIT_CONFIG_VALUE_0: 8 CMAKE_C_COMPILER_LAUNCHER: sccache CMAKE_CXX_COMPILER_LAUNCHER: sccache steps: @@ -151,7 +147,11 @@ jobs: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 with: persist-credentials: false - submodules: recursive + submodules: false + - name: Fetch and verify native gitlinks in parallel + run: | + git config --global --add safe.directory "$PWD" + python3 Tools/checkout_native.py - name: Restore compiler objects (never test results or applications) uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 with: diff --git a/.github/workflows/switch2kit-macos.yml b/.github/workflows/switch2kit-macos.yml index d6dc805ad613..59efbf0031f8 100644 --- a/.github/workflows/switch2kit-macos.yml +++ b/.github/workflows/switch2kit-macos.yml @@ -27,16 +27,15 @@ jobs: # One cache archive per job, not network requests for every object. SCCACHE_DIR: ${{ github.workspace }}/.ci-cache/sccache SCCACHE_CACHE_SIZE: 2G - GIT_CONFIG_COUNT: 1 - GIT_CONFIG_KEY_0: submodule.fetchJobs - GIT_CONFIG_VALUE_0: 8 CMAKE_C_COMPILER_LAUNCHER: sccache CMAKE_CXX_COMPILER_LAUNCHER: sccache steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 with: - submodules: recursive + submodules: false persist-credentials: false + - name: Fetch and verify native gitlinks in parallel + run: python3 Tools/checkout_native.py - name: Restore compiler objects (never test results or applications) uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 with: diff --git a/.github/workflows/switch2kit-windows.yml b/.github/workflows/switch2kit-windows.yml index 65c1a63c7dd2..4abca1b12a1a 100644 --- a/.github/workflows/switch2kit-windows.yml +++ b/.github/workflows/switch2kit-windows.yml @@ -17,16 +17,15 @@ jobs: # One cache archive per job, not network requests for every object. SCCACHE_DIR: ${{ github.workspace }}/.ci-cache/sccache SCCACHE_CACHE_SIZE: 2G - GIT_CONFIG_COUNT: 1 - GIT_CONFIG_KEY_0: submodule.fetchJobs - GIT_CONFIG_VALUE_0: 8 CMAKE_C_COMPILER_LAUNCHER: sccache CMAKE_CXX_COMPILER_LAUNCHER: sccache steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 with: persist-credentials: false - submodules: recursive + submodules: false + - name: Fetch and verify native gitlinks in parallel + run: python Tools/checkout_native.py - name: Restore compiler objects (never test results or applications) uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 with: diff --git a/Tools/checkout_native.py b/Tools/checkout_native.py new file mode 100644 index 000000000000..f59d84d0be1e --- /dev/null +++ b/Tools/checkout_native.py @@ -0,0 +1,80 @@ +#!/usr/bin/env python3 +# Copyright 2026 Dolphin Emulator Project +# SPDX-License-Identifier: GPL-2.0-or-later +"""Fetch the native build's exact gitlinks concurrently, including pinned commits.""" +import configparser +from concurrent.futures import ThreadPoolExecutor +from pathlib import Path, PurePosixPath +import subprocess +import sys + + +def git(root, *arguments): + result = subprocess.run(['git', '-C', str(root), *arguments], check=True, + stdout=subprocess.PIPE, stderr=subprocess.STDOUT, + text=True, timeout=120) + return result.stdout + + +def native_modules(root, platform): + if platform not in ('linux', 'darwin', 'win32'): + raise ValueError(f'Unsupported native host: {platform}') + config = configparser.ConfigParser(interpolation=None) + with (root / '.gitmodules').open() as file: + config.read_file(file) + modules = [] + for section in config.sections(): + name = section.removeprefix('submodule "').removesuffix('"') + path = config[section]['path'] + relative = PurePosixPath(path) + if relative.is_absolute() or '..' in relative.parts: + raise ValueError(f'Unsafe submodule path: {path}') + active = path != 'Externals/libadrenotools' # Android only. + if platform != 'win32' and path in ('Externals/Qt', 'Externals/FFmpeg-bin'): + active = False # Windows binaries, not POSIX build dependencies. + modules.append((name, path, active)) + return modules + + +def checkout(root, platform): + modules = native_modules(root, platform) + paths = [path for _, path, active in modules if active] + if not paths: + raise ValueError('No native dependencies selected') + # Serialize parent configuration writes before starting independent workers. + # The normal build helpers also respect these active/inactive module settings. + for name, _, active in modules: + git(root, 'config', f'submodule.{name}.active', str(active).lower()) + git(root, 'submodule', 'init', '--', *paths) + entries = git(root, 'ls-tree', '-z', 'HEAD', '--', *paths).split('\0') + pins = {} + for entry in filter(None, entries): + metadata, path = entry.split('\t', 1) + mode, kind, sha = metadata.split() + if mode != '160000' or kind != 'commit': + raise ValueError(f'Expected a pinned gitlink: {path}') + pins[path] = sha + if set(pins) != set(paths): + raise ValueError('Selected dependencies do not match committed gitlinks') + + def update(path): + # --jobs only parallelizes the initial clones; fetching older gitlink + # commits afterwards was still serial. Parallelize each complete update. + output = git(root, 'submodule', 'update', '--init', '--recursive', + '--depth=1', '--checkout', '--', path) + actual = git(root / path, 'rev-parse', 'HEAD').strip() + if actual != pins[path]: + raise ValueError(f'{path}: expected {pins[path]}, got {actual}') + print(f'{path}: {actual}', flush=True) + return output + + with ThreadPoolExecutor(max_workers=8) as workers: + list(workers.map(update, paths)) + + +if __name__ == '__main__': + try: + checkout(Path(__file__).resolve().parents[1], sys.platform) + except subprocess.CalledProcessError as error: + print(error.stdout, file=sys.stderr) + raise SystemExit(error.returncode) diff --git a/Tools/ci-native.cmake b/Tools/ci-native.cmake index b3aefcfedeb2..603031327a57 100644 --- a/Tools/ci-native.cmake +++ b/Tools/ci-native.cmake @@ -10,6 +10,13 @@ if(NOT "$ENV{GITHUB_EVENT_NAME}" STREQUAL "workflow_dispatch") if(MSVC) set(CMAKE_C_FLAGS_RELEASE "/Od /Ob0 /DNDEBUG /Z7") set(CMAKE_CXX_FLAGS_RELEASE "/Od /Ob0 /DNDEBUG /Z7") + # Child project() calls can load the upstream override again. Directory + # options follow configuration flags, so the actual child compiler commands + # must also end in /Od /Ob0. Keep CRT selection, defines and warnings intact. + add_compile_options( + "$<$,$>:/Od>" + "$<$,$>:/Ob0>" + ) elseif(APPLE) set(CMAKE_C_FLAGS_RELEASE "-O0 -DNDEBUG") set(CMAKE_CXX_FLAGS_RELEASE "-O0 -DNDEBUG") @@ -18,6 +25,14 @@ endif() message(STATUS "Native CI C Release flags: ${CMAKE_C_FLAGS_RELEASE}") message(STATUS "Native CI C++ Release flags: ${CMAKE_CXX_FLAGS_RELEASE}") +# Configure probes compile once and almost never hit the object cache. Do not +# route hundreds of feature checks through another compiler-wrapper process. +# Restore launchers on actual build targets after configuration, below. +set(_switch2kit_c_launcher "${CMAKE_C_COMPILER_LAUNCHER}") +set(_switch2kit_cxx_launcher "${CMAKE_CXX_COMPILER_LAUNCHER}") +set(CMAKE_C_COMPILER_LAUNCHER "") +set(CMAKE_CXX_COMPILER_LAUNCHER "") + # Windows already shares upstream's PCH. On POSIX, compile the same upstream # header once per target instead of parsing it for every translation unit. # Per-target PCHs preserve each target's own defines, include paths and flags. @@ -58,4 +73,27 @@ function(switch2kit_ci_precompile_headers) "$<$:>") endif() endfunction() + +function(switch2kit_ci_cache_targets directory) + get_property(children DIRECTORY "${directory}" PROPERTY SUBDIRECTORIES) + foreach(child IN LISTS children) + switch2kit_ci_cache_targets("${child}") + endforeach() + get_property(targets DIRECTORY "${directory}" PROPERTY BUILDSYSTEM_TARGETS) + foreach(target IN LISTS targets) + get_target_property(kind ${target} TYPE) + if(kind MATCHES "^(EXECUTABLE|STATIC_LIBRARY|SHARED_LIBRARY|MODULE_LIBRARY|OBJECT_LIBRARY)$") + get_target_property(pch ${target} PRECOMPILE_HEADERS) + get_target_property(reuse_pch ${target} PRECOMPILE_HEADERS_REUSE_FROM) + # Native PCH consumption must not be turned back into expensive per-file + # preprocessing. Other targets retain the bounded compiler-object cache. + if(NOT pch AND NOT reuse_pch) + set_target_properties(${target} PROPERTIES + C_COMPILER_LAUNCHER "${_switch2kit_c_launcher}" + CXX_COMPILER_LAUNCHER "${_switch2kit_cxx_launcher}") + endif() + endif() + endforeach() +endfunction() cmake_language(DEFER CALL switch2kit_ci_precompile_headers) +cmake_language(DEFER CALL switch2kit_ci_cache_targets "${PROJECT_SOURCE_DIR}") diff --git a/Tools/test_native_ci_setup.py b/Tools/test_native_ci_setup.py new file mode 100644 index 000000000000..300703eda144 --- /dev/null +++ b/Tools/test_native_ci_setup.py @@ -0,0 +1,131 @@ +# Copyright 2026 Dolphin Emulator Project +# SPDX-License-Identifier: GPL-2.0-or-later +"""Setup regressions loaded by the existing Switch2Kit CTest CI test group.""" +import json +import os +from pathlib import Path +import subprocess +import tempfile +import unittest +from unittest.mock import patch + +from checkout_native import checkout, git, native_modules + + +class NativeCheckoutTests(unittest.TestCase): + def test_only_foreign_binary_and_android_modules_are_excluded(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + paths = ('Externals/Qt', 'Externals/FFmpeg-bin', 'Externals/libadrenotools', + 'Externals/Switch2Kit', 'Externals/SDL/SDL', 'new-dependency') + (root / '.gitmodules').write_text(''.join( + f'[submodule "{path}"]\npath = {path}\nurl = example\n' for path in paths)) + for platform in ('linux', 'darwin', 'win32'): + selected = {path for _, path, active in native_modules(root, platform) if active} + self.assertEqual(selected, set(paths) - {'Externals/libadrenotools'} - + (set() if platform == 'win32' else {'Externals/Qt', 'Externals/FFmpeg-bin'})) + with self.assertRaises(ValueError): + native_modules(root, 'unsupported') + (root / '.gitmodules').write_text('[submodule "bad"]\npath = ../escape\n') + with self.assertRaises(ValueError): + native_modules(root, 'linux') + + def test_parallel_updates_use_committed_pins_not_remote_heads(self): + # Real local Git repositories: no network or timing-dependent sleeps. + with tempfile.TemporaryDirectory() as directory, patch.dict(os.environ, { + 'GIT_CONFIG_COUNT': '1', 'GIT_CONFIG_KEY_0': 'protocol.file.allow', + 'GIT_CONFIG_VALUE_0': 'always'}): + root = Path(directory) / 'parent' + origin = Path(directory) / 'dependency' + for repo in (root, origin): + repo.mkdir() + git(repo, 'init', '-q') + git(repo, 'config', 'user.name', 'Test') + git(repo, 'config', 'user.email', 'test@example.invalid') + (origin / 'value').write_text('pinned\n') + git(origin, 'add', '.') + git(origin, 'commit', '-qm', 'pin') + pin = git(origin, 'rev-parse', 'HEAD').strip() + (origin / 'value').write_text('not the pinned version\n') + git(origin, 'commit', '-qam', 'remote head') + paths = ('Externals/first', 'Externals/second with space') + (root / '.gitmodules').write_text(''.join( + f'[submodule "{path}"]\npath = {path}\nurl = {origin.as_uri()}\n' for path in paths)) + git(root, 'add', '.gitmodules') + for path in paths: + git(root, 'update-index', '--add', '--cacheinfo', f'160000,{pin},{path}') + git(root, 'commit', '-qm', 'parent') + checkout(root, 'linux') + for path in paths: + self.assertEqual(git(root / path, 'rev-parse', 'HEAD').strip(), pin) + self.assertEqual((root / path / 'value').read_text(), 'pinned\n') + # No matching committed gitlink must be a failure, not an empty pass. + (root / '.gitmodules').write_text('[submodule "missing"]\npath = missing\nurl = bad\n') + with self.assertRaises((ValueError, subprocess.CalledProcessError)): + checkout(root, 'linux') + + +class NativeCacheSetupTests(unittest.TestCase): + def test_child_project_cannot_undo_msvc_smoke_options(self): + repository = Path(__file__).resolve().parents[1] + for event in ('pull_request', 'workflow_dispatch'): + with self.subTest(event=event), tempfile.TemporaryDirectory() as directory: + root = Path(directory) + (root / 'child').mkdir() + (root / 'CMakeLists.txt').write_text('cmake_minimum_required(VERSION 3.25)\n' + 'set(MSVC TRUE)\nset(CMAKE_CXX_COMPILER_ID MSVC)\n' + 'project(dolphin-emu LANGUAGES NONE)\nadd_subdirectory(child)\n') + (root / 'child/CMakeLists.txt').write_text( + 'project(dependency LANGUAGES NONE)\n' + f'include("{repository}/CMake/FlagsOverride.cmake")\n' + 'get_directory_property(options COMPILE_OPTIONS)\n' + 'file(WRITE "${CMAKE_BINARY_DIR}/child.txt" "${CMAKE_CXX_FLAGS_RELEASE}\\n${options}")\n') + result = subprocess.run(['cmake', '-S', str(root), '-B', str(root / 'build'), + f'-DCMAKE_PROJECT_dolphin-emu_INCLUDE={repository}/Tools/ci-native.cmake'], + env=dict(os.environ, GITHUB_ACTIONS='true', GITHUB_EVENT_NAME=event), + capture_output=True, text=True, timeout=10) + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + flags, options = (root / 'build/child.txt').read_text().split('\n', 1) + self.assertEqual(flags, '/O2 /DNDEBUG /Z7') + if event == 'pull_request': + self.assertIn(':/Od>', options) + self.assertIn(':/Ob0>', options) + else: + self.assertEqual(options, '') + + def test_cache_is_not_used_by_probes_or_pch_consumers(self): + hook = Path(__file__).resolve().parent / 'ci-native.cmake' + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + (root / 'Source/PCH').mkdir(parents=True) + (root / 'Source/PCH/pch.h').write_text('#include \n') + (root / 'value.cpp').write_text('int value() { return 0; }\n') + (root / 'plain.c').write_text('int plain(void) { return 0; }\n') + (root / 'CMakeLists.txt').write_text(''' +cmake_minimum_required(VERSION 3.25) +project(dolphin-emu LANGUAGES C CXX) +if(CMAKE_C_COMPILER_LAUNCHER OR CMAKE_CXX_COMPILER_LAUNCHER) + message(FATAL_ERROR "Configure probes must not use the cache wrapper") +endif() +include(CheckCSourceCompiles) +check_c_source_compiles("int main(void) { return 0; }" HAVE_WORKING_C) +if(NOT HAVE_WORKING_C) + message(FATAL_ERROR "The real compiler probe must still succeed") +endif() +add_library(common OBJECT value.cpp) +add_library(ordinary OBJECT plain.c) +file(GENERATE OUTPUT "${CMAKE_BINARY_DIR}/launchers.json" CONTENT + "[\\"$\\",\\"$\\"]") +''') + env = dict(os.environ, GITHUB_ACTIONS='true', GITHUB_EVENT_NAME='pull_request', + CC='clang', CXX='clang++') + command = ['cmake', '-S', str(root), '-B', str(root / 'build'), '-G', 'Ninja', + f'-DCMAKE_PROJECT_dolphin-emu_INCLUDE={hook}', + '-DCMAKE_C_COMPILER_LAUNCHER=cmake;-E;env', + '-DCMAKE_CXX_COMPILER_LAUNCHER=cmake;-E;env'] + result = subprocess.run(command, env=env, capture_output=True, text=True, timeout=10) + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + self.assertEqual(json.loads((root / 'build/launchers.json').read_text()), ['', 'cmake;-E;env']) + result = subprocess.run(['cmake', '--build', str(root / 'build'), '--parallel', '2'], + capture_output=True, text=True, timeout=10) + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) diff --git a/Tools/test_switch2kit_ci.py b/Tools/test_switch2kit_ci.py index b1e859109d46..150ee58ac938 100644 --- a/Tools/test_switch2kit_ci.py +++ b/Tools/test_switch2kit_ci.py @@ -11,6 +11,7 @@ from unittest.mock import patch from switch2kit_ci import TARGETS, changed_paths, select_checks +from test_native_ci_setup import NativeCacheSetupTests, NativeCheckoutTests NATIVE = {'macos', 'linux', 'windows'} From 5ac1683db1610dc42ebbae4c9c69a39c2b3eee56 Mon Sep 17 00:00:00 2001 From: Johnny D Date: Mon, 21 Sep 2026 12:53:04 -0400 Subject: [PATCH 13/21] ci: isolate compiler caches and remove native setup bottlenecks Keep native configure and nested SDK/sanitizer probes outside compiler-object caches, explicitly clear inherited PCH launchers, and recognize upstream's manual MSVC build_pch/use_pch implementation. Preserve its ordering edges. Reduce automatic Linux compile cost with optimized -O1 builds and POSIX line-level debug information. Extend per-target upstream PCH use and let CMake optimize redundant static/object-library ordering without removing translation units or generated-source dependencies. Manual artifacts retain upstream Release optimization and debug defaults. Verify already-fetched recursive gitlinks in CI helpers rather than repeating unfiltered updates. Avoid unrelated Homebrew dependent upgrades and repeated Homebrew/toolchain-selector processes. Use a container-visible Linux object cache path and a fresh cache namespace for cold validation. Add executable CMake/cache/PCH/generated-source/CTest and recursive local-Git regressions to ordinary CTest. Existing upstream tests, SDK regressions, sanitizers, native archive launch checks, runners and job budgets stay intact. AI-assisted; native acceptance must be established on this exact PR head. --- .github/workflows/switch2kit-linux.yml | 24 +- .github/workflows/switch2kit-macos.yml | 28 ++- .github/workflows/switch2kit-windows.yml | 10 +- Source/UnitTests/Switch2Kit/CMakeLists.txt | 1 + Tools/build-switch2kit-linux.sh | 6 +- Tools/build-switch2kit-windows.ps1 | 6 +- Tools/checkout_native.py | 56 ++++- Tools/ci-native.cmake | 60 ++++- Tools/test_native_ci_execution.py | 266 +++++++++++++++++++++ 9 files changed, 409 insertions(+), 48 deletions(-) create mode 100644 Tools/test_native_ci_execution.py diff --git a/.github/workflows/switch2kit-linux.yml b/.github/workflows/switch2kit-linux.yml index 9209b949c036..029015020ea9 100644 --- a/.github/workflows/switch2kit-linux.yml +++ b/.github/workflows/switch2kit-linux.yml @@ -16,10 +16,10 @@ jobs: env: DEBIAN_FRONTEND: noninteractive # One cache archive per job, not network requests for every object. - SCCACHE_DIR: ${{ github.workspace }}/.ci-cache/sccache + SCCACHE_DIR: /github/home/.cache/sccache SCCACHE_CACHE_SIZE: 2G - CMAKE_C_COMPILER_LAUNCHER: sccache - CMAKE_CXX_COMPILER_LAUNCHER: sccache + # Configure probes and standalone sanitizer/SDK tests must compile directly. + S2K_CI_COMPILER_CACHE: sccache steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 with: @@ -42,10 +42,10 @@ jobs: uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 with: path: ${{ env.SCCACHE_DIR }} - key: s2k-objects-v2-linux-clang19-${{ github.event_name == 'workflow_dispatch' && 'release' || 'smoke' }}-${{ github.sha }}-${{ github.job }} + key: s2k-objects-v3-linux-clang19-${{ github.event_name == 'workflow_dispatch' && 'release' || 'smoke' }}-${{ github.sha }}-${{ github.job }} restore-keys: | - s2k-objects-v2-linux-clang19-${{ github.event_name == 'workflow_dispatch' && 'release' || 'smoke' }}-${{ github.sha }}- - s2k-objects-v2-linux-clang19-${{ github.event_name == 'workflow_dispatch' && 'release' || 'smoke' }}- + s2k-objects-v3-linux-clang19-${{ github.event_name == 'workflow_dispatch' && 'release' || 'smoke' }}-${{ github.sha }}- + s2k-objects-v3-linux-clang19-${{ github.event_name == 'workflow_dispatch' && 'release' || 'smoke' }}- - name: Cache native compiler results (tests still run) uses: Mozilla-Actions/sccache-action@7d986dd989559c6ecdb630a3fd2557667be217ad with: @@ -115,10 +115,10 @@ jobs: env: DEBIAN_FRONTEND: noninteractive # One cache archive per job, not network requests for every object. - SCCACHE_DIR: ${{ github.workspace }}/.ci-cache/sccache + SCCACHE_DIR: /github/home/.cache/sccache SCCACHE_CACHE_SIZE: 2G - CMAKE_C_COMPILER_LAUNCHER: sccache - CMAKE_CXX_COMPILER_LAUNCHER: sccache + # Configure probes and standalone sanitizer/SDK tests must compile directly. + S2K_CI_COMPILER_CACHE: sccache steps: - name: Check shared memory before installing dependencies or compiling shell: bash @@ -156,10 +156,10 @@ jobs: uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 with: path: ${{ env.SCCACHE_DIR }} - key: s2k-objects-v2-linux-clang19-${{ github.event_name == 'workflow_dispatch' && 'release' || 'smoke' }}-${{ github.sha }}-${{ github.job }} + key: s2k-objects-v3-linux-clang19-${{ github.event_name == 'workflow_dispatch' && 'release' || 'smoke' }}-${{ github.sha }}-${{ github.job }} restore-keys: | - s2k-objects-v2-linux-clang19-${{ github.event_name == 'workflow_dispatch' && 'release' || 'smoke' }}-${{ github.sha }}- - s2k-objects-v2-linux-clang19-${{ github.event_name == 'workflow_dispatch' && 'release' || 'smoke' }}- + s2k-objects-v3-linux-clang19-${{ github.event_name == 'workflow_dispatch' && 'release' || 'smoke' }}-${{ github.sha }}- + s2k-objects-v3-linux-clang19-${{ github.event_name == 'workflow_dispatch' && 'release' || 'smoke' }}- - name: Cache native compiler results (tests still run) uses: Mozilla-Actions/sccache-action@7d986dd989559c6ecdb630a3fd2557667be217ad with: diff --git a/.github/workflows/switch2kit-macos.yml b/.github/workflows/switch2kit-macos.yml index 59efbf0031f8..28093dcb7ecc 100644 --- a/.github/workflows/switch2kit-macos.yml +++ b/.github/workflows/switch2kit-macos.yml @@ -23,12 +23,15 @@ jobs: env: HOMEBREW_NO_AUTO_UPDATE: 1 HOMEBREW_NO_INSTALL_CLEANUP: 1 + # Do not upgrade unrelated installed Qt modules after installing qtbase. + # Required components and the complete relocated application are validated below. + HOMEBREW_NO_INSTALLED_DEPENDENTS_CHECK: 1 DEVELOPER_DIR: /Applications/Xcode_26.3.app/Contents/Developer # One cache archive per job, not network requests for every object. SCCACHE_DIR: ${{ github.workspace }}/.ci-cache/sccache SCCACHE_CACHE_SIZE: 2G - CMAKE_C_COMPILER_LAUNCHER: sccache - CMAKE_CXX_COMPILER_LAUNCHER: sccache + # Configure probes and standalone sanitizer/SDK tests must compile directly. + S2K_CI_COMPILER_CACHE: sccache steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 with: @@ -40,10 +43,10 @@ jobs: uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 with: path: ${{ env.SCCACHE_DIR }} - key: s2k-objects-v2-${{ matrix.os }}-${{ matrix.arch }}-xcode26.3-${{ github.event_name == 'workflow_dispatch' && 'release' || 'smoke' }}-${{ github.sha }}-${{ github.job }} + key: s2k-objects-v3-${{ matrix.os }}-${{ matrix.arch }}-xcode26.3-${{ github.event_name == 'workflow_dispatch' && 'release' || 'smoke' }}-${{ github.sha }}-${{ github.job }} restore-keys: | - s2k-objects-v2-${{ matrix.os }}-${{ matrix.arch }}-xcode26.3-${{ github.event_name == 'workflow_dispatch' && 'release' || 'smoke' }}-${{ github.sha }}- - s2k-objects-v2-${{ matrix.os }}-${{ matrix.arch }}-xcode26.3-${{ github.event_name == 'workflow_dispatch' && 'release' || 'smoke' }}- + s2k-objects-v3-${{ matrix.os }}-${{ matrix.arch }}-xcode26.3-${{ github.event_name == 'workflow_dispatch' && 'release' || 'smoke' }}-${{ github.sha }}- + s2k-objects-v3-${{ matrix.os }}-${{ matrix.arch }}-xcode26.3-${{ github.event_name == 'workflow_dispatch' && 'release' || 'smoke' }}- - name: Cache native compiler results (tests still run) uses: Mozilla-Actions/sccache-action@7d986dd989559c6ecdb630a3fd2557667be217ad with: @@ -51,9 +54,10 @@ jobs: - name: Install build dependencies run: | set -euo pipefail + installed="$(brew list --formula --versions | awk '{print $1}')" missing=() for formula in cmake ninja nasm automake libtool qtbase qtsvg; do - brew list --versions "$formula" >/dev/null 2>&1 || missing+=("$formula") + grep -Fxq "$formula" <<< "$installed" || missing+=("$formula") done if [ "${#missing[@]}" -ne 0 ]; then brew install "${missing[@]}" @@ -61,6 +65,14 @@ jobs: # Split Qt formulae share Homebrew's linked installation prefix. # A qtbase keg is not a complete Qt prefix: Svg lives in qtsvg. qt_prefix="$(brew --prefix)" + echo "S2K_QT_PREFIX=$qt_prefix" >> "$GITHUB_ENV" + # Resolve the selected Xcode compiler once, rather than invoking the + # /usr/bin toolchain selector for every probe and translation unit. + cc="$(xcrun --find clang)" + cxx="$(xcrun --find clang++)" + test -x "$cc" && test -x "$cxx" + printf 'CC=%s\nCXX=%s\nOBJC=%s\nOBJCXX=%s\n' \ + "$cc" "$cxx" "$cc" "$cxx" >> "$GITHUB_ENV" for component in Qt6 Qt6Svg; do test -f "$qt_prefix/lib/cmake/$component/${component}Config.cmake" || { echo "::error::Missing linked Qt package: $component in $qt_prefix" >&2 @@ -86,8 +98,8 @@ jobs: -DCMAKE_OSX_ARCHITECTURES="$ARCH" \ -DENABLE_SWITCH2KIT=ON -DENABLE_QT=ON -DENABLE_SDL=ON \ -DUSE_SYSTEM_SDL3=OFF \ - -DCMAKE_PREFIX_PATH="$(brew --prefix)" \ - -DQt6_DIR="$(brew --prefix)/lib/cmake/Qt6" \ + -DCMAKE_PREFIX_PATH="$S2K_QT_PREFIX" \ + -DQt6_DIR="$S2K_QT_PREFIX/lib/cmake/Qt6" \ -DENABLE_VULKAN=OFF -DENABLE_TESTS=OFF -DPOSTPROCESS_BUNDLE=ON \ 2>&1 | tee switch2kit-configure.log cmake --build build-switch2kit --target dolphin-emu --parallel "$(sysctl -n hw.logicalcpu)" \ diff --git a/.github/workflows/switch2kit-windows.yml b/.github/workflows/switch2kit-windows.yml index 4abca1b12a1a..87d442ad9e27 100644 --- a/.github/workflows/switch2kit-windows.yml +++ b/.github/workflows/switch2kit-windows.yml @@ -17,8 +17,8 @@ jobs: # One cache archive per job, not network requests for every object. SCCACHE_DIR: ${{ github.workspace }}/.ci-cache/sccache SCCACHE_CACHE_SIZE: 2G - CMAKE_C_COMPILER_LAUNCHER: sccache - CMAKE_CXX_COMPILER_LAUNCHER: sccache + # Configure probes and standalone sanitizer/SDK tests must compile directly. + S2K_CI_COMPILER_CACHE: sccache steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 with: @@ -30,10 +30,10 @@ jobs: uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 with: path: ${{ env.SCCACHE_DIR }} - key: s2k-objects-v2-windows-vs2026-${{ github.event_name == 'workflow_dispatch' && 'release' || 'smoke' }}-${{ github.sha }}-${{ github.job }} + key: s2k-objects-v3-windows-vs2026-${{ github.event_name == 'workflow_dispatch' && 'release' || 'smoke' }}-${{ github.sha }}-${{ github.job }} restore-keys: | - s2k-objects-v2-windows-vs2026-${{ github.event_name == 'workflow_dispatch' && 'release' || 'smoke' }}-${{ github.sha }}- - s2k-objects-v2-windows-vs2026-${{ github.event_name == 'workflow_dispatch' && 'release' || 'smoke' }}- + s2k-objects-v3-windows-vs2026-${{ github.event_name == 'workflow_dispatch' && 'release' || 'smoke' }}-${{ github.sha }}- + s2k-objects-v3-windows-vs2026-${{ github.event_name == 'workflow_dispatch' && 'release' || 'smoke' }}- - name: Cache native compiler results (tests still run) uses: Mozilla-Actions/sccache-action@7d986dd989559c6ecdb630a3fd2557667be217ad with: diff --git a/Source/UnitTests/Switch2Kit/CMakeLists.txt b/Source/UnitTests/Switch2Kit/CMakeLists.txt index e25dbb7a7c0f..a6b676a479f9 100644 --- a/Source/UnitTests/Switch2Kit/CMakeLists.txt +++ b/Source/UnitTests/Switch2Kit/CMakeLists.txt @@ -16,6 +16,7 @@ function(add_switch2kit_test name script) endfunction() add_switch2kit_test(ChangeSelection test_switch2kit_ci.py) +add_switch2kit_test(NativeExecution test_native_ci_execution.py) add_switch2kit_test(Integration test_switch2kit.py) add_switch2kit_test(AutoConnect test_switch2kit_autoconnect.py) add_switch2kit_test(Mapping test_switch2kit_mapping.py --sanitize) diff --git a/Tools/build-switch2kit-linux.sh b/Tools/build-switch2kit-linux.sh index 808f785c1d15..fa38bb0aea99 100755 --- a/Tools/build-switch2kit-linux.sh +++ b/Tools/build-switch2kit-linux.sh @@ -9,7 +9,11 @@ done # Remaining arguments are ordinary CMake configure arguments, as in the Windows helper. run=false if [[ ${1:-} == --run ]]; then run=true; shift; fi -git submodule update --init --recursive +if [[ ${GITHUB_ACTIONS:-} == true ]]; then + python3 Tools/checkout_native.py --verify +else + git submodule update --init --recursive +fi cmake -S . -B build-switch2kit -G Ninja \ -DCMAKE_BUILD_TYPE=Release -DENABLE_SWITCH2KIT=ON \ -DENABLE_SDL=ON -DENABLE_QT=ON -DUSE_SYSTEM_SDL3=OFF \ diff --git a/Tools/build-switch2kit-windows.ps1 b/Tools/build-switch2kit-windows.ps1 index 63c2f123e04c..c5d8fdf7fe27 100644 --- a/Tools/build-switch2kit-windows.ps1 +++ b/Tools/build-switch2kit-windows.ps1 @@ -39,7 +39,11 @@ Write-Output "Visual Studio: $vs" Write-Output "Swift target: $($target.target.triple)" # Keep runtime lookup local to this process and its launched application. $env:PATH = (($target.paths.runtimeLibraryPaths | Where-Object { Test-Path $_ }) -join ';') + ';' + $env:PATH -git submodule update --init --recursive +if ($env:GITHUB_ACTIONS -eq 'true') { + python Tools/checkout_native.py --verify +} else { + git submodule update --init --recursive +} if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } cmake -S . -B build-switch2kit-windows -G Ninja -DCMAKE_BUILD_TYPE=Release ` -DENABLE_SWITCH2KIT=ON -DENABLE_SDL=ON -DENABLE_QT=ON -DUSE_SYSTEM_SDL3=OFF ` diff --git a/Tools/checkout_native.py b/Tools/checkout_native.py index f59d84d0be1e..a0b4ef210e57 100644 --- a/Tools/checkout_native.py +++ b/Tools/checkout_native.py @@ -2,6 +2,7 @@ # Copyright 2026 Dolphin Emulator Project # SPDX-License-Identifier: GPL-2.0-or-later """Fetch the native build's exact gitlinks concurrently, including pinned commits.""" +import argparse import configparser from concurrent.futures import ThreadPoolExecutor from pathlib import Path, PurePosixPath @@ -42,20 +43,11 @@ def checkout(root, platform): if not paths: raise ValueError('No native dependencies selected') # Serialize parent configuration writes before starting independent workers. - # The normal build helpers also respect these active/inactive module settings. + # Build helpers use --verify in CI rather than fetching unrelated modules. for name, _, active in modules: git(root, 'config', f'submodule.{name}.active', str(active).lower()) git(root, 'submodule', 'init', '--', *paths) - entries = git(root, 'ls-tree', '-z', 'HEAD', '--', *paths).split('\0') - pins = {} - for entry in filter(None, entries): - metadata, path = entry.split('\t', 1) - mode, kind, sha = metadata.split() - if mode != '160000' or kind != 'commit': - raise ValueError(f'Expected a pinned gitlink: {path}') - pins[path] = sha - if set(pins) != set(paths): - raise ValueError('Selected dependencies do not match committed gitlinks') + pins = native_pins(root, paths) def update(path): # --jobs only parallelizes the initial clones; fetching older gitlink @@ -72,9 +64,49 @@ def update(path): list(workers.map(update, paths)) +def native_pins(root, paths): + entries = git(root, 'ls-tree', '-z', 'HEAD', '--', *paths).split('\0') + pins = {} + for entry in filter(None, entries): + metadata, path = entry.split('\t', 1) + mode, kind, sha = metadata.split() + if mode != '160000' or kind != 'commit': + raise ValueError(f'Expected a pinned gitlink: {path}') + pins[path] = sha + if set(pins) != set(paths): + raise ValueError('Selected dependencies do not match committed gitlinks') + + return pins + + +def verify_checkout(root, platform): + """Reject missing/stale pins without fetching, including nested gitlinks.""" + paths = [path for _, path, active in native_modules(root, platform) if active] + if not paths: + raise ValueError('No native dependencies selected') + pins = native_pins(root, paths) + for path, expected in pins.items(): + # git -C on an uninitialized submodule can resolve the parent repository. + # Require the submodule's own .git before comparing its HEAD. + if not (root / path / '.git').exists(): + raise ValueError(f'Uninitialized native dependency: {path}') + actual = git(root / path, 'rev-parse', 'HEAD').strip() + if actual != expected: + raise ValueError(f'{path}: expected {expected}, got {actual}') + status = git(root, 'submodule', 'status', '--recursive', '--', *paths) + for line in status.splitlines(): + if not line.startswith(' '): + raise ValueError(f'Uninitialized or stale recursive gitlink: {line}') + print(f'Verified {len(pins)} native dependencies and their recursive gitlinks', flush=True) + + if __name__ == '__main__': + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--verify', action='store_true', help='Verify pins without fetching') + args = parser.parse_args() try: - checkout(Path(__file__).resolve().parents[1], sys.platform) + action = verify_checkout if args.verify else checkout + action(Path(__file__).resolve().parents[1], sys.platform) except subprocess.CalledProcessError as error: print(error.stdout, file=sys.stderr) raise SystemExit(error.returncode) diff --git a/Tools/ci-native.cmake b/Tools/ci-native.cmake index 603031327a57..5feca50196fa 100644 --- a/Tools/ci-native.cmake +++ b/Tools/ci-native.cmake @@ -20,18 +20,35 @@ if(NOT "$ENV{GITHUB_EVENT_NAME}" STREQUAL "workflow_dispatch") elseif(APPLE) set(CMAKE_C_FLAGS_RELEASE "-O0 -DNDEBUG") set(CMAKE_CXX_FLAGS_RELEASE "-O0 -DNDEBUG") + elseif(CMAKE_SYSTEM_NAME STREQUAL "Linux") + # Keep the complete upstream Linux suite optimized. Explicit artifacts keep + # the upstream Release optimizer; automatic qualification uses -O1. + set(CMAKE_C_FLAGS_RELEASE "-O1 -DNDEBUG") + set(CMAKE_CXX_FLAGS_RELEASE "-O1 -DNDEBUG") endif() endif() message(STATUS "Native CI C Release flags: ${CMAKE_C_FLAGS_RELEASE}") message(STATUS "Native CI C++ Release flags: ${CMAKE_CXX_FLAGS_RELEASE}") -# Configure probes compile once and almost never hit the object cache. Do not -# route hundreds of feature checks through another compiler-wrapper process. -# Restore launchers on actual build targets after configuration, below. +# Use a CI-specific variable, not CMAKE_*_COMPILER_LAUNCHER in the job's +# environment: those standard variables also reach every nested SDK/fixture +# configuration and try_compile project. Clearing a normal variable alone does +# not clear that environment or its cache entry. Keep actual feature/link probes. set(_switch2kit_c_launcher "${CMAKE_C_COMPILER_LAUNCHER}") set(_switch2kit_cxx_launcher "${CMAKE_CXX_COMPILER_LAUNCHER}") -set(CMAKE_C_COMPILER_LAUNCHER "") -set(CMAKE_CXX_COMPILER_LAUNCHER "") +if(DEFINED ENV{S2K_CI_COMPILER_CACHE}) + if(NOT _switch2kit_c_launcher) + set(_switch2kit_c_launcher "$ENV{S2K_CI_COMPILER_CACHE}") + endif() + if(NOT _switch2kit_cxx_launcher) + set(_switch2kit_cxx_launcher "$ENV{S2K_CI_COMPILER_CACHE}") + endif() +endif() +foreach(language C CXX) + set(CMAKE_${language}_COMPILER_LAUNCHER "" CACHE STRING "Native CI target-only cache" FORCE) + set(CMAKE_${language}_COMPILER_LAUNCHER "") + unset(ENV{CMAKE_${language}_COMPILER_LAUNCHER}) +endforeach() # Windows already shares upstream's PCH. On POSIX, compile the same upstream # header once per target instead of parsing it for every translation unit. @@ -42,7 +59,9 @@ function(switch2kit_ci_precompile_headers) if(MSVC) return() endif() - foreach(target common audiocommon inputcommon videocommon discio core dolphin-emu) + foreach(target common audiocommon inputcommon videocommon discio core uicommon + videoogl videonull videosoftware videometal videovulkan + dolphin-tool dolphin-emu) if(TARGET ${target}) # Upstream gives some files different flags (e.g. ARM crypto ISA flags). # They still compile normally; a target-wide PCH cannot represent those @@ -85,13 +104,36 @@ function(switch2kit_ci_cache_targets directory) if(kind MATCHES "^(EXECUTABLE|STATIC_LIBRARY|SHARED_LIBRARY|MODULE_LIBRARY|OBJECT_LIBRARY)$") get_target_property(pch ${target} PRECOMPILE_HEADERS) get_target_property(reuse_pch ${target} PRECOMPILE_HEADERS_REUSE_FROM) - # Native PCH consumption must not be turned back into expensive per-file - # preprocessing. Other targets retain the bounded compiler-object cache. - if(NOT pch AND NOT reuse_pch) + get_target_property(links ${target} LINK_LIBRARIES) + # Source/PCH implements MSVC's /Yc and /Yu manually, through build_pch + # and the use_pch interface. PRECOMPILE_HEADERS does not describe it. + set(manual_pch FALSE) + if(MSVC AND (target STREQUAL "build_pch" OR "use_pch" IN_LIST links)) + set(manual_pch TRUE) + endif() + if(pch OR reuse_pch OR manual_pch) + # Explicitly clear inherited launchers as well. Merely doing nothing + # left SDL and child-project PCH consumers going through sccache. + set_target_properties(${target} PROPERTIES + C_COMPILER_LAUNCHER "" CXX_COMPILER_LAUNCHER "") + else() set_target_properties(${target} PROPERTIES C_COMPILER_LAUNCHER "${_switch2kit_c_launcher}" CXX_COMPILER_LAUNCHER "${_switch2kit_cxx_launcher}") endif() + if(kind MATCHES "^(STATIC_LIBRARY|OBJECT_LIBRARY)$" AND NOT manual_pch) + # Keep manual MSVC PCH ordering opaque to this optimization. + # CMake removes only unnecessary ordering edges; generated sources, + # custom commands and explicit dependencies remain prerequisites. + set_property(TARGET ${target} PROPERTY OPTIMIZE_DEPENDENCIES ON) + endif() + if(NOT MSVC AND NOT "$ENV{GITHUB_EVENT_NAME}" STREQUAL "workflow_dispatch") + # Upstream appends -ggdb even in Release. Keep line-level backtraces, + # without emitting full type debug information for every smoke object. + # Append after upstream initialization; never change sanitizer options. + target_compile_options(${target} PRIVATE + "$<$,$>:-g1>") + endif() endif() endforeach() endfunction() diff --git a/Tools/test_native_ci_execution.py b/Tools/test_native_ci_execution.py new file mode 100644 index 000000000000..dcec0db365ae --- /dev/null +++ b/Tools/test_native_ci_execution.py @@ -0,0 +1,266 @@ +#!/usr/bin/env python3 +# Copyright 2026 Dolphin Emulator Project +# SPDX-License-Identifier: GPL-2.0-or-later +"""Execute the native-CI configure/cache/PCH boundaries, without Dolphin or Swift.""" +import json +import os +from pathlib import Path +import shutil +import subprocess +import sys +import tempfile +import textwrap +import unittest + +import checkout_native + +PRESET = Path(__file__).with_name('ci-native.cmake').resolve() + + +def run(*args, cwd, env=None, ok=True): + result = subprocess.run([str(arg) for arg in args], cwd=cwd, env=env, + text=True, stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, timeout=15) + if ok and result.returncode: + raise AssertionError(f'{args}:\n{result.stdout}') + return result + + +def write(root, name, content): + path = root / name + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(textwrap.dedent(content), encoding='utf-8') + return path + + +class NativeExecutionTests(unittest.TestCase): + def test_real_probes_pch_generated_sources_and_test_execution(self): + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + wrapper = write(root, 'cache.py', '''\ + import json, os, sys + with open(os.environ['CACHE_CALLS'], 'a') as out: + out.write(json.dumps([os.getcwd(), *sys.argv[1:]]) + '\\n') + os.execv(sys.argv[1], sys.argv[1:]) + ''') + launcher = f'{sys.executable};{wrapper}' + calls = root / 'cache-calls.jsonl' + env = dict(os.environ, GITHUB_ACTIONS='true', CACHE_CALLS=str(calls), + S2K_CI_COMPILER_CACHE=launcher, + CMAKE_C_COMPILER_LAUNCHER=launcher, + CMAKE_CXX_COMPILER_LAUNCHER=launcher) + # Use the selected host compiler, not a fake successful compiler. + cc = os.environ.get('CC') or shutil.which('cc') + cxx = os.environ.get('CXX') or shutil.which('c++') + self.assertTrue(cc and cxx, 'A real C and C++ compiler is required') + write(root, 'Source/PCH/pch.h', '#pragma once\n#include \n') + write(root, 'common.cpp', 'int common_value() { return 10; }\n') + write(root, 'special.cpp', '''\ + #ifndef PER_SOURCE + #error per-source options lost + #endif + int special_value() { return 11; } + ''') + write(root, 'generated.cpp.in', 'int generated_value() { return 12; }\n') + write(root, 'plain.c', '''\ + #ifndef NDEBUG + #error Release defines lost + #endif + #if defined(__linux__) && !defined(__OPTIMIZE__) + #error Linux must remain optimized + #endif + int plain_value(void) { return 9; } + ''') + write(root, 'external.cpp', 'int external_value() { return 0; }\n') + write(root, 'main.cpp', '''\ + #include + extern "C" int plain_value(void); + int common_value(); int special_value(); int generated_value(); + int external_value(); + int main() { + std::ofstream("executed.txt", std::ios::app) << "executed\\n"; + return plain_value() + common_value() + special_value() + + generated_value() + external_value() == 42 ? 0 : 1; + } + ''') + # A separate CMake process models SDK and try_compile environments. + write(root, 'probe/CMakeLists.txt', '''\ + cmake_minimum_required(VERSION 3.25) + project(NestedProbe LANGUAGES C CXX) + include(CheckCSourceCompiles) + include(CheckCXXSourceCompiles) + check_c_source_compiles("int main(void) { return 0; }" REAL_C) + check_cxx_source_compiles("int main() { return 0; }" REAL_CXX) + check_c_source_compiles("extern int missing_symbol(void); int main(void) { return missing_symbol(); }" BAD_LINK) + if(NOT REAL_C OR NOT REAL_CXX OR BAD_LINK) + message(FATAL_ERROR "Real configure/link probes were bypassed") + endif() + ''') + write(root, 'child/CMakeLists.txt', '''\ + project(Child LANGUAGES C CXX) + include(CheckCSourceCompiles) + check_c_source_compiles("int main(void) { return 0; }" CHILD_C) + if(NOT CHILD_C) + message(FATAL_ERROR "Child probe failed") + endif() + add_library(plain STATIC ../plain.c) + ''') + write(root, 'CMakeLists.txt', '''\ + cmake_minimum_required(VERSION 3.25) + project(dolphin-emu LANGUAGES C CXX) + add_subdirectory(child) + execute_process(COMMAND "${CMAKE_COMMAND}" -S "${CMAKE_SOURCE_DIR}/probe" + -B "${CMAKE_BINARY_DIR}/probe" -G Ninja + "-DCMAKE_C_COMPILER=${CMAKE_C_COMPILER}" + "-DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER}" + RESULT_VARIABLE probe_result OUTPUT_VARIABLE probe_output ERROR_VARIABLE probe_error) + if(probe_result) + message(FATAL_ERROR "Nested probe failed: ${probe_output}${probe_error}") + endif() + add_custom_command(OUTPUT "${CMAKE_BINARY_DIR}/generated.cpp" + COMMAND "${CMAKE_COMMAND}" -E copy "${CMAKE_SOURCE_DIR}/generated.cpp.in" + "${CMAKE_BINARY_DIR}/generated.cpp" DEPENDS generated.cpp.in) + add_library(common STATIC common.cpp special.cpp "${CMAKE_BINARY_DIR}/generated.cpp") + set_source_files_properties(special.cpp PROPERTIES COMPILE_OPTIONS -DPER_SOURCE=1) + # Model a child resetting launchers, like SDL's own setup. + set_property(TARGET common PROPERTY CXX_COMPILER_LAUNCHER "$ENV{S2K_CI_COMPILER_CACHE}") + add_library(external_pch STATIC external.cpp) + target_precompile_headers(external_pch PRIVATE "${CMAKE_SOURCE_DIR}/Source/PCH/pch.h") + set_property(TARGET external_pch PROPERTY CXX_COMPILER_LAUNCHER "$ENV{S2K_CI_COMPILER_CACHE}") + add_executable(check main.cpp) + target_link_libraries(check PRIVATE plain common external_pch) + # Upstream adds this after project(). The CI setting must follow it. + target_compile_options(common PRIVATE -ggdb) + enable_testing() + add_test(NAME actual_execution COMMAND check) + ''') + for event, expected_opt in [('pull_request', '-O1'), ('workflow_dispatch', '-O3')]: + with self.subTest(event=event): + env['GITHUB_EVENT_NAME'] = event + build = root / event + calls.unlink(missing_ok=True) + configure = ['cmake', '-S', root, '-B', build, '-G', 'Ninja', + '-DCMAKE_BUILD_TYPE=Release', '-DCMAKE_EXPORT_COMPILE_COMMANDS=ON', + f'-DCMAKE_C_COMPILER={cc}', f'-DCMAKE_CXX_COMPILER={cxx}', + f'-DCMAKE_PROJECT_dolphin-emu_INCLUDE={PRESET}'] + run(*configure, cwd=root, env=env) + # Legacy standard launchers can reach root ABI detection, + # which precedes project()'s include. They must not reach any + # subsequent feature probe or nested configure. Workflows now + # set only S2K_CI_COMPILER_CACHE, avoiding even these two calls. + initial = [json.loads(line) for line in calls.read_text().splitlines()] if calls.exists() else [] + for request in initial: + self.assertTrue(request[0].startswith(str(build / 'CMakeFiles'))) + self.assertIn(Path(request[-1]).name, + ['CMakeCCompilerABI.c', 'CMakeCXXCompilerABI.cpp']) + calls.unlink(missing_ok=True) + commands = json.loads((build / 'compile_commands.json').read_text()) + c_command = next(c['command'] for c in commands if c['file'].endswith('plain.c')) + if sys.platform.startswith('linux'): + self.assertIn(expected_opt, c_command) + common_command = next(c['command'] for c in commands if c['file'].endswith('common.cpp')) + self.assertIn('cmake_pch', common_command) + if event == 'pull_request': + self.assertLess(common_command.index('-ggdb'), common_command.index('-g1')) + else: + self.assertNotIn('-g1', common_command) + special = next(c['command'] for c in commands if c['file'].endswith('special.cpp')) + self.assertNotIn('cmake_pch', special) + run('cmake', '--build', build, '--parallel', '2', cwd=root, env=env) + requests = [json.loads(line) for line in calls.read_text().splitlines()] + self.assertTrue(any('plain.c' in ' '.join(req) for req in requests)) + self.assertFalse(any('cmake_pch' in ' '.join(req) or + 'common.cpp' in ' '.join(req) or + 'external.cpp' in ' '.join(req) for req in requests)) + for _ in range(2): + run('cmake', '--build', build, '--parallel', '2', cwd=root, env=env) + run('ctest', '--test-dir', build, '--output-on-failure', '--no-tests=error', + cwd=root, env=env) + self.assertEqual((build / 'executed.txt').read_text().splitlines(), + ['executed', 'executed']) + # Reconfiguring must not resurrect a cached launcher in probes. + calls.unlink() + run(*configure, cwd=root, env=env) + self.assertFalse(calls.exists()) + + def test_manual_msvc_pch_is_not_a_cmake_pch_property(self): + # This exercises target wiring on POSIX. The real Windows job remains + # responsible for compiling/linking the upstream /Yc and /Yu commands. + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + write(root, 'part.cpp', 'int part;\n') + write(root, 'CMakeLists.txt', '''\ + cmake_minimum_required(VERSION 3.25) + project(dolphin-emu LANGUAGES CXX) + set(MSVC TRUE) + add_library(build_pch STATIC part.cpp) + add_library(use_pch INTERFACE) + add_dependencies(use_pch build_pch) + add_library(core STATIC part.cpp) + target_link_libraries(core PRIVATE use_pch) + add_library(ordinary STATIC part.cpp) + foreach(t build_pch core ordinary) + set_target_properties(${t} PROPERTIES LINKER_LANGUAGE CXX + C_COMPILER_LAUNCHER bad-inherited CXX_COMPILER_LAUNCHER bad-inherited) + file(GENERATE OUTPUT "${CMAKE_BINARY_DIR}/${t}.txt" CONTENT + "$|$|$") + endforeach() + ''') + env = dict(os.environ, GITHUB_ACTIONS='true', GITHUB_EVENT_NAME='pull_request', + S2K_CI_COMPILER_CACHE='cmake;-E;env') + run('cmake', '-S', root, '-B', root / 'build', '-G', 'Ninja', + f'-DCMAKE_PROJECT_dolphin-emu_INCLUDE={PRESET}', + '-DCMAKE_C_COMPILER_LAUNCHER=cmake;-E;env', + '-DCMAKE_CXX_COMPILER_LAUNCHER=cmake;-E;env', cwd=root, env=env) + for target in ['core', 'build_pch']: + self.assertEqual((root / 'build' / f'{target}.txt').read_text(), '||') + self.assertEqual((root / 'build/ordinary.txt').read_text(), + 'cmake;-E;env|cmake;-E;env|ON') + + +class NativePinVerificationTests(unittest.TestCase): + def test_local_recursive_gitlinks_reject_missing_and_stale_pins(self): + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + env = dict(os.environ, GIT_ALLOW_PROTOCOL='file', + GIT_AUTHOR_NAME='Fixture', GIT_AUTHOR_EMAIL='fixture@example.invalid', + GIT_COMMITTER_NAME='Fixture', GIT_COMMITTER_EMAIL='fixture@example.invalid') + + def git(where, *args): + return run('git', '-C', where, *args, cwd=root, env=env).stdout.strip() + + leaf, library, parent = [root / name for name in ('leaf', 'library', 'parent')] + for repo in (leaf, library, parent): + repo.mkdir() + git(repo, 'init', '-q') + write(repo, 'version', 'one\n') + git(repo, 'add', '.') + git(repo, 'commit', '-qm', 'one') + old_leaf = git(leaf, 'rev-parse', 'HEAD') + write(leaf, 'version', 'two\n') + git(leaf, 'commit', '-qam', 'two') + new_leaf = git(leaf, 'rev-parse', 'HEAD') + git(library, 'submodule', 'add', '-q', str(leaf), 'nested') + git(library / 'nested', 'checkout', '-q', old_leaf) + git(library, 'commit', '-qam', 'pin nested') + git(parent, 'submodule', 'add', '-q', str(library), 'Externals/UnknownDependency') + git(parent, 'commit', '-qam', 'pin native') + native = parent / 'Externals/UnknownDependency' + with self.assertRaisesRegex(ValueError, 'recursive gitlink'): + checkout_native.verify_checkout(parent, 'linux') + git(parent, 'submodule', 'update', '--init', '--recursive') + checkout_native.verify_checkout(parent, 'linux') + git(native / 'nested', 'checkout', '-q', new_leaf) + with self.assertRaisesRegex(ValueError, 'recursive gitlink'): + checkout_native.verify_checkout(parent, 'linux') + git(native / 'nested', 'checkout', '-q', old_leaf) + git(native, 'checkout', '-q', 'HEAD~1') + with self.assertRaisesRegex(ValueError, 'expected .* got'): + checkout_native.verify_checkout(parent, 'linux') + git(parent, 'submodule', 'deinit', '-f', '--all') + with self.assertRaisesRegex(ValueError, 'Uninitialized native dependency'): + checkout_native.verify_checkout(parent, 'linux') + + +if __name__ == '__main__': + unittest.main() From 7037ba16b8f7ba2ff7a2cef212612f953e6325fb Mon Sep 17 00:00:00 2001 From: Johnny D Date: Mon, 21 Sep 2026 12:57:21 -0400 Subject: [PATCH 14/21] test: verify the optimized Linux CI preset and isolate fixture caches Retain the PCH/Release assertions with the automatic Linux -O1 preset. Keep standalone scope fixtures independent of the native job's private compiler-cache environment. All 16 change-selection/setup tests pass. AI-assisted. --- Tools/test_switch2kit_ci.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/Tools/test_switch2kit_ci.py b/Tools/test_switch2kit_ci.py index 150ee58ac938..65e22546ece2 100644 --- a/Tools/test_switch2kit_ci.py +++ b/Tools/test_switch2kit_ci.py @@ -95,7 +95,8 @@ class NativeBuildSetupTests(unittest.TestCase): def configure(self, root, event='pull_request', **extra_env): env = dict(os.environ, GITHUB_ACTIONS='true', GITHUB_EVENT_NAME=event, **extra_env) # Fixtures exercise CMake, not the surrounding native job's cache server. - for name in ('CMAKE_C_COMPILER_LAUNCHER', 'CMAKE_CXX_COMPILER_LAUNCHER'): + for name in ('CMAKE_C_COMPILER_LAUNCHER', 'CMAKE_CXX_COMPILER_LAUNCHER', + 'S2K_CI_COMPILER_CACHE'): env.pop(name, None) result = subprocess.run(['cmake', '-S', str(root), '-B', str(root / 'build'), '-G', 'Ninja', '-DCMAKE_BUILD_TYPE=Release', @@ -177,7 +178,7 @@ def test_pch_keeps_per_target_flags_and_linux_optimization(self): self.assertIn('-std=c++20', special) cpp = next(entry['command'] for entry in commands if entry['file'].endswith('value.cpp')) self.assertIn('cmake_pch.hxx', cpp) - self.assertIn('-O0' if sys.platform == 'darwin' else '-O3', cpp) + self.assertIn('-O0' if sys.platform == 'darwin' else '-O1', cpp) self.assertIn('-DNDEBUG', cpp) def test_ci_build_acceleration_changes_select_all_native_platforms(self): From 385b5e44739dc08fa2a06408164e03316c3afa4e Mon Sep 17 00:00:00 2001 From: Johnny D Date: Mon, 21 Sep 2026 13:31:44 -0400 Subject: [PATCH 15/21] ci: accelerate real native probes and shader compilation Use the existing runner's LLD for automatic Windows links and real Release configure probes, while retaining MSVC compilation, the upstream manual PCH, all feature/link checks, and explicit artifact defaults. Compile glslang's own pinned parser PCH rather than repeatedly parsing it for every source. Restore normal upstream dependency ordering instead of the unproven eager static-library scheduling optimization. Resolve and validate SDKROOT together with direct Xcode compiler paths so standalone sanitizer builds retain the complete macOS SDK. Add a real C++23 atomic compile/link/run toolchain probe before native configuration. Report active Ninja workers and elapsed build time on the same runners. Existing local setup and executable CMake/Git regressions pass with GCC, Clang, and sanitizer-enabled fixtures. Full native acceptance remains a requirement of this exact PR head; no budgets or coverage were reduced. AI-assisted; human review required. --- .github/workflows/switch2kit-linux.yml | 2 ++ .github/workflows/switch2kit-macos.yml | 15 ++++++++--- .github/workflows/switch2kit-windows.yml | 1 + Tools/ci-native.cmake | 33 +++++++++++++++++++----- Tools/test_native_ci_execution.py | 4 ++- 5 files changed, 45 insertions(+), 10 deletions(-) diff --git a/.github/workflows/switch2kit-linux.yml b/.github/workflows/switch2kit-linux.yml index 029015020ea9..d92de94aa5c0 100644 --- a/.github/workflows/switch2kit-linux.yml +++ b/.github/workflows/switch2kit-linux.yml @@ -18,6 +18,7 @@ jobs: # One cache archive per job, not network requests for every object. SCCACHE_DIR: /github/home/.cache/sccache SCCACHE_CACHE_SIZE: 2G + NINJA_STATUS: '[%f/%t %r running, %e sec] ' # Configure probes and standalone sanitizer/SDK tests must compile directly. S2K_CI_COMPILER_CACHE: sccache steps: @@ -117,6 +118,7 @@ jobs: # One cache archive per job, not network requests for every object. SCCACHE_DIR: /github/home/.cache/sccache SCCACHE_CACHE_SIZE: 2G + NINJA_STATUS: '[%f/%t %r running, %e sec] ' # Configure probes and standalone sanitizer/SDK tests must compile directly. S2K_CI_COMPILER_CACHE: sccache steps: diff --git a/.github/workflows/switch2kit-macos.yml b/.github/workflows/switch2kit-macos.yml index 28093dcb7ecc..f5996623c0bc 100644 --- a/.github/workflows/switch2kit-macos.yml +++ b/.github/workflows/switch2kit-macos.yml @@ -30,6 +30,7 @@ jobs: # One cache archive per job, not network requests for every object. SCCACHE_DIR: ${{ github.workspace }}/.ci-cache/sccache SCCACHE_CACHE_SIZE: 2G + NINJA_STATUS: '[%f/%t %r running, %e sec] ' # Configure probes and standalone sanitizer/SDK tests must compile directly. S2K_CI_COMPILER_CACHE: sccache steps: @@ -71,8 +72,16 @@ jobs: cc="$(xcrun --find clang)" cxx="$(xcrun --find clang++)" test -x "$cc" && test -x "$cxx" - printf 'CC=%s\nCXX=%s\nOBJC=%s\nOBJCXX=%s\n' \ - "$cc" "$cxx" "$cc" "$cxx" >> "$GITHUB_ENV" + # Direct Xcode compiler paths do not inherit xcrun's SDK selection. + # Standalone sanitizer tests need the same headers and libraries too. + sdk="$(xcrun --sdk macosx --show-sdk-path)" + test -d "$sdk" + printf 'CC=%s\nCXX=%s\nOBJC=%s\nOBJCXX=%s\nSDKROOT=%s\n' \ + "$cc" "$cxx" "$cc" "$cxx" "$sdk" >> "$GITHUB_ENV" + printf '%s\n' '#include ' \ + 'int main() { std::atomic value{42}; return value.load() == 42 ? 0 : 1; }' | \ + SDKROOT="$sdk" "$cxx" -std=c++23 -x c++ - -o "$RUNNER_TEMP/s2k-toolchain-probe" + "$RUNNER_TEMP/s2k-toolchain-probe" for component in Qt6 Qt6Svg; do test -f "$qt_prefix/lib/cmake/$component/${component}Config.cmake" || { echo "::error::Missing linked Qt package: $component in $qt_prefix" >&2 @@ -95,7 +104,7 @@ jobs: cmake -S . -B build-switch2kit -G Ninja \ -DCMAKE_BUILD_TYPE=Release -DCMAKE_OSX_DEPLOYMENT_TARGET=15.0 \ "-DCMAKE_PROJECT_dolphin-emu_INCLUDE=$PWD/Tools/ci-native.cmake" \ - -DCMAKE_OSX_ARCHITECTURES="$ARCH" \ + -DCMAKE_OSX_ARCHITECTURES="$ARCH" -DCMAKE_OSX_SYSROOT="$SDKROOT" \ -DENABLE_SWITCH2KIT=ON -DENABLE_QT=ON -DENABLE_SDL=ON \ -DUSE_SYSTEM_SDL3=OFF \ -DCMAKE_PREFIX_PATH="$S2K_QT_PREFIX" \ diff --git a/.github/workflows/switch2kit-windows.yml b/.github/workflows/switch2kit-windows.yml index 87d442ad9e27..3c03e7081cdc 100644 --- a/.github/workflows/switch2kit-windows.yml +++ b/.github/workflows/switch2kit-windows.yml @@ -17,6 +17,7 @@ jobs: # One cache archive per job, not network requests for every object. SCCACHE_DIR: ${{ github.workspace }}/.ci-cache/sccache SCCACHE_CACHE_SIZE: 2G + NINJA_STATUS: '[%f/%t %r running, %e sec] ' # Configure probes and standalone sanitizer/SDK tests must compile directly. S2K_CI_COMPILER_CACHE: sccache steps: diff --git a/Tools/ci-native.cmake b/Tools/ci-native.cmake index 5feca50196fa..9dd41e79935d 100644 --- a/Tools/ci-native.cmake +++ b/Tools/ci-native.cmake @@ -30,6 +30,24 @@ endif() message(STATUS "Native CI C Release flags: ${CMAKE_C_FLAGS_RELEASE}") message(STATUS "Native CI C++ Release flags: ${CMAKE_CXX_FLAGS_RELEASE}") +# Keep all configure probes as real compilations AND links. On Windows, use +# the runner's native LLVM linker for automatic smoke builds, including probes; +# MSVC still compiles every C/C++ source and owns Dolphin's shared PCH. +if(CMAKE_HOST_WIN32 AND MSVC AND NOT "$ENV{GITHUB_EVENT_NAME}" STREQUAL "workflow_dispatch") + if(CMAKE_VERSION VERSION_LESS 3.29) + message(FATAL_ERROR "Native Windows CI requires CMake 3.29 or newer for LLD") + endif() + find_program(CMAKE_LINKER_LLD NAMES lld-link HINTS "$ENV{ProgramFiles}/LLVM/bin" REQUIRED) + set(CMAKE_C_USING_LINKER_LLD "${CMAKE_LINKER_LLD}") + set(CMAKE_CXX_USING_LINKER_LLD "${CMAKE_LINKER_LLD}") + list(APPEND CMAKE_TRY_COMPILE_PLATFORM_VARIABLES CMAKE_LINKER_LLD) + set(CMAKE_LINKER_TYPE LLD) + # Test the same runtime configuration as the real application, not a separate + # Debug link with an incremental PDB/manifest cycle for every tiny probe. + set(CMAKE_TRY_COMPILE_CONFIGURATION Release) + message(STATUS "Native CI Windows linker: ${CMAKE_LINKER_LLD}") +endif() + # Use a CI-specific variable, not CMAKE_*_COMPILER_LAUNCHER in the job's # environment: those standard variables also reach every nested SDK/fixture # configuration and try_compile project. Clearing a normal variable alone does @@ -56,6 +74,13 @@ endforeach() # Do not use unity builds (which change translation-unit boundaries), touch the # production sources, replace libraries, or exclude a source from compilation. function(switch2kit_ci_precompile_headers) + # The pinned shader compiler supplies its own PCH for these exact sources. + # Its parser headers dominated cold compilation; compile that same upstream + # header once, without unity builds or changing the shader compiler's sources. + if(TARGET glslang) + get_target_property(glslang_source glslang SOURCE_DIR) + target_precompile_headers(glslang PRIVATE "${glslang_source}/MachineIndependent/pch.h") + endif() if(MSVC) return() endif() @@ -121,12 +146,8 @@ function(switch2kit_ci_cache_targets directory) C_COMPILER_LAUNCHER "${_switch2kit_c_launcher}" CXX_COMPILER_LAUNCHER "${_switch2kit_cxx_launcher}") endif() - if(kind MATCHES "^(STATIC_LIBRARY|OBJECT_LIBRARY)$" AND NOT manual_pch) - # Keep manual MSVC PCH ordering opaque to this optimization. - # CMake removes only unnecessary ordering edges; generated sources, - # custom commands and explicit dependencies remain prerequisites. - set_property(TARGET ${target} PROPERTY OPTIMIZE_DEPENDENCIES ON) - endif() + # Preserve upstream dependency/PCH ordering rather than retaining an + # unproven eager-scheduling optimization on the standard runners. if(NOT MSVC AND NOT "$ENV{GITHUB_EVENT_NAME}" STREQUAL "workflow_dispatch") # Upstream appends -ggdb even in Release. Keep line-level backtraces, # without emitting full type debug information for every smoke object. diff --git a/Tools/test_native_ci_execution.py b/Tools/test_native_ci_execution.py index dcec0db365ae..5682995fc9df 100644 --- a/Tools/test_native_ci_execution.py +++ b/Tools/test_native_ci_execution.py @@ -72,6 +72,7 @@ def test_real_probes_pch_generated_sources_and_test_execution(self): int plain_value(void) { return 9; } ''') write(root, 'external.cpp', 'int external_value() { return 0; }\n') + write(root, 'MachineIndependent/pch.h', '#pragma once\n#include \n') write(root, 'main.cpp', '''\ #include extern "C" int plain_value(void); @@ -124,6 +125,7 @@ def test_real_probes_pch_generated_sources_and_test_execution(self): set_source_files_properties(special.cpp PROPERTIES COMPILE_OPTIONS -DPER_SOURCE=1) # Model a child resetting launchers, like SDL's own setup. set_property(TARGET common PROPERTY CXX_COMPILER_LAUNCHER "$ENV{S2K_CI_COMPILER_CACHE}") + add_library(glslang STATIC external.cpp) add_library(external_pch STATIC external.cpp) target_precompile_headers(external_pch PRIVATE "${CMAKE_SOURCE_DIR}/Source/PCH/pch.h") set_property(TARGET external_pch PROPERTY CXX_COMPILER_LAUNCHER "$ENV{S2K_CI_COMPILER_CACHE}") @@ -215,7 +217,7 @@ def test_manual_msvc_pch_is_not_a_cmake_pch_property(self): for target in ['core', 'build_pch']: self.assertEqual((root / 'build' / f'{target}.txt').read_text(), '||') self.assertEqual((root / 'build/ordinary.txt').read_text(), - 'cmake;-E;env|cmake;-E;env|ON') + 'cmake;-E;env|cmake;-E;env|') class NativePinVerificationTests(unittest.TestCase): From e8056e38963fa5f23c968744ba7bd9aab13cb21f Mon Sep 17 00:00:00 2001 From: Johnny D Date: Mon, 21 Sep 2026 14:18:32 -0400 Subject: [PATCH 16/21] ci: finalize native flags and accelerate upstream test compilation Apply automatic MSVC Release options after target initialization, grouping the final /Od /Ob0 pair so CMake option deduplication cannot discard it. Keep explicit Release artifacts and all upstream runtime/debug flags intact. Use CMake's dependency optimizer for automatic static/object targets, retaining explicit dependencies, generated sources, custom-command side effects and final link inputs. Extend per-target PCH to upstream test object libraries without changing translation units or selected tests. Extend the existing executable CMake regressions to cover final compiler option order, explicit artifact defaults, native test-library PCH builds, and the optimized Ninja compile-order edge. GCC and Clang fixture runs both pass; full native/platform acceptance remains required in CI. No job, runner, time budget, SDK pin, production controller behavior, sanitizer check, upstream unit test or archive-launch requirement changes. AI-assisted; human review required. --- Tools/ci-native.cmake | 61 +++++++++++++++++++++++++------ Tools/test_native_ci_execution.py | 46 ++++++++++++++++++++++- 2 files changed, 93 insertions(+), 14 deletions(-) diff --git a/Tools/ci-native.cmake b/Tools/ci-native.cmake index 9dd41e79935d..db11e8638d2a 100644 --- a/Tools/ci-native.cmake +++ b/Tools/ci-native.cmake @@ -10,13 +10,8 @@ if(NOT "$ENV{GITHUB_EVENT_NAME}" STREQUAL "workflow_dispatch") if(MSVC) set(CMAKE_C_FLAGS_RELEASE "/Od /Ob0 /DNDEBUG /Z7") set(CMAKE_CXX_FLAGS_RELEASE "/Od /Ob0 /DNDEBUG /Z7") - # Child project() calls can load the upstream override again. Directory - # options follow configuration flags, so the actual child compiler commands - # must also end in /Od /Ob0. Keep CRT selection, defines and warnings intact. - add_compile_options( - "$<$,$>:/Od>" - "$<$,$>:/Ob0>" - ) + # Child project() calls and target options can override configuration flags. + # Apply the final Release options to completed targets below, not here. elseif(APPLE) set(CMAKE_C_FLAGS_RELEASE "-O0 -DNDEBUG") set(CMAKE_CXX_FLAGS_RELEASE "-O0 -DNDEBUG") @@ -73,6 +68,18 @@ endforeach() # Per-target PCHs preserve each target's own defines, include paths and flags. # Do not use unity builds (which change translation-unit boundaries), touch the # production sources, replace libraries, or exclude a source from compilation. +# Upstream splits its unit suite into object libraries. Accelerate their C++ +# parsing too, rather than only the two sources in the final tests executable. +function(switch2kit_ci_test_targets directory output) + get_property(targets DIRECTORY "${directory}" PROPERTY BUILDSYSTEM_TARGETS) + get_property(children DIRECTORY "${directory}" PROPERTY SUBDIRECTORIES) + foreach(child IN LISTS children) + switch2kit_ci_test_targets("${child}" child_targets) + list(APPEND targets ${child_targets}) + endforeach() + set(${output} "${targets}" PARENT_SCOPE) +endfunction() + function(switch2kit_ci_precompile_headers) # The pinned shader compiler supplies its own PCH for these exact sources. # Its parser headers dominated cold compilation; compile that same upstream @@ -84,10 +91,25 @@ function(switch2kit_ci_precompile_headers) if(MSVC) return() endif() - foreach(target common audiocommon inputcommon videocommon discio core uicommon - videoogl videonull videosoftware videometal videovulkan - dolphin-tool dolphin-emu) + set(pch_targets common audiocommon inputcommon videocommon discio core uicommon + videoogl videonull videosoftware videometal videovulkan + dolphin-tool dolphin-emu) + if(TARGET tests) + get_target_property(test_dir tests SOURCE_DIR) + switch2kit_ci_test_targets("${test_dir}" test_targets) + list(APPEND pch_targets ${test_targets}) + endif() + foreach(target IN LISTS pch_targets) if(TARGET ${target}) + get_target_property(kind ${target} TYPE) + if(NOT kind MATCHES "^(EXECUTABLE|STATIC_LIBRARY|SHARED_LIBRARY|MODULE_LIBRARY|OBJECT_LIBRARY)$") + continue() + endif() + get_target_property(existing_pch ${target} PRECOMPILE_HEADERS) + get_target_property(reuse_pch ${target} PRECOMPILE_HEADERS_REUSE_FROM) + if(existing_pch OR reuse_pch) + continue() + endif() # Upstream gives some files different flags (e.g. ARM crypto ISA flags). # They still compile normally; a target-wide PCH cannot represent those # per-source options. Resolve properties in the target's source directory. @@ -146,8 +168,23 @@ function(switch2kit_ci_cache_targets directory) C_COMPILER_LAUNCHER "${_switch2kit_c_launcher}" CXX_COMPILER_LAUNCHER "${_switch2kit_cxx_launcher}") endif() - # Preserve upstream dependency/PCH ordering rather than retaining an - # unproven eager-scheduling optimization on the standard runners. + if(NOT "$ENV{GITHUB_EVENT_NAME}" STREQUAL "workflow_dispatch") + # Let CMake remove only unnecessary compile-order edges between static + # and object libraries. It retains explicit add_dependencies(), generated + # source prerequisites, custom-command side effects and final link inputs. + # Never clear MANUALLY_ADDED_DEPENDENCIES or edit Ninja's generated graph. + if(kind MATCHES "^(STATIC_LIBRARY|OBJECT_LIBRARY)$") + set_property(TARGET ${target} PROPERTY OPTIMIZE_DEPENDENCIES ON) + endif() + if(MSVC) + # SHELL keeps this final pair together even when an inherited /Od or + # /Ob0 appeared earlier: CMake otherwise de-duplicates the last copy. + # Apply to the PCH producer and consumers alike; keep Debug, CRT, + # warning, architecture, debug-information and sanitizer flags intact. + target_compile_options(${target} PRIVATE + "$<$,$>:SHELL:/Od /Ob0>") + endif() + endif() if(NOT MSVC AND NOT "$ENV{GITHUB_EVENT_NAME}" STREQUAL "workflow_dispatch") # Upstream appends -ggdb even in Release. Keep line-level backtraces, # without emitting full type debug information for every smoke object. diff --git a/Tools/test_native_ci_execution.py b/Tools/test_native_ci_execution.py index 5682995fc9df..0ddcb7ee226c 100644 --- a/Tools/test_native_ci_execution.py +++ b/Tools/test_native_ci_execution.py @@ -55,6 +55,15 @@ def test_real_probes_pch_generated_sources_and_test_execution(self): self.assertTrue(cc and cxx, 'A real C and C++ compiler is required') write(root, 'Source/PCH/pch.h', '#pragma once\n#include \n') write(root, 'common.cpp', 'int common_value() { return 10; }\n') + write(root, 'Source/UnitTests/CMakeLists.txt', '''\ + add_library(upstream_cases OBJECT case.cpp) + add_executable(tests EXCLUDE_FROM_ALL main.cpp) + target_link_libraries(tests PRIVATE upstream_cases) + add_custom_target(unittests DEPENDS tests) + ''') + write(root, 'Source/UnitTests/case.cpp', 'int tested_value() { return 42; }\n') + write(root, 'Source/UnitTests/main.cpp', + 'int tested_value(); int main() { return tested_value() == 42 ? 0 : 1; }\n') write(root, 'special.cpp', '''\ #ifndef PER_SOURCE #error per-source options lost @@ -110,6 +119,7 @@ def test_real_probes_pch_generated_sources_and_test_execution(self): cmake_minimum_required(VERSION 3.25) project(dolphin-emu LANGUAGES C CXX) add_subdirectory(child) + add_subdirectory(Source/UnitTests) execute_process(COMMAND "${CMAKE_COMMAND}" -S "${CMAKE_SOURCE_DIR}/probe" -B "${CMAKE_BINARY_DIR}/probe" -G Ninja "-DCMAKE_C_COMPILER=${CMAKE_C_COMPILER}" @@ -122,6 +132,7 @@ def test_real_probes_pch_generated_sources_and_test_execution(self): COMMAND "${CMAKE_COMMAND}" -E copy "${CMAKE_SOURCE_DIR}/generated.cpp.in" "${CMAKE_BINARY_DIR}/generated.cpp" DEPENDS generated.cpp.in) add_library(common STATIC common.cpp special.cpp "${CMAKE_BINARY_DIR}/generated.cpp") + target_link_libraries(common PRIVATE plain) set_source_files_properties(special.cpp PROPERTIES COMPILE_OPTIONS -DPER_SOURCE=1) # Model a child resetting launchers, like SDL's own setup. set_property(TARGET common PROPERTY CXX_COMPILER_LAUNCHER "$ENV{S2K_CI_COMPILER_CACHE}") @@ -162,6 +173,14 @@ def test_real_probes_pch_generated_sources_and_test_execution(self): self.assertIn(expected_opt, c_command) common_command = next(c['command'] for c in commands if c['file'].endswith('common.cpp')) self.assertIn('cmake_pch', common_command) + case_command = next(c['command'] for c in commands if c['file'].endswith('case.cpp')) + self.assertIn('cmake_pch', case_command) + order = run('ninja', '-C', build, '-t', 'query', + 'cmake_object_order_depends_target_common', cwd=root, env=env).stdout + if event == 'pull_request': + self.assertNotIn('cmake_object_order_depends_target_plain', order) + else: + self.assertIn('cmake_object_order_depends_target_plain', order) if event == 'pull_request': self.assertLess(common_command.index('-ggdb'), common_command.index('-g1')) else: @@ -174,6 +193,9 @@ def test_real_probes_pch_generated_sources_and_test_execution(self): self.assertFalse(any('cmake_pch' in ' '.join(req) or 'common.cpp' in ' '.join(req) or 'external.cpp' in ' '.join(req) for req in requests)) + run('cmake', '--build', build, '--target', 'unittests', '--parallel', '2', + cwd=root, env=env) + run(build / 'Source/UnitTests/tests', cwd=root, env=env) for _ in range(2): run('cmake', '--build', build, '--parallel', '2', cwd=root, env=env) run('ctest', '--test-dir', build, '--output-on-failure', '--no-tests=error', @@ -195,6 +217,8 @@ def test_manual_msvc_pch_is_not_a_cmake_pch_property(self): cmake_minimum_required(VERSION 3.25) project(dolphin-emu LANGUAGES CXX) set(MSVC TRUE) + # Earlier /Od values used to be retained by option deduplication. + add_compile_options(/Od /Ob0) add_library(build_pch STATIC part.cpp) add_library(use_pch INTERFACE) add_dependencies(use_pch build_pch) @@ -202,6 +226,7 @@ def test_manual_msvc_pch_is_not_a_cmake_pch_property(self): target_link_libraries(core PRIVATE use_pch) add_library(ordinary STATIC part.cpp) foreach(t build_pch core ordinary) + target_compile_options(${t} PRIVATE "$<$:/O2>" "$<$:/Ob2>") set_target_properties(${t} PROPERTIES LINKER_LANGUAGE CXX C_COMPILER_LAUNCHER bad-inherited CXX_COMPILER_LAUNCHER bad-inherited) file(GENERATE OUTPUT "${CMAKE_BINARY_DIR}/${t}.txt" CONTENT @@ -212,12 +237,29 @@ def test_manual_msvc_pch_is_not_a_cmake_pch_property(self): S2K_CI_COMPILER_CACHE='cmake;-E;env') run('cmake', '-S', root, '-B', root / 'build', '-G', 'Ninja', f'-DCMAKE_PROJECT_dolphin-emu_INCLUDE={PRESET}', + '-DCMAKE_BUILD_TYPE=Release', '-DCMAKE_EXPORT_COMPILE_COMMANDS=ON', '-DCMAKE_C_COMPILER_LAUNCHER=cmake;-E;env', '-DCMAKE_CXX_COMPILER_LAUNCHER=cmake;-E;env', cwd=root, env=env) for target in ['core', 'build_pch']: - self.assertEqual((root / 'build' / f'{target}.txt').read_text(), '||') + self.assertEqual((root / 'build' / f'{target}.txt').read_text(), '||ON') self.assertEqual((root / 'build/ordinary.txt').read_text(), - 'cmake;-E;env|cmake;-E;env|') + 'cmake;-E;env|cmake;-E;env|ON') + commands = json.loads((root / 'build/compile_commands.json').read_text()) + for command in commands: + options = command['command'].split() + self.assertEqual([o for o in options if o in ('/Od', '/O1', '/O2')][-1], '/Od') + self.assertEqual([o for o in options if o in ('/Ob0', '/Ob1', '/Ob2')][-1], '/Ob0') + # Explicit release artifacts must retain the later upstream optimizer. + env['GITHUB_EVENT_NAME'] = 'workflow_dispatch' + run('cmake', '-S', root, '-B', root / 'manual', '-G', 'Ninja', + f'-DCMAKE_PROJECT_dolphin-emu_INCLUDE={PRESET}', + '-DCMAKE_BUILD_TYPE=Release', '-DCMAKE_EXPORT_COMPILE_COMMANDS=ON', cwd=root, env=env) + commands = json.loads((root / 'manual/compile_commands.json').read_text()) + for command in commands: + options = command['command'].split() + self.assertEqual([o for o in options if o in ('/Od', '/O1', '/O2')][-1], '/O2') + self.assertEqual([o for o in options if o in ('/Ob0', '/Ob1', '/Ob2')][-1], '/Ob2') + class NativePinVerificationTests(unittest.TestCase): From 6ed92ae034291d75d5248b317ef9b69acb68af3b Mon Sep 17 00:00:00 2001 From: Johnny D Date: Mon, 21 Sep 2026 14:24:09 -0400 Subject: [PATCH 17/21] ci: share equivalent upstream test PCHs and preserve setup contract Avoid generating one PCH for every one-source upstream test target. Reuse an object-test producer only when target settings, directory compiler flags and dependency usage requirements match; isolate differing definitions. Never reuse the final test executable, which would introduce a cycle. Keep the early MSVC child-directory options expected by the setup hook, while the final grouped target options remain authoritative. This fixes the setup-contract failure reported by automatic run 35637581972 without removing or weakening that regression. Existing GCC and Clang executable regressions pass, including shared-PCH reuse and isolation of differing compile definitions. Native acceptance and the complete upstream suite remain mandatory on the unchanged jobs. AI-assisted; human review required. --- Tools/ci-native.cmake | 48 ++++++++++++++++++++++++++++--- Tools/test_native_ci_execution.py | 22 +++++++++++--- 2 files changed, 62 insertions(+), 8 deletions(-) diff --git a/Tools/ci-native.cmake b/Tools/ci-native.cmake index db11e8638d2a..583aa5cfb16c 100644 --- a/Tools/ci-native.cmake +++ b/Tools/ci-native.cmake @@ -10,8 +10,12 @@ if(NOT "$ENV{GITHUB_EVENT_NAME}" STREQUAL "workflow_dispatch") if(MSVC) set(CMAKE_C_FLAGS_RELEASE "/Od /Ob0 /DNDEBUG /Z7") set(CMAKE_CXX_FLAGS_RELEASE "/Od /Ob0 /DNDEBUG /Z7") - # Child project() calls and target options can override configuration flags. - # Apply the final Release options to completed targets below, not here. + # Preserve the setup hook's child-directory options. The deferred target + # pass below is authoritative when later target options would override them. + add_compile_options( + "$<$,$>:/Od>" + "$<$,$>:/Ob0>" + ) elseif(APPLE) set(CMAKE_C_FLAGS_RELEASE "-O0 -DNDEBUG") set(CMAKE_CXX_FLAGS_RELEASE "-O0 -DNDEBUG") @@ -130,8 +134,44 @@ function(switch2kit_ci_precompile_headers) endif() endforeach() endforeach() - target_precompile_headers(${target} PRIVATE - "$<$:${PROJECT_SOURCE_DIR}/Source/PCH/pch.h>") + # Most upstream cases are one-source object libraries. Share a PCH only + # across cases with the same target properties, directory flags and exact + # dependency usage requirements. Reusing the tests executable would create + # a dependency cycle: it links these cases. Use the first matching case. + set(pch_key "") + if(kind STREQUAL "OBJECT_LIBRARY" AND target IN_LIST test_targets) + set(signature "") + foreach(property COMPILE_OPTIONS COMPILE_DEFINITIONS INCLUDE_DIRECTORIES + LINK_LIBRARIES CXX_STANDARD CXX_STANDARD_REQUIRED CXX_EXTENSIONS + POSITION_INDEPENDENT_CODE CXX_VISIBILITY_PRESET + VISIBILITY_INLINES_HIDDEN COMPILE_FLAGS AUTOMOC AUTOUIC) + get_target_property(value ${target} ${property}) + string(APPEND signature "|${property}=${value}") + endforeach() + foreach(variable CMAKE_CXX_FLAGS CMAKE_CXX_FLAGS_DEBUG CMAKE_CXX_FLAGS_RELEASE + CMAKE_CXX_FLAGS_RELWITHDEBINFO CMAKE_CXX_FLAGS_MINSIZEREL + CMAKE_INCLUDE_CURRENT_DIR CMAKE_CXX_SCAN_FOR_MODULES) + get_directory_property(value DIRECTORY "${source_dir}" DEFINITION ${variable}) + string(APPEND signature "|${variable}=${value}") + if(variable STREQUAL "CMAKE_INCLUDE_CURRENT_DIR" AND value) + string(APPEND signature "|source=${source_dir}") + endif() + endforeach() + # Target-context expressions may evaluate differently despite matching + # strings. Such targets retain their own PCH instead of assuming equality. + if(NOT signature MATCHES "TARGET_PROPERTY|TARGET_NAME|TARGET_OBJECTS") + string(SHA256 pch_key "${signature}") + endif() + endif() + if(pch_key AND DEFINED pch_provider_${pch_key}) + target_precompile_headers(${target} REUSE_FROM ${pch_provider_${pch_key}}) + else() + target_precompile_headers(${target} PRIVATE + "$<$:${PROJECT_SOURCE_DIR}/Source/PCH/pch.h>") + if(pch_key) + set(pch_provider_${pch_key} ${target}) + endif() + endif() endif() endforeach() if(TARGET dolphin-emu AND ENABLE_QT) diff --git a/Tools/test_native_ci_execution.py b/Tools/test_native_ci_execution.py index 0ddcb7ee226c..532df4c1179b 100644 --- a/Tools/test_native_ci_execution.py +++ b/Tools/test_native_ci_execution.py @@ -53,17 +53,26 @@ def test_real_probes_pch_generated_sources_and_test_execution(self): cc = os.environ.get('CC') or shutil.which('cc') cxx = os.environ.get('CXX') or shutil.which('c++') self.assertTrue(cc and cxx, 'A real C and C++ compiler is required') - write(root, 'Source/PCH/pch.h', '#pragma once\n#include \n') + write(root, 'Source/PCH/pch.h', '#pragma once\n#include \n' + '#ifdef CASE_VALUE\nconstexpr int value_from_pch = CASE_VALUE;\n#endif\n') write(root, 'common.cpp', 'int common_value() { return 10; }\n') write(root, 'Source/UnitTests/CMakeLists.txt', '''\ add_library(upstream_cases OBJECT case.cpp) + add_library(matching_cases OBJECT matching.cpp) + add_library(different_cases OBJECT different.cpp) + target_compile_definitions(upstream_cases PRIVATE CASE_VALUE=10) + target_compile_definitions(matching_cases PRIVATE CASE_VALUE=10) + target_compile_definitions(different_cases PRIVATE CASE_VALUE=22) add_executable(tests EXCLUDE_FROM_ALL main.cpp) - target_link_libraries(tests PRIVATE upstream_cases) + target_link_libraries(tests PRIVATE upstream_cases matching_cases different_cases) add_custom_target(unittests DEPENDS tests) ''') - write(root, 'Source/UnitTests/case.cpp', 'int tested_value() { return 42; }\n') + write(root, 'Source/UnitTests/case.cpp', 'int tested_value() { return value_from_pch; }\n') + write(root, 'Source/UnitTests/matching.cpp', 'int matching_value() { return value_from_pch; }\n') + write(root, 'Source/UnitTests/different.cpp', 'int different_value() { return value_from_pch; }\n') write(root, 'Source/UnitTests/main.cpp', - 'int tested_value(); int main() { return tested_value() == 42 ? 0 : 1; }\n') + 'int tested_value(); int matching_value(); int different_value(); ' + 'int main() { return tested_value() + matching_value() + different_value() == 42 ? 0 : 1; }\n') write(root, 'special.cpp', '''\ #ifndef PER_SOURCE #error per-source options lost @@ -175,6 +184,11 @@ def test_real_probes_pch_generated_sources_and_test_execution(self): self.assertIn('cmake_pch', common_command) case_command = next(c['command'] for c in commands if c['file'].endswith('case.cpp')) self.assertIn('cmake_pch', case_command) + matching = next(c['command'] for c in commands if c['file'].endswith('matching.cpp')) + different = next(c['command'] for c in commands if c['file'].endswith('different.cpp')) + self.assertIn('upstream_cases.dir/cmake_pch', matching) + self.assertIn('different_cases.dir/cmake_pch', different) + self.assertNotIn('upstream_cases.dir/cmake_pch', different) order = run('ninja', '-C', build, '-t', 'query', 'cmake_object_order_depends_target_common', cwd=root, env=env).stdout if event == 'pull_request': From 705c580b297621653b61d3e297fa1a3323e81341 Mon Sep 17 00:00:00 2001 From: Johnny D Date: Mon, 21 Sep 2026 15:15:45 -0400 Subject: [PATCH 18/21] ci: apply the automatic Apple profile to late Objective-C languages Finalize Release target options after Objective-C and Objective-C++ are enabled, so automatic native builds do not silently retain -O3 for those sources. Preserve explicit Release artifacts, Debug settings, definitions, sanitizer instrumentation, and all existing native acceptance checks. Add a real Clang Objective-C/C++ compile, sanitizer link, and CTest execution regression across automatic Release, explicit Release, and Debug in one reconfigured build tree. All four native execution regressions pass locally with GCC and Clang; native platform acceptance is still required. AI-assisted; human review required. --- Tools/ci-native.cmake | 7 +++ Tools/test_native_ci_execution.py | 72 +++++++++++++++++++++++++++++++ 2 files changed, 79 insertions(+) diff --git a/Tools/ci-native.cmake b/Tools/ci-native.cmake index 583aa5cfb16c..17619b19d2f0 100644 --- a/Tools/ci-native.cmake +++ b/Tools/ci-native.cmake @@ -226,6 +226,13 @@ function(switch2kit_ci_cache_targets directory) endif() endif() if(NOT MSVC AND NOT "$ENV{GITHUB_EVENT_NAME}" STREQUAL "workflow_dispatch") + if(APPLE) + # Objective-C languages are enabled after the project hook. Their + # Release flags otherwise retain -O3 even when C/C++ use -O0. + target_compile_options(${target} PRIVATE + "$<$,$>:-O0>" + "$<$,$>:-g1>") + endif() # Upstream appends -ggdb even in Release. Keep line-level backtraces, # without emitting full type debug information for every smoke object. # Append after upstream initialization; never change sanitizer options. diff --git a/Tools/test_native_ci_execution.py b/Tools/test_native_ci_execution.py index 532df4c1179b..e7d4e20e23e8 100644 --- a/Tools/test_native_ci_execution.py +++ b/Tools/test_native_ci_execution.py @@ -221,6 +221,78 @@ def test_real_probes_pch_generated_sources_and_test_execution(self): run(*configure, cwd=root, env=env) self.assertFalse(calls.exists()) + def test_late_objective_c_languages_preserve_profiles_and_sanitizers(self): + # Compile real Objective-C languages with host Clang; select the Apple + # policy without pretending this fixture validates macOS frameworks. + cc, cxx = shutil.which('clang'), shutil.which('clang++') + self.assertTrue(cc and cxx, 'Clang is required for Objective-C CI regressions') + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + write(root, 'part.m', 'int objc_value(void) { return 20; }\n') + write(root, 'part.mm', '''\ + #include + int objcxx_value() { std::array a{11, 11}; return a[0] + a[1]; } + ''') + write(root, 'main.cpp', '''\ + extern "C" int objc_value(void); + int objcxx_value(); + int main() { return objc_value() + objcxx_value() == 42 ? 0 : 1; } + ''') + write(root, 'CMakeLists.txt', '''\ + cmake_minimum_required(VERSION 3.25) + project(dolphin-emu LANGUAGES C CXX) + set(APPLE TRUE) + enable_language(OBJC OBJCXX) + add_compile_options(-ggdb) + add_library(parts STATIC part.m part.mm) + add_executable(check main.cpp) + target_link_libraries(check PRIVATE parts) + enable_testing() + add_test(NAME language_execution COMMAND check) + ''') + for event, config in [('pull_request', 'Release'), + ('workflow_dispatch', 'Release'), + ('pull_request', 'Debug')]: + with self.subTest(event=event, config=config): + env = dict(os.environ, GITHUB_ACTIONS='true', GITHUB_EVENT_NAME=event) + for variable in ('S2K_CI_COMPILER_CACHE', 'CMAKE_C_COMPILER_LAUNCHER', + 'CMAKE_CXX_COMPILER_LAUNCHER'): + env.pop(variable, None) + # Reconfigure the same tree to detect sticky CI-only flags + # without repeating unchanged compiler ABI detection. + build = root / 'build' + flags = '-fsanitize=address,undefined -fno-omit-frame-pointer' + configure = ['cmake', '-S', root, '-B', build, '-G', 'Ninja', + f'-DCMAKE_BUILD_TYPE={config}', '-DCMAKE_EXPORT_COMPILE_COMMANDS=ON', + f'-DCMAKE_PROJECT_dolphin-emu_INCLUDE={PRESET}', + f'-DCMAKE_C_COMPILER={cc}', f'-DCMAKE_CXX_COMPILER={cxx}', + f'-DCMAKE_OBJC_COMPILER={cc}', f'-DCMAKE_OBJCXX_COMPILER={cxx}'] + configure += [f'-DCMAKE_{language}_FLAGS={flags}' + for language in ('C', 'CXX', 'OBJC', 'OBJCXX')] + run(*configure, cwd=root, env=env) + commands = json.loads((build / 'compile_commands.json').read_text()) + self.assertEqual(len(commands), 3) + for command in commands: + options = command['command'].split() + self.assertIn('-fsanitize=address,undefined', options) + self.assertIn('-fno-omit-frame-pointer', options) + if config == 'Release': + optimization = [arg for arg in options + if arg in ['-O0', '-O1', '-O2', '-O3', '-Os', '-Oz']] + self.assertEqual(optimization[-1], + '-O0' if event == 'pull_request' else '-O3') + self.assertIn('-DNDEBUG', options) + else: + self.assertNotIn('-O0', options) + self.assertNotIn('-DNDEBUG', options) + if config == 'Release' and event == 'pull_request': + self.assertLess(options.index('-ggdb'), options.index('-g1')) + else: + self.assertNotIn('-g1', options) + run('cmake', '--build', build, '--parallel', '2', cwd=root, env=env) + run('ctest', '--test-dir', build, '--output-on-failure', '--no-tests=error', + cwd=root, env=env) + def test_manual_msvc_pch_is_not_a_cmake_pch_property(self): # This exercises target wiring on POSIX. The real Windows job remains # responsible for compiling/linking the upstream /Yc and /Yu commands. From b3b17aea78c89ae2599b9e9dcec98f6fcf8f09e1 Mon Sep 17 00:00:00 2001 From: Johnny D Date: Mon, 21 Sep 2026 15:38:55 -0400 Subject: [PATCH 19/21] ci: retain compiler objects after timeouts and overlap upstream test compilation Split compiler-object restore/save using the existing pinned cache action. Save valid objects even after failed or cancelled native validation, under unique run/attempt keys so reruns can advance an immutable cache. Never cache build trees, applications, configure answers or test results. Preserve all job results and ten-minute automatic acceptance budgets. Collect native diagnostics on cancellation as well as failure. Build the disabled Linux application and complete upstream tests executable in one Ninja graph, then execute the unchanged mandatory unittests target. This avoids a separate serial tail of test-object compilation without removing any source, test, sanitizer, dependency or launch assertion. Parsed before/after workflow validation confirms unchanged runners, budgets, environments and acceptance commands except the combined Linux build. Native acceptance remains required on this exact revision. AI-assisted; human review required. --- .github/workflows/switch2kit-linux.yml | 36 +++++++++++++++++++----- .github/workflows/switch2kit-macos.yml | 16 +++++++++-- .github/workflows/switch2kit-windows.yml | 16 +++++++++-- 3 files changed, 55 insertions(+), 13 deletions(-) diff --git a/.github/workflows/switch2kit-linux.yml b/.github/workflows/switch2kit-linux.yml index d92de94aa5c0..ac6ec858c327 100644 --- a/.github/workflows/switch2kit-linux.yml +++ b/.github/workflows/switch2kit-linux.yml @@ -40,14 +40,16 @@ jobs: 'static_assert(std::expected(42).value() == 42);' | \ /usr/bin/clang++-19 -std=c++23 -x c++ -fsyntax-only - - name: Restore compiler objects (never test results or applications) - uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 + id: compiler-objects + uses: actions/cache/restore@0057852bfaa89a56745cba8c7296529d2fc39830 with: path: ${{ env.SCCACHE_DIR }} - key: s2k-objects-v3-linux-clang19-${{ github.event_name == 'workflow_dispatch' && 'release' || 'smoke' }}-${{ github.sha }}-${{ github.job }} + key: s2k-objects-v3-linux-clang19-${{ github.event_name == 'workflow_dispatch' && 'release' || 'smoke' }}-${{ github.sha }}-${{ github.job }}-${{ github.run_id }}-${{ github.run_attempt }} restore-keys: | s2k-objects-v3-linux-clang19-${{ github.event_name == 'workflow_dispatch' && 'release' || 'smoke' }}-${{ github.sha }}- s2k-objects-v3-linux-clang19-${{ github.event_name == 'workflow_dispatch' && 'release' || 'smoke' }}- - name: Cache native compiler results (tests still run) + id: compiler-cache uses: Mozilla-Actions/sccache-action@7d986dd989559c6ecdb630a3fd2557667be217ad with: version: v0.10.0 @@ -94,7 +96,7 @@ jobs: retention-days: 14 if-no-files-found: error - name: Native diagnostics - if: failure() || github.event_name == 'workflow_dispatch' + if: failure() || cancelled() || github.event_name == 'workflow_dispatch' uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 with: name: Dolphin-Switch2Kit-linux-diagnostics @@ -102,6 +104,14 @@ jobs: linux-*.log linux-launch.json retention-days: 7 + # Keep valid compiler objects even when a later build/test is cancelled. + # Never save build trees, applications, configuration answers or test results. + - name: Save compiler objects after native validation + if: always() && steps.compiler-cache.outcome == 'success' + uses: actions/cache/save@0057852bfaa89a56745cba8c7296529d2fc39830 + with: + path: ${{ env.SCCACHE_DIR }} + key: ${{ steps.compiler-objects.outputs.cache-primary-key }} disabled: needs: linux @@ -155,14 +165,16 @@ jobs: git config --global --add safe.directory "$PWD" python3 Tools/checkout_native.py - name: Restore compiler objects (never test results or applications) - uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 + id: compiler-objects + uses: actions/cache/restore@0057852bfaa89a56745cba8c7296529d2fc39830 with: path: ${{ env.SCCACHE_DIR }} - key: s2k-objects-v3-linux-clang19-${{ github.event_name == 'workflow_dispatch' && 'release' || 'smoke' }}-${{ github.sha }}-${{ github.job }} + key: s2k-objects-v3-linux-clang19-${{ github.event_name == 'workflow_dispatch' && 'release' || 'smoke' }}-${{ github.sha }}-${{ github.job }}-${{ github.run_id }}-${{ github.run_attempt }} restore-keys: | s2k-objects-v3-linux-clang19-${{ github.event_name == 'workflow_dispatch' && 'release' || 'smoke' }}-${{ github.sha }}- s2k-objects-v3-linux-clang19-${{ github.event_name == 'workflow_dispatch' && 'release' || 'smoke' }}- - name: Cache native compiler results (tests still run) + id: compiler-cache uses: Mozilla-Actions/sccache-action@7d986dd989559c6ecdb630a3fd2557667be217ad with: version: v0.10.0 @@ -178,7 +190,9 @@ jobs: -DENABLE_SWITCH2KIT=OFF -DENABLE_SDL=ON -DENABLE_QT=ON \ -DUSE_SYSTEM_SDL3=OFF -DENABLE_TESTS=ON -DENABLE_VULKAN=OFF \ -DENABLE_AUTOUPDATE=OFF 2>&1 | tee disabled-configure.log - cmake --build build-switch2kit --target dolphin-emu --parallel "$(nproc)" 2>&1 | tee disabled-build.log + # Let Ninja schedule the application and all upstream test objects together. + cmake --build build-switch2kit --target dolphin-emu tests --parallel "$(nproc)" 2>&1 | tee disabled-build.log + # Building tests does not execute them; retain the complete CTest target. cmake --build build-switch2kit --target unittests --parallel "$(nproc)" 2>&1 | tee upstream-unittests.log ldd build-switch2kit/Binaries/dolphin-emu | tee disabled-dependencies.log ! grep -q 'not found' disabled-dependencies.log @@ -193,7 +207,7 @@ jobs: if: always() run: sccache --show-stats - name: Backend-disabled build diagnostics - if: failure() || github.event_name == 'workflow_dispatch' + if: failure() || cancelled() || github.event_name == 'workflow_dispatch' uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 with: name: Dolphin-without-Switch2Kit-linux-diagnostics @@ -201,3 +215,11 @@ jobs: disabled-*.log upstream-unittests.log retention-days: 7 + # Keep valid compiler objects even when a later build/test is cancelled. + # Never save build trees, applications, configuration answers or test results. + - name: Save compiler objects after native validation + if: always() && steps.compiler-cache.outcome == 'success' + uses: actions/cache/save@0057852bfaa89a56745cba8c7296529d2fc39830 + with: + path: ${{ env.SCCACHE_DIR }} + key: ${{ steps.compiler-objects.outputs.cache-primary-key }} diff --git a/.github/workflows/switch2kit-macos.yml b/.github/workflows/switch2kit-macos.yml index f5996623c0bc..981020c0285c 100644 --- a/.github/workflows/switch2kit-macos.yml +++ b/.github/workflows/switch2kit-macos.yml @@ -41,14 +41,16 @@ jobs: - name: Fetch and verify native gitlinks in parallel run: python3 Tools/checkout_native.py - name: Restore compiler objects (never test results or applications) - uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 + id: compiler-objects + uses: actions/cache/restore@0057852bfaa89a56745cba8c7296529d2fc39830 with: path: ${{ env.SCCACHE_DIR }} - key: s2k-objects-v3-${{ matrix.os }}-${{ matrix.arch }}-xcode26.3-${{ github.event_name == 'workflow_dispatch' && 'release' || 'smoke' }}-${{ github.sha }}-${{ github.job }} + key: s2k-objects-v3-${{ matrix.os }}-${{ matrix.arch }}-xcode26.3-${{ github.event_name == 'workflow_dispatch' && 'release' || 'smoke' }}-${{ github.sha }}-${{ github.job }}-${{ github.run_id }}-${{ github.run_attempt }} restore-keys: | s2k-objects-v3-${{ matrix.os }}-${{ matrix.arch }}-xcode26.3-${{ github.event_name == 'workflow_dispatch' && 'release' || 'smoke' }}-${{ github.sha }}- s2k-objects-v3-${{ matrix.os }}-${{ matrix.arch }}-xcode26.3-${{ github.event_name == 'workflow_dispatch' && 'release' || 'smoke' }}- - name: Cache native compiler results (tests still run) + id: compiler-cache uses: Mozilla-Actions/sccache-action@7d986dd989559c6ecdb630a3fd2557667be217ad with: version: v0.10.0 @@ -145,7 +147,7 @@ jobs: switch2kit-validation.json if-no-files-found: error - name: Build diagnostics - if: failure() || github.event_name == 'workflow_dispatch' + if: failure() || cancelled() || github.event_name == 'workflow_dispatch' uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 with: name: Switch2Kit-${{ matrix.arch }}-diagnostics @@ -155,3 +157,11 @@ jobs: switch2kit-validation.json build-switch2kit/CMakeCache.txt build-switch2kit/CMakeFiles/CMakeConfigureLog.yaml + # Keep valid compiler objects even when a later build/test is cancelled. + # Never save build trees, applications, configuration answers or test results. + - name: Save compiler objects after native validation + if: always() && steps.compiler-cache.outcome == 'success' + uses: actions/cache/save@0057852bfaa89a56745cba8c7296529d2fc39830 + with: + path: ${{ env.SCCACHE_DIR }} + key: ${{ steps.compiler-objects.outputs.cache-primary-key }} diff --git a/.github/workflows/switch2kit-windows.yml b/.github/workflows/switch2kit-windows.yml index 3c03e7081cdc..253dca84b31f 100644 --- a/.github/workflows/switch2kit-windows.yml +++ b/.github/workflows/switch2kit-windows.yml @@ -28,14 +28,16 @@ jobs: - name: Fetch and verify native gitlinks in parallel run: python Tools/checkout_native.py - name: Restore compiler objects (never test results or applications) - uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 + id: compiler-objects + uses: actions/cache/restore@0057852bfaa89a56745cba8c7296529d2fc39830 with: path: ${{ env.SCCACHE_DIR }} - key: s2k-objects-v3-windows-vs2026-${{ github.event_name == 'workflow_dispatch' && 'release' || 'smoke' }}-${{ github.sha }}-${{ github.job }} + key: s2k-objects-v3-windows-vs2026-${{ github.event_name == 'workflow_dispatch' && 'release' || 'smoke' }}-${{ github.sha }}-${{ github.job }}-${{ github.run_id }}-${{ github.run_attempt }} restore-keys: | s2k-objects-v3-windows-vs2026-${{ github.event_name == 'workflow_dispatch' && 'release' || 'smoke' }}-${{ github.sha }}- s2k-objects-v3-windows-vs2026-${{ github.event_name == 'workflow_dispatch' && 'release' || 'smoke' }}- - name: Cache native compiler results (tests still run) + id: compiler-cache uses: Mozilla-Actions/sccache-action@7d986dd989559c6ecdb630a3fd2557667be217ad with: version: v0.10.0 @@ -85,7 +87,7 @@ jobs: retention-days: 14 if-no-files-found: error - name: Native diagnostics - if: failure() || github.event_name == 'workflow_dispatch' + if: failure() || cancelled() || github.event_name == 'workflow_dispatch' uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 with: name: Dolphin-Switch2Kit-windows-diagnostics @@ -93,3 +95,11 @@ jobs: windows-*.log windows-launch.json retention-days: 7 + # Keep valid compiler objects even when a later build/test is cancelled. + # Never save build trees, applications, configuration answers or test results. + - name: Save compiler objects after native validation + if: always() && steps.compiler-cache.outcome == 'success' + uses: actions/cache/save@0057852bfaa89a56745cba8c7296529d2fc39830 + with: + path: ${{ env.SCCACHE_DIR }} + key: ${{ steps.compiler-objects.outputs.cache-primary-key }} From b51491b885853316a6fdb64d834fe6f9c59045e3 Mon Sep 17 00:00:00 2001 From: Johnny D Date: Mon, 21 Sep 2026 16:06:31 -0400 Subject: [PATCH 20/21] ci: remove Qt frontend serialization and precompile the Windows Qt headers Give DolphinQt its own MSVC PCH with its existing RTTI and Qt definitions, without changing the core's manual shared PCH. Schedule Qt code generation without waiting for compiled libraries, retaining explicit staging targets, existing autogen dependencies and the generated achievement header. Keep feature probes as real executable links while omitting only their unneeded Windows PDB and manifest work; application link settings remain unchanged. Record CMake profiling and build-graph diagnostics for the remaining Intel macOS configure-time bottleneck. Extend the existing native wiring regression for Qt PCH/cache isolation, generated-header and explicit dependency preservation, and manual artifact settings. All four local GCC/Clang regressions pass (11-12 seconds); the prior helper fails the added regression. Native acceptance remains required. No runners, budgets, translation units, tests, SDK pins or launch checks were removed or relaxed. AI-assisted; human review required. --- .github/workflows/switch2kit-macos.yml | 8 +++++- Tools/ci-native.cmake | 40 ++++++++++++++++++++++++++ Tools/test_native_ci_execution.py | 22 ++++++++++++-- 3 files changed, 67 insertions(+), 3 deletions(-) diff --git a/.github/workflows/switch2kit-macos.yml b/.github/workflows/switch2kit-macos.yml index 981020c0285c..51d5afd75122 100644 --- a/.github/workflows/switch2kit-macos.yml +++ b/.github/workflows/switch2kit-macos.yml @@ -103,7 +103,10 @@ jobs: run: | set -euo pipefail test "$(uname -m)" = "$ARCH" - cmake -S . -B build-switch2kit -G Ninja \ + ps -axo pid,pcpu,comm | sort -rn -k2 | sed -n '1,15p' + cmake --profiling-format=google-trace \ + --profiling-output="$PWD/switch2kit-configure-profile.json" \ + -S . -B build-switch2kit -G Ninja \ -DCMAKE_BUILD_TYPE=Release -DCMAKE_OSX_DEPLOYMENT_TARGET=15.0 \ "-DCMAKE_PROJECT_dolphin-emu_INCLUDE=$PWD/Tools/ci-native.cmake" \ -DCMAKE_OSX_ARCHITECTURES="$ARCH" -DCMAKE_OSX_SYSROOT="$SDKROOT" \ @@ -155,6 +158,9 @@ jobs: switch2kit-*.log switch2kit-*.sample switch2kit-validation.json + switch2kit-configure-profile.json + build-switch2kit/build.ninja + build-switch2kit/compile_commands.json build-switch2kit/CMakeCache.txt build-switch2kit/CMakeFiles/CMakeConfigureLog.yaml # Keep valid compiler objects even when a later build/test is cancelled. diff --git a/Tools/ci-native.cmake b/Tools/ci-native.cmake index 17619b19d2f0..573c3c332c9a 100644 --- a/Tools/ci-native.cmake +++ b/Tools/ci-native.cmake @@ -44,6 +44,10 @@ if(CMAKE_HOST_WIN32 AND MSVC AND NOT "$ENV{GITHUB_EVENT_NAME}" STREQUAL "workflo # Test the same runtime configuration as the real application, not a separate # Debug link with an incremental PDB/manifest cycle for every tiny probe. set(CMAKE_TRY_COMPILE_CONFIGURATION Release) + # Upstream appends /DEBUG to every executable link, including feature probes. + # Tiny configure executables still compile and link, but need neither a PDB + # nor an application manifest. These check-only options never reach targets. + list(APPEND CMAKE_REQUIRED_LINK_OPTIONS /DEBUG:NONE /MANIFEST:NO) message(STATUS "Native CI Windows linker: ${CMAKE_LINKER_LLD}") endif() @@ -92,7 +96,43 @@ function(switch2kit_ci_precompile_headers) get_target_property(glslang_source glslang SOURCE_DIR) target_precompile_headers(glslang PRIVATE "${glslang_source}/MachineIndependent/pch.h") endif() + if(TARGET dolphin-emu AND ENABLE_QT AND + NOT "$ENV{GITHUB_EVENT_NAME}" STREQUAL "workflow_dispatch") + # moc reads the Qt target's sources; it need not wait for every linked + # library to finish compiling. Preserve explicit generated/staging targets + # and any existing autogen dependencies. CMake still owns source generation + # and the application's compile/link dependency graph. + get_target_property(qt_dependencies dolphin-emu MANUALLY_ADDED_DEPENDENCIES) + if(qt_dependencies) + set_property(TARGET dolphin-emu APPEND PROPERTY AUTOGEN_TARGET_DEPENDS + ${qt_dependencies}) + endif() + # This upstream generated header is a core source, not a Qt source. Keep + # the file prerequisite without waiting for core's compiled static library. + if(TARGET core) + get_target_property(core_binary core BINARY_DIR) + get_target_property(core_sources core SOURCES) + if("AchievementApprovedHash.h" IN_LIST core_sources) + set_property(TARGET dolphin-emu APPEND PROPERTY AUTOGEN_TARGET_DEPENDS + "${core_binary}/AchievementApprovedHash.h") + endif() + endif() + set_property(TARGET dolphin-emu PROPERTY AUTOGEN_ORIGIN_DEPENDS OFF) + endif() if(MSVC) + # DolphinQt enables RTTI and does not use the core's shared /GR- PCH. + # Give it its own PCH, with its actual Qt flags, as on POSIX. Do not mix + # CMake's PCH with an existing manual or target-provided implementation. + if(TARGET dolphin-emu AND ENABLE_QT) + get_target_property(qt_links dolphin-emu LINK_LIBRARIES) + get_target_property(qt_pch dolphin-emu PRECOMPILE_HEADERS) + get_target_property(qt_reuse_pch dolphin-emu PRECOMPILE_HEADERS_REUSE_FROM) + if(NOT "use_pch" IN_LIST qt_links AND NOT qt_pch AND NOT qt_reuse_pch) + target_precompile_headers(dolphin-emu PRIVATE + "$<$:${PROJECT_SOURCE_DIR}/Source/PCH/pch.h>" + "$<$:>") + endif() + endif() return() endif() set(pch_targets common audiocommon inputcommon videocommon discio core uicommon diff --git a/Tools/test_native_ci_execution.py b/Tools/test_native_ci_execution.py index e7d4e20e23e8..6a906cedac20 100644 --- a/Tools/test_native_ci_execution.py +++ b/Tools/test_native_ci_execution.py @@ -299,6 +299,7 @@ def test_manual_msvc_pch_is_not_a_cmake_pch_property(self): with tempfile.TemporaryDirectory() as temporary: root = Path(temporary) write(root, 'part.cpp', 'int part;\n') + write(root, 'Source/PCH/pch.h', '#pragma once\n') write(root, 'CMakeLists.txt', '''\ cmake_minimum_required(VERSION 3.25) project(dolphin-emu LANGUAGES CXX) @@ -308,10 +309,20 @@ def test_manual_msvc_pch_is_not_a_cmake_pch_property(self): add_library(build_pch STATIC part.cpp) add_library(use_pch INTERFACE) add_dependencies(use_pch build_pch) - add_library(core STATIC part.cpp) + add_custom_command(OUTPUT AchievementApprovedHash.h + COMMAND "${CMAKE_COMMAND}" -E touch AchievementApprovedHash.h) + add_library(core STATIC part.cpp AchievementApprovedHash.h) target_link_libraries(core PRIVATE use_pch) add_library(ordinary STATIC part.cpp) - foreach(t build_pch core ordinary) + set(ENABLE_QT ON) + add_executable(dolphin-emu part.cpp) + # This fixture checks target wiring, not a simulated Qt build. + # The native Windows job compiles the actual Qt headers and moc. + add_dependencies(dolphin-emu build_pch) + set_property(TARGET dolphin-emu PROPERTY AUTOGEN_TARGET_DEPENDS ordinary) + file(GENERATE OUTPUT "${CMAKE_BINARY_DIR}/qt.txt" CONTENT + "$|$|$") + foreach(t build_pch core ordinary dolphin-emu) target_compile_options(${t} PRIVATE "$<$:/O2>" "$<$:/Ob2>") set_target_properties(${t} PROPERTIES LINKER_LANGUAGE CXX C_COMPILER_LAUNCHER bad-inherited CXX_COMPILER_LAUNCHER bad-inherited) @@ -330,6 +341,12 @@ def test_manual_msvc_pch_is_not_a_cmake_pch_property(self): self.assertEqual((root / 'build' / f'{target}.txt').read_text(), '||ON') self.assertEqual((root / 'build/ordinary.txt').read_text(), 'cmake;-E;env|cmake;-E;env|ON') + self.assertEqual((root / 'build/dolphin-emu.txt').read_text(), '||') + qt = (root / 'build/qt.txt').read_text() + self.assertIn('Source/PCH/pch.h', qt) + self.assertIn('QtWidgets', qt) + self.assertIn('|OFF|ordinary;build_pch;', qt) + self.assertTrue(qt.endswith('/build/AchievementApprovedHash.h')) commands = json.loads((root / 'build/compile_commands.json').read_text()) for command in commands: options = command['command'].split() @@ -340,6 +357,7 @@ def test_manual_msvc_pch_is_not_a_cmake_pch_property(self): run('cmake', '-S', root, '-B', root / 'manual', '-G', 'Ninja', f'-DCMAKE_PROJECT_dolphin-emu_INCLUDE={PRESET}', '-DCMAKE_BUILD_TYPE=Release', '-DCMAKE_EXPORT_COMPILE_COMMANDS=ON', cwd=root, env=env) + self.assertTrue((root / 'manual/qt.txt').read_text().endswith('|ON|ordinary'), (root / 'manual/qt.txt').read_text()) commands = json.loads((root / 'manual/compile_commands.json').read_text()) for command in commands: options = command['command'].split() From 5001d9b733c7be6ea6ca5d9c1138c16b9a275405 Mon Sep 17 00:00:00 2001 From: Johnny D Date: Mon, 21 Sep 2026 17:23:53 -0400 Subject: [PATCH 21/21] ci: recover Intel runner CPU and overlap the complete native build graph Apply the runner-images PerfPowerServices workaround on the disposable Intel macOS runner: the retained process profile showed it consuming 98.5% CPU. Keep Apple Silicon and developer machines untouched. Build the real Windows SDK and complete application in one automatic Ninja graph instead of leaving native compilation idle during the 79-second SDK phase. Preserve SDK dependency ordering, fatal errors, a final incremental application check, and the developer/explicit-artifact SDK-first path. Submit Qt autogen and the expensive Linux emitter test early within the same complete graph, then execute the unchanged upstream unittests target. Use Ninja's CPU+2 queue headroom on the existing standard runners; do not add CPUs, paid runners, shards, jobs, skipped sources or reduced test coverage. Select the installed native LLVM 19 ELF linker for automatic Linux Clang application/test links and real executable probes through a private CI-only setting. Preserve explicit artifact, GCC and nested SDK defaults. Positive compile/link/run and missing-symbol negative controls pass for automatic and explicit modes. Existing four native execution regressions pass with GCC and Clang, including sanitizer, generated-source/PCH and recursive-pin checks. Start v4 compiler-object caches to measure the initial native run cold rather than hide a cold failure behind prior objects. All ten-minute automatic job budgets, optimized Linux flags, mandatory SDK/sanitizer/upstream/GUI archive checks, dependency pins, controller behavior and user-data safeguards remain. Exact-head native acceptance is still required. AI-assisted; human review required. --- .github/workflows/switch2kit-linux.yml | 29 +++++++++++++++--------- .github/workflows/switch2kit-macos.yml | 22 ++++++++++++++---- .github/workflows/switch2kit-windows.yml | 6 ++--- Tools/build-switch2kit-windows.ps1 | 21 ++++++++++++----- Tools/ci-native.cmake | 15 ++++++++++++ 5 files changed, 69 insertions(+), 24 deletions(-) diff --git a/.github/workflows/switch2kit-linux.yml b/.github/workflows/switch2kit-linux.yml index ac6ec858c327..ee3ffda1407a 100644 --- a/.github/workflows/switch2kit-linux.yml +++ b/.github/workflows/switch2kit-linux.yml @@ -21,6 +21,7 @@ jobs: NINJA_STATUS: '[%f/%t %r running, %e sec] ' # Configure probes and standalone sanitizer/SDK tests must compile directly. S2K_CI_COMPILER_CACHE: sccache + S2K_CI_LINUX_LINKER: /usr/bin/ld.lld-19 steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 with: @@ -29,7 +30,7 @@ jobs: - name: Install native dependencies run: | apt-get update - apt-get install -y --no-install-recommends build-essential clang clang-19 cmake ninja-build python3 pkg-config qt6-base-dev qt6-base-private-dev qt6-svg-dev libbluetooth-dev libevdev-dev libudev-dev libusb-1.0-0-dev libasound2-dev libpulse-dev libx11-dev libxi-dev libxrandr-dev libegl1-mesa-dev libgl1-mesa-dev libsystemd0 dbus xvfb xauth openbox wmctrl x11-utils + apt-get install -y --no-install-recommends build-essential clang clang-19 lld-19 cmake ninja-build python3 pkg-config qt6-base-dev qt6-base-private-dev qt6-svg-dev libbluetooth-dev libevdev-dev libudev-dev libusb-1.0-0-dev libasound2-dev libpulse-dev libx11-dev libxi-dev libxrandr-dev libegl1-mesa-dev libgl1-mesa-dev libsystemd0 dbus xvfb xauth openbox wmctrl x11-utils git config --global --add safe.directory "$PWD" - name: Fetch and verify native gitlinks in parallel run: python3 Tools/checkout_native.py @@ -44,10 +45,10 @@ jobs: uses: actions/cache/restore@0057852bfaa89a56745cba8c7296529d2fc39830 with: path: ${{ env.SCCACHE_DIR }} - key: s2k-objects-v3-linux-clang19-${{ github.event_name == 'workflow_dispatch' && 'release' || 'smoke' }}-${{ github.sha }}-${{ github.job }}-${{ github.run_id }}-${{ github.run_attempt }} + key: s2k-objects-v4-linux-clang19-${{ github.event_name == 'workflow_dispatch' && 'release' || 'smoke' }}-${{ github.sha }}-${{ github.job }}-${{ github.run_id }}-${{ github.run_attempt }} restore-keys: | - s2k-objects-v3-linux-clang19-${{ github.event_name == 'workflow_dispatch' && 'release' || 'smoke' }}-${{ github.sha }}- - s2k-objects-v3-linux-clang19-${{ github.event_name == 'workflow_dispatch' && 'release' || 'smoke' }}- + s2k-objects-v4-linux-clang19-${{ github.event_name == 'workflow_dispatch' && 'release' || 'smoke' }}-${{ github.sha }}- + s2k-objects-v4-linux-clang19-${{ github.event_name == 'workflow_dispatch' && 'release' || 'smoke' }}- - name: Cache native compiler results (tests still run) id: compiler-cache uses: Mozilla-Actions/sccache-action@7d986dd989559c6ecdb630a3fd2557667be217ad @@ -57,7 +58,9 @@ jobs: shell: bash run: | set -o pipefail - S2K_BUILD_JOBS="$(nproc)" bash Tools/build-switch2kit-linux.sh \ + jobs="$(nproc)" + if [ "$GITHUB_EVENT_NAME" != workflow_dispatch ]; then jobs=$((jobs + 2)); fi + S2K_BUILD_JOBS="$jobs" bash Tools/build-switch2kit-linux.sh \ -DCMAKE_C_COMPILER=/usr/bin/clang-19 -DCMAKE_CXX_COMPILER=/usr/bin/clang++-19 \ "-DCMAKE_PROJECT_dolphin-emu_INCLUDE=$PWD/Tools/ci-native.cmake" \ 2>&1 | tee linux-build.log @@ -131,6 +134,7 @@ jobs: NINJA_STATUS: '[%f/%t %r running, %e sec] ' # Configure probes and standalone sanitizer/SDK tests must compile directly. S2K_CI_COMPILER_CACHE: sccache + S2K_CI_LINUX_LINKER: /usr/bin/ld.lld-19 steps: - name: Check shared memory before installing dependencies or compiling shell: bash @@ -145,7 +149,7 @@ jobs: - name: Install ordinary native dependencies without Swift run: | apt-get update - apt-get install -y --no-install-recommends git ca-certificates build-essential clang clang-19 cmake ninja-build python3 pkg-config qt6-base-dev qt6-base-private-dev qt6-svg-dev libbluetooth-dev libevdev-dev libudev-dev libusb-1.0-0-dev libasound2-dev libpulse-dev libx11-dev libxi-dev libxrandr-dev libegl1-mesa-dev libgl1-mesa-dev libsystemd0 dbus + apt-get install -y --no-install-recommends git ca-certificates build-essential clang clang-19 lld-19 cmake ninja-build python3 pkg-config qt6-base-dev qt6-base-private-dev qt6-svg-dev libbluetooth-dev libevdev-dev libudev-dev libusb-1.0-0-dev libasound2-dev libpulse-dev libx11-dev libxi-dev libxrandr-dev libegl1-mesa-dev libgl1-mesa-dev libsystemd0 dbus if command -v swift || command -v swiftc; then echo 'The backend-disabled qualification image must not contain Swift.' >&2 exit 1 @@ -169,10 +173,10 @@ jobs: uses: actions/cache/restore@0057852bfaa89a56745cba8c7296529d2fc39830 with: path: ${{ env.SCCACHE_DIR }} - key: s2k-objects-v3-linux-clang19-${{ github.event_name == 'workflow_dispatch' && 'release' || 'smoke' }}-${{ github.sha }}-${{ github.job }}-${{ github.run_id }}-${{ github.run_attempt }} + key: s2k-objects-v4-linux-clang19-${{ github.event_name == 'workflow_dispatch' && 'release' || 'smoke' }}-${{ github.sha }}-${{ github.job }}-${{ github.run_id }}-${{ github.run_attempt }} restore-keys: | - s2k-objects-v3-linux-clang19-${{ github.event_name == 'workflow_dispatch' && 'release' || 'smoke' }}-${{ github.sha }}- - s2k-objects-v3-linux-clang19-${{ github.event_name == 'workflow_dispatch' && 'release' || 'smoke' }}- + s2k-objects-v4-linux-clang19-${{ github.event_name == 'workflow_dispatch' && 'release' || 'smoke' }}-${{ github.sha }}- + s2k-objects-v4-linux-clang19-${{ github.event_name == 'workflow_dispatch' && 'release' || 'smoke' }}- - name: Cache native compiler results (tests still run) id: compiler-cache uses: Mozilla-Actions/sccache-action@7d986dd989559c6ecdb630a3fd2557667be217ad @@ -190,8 +194,11 @@ jobs: -DENABLE_SWITCH2KIT=OFF -DENABLE_SDL=ON -DENABLE_QT=ON \ -DUSE_SYSTEM_SDL3=OFF -DENABLE_TESTS=ON -DENABLE_VULKAN=OFF \ -DENABLE_AUTOUPDATE=OFF 2>&1 | tee disabled-configure.log - # Let Ninja schedule the application and all upstream test objects together. - cmake --build build-switch2kit --target dolphin-emu tests --parallel "$(nproc)" 2>&1 | tee disabled-build.log + # Submit the expensive emitter case and moc first, in the same full + # application/test graph. They must not become a serial compile tail. + jobs="$(nproc)" + if [ "$GITHUB_EVENT_NAME" != workflow_dispatch ]; then jobs=$((jobs + 2)); fi + cmake --build build-switch2kit --target x64EmitterTest dolphin-emu_autogen tests dolphin-emu --parallel "$jobs" 2>&1 | tee disabled-build.log # Building tests does not execute them; retain the complete CTest target. cmake --build build-switch2kit --target unittests --parallel "$(nproc)" 2>&1 | tee upstream-unittests.log ldd build-switch2kit/Binaries/dolphin-emu | tee disabled-dependencies.log diff --git a/.github/workflows/switch2kit-macos.yml b/.github/workflows/switch2kit-macos.yml index 51d5afd75122..8d530fcdd5ce 100644 --- a/.github/workflows/switch2kit-macos.yml +++ b/.github/workflows/switch2kit-macos.yml @@ -34,6 +34,18 @@ jobs: # Configure probes and standalone sanitizer/SDK tests must compile directly. S2K_CI_COMPILER_CACHE: sccache steps: + - name: Restore Intel runner CPU availability + if: matrix.arch == 'x86_64' + run: | + set -euo pipefail + # The retained runner profile showed PerfPowerServices using a core. + # Use the runner-images workaround, scoped to this disposable Intel VM. + # https://github.com/actions/runner-images/issues/13358 + ps -axo pid,pcpu,comm | sort -rn -k2 | sed -n '1,15p' + sudo defaults -currentHost write /Library/Preferences/com.apple.powerlogd SMCMonitorCadence 0 + if pgrep -x PerfPowerServices >/dev/null; then + sudo killall PerfPowerServices + fi - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 with: submodules: false @@ -45,10 +57,10 @@ jobs: uses: actions/cache/restore@0057852bfaa89a56745cba8c7296529d2fc39830 with: path: ${{ env.SCCACHE_DIR }} - key: s2k-objects-v3-${{ matrix.os }}-${{ matrix.arch }}-xcode26.3-${{ github.event_name == 'workflow_dispatch' && 'release' || 'smoke' }}-${{ github.sha }}-${{ github.job }}-${{ github.run_id }}-${{ github.run_attempt }} + key: s2k-objects-v4-${{ matrix.os }}-${{ matrix.arch }}-xcode26.3-${{ github.event_name == 'workflow_dispatch' && 'release' || 'smoke' }}-${{ github.sha }}-${{ github.job }}-${{ github.run_id }}-${{ github.run_attempt }} restore-keys: | - s2k-objects-v3-${{ matrix.os }}-${{ matrix.arch }}-xcode26.3-${{ github.event_name == 'workflow_dispatch' && 'release' || 'smoke' }}-${{ github.sha }}- - s2k-objects-v3-${{ matrix.os }}-${{ matrix.arch }}-xcode26.3-${{ github.event_name == 'workflow_dispatch' && 'release' || 'smoke' }}- + s2k-objects-v4-${{ matrix.os }}-${{ matrix.arch }}-xcode26.3-${{ github.event_name == 'workflow_dispatch' && 'release' || 'smoke' }}-${{ github.sha }}- + s2k-objects-v4-${{ matrix.os }}-${{ matrix.arch }}-xcode26.3-${{ github.event_name == 'workflow_dispatch' && 'release' || 'smoke' }}- - name: Cache native compiler results (tests still run) id: compiler-cache uses: Mozilla-Actions/sccache-action@7d986dd989559c6ecdb630a3fd2557667be217ad @@ -116,7 +128,9 @@ jobs: -DQt6_DIR="$S2K_QT_PREFIX/lib/cmake/Qt6" \ -DENABLE_VULKAN=OFF -DENABLE_TESTS=OFF -DPOSTPROCESS_BUNDLE=ON \ 2>&1 | tee switch2kit-configure.log - cmake --build build-switch2kit --target dolphin-emu --parallel "$(sysctl -n hw.logicalcpu)" \ + jobs="$(sysctl -n hw.logicalcpu)" + if [ "$GITHUB_EVENT_NAME" != workflow_dispatch ]; then jobs=$((jobs + 2)); fi + cmake --build build-switch2kit --target dolphin-emu_autogen dolphin-emu --parallel "$jobs" \ 2>&1 | tee switch2kit-build.log - name: Compiler cache statistics if: always() diff --git a/.github/workflows/switch2kit-windows.yml b/.github/workflows/switch2kit-windows.yml index 253dca84b31f..7ffd124333db 100644 --- a/.github/workflows/switch2kit-windows.yml +++ b/.github/workflows/switch2kit-windows.yml @@ -32,10 +32,10 @@ jobs: uses: actions/cache/restore@0057852bfaa89a56745cba8c7296529d2fc39830 with: path: ${{ env.SCCACHE_DIR }} - key: s2k-objects-v3-windows-vs2026-${{ github.event_name == 'workflow_dispatch' && 'release' || 'smoke' }}-${{ github.sha }}-${{ github.job }}-${{ github.run_id }}-${{ github.run_attempt }} + key: s2k-objects-v4-windows-vs2026-${{ github.event_name == 'workflow_dispatch' && 'release' || 'smoke' }}-${{ github.sha }}-${{ github.job }}-${{ github.run_id }}-${{ github.run_attempt }} restore-keys: | - s2k-objects-v3-windows-vs2026-${{ github.event_name == 'workflow_dispatch' && 'release' || 'smoke' }}-${{ github.sha }}- - s2k-objects-v3-windows-vs2026-${{ github.event_name == 'workflow_dispatch' && 'release' || 'smoke' }}- + s2k-objects-v4-windows-vs2026-${{ github.event_name == 'workflow_dispatch' && 'release' || 'smoke' }}-${{ github.sha }}- + s2k-objects-v4-windows-vs2026-${{ github.event_name == 'workflow_dispatch' && 'release' || 'smoke' }}- - name: Cache native compiler results (tests still run) id: compiler-cache uses: Mozilla-Actions/sccache-action@7d986dd989559c6ecdb630a3fd2557667be217ad diff --git a/Tools/build-switch2kit-windows.ps1 b/Tools/build-switch2kit-windows.ps1 index c5d8fdf7fe27..c844f9573a88 100644 --- a/Tools/build-switch2kit-windows.ps1 +++ b/Tools/build-switch2kit-windows.ps1 @@ -50,13 +50,22 @@ cmake -S . -B build-switch2kit-windows -G Ninja -DCMAKE_BUILD_TYPE=Release ` -DENABLE_TESTS=OFF -DENABLE_CLI_TOOL=OFF -DENABLE_AUTOUPDATE=OFF ` "-DSWITCH2KIT_SWIFT=$swift" "-DSWITCH2KIT_SWIFTC=$swiftc" @CMakeArgs if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } -# Compile the real Swift/C++ WinRT library first. Fail on toolchain/SDK -# incompatibility before spending time on Dolphin's other dependencies; do not -# bypass the Microsoft STL or Dolphin compiler guards. This target is reused by -# the application build, with the same SwiftPM configuration and scratch path. -cmake --build build-switch2kit-windows --target Switch2KitCBuild --parallel ([Environment]::ProcessorCount) +# Preserve the developer/explicit-artifact SDK-first failure path. Automatic +# builds request the SDK and complete application in the same Ninja graph, so +# independent native objects can compile while Swift runs. The existing graph +# still orders every SDK consumer and staging step, and either failure is fatal. +$parallelTargets = @() +$buildJobs = [Environment]::ProcessorCount +if ($env:GITHUB_ACTIONS -eq 'true' -and $env:GITHUB_EVENT_NAME -ne 'workflow_dispatch') { + $parallelTargets = @('dolphin-emu_autogen', 'dolphin-emu') + # Match Ninja's normal CPU+2 queue headroom on the same standard runner. + $buildJobs += 2 +} +cmake --build build-switch2kit-windows --target Switch2KitCBuild @parallelTargets --parallel $buildJobs if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } -cmake --build build-switch2kit-windows --target dolphin-emu --parallel ([Environment]::ProcessorCount) +# This is an incremental verification after an automatic combined build, not a +# second SDK scratch tree or a cached application. Developer behavior is unchanged. +cmake --build build-switch2kit-windows --target dolphin-emu --parallel $buildJobs if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } $app = Join-Path $PWD 'build-switch2kit-windows/Binaries/Dolphin.exe' if (-not (Test-Path $app) -or -not (Test-Path (Join-Path (Split-Path $app) 'Switch2KitC.dll'))) { diff --git a/Tools/ci-native.cmake b/Tools/ci-native.cmake index 573c3c332c9a..bd35e8ecb9f1 100644 --- a/Tools/ci-native.cmake +++ b/Tools/ci-native.cmake @@ -51,6 +51,21 @@ if(CMAKE_HOST_WIN32 AND MSVC AND NOT "$ENV{GITHUB_EVENT_NAME}" STREQUAL "workflo message(STATUS "Native CI Windows linker: ${CMAKE_LINKER_LLD}") endif() +# Clang's native ELF linker avoids the serial GNU ld tail for the complete +# Linux application and upstream test executable. Keep real executable feature +# probes, and leave explicit artifacts, nested SDK builds and GCC unchanged. +if(CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_CXX_COMPILER_ID STREQUAL "Clang" + AND DEFINED ENV{S2K_CI_LINUX_LINKER} + AND NOT "$ENV{GITHUB_EVENT_NAME}" STREQUAL "workflow_dispatch") + set(_switch2kit_linux_lld "$ENV{S2K_CI_LINUX_LINKER}") + if(NOT IS_ABSOLUTE "${_switch2kit_linux_lld}" OR NOT EXISTS "${_switch2kit_linux_lld}") + message(FATAL_ERROR "Native CI requires the explicitly selected Linux linker") + endif() + add_link_options("-fuse-ld=${_switch2kit_linux_lld}") + list(APPEND CMAKE_REQUIRED_LINK_OPTIONS "-fuse-ld=${_switch2kit_linux_lld}") + message(STATUS "Native CI Linux linker: ${_switch2kit_linux_lld}") +endif() + # Use a CI-specific variable, not CMAKE_*_COMPILER_LAUNCHER in the job's # environment: those standard variables also reach every nested SDK/fixture # configuration and try_compile project. Clearing a normal variable alone does