From 0c738f2d2a16704940122d3a58a4c264df80c5a9 Mon Sep 17 00:00:00 2001 From: Baptiste LAFOURCADE Date: Mon, 12 Oct 2026 15:51:43 +0200 Subject: [PATCH 1/3] ci(cli): split the four slowest mutation scopes into parallel shards framework, kernel, tools and telemetry took 30 to 45 minutes each in one job, and set the wall time of every pull request. Each now runs as 3 or 4 legs in parallel: a deterministic split by file weight (heaviest first, to the lightest leg, ties by path), one incremental file and sandbox per leg, the full stryker.conf.json otherwise. A scoring job merges the legs' reports and applies the same score and floor as one run; it fails when a leg is missing, mutated other files than the split gives it at this commit, or the legs hold no mutant, and it runs even when a leg crashed. The canary and every other scope stay one job, and running a scope locally is unchanged. Measured on tools-codex, split in two for the proof: one run and two legs give the same 444 detected, 21 undetected and survivors, score 95.5 both ways. The incremental cache keys now end their scope name with --, so tools can never restore tools-codex's file. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01VWNxk63AGKkqE8HRqHLjGp --- .github/workflows/cli-ci.yml | 77 ++++++-- cli/mutation-scopes.json | 12 +- cli/scripts/mutation-scopes-to-run.d.mts | 4 + cli/scripts/mutation-scopes-to-run.mjs | 19 +- cli/scripts/mutation-shards.d.mts | 37 ++++ cli/scripts/mutation-shards.mjs | 85 ++++++++ cli/scripts/run-mutation.d.mts | 10 + cli/scripts/run-mutation.mjs | 123 +++++++++++- cli/tests/architecture/helpers.ts | 11 +- .../architecture/mutation-shards.arch.test.ts | 184 ++++++++++++++++++ .../cli-ci-gate-covers-every-job.test.js | 7 +- 11 files changed, 532 insertions(+), 37 deletions(-) create mode 100644 cli/scripts/mutation-shards.d.mts create mode 100644 cli/scripts/mutation-shards.mjs create mode 100644 cli/tests/architecture/mutation-shards.arch.test.ts diff --git a/.github/workflows/cli-ci.yml b/.github/workflows/cli-ci.yml index b7e2ca1bc..7c94687db 100644 --- a/.github/workflows/cli-ci.yml +++ b/.github/workflows/cli-ci.yml @@ -35,7 +35,8 @@ jobs: outputs: relevant: ${{ steps.filter.outputs.relevant }} trusted_promotion: ${{ steps.promotion.outputs.trusted }} - mutation_scopes: ${{ steps.mutation.outputs.scopes }} + mutation_legs: ${{ steps.mutation.outputs.legs }} + mutation_sharded: ${{ steps.mutation.outputs.sharded }} steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: @@ -183,7 +184,8 @@ jobs: run: | set -euo pipefail if [[ "${{ steps.promotion.outputs.trusted }}" == "true" ]]; then - scopes='[]' + legs='[]' + sharded='[]' else if [[ "${{ github.event_name }}" == "pull_request" ]]; then BASE="${{ github.event.pull_request.base.sha }}" @@ -199,10 +201,13 @@ jobs: changed="$(git diff --name-only "$BASE" "$HEAD")" all=false fi - scopes="$(ALL="$all" CHANGED="$changed" node cli/scripts/mutation-scopes-to-run.mjs)" + legs="$(ALL="$all" CHANGED="$changed" node cli/scripts/mutation-scopes-to-run.mjs)" + sharded="$(ALL="$all" CHANGED="$changed" OUTPUT=sharded node cli/scripts/mutation-scopes-to-run.mjs)" fi - echo "mutation scopes: $scopes" - echo "scopes=$scopes" >> "$GITHUB_OUTPUT" + echo "mutation legs: $legs" + echo "sharded scopes: $sharded" + echo "legs=$legs" >> "$GITHUB_OUTPUT" + echo "sharded=$sharded" >> "$GITHUB_OUTPUT" cli-typecheck: name: cli / Typecheck @@ -583,15 +588,21 @@ jobs: # nothing relevant. The one check the branch rulesets require — without it a pull request # merges with the real jobs red, because nothing names them. cli-mutation: - name: cli / Mutation (${{ matrix.scope }}) + # A heavy scope runs as several legs in parallel, each mutating its share of the files; the + # job below scores it once over every leg. Any other scope is one leg and gates itself. + name: cli / Mutation (${{ matrix.scope }}${{ matrix.of > 1 && format(' {0}/{1}', matrix.shard, matrix.of) || '' }}) needs: [changes] - if: needs.changes.outputs.mutation_scopes != '[]' + if: needs.changes.outputs.mutation_legs != '[]' runs-on: ubuntu-latest timeout-minutes: 90 strategy: fail-fast: false matrix: - scope: ${{ fromJSON(needs.changes.outputs.mutation_scopes) }} + include: ${{ fromJSON(needs.changes.outputs.mutation_legs) }} + env: + # `--` closes the name: `tools` must never restore `tools-codex`'s file. + LEG: ${{ matrix.of > 1 && format('{0}-shard-{1}-of-{2}', matrix.scope, matrix.shard, matrix.of) || matrix.scope }} + LEG_DIR: cli/reports/mutation/${{ matrix.of > 1 && format('{0}/shard-{1}-of-{2}', matrix.scope, matrix.shard, matrix.of) || matrix.scope }} steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - name: Install pnpm @@ -608,19 +619,51 @@ jobs: - run: cd cli && pnpm install --frozen-lockfile # The incremental file is what keeps a run to the mutants a change can move; the newest # one this branch or its base saved is restored, and this run's is saved whatever the score. - - name: Restore this scope's incremental file + - name: Restore this leg's incremental file uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 with: - path: cli/reports/mutation/${{ matrix.scope }}/incremental.json - key: mutation-${{ matrix.scope }}-${{ github.run_id }} - restore-keys: mutation-${{ matrix.scope }}- - - run: cd cli && node scripts/run-mutation.mjs "${{ matrix.scope }}" ${{ github.event_name == 'schedule' && '--force' || '' }} - - name: Save this scope's incremental file + path: ${{ env.LEG_DIR }}/incremental.json + key: mutation--${{ env.LEG }}--${{ github.run_id }} + restore-keys: mutation--${{ env.LEG }}-- + - run: cd cli && node scripts/run-mutation.mjs "${{ matrix.scope }}" ${{ matrix.of > 1 && format('--shard {0}/{1}', matrix.shard, matrix.of) || '' }} ${{ github.event_name == 'schedule' && '--force' || '' }} + - name: Save this leg's incremental file if: always() uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 with: - path: cli/reports/mutation/${{ matrix.scope }}/incremental.json - key: mutation-${{ matrix.scope }}-${{ github.run_id }} + path: ${{ env.LEG_DIR }}/incremental.json + key: mutation--${{ env.LEG }}--${{ github.run_id }} + - name: Hand this shard's report to the scoring job + if: matrix.of > 1 + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: mutation-${{ matrix.scope }}-shard-${{ matrix.shard }}-of-${{ matrix.of }} + path: | + ${{ env.LEG_DIR }}/manifest.json + ${{ env.LEG_DIR }}/mutation.json + if-no-files-found: error + retention-days: 1 + + cli-mutation-score: + name: cli / Mutation (${{ matrix.scope }}) + needs: [changes, cli-mutation] + # Always, so a crashed or cancelled shard is scored as missing instead of skipped. + if: always() && needs.changes.outputs.mutation_sharded != '[]' && needs.changes.outputs.mutation_sharded != '' + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + scope: ${{ fromJSON(needs.changes.outputs.mutation_sharded) }} + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 + with: + node-version: "22" + - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 + with: + pattern: mutation-${{ matrix.scope }}-shard-* + path: shards + - name: Score the scope over every shard, against its floor + run: node cli/scripts/run-mutation.mjs "${{ matrix.scope }}" --aggregate "$GITHUB_WORKSPACE/shards" gate: name: cli / gate @@ -639,6 +682,7 @@ jobs: - identifier-join - cli-jscpd - cli-mutation + - cli-mutation-score - kanban-checks - windows steps: @@ -660,6 +704,7 @@ jobs: "${{ needs.identifier-join.result }}" \ "${{ needs.cli-jscpd.result }}" \ "${{ needs.cli-mutation.result }}" \ + "${{ needs.cli-mutation-score.result }}" \ "${{ needs.kanban-checks.result }}" \ "${{ needs.windows.result }}"; do if [[ "$result" == "failure" || "$result" == "cancelled" ]]; then diff --git a/cli/mutation-scopes.json b/cli/mutation-scopes.json index 4c6938794..17900ee5e 100644 --- a/cli/mutation-scopes.json +++ b/cli/mutation-scopes.json @@ -3,7 +3,8 @@ "scopes": { "kernel": { "mutate": "src/kernel/**/*.ts", - "break": 95 + "break": 95, + "shards": 3 }, "tools": { "mutate": [ @@ -15,11 +16,13 @@ "!src/contexts/tools/domain/profiles/kilo/**/*.ts", "!src/contexts/tools/domain/profiles/opencode/**/*.ts" ], - "break": 94 + "break": 94, + "shards": 4 }, "telemetry": { "mutate": "src/contexts/telemetry/**/*.ts", - "break": 93 + "break": 93, + "shards": 3 }, "translate": { "mutate": "src/contexts/translate/**/*.ts", @@ -31,7 +34,8 @@ }, "framework": { "mutate": "src/contexts/framework/**/*.ts", - "break": 93 + "break": 93, + "shards": 4 }, "presentation": { "mutate": "src/presentation/**/*.ts", diff --git a/cli/scripts/mutation-scopes-to-run.d.mts b/cli/scripts/mutation-scopes-to-run.d.mts index 45eb1961e..a7bd03703 100644 --- a/cli/scripts/mutation-scopes-to-run.d.mts +++ b/cli/scripts/mutation-scopes-to-run.d.mts @@ -6,3 +6,7 @@ export function scopesToRun( scopes: Readonly>, options?: { readonly all?: boolean } ): string[]; +export function legsOf( + names: readonly string[], + scopes: Readonly> +): { scope: string; shard: number; of: number }[]; diff --git a/cli/scripts/mutation-scopes-to-run.mjs b/cli/scripts/mutation-scopes-to-run.mjs index e693bc684..2eb440cfa 100644 --- a/cli/scripts/mutation-scopes-to-run.mjs +++ b/cli/scripts/mutation-scopes-to-run.mjs @@ -12,6 +12,7 @@ export const HARNESS = [ "cli/vitest.mutation.config.ts", "cli/scripts/run-mutation.mjs", "cli/scripts/mutation-scopes-to-run.mjs", + "cli/scripts/mutation-shards.mjs", "cli/package.json", "cli/pnpm-lock.yaml", ".github/workflows/cli-ci.yml", @@ -36,10 +37,22 @@ export function scopesToRun(changed, scopes, { all = false } = {}) { .map(([name]) => name); } +/** The CI legs for `names`: one per shard of a sharded scope, one for any other. */ +export function legsOf(names, scopes) { + return names.flatMap((scope) => { + const of = scopes[scope].shards ?? 1; + return Array.from({ length: of }, (_, i) => ({ scope, shard: i + 1, of })); + }); +} + if (process.argv[1] !== undefined && resolve(process.argv[1]) === fileURLToPath(import.meta.url)) { const scopes = JSON.parse(readFileSync(join(CLI_ROOT, "mutation-scopes.json"), "utf8")).scopes; const changed = (process.env.CHANGED ?? "").split("\n").filter(Boolean); - process.stdout.write( - JSON.stringify(scopesToRun(changed, scopes, { all: process.env.ALL === "true" })) - ); + const names = scopesToRun(changed, scopes, { all: process.env.ALL === "true" }); + // `legs` feeds the mutation matrix; `sharded` the jobs that score a split scope once. + const output = + process.env.OUTPUT === "sharded" + ? names.filter((name) => (scopes[name].shards ?? 1) > 1) + : legsOf(names, scopes); + process.stdout.write(JSON.stringify(output)); } diff --git a/cli/scripts/mutation-shards.d.mts b/cli/scripts/mutation-shards.d.mts new file mode 100644 index 000000000..704723041 --- /dev/null +++ b/cli/scripts/mutation-shards.d.mts @@ -0,0 +1,37 @@ +import type { MutationReport } from "./run-mutation.mjs"; + +type Mutate = string | readonly string[]; +type WeightOf = (path: string) => number; + +export function linesOf(path: string): number; +export function matchesGlob(glob: string, path: string): boolean; +export function inScope(mutate: Mutate, path: string): boolean; +export function scopeFiles(mutate: Mutate, files: readonly string[]): string[]; +export function partition( + mutate: Mutate, + files: readonly string[], + of: number, + weightOf?: WeightOf +): string[][]; +export function shardFiles( + mutate: Mutate, + files: readonly string[], + shard: number, + of: number, + weightOf?: WeightOf +): string[]; +export function mergeReports(reports: readonly MutationReport[]): Required; +export function aggregateVerdict(input: { + readonly mutate: Mutate; + readonly files: readonly string[]; + readonly of: number; + readonly shards: readonly { + readonly manifest: { + readonly shard: number; + readonly of: number; + readonly files: readonly string[]; + }; + readonly report: MutationReport; + }[]; + readonly weightOf?: WeightOf; +}): string | null; diff --git a/cli/scripts/mutation-shards.mjs b/cli/scripts/mutation-shards.mjs new file mode 100644 index 000000000..8bc1806a5 --- /dev/null +++ b/cli/scripts/mutation-shards.mjs @@ -0,0 +1,85 @@ +#!/usr/bin/env node +import { readFileSync } from "node:fs"; +import { dirname, join, resolve } from "node:path"; +import { fileURLToPath } from "node:url"; + +const CLI_ROOT = resolve(dirname(fileURLToPath(import.meta.url)), ".."); + +/** A file's weight in a shard: its line count, which the number of its mutants follows. */ +export function linesOf(path) { + return readFileSync(join(CLI_ROOT, path), "utf8").split("\n").length; +} + +/** A `mutate` glob as `mutation-scopes.json` writes them: `**` crosses directories, `*` does not. */ +export function matchesGlob(glob, path) { + const pattern = glob + .split("/**/") + .map((segment) => segment.split("*").map(escapeGlobLiteral).join("[^/]*")) + .join("/(?:.*/)?"); + return new RegExp(`^${pattern}$`).test(path); +} + +function escapeGlobLiteral(literal) { + return literal.replace(/[.+?^${}()|[\]\\]/g, "\\$&"); +} + +/** Whether a scope's `mutate` takes `path`: one of its globs matches and no `!` glob does. */ +export function inScope(mutate, path) { + const globs = [mutate].flat(); + return ( + globs.some((glob) => !glob.startsWith("!") && matchesGlob(glob, path)) && + !globs.some((glob) => glob.startsWith("!") && matchesGlob(glob.slice(1), path)) + ); +} + +/** The scope's files among `files`, sorted. */ +export function scopeFiles(mutate, files) { + return files.filter((path) => inScope(mutate, path)).sort(); +} + +/** The scope's files cut into `of` shards of about the same weight: heaviest file first, each + * to the lightest shard so far, ties broken by path and by shard number. The same files at the + * same weights give the same shards on every machine. */ +export function partition(mutate, files, of, weightOf = linesOf) { + const weighted = scopeFiles(mutate, files) + .map((path) => ({ path, weight: weightOf(path) })) + .sort((a, b) => b.weight - a.weight || (a.path < b.path ? -1 : 1)); + const shards = Array.from({ length: of }, () => ({ weight: 0, files: [] })); + for (const { path, weight } of weighted) { + const lightest = shards.reduce((min, shard) => (shard.weight < min.weight ? shard : min)); + lightest.files.push(path); + lightest.weight += weight; + } + return shards.map((shard) => shard.files.sort()); +} + +/** The files shard `shard` (from 1) of `of` mutates. */ +export function shardFiles(mutate, files, shard, of, weightOf = linesOf) { + return partition(mutate, files, of, weightOf)[shard - 1]; +} + +/** The shards' reports as one: each file is in exactly one shard, so nothing overlaps. */ +export function mergeReports(reports) { + return { files: Object.assign({}, ...reports.map((report) => report.files ?? {})) }; +} + +/** Why a sharded scope cannot be scored, or null. Every shard must be there, each must have + * mutated exactly what the partition gives it at this commit, and something must have been + * mutated: a missing or drifted shard would lift the score by leaving mutants out. */ +export function aggregateVerdict({ mutate, files, of, shards, weightOf = linesOf }) { + for (let shard = 1; shard <= of; shard += 1) { + const found = shards.find((candidate) => candidate.manifest.shard === shard); + if (found === undefined) return `shard ${shard} of ${of} is missing`; + if (found.manifest.of !== of) + return `shard ${shard} was cut in ${found.manifest.of}, not ${of}`; + const expected = shardFiles(mutate, files, shard, of, weightOf); + const mutated = [...found.manifest.files].sort(); + if (JSON.stringify(mutated) !== JSON.stringify(expected)) { + return `shard ${shard} of ${of} mutated other files than the partition gives it`; + } + } + const mutants = Object.values(mergeReports(shards.map((s) => s.report)).files).flatMap( + (file) => file.mutants + ).length; + return mutants === 0 ? "the shards hold no mutant" : null; +} diff --git a/cli/scripts/run-mutation.d.mts b/cli/scripts/run-mutation.d.mts index 699f3a40f..2eaf44979 100644 --- a/cli/scripts/run-mutation.d.mts +++ b/cli/scripts/run-mutation.d.mts @@ -1,6 +1,8 @@ export interface MutationScope { readonly mutate: string | readonly string[]; readonly break: number; + /** CI legs the scope is split into; one when absent. */ + readonly shards?: number; } export interface MutationReport { @@ -36,3 +38,11 @@ export function canarySelectionVerdict(report: MutationReport): string | null; export function changedRanges(diff: string): string[]; export function changedArgs(ranges: readonly string[]): string[]; export function survivorsOf(report: MutationReport): string[]; +export function shardConfig( + base: Readonly>, + shard: { readonly files: readonly string[]; readonly dir: string; readonly tempDirName: string } +): Record; +export function parseShard( + text: string | undefined, + declared: { readonly shards?: number } +): { shard: number; of: number }; diff --git a/cli/scripts/run-mutation.mjs b/cli/scripts/run-mutation.mjs index 178e2ca77..11d50cdf4 100644 --- a/cli/scripts/run-mutation.mjs +++ b/cli/scripts/run-mutation.mjs @@ -1,8 +1,18 @@ #!/usr/bin/env node import { spawnSync } from "node:child_process"; -import { existsSync, mkdirSync, readFileSync, renameSync, rmSync, writeFileSync } from "node:fs"; -import { dirname, join, resolve } from "node:path"; +import { + existsSync, + mkdirSync, + readdirSync, + readFileSync, + renameSync, + rmSync, + statSync, + writeFileSync, +} from "node:fs"; +import { dirname, join, relative, resolve } from "node:path"; import { fileURLToPath } from "node:url"; +import { aggregateVerdict, mergeReports, shardFiles } from "./mutation-shards.mjs"; const CLI_ROOT = resolve(dirname(fileURLToPath(import.meta.url)), ".."); const REPORT_ROOT = join(CLI_ROOT, "reports", "mutation"); @@ -33,6 +43,33 @@ export function strykerArgs(scope, scopes, { force = false } = {}) { return args; } +/** The config one shard runs under: the whole `stryker.conf.json`, with only what it mutates, + * where it keeps its incremental file, its sandbox and its reports made its own, so two shards + * never share a file and each measures exactly what one run would have measured of its files. */ +export function shardConfig(base, { files, dir, tempDirName }) { + return { + ...base, + mutate: files, + incremental: true, + incrementalFile: `${dir}/incremental.json`, + tempDirName, + htmlReporter: { ...base.htmlReporter, fileName: `${dir}/report.html` }, + jsonReporter: { ...base.jsonReporter, fileName: `${dir}/mutation.json` }, + }; +} + +/** `i/n`, with `n` the shards the scope declares: a leg cut otherwise measures another split. */ +export function parseShard(text, declared) { + const match = /^(\d+)\/(\d+)$/.exec(text ?? ""); + const of = declared.shards ?? 1; + if (match === null) throw new Error(`--shard takes i/n, got "${text}"`); + const [shard, n] = [Number(match[1]), Number(match[2])]; + if (n !== of || shard < 1 || shard > of) { + throw new Error(`--shard ${text} does not fit the ${of} shard(s) the scope declares`); + } + return { shard, of }; +} + /** Stryker's own score: detected (killed, timed out) over detected plus undetected (survived, * uncovered); an ignored or errored mutant counts on neither side. No mutant scores zero. */ export function scoreOf(report) { @@ -145,7 +182,7 @@ function pruneIncrementalFile(path) { function usage(problem, scopes) { console.error( - `${problem}\n\nUsage: node scripts/run-mutation.mjs [--force]\n node scripts/run-mutation.mjs --changed []` + `${problem}\n\nUsage: node scripts/run-mutation.mjs [--force]\n node scripts/run-mutation.mjs --shard / [--force] (CI)\n node scripts/run-mutation.mjs --aggregate (CI)\n node scripts/run-mutation.mjs --changed []` ); console.error(`Scopes: ${Object.keys(scopes).join(", ")}`); process.exit(1); @@ -190,6 +227,82 @@ function mainChanged(base = "origin/next") { for (const survivor of survivorsOf(report)) console.log(` survived: ${survivor}`); } +function sources() { + const out = []; + const walk = (dir) => { + for (const entry of readdirSync(dir)) { + const full = join(dir, entry); + if (statSync(full).isDirectory()) walk(full); + else if (entry.endsWith(".ts")) out.push(relative(CLI_ROOT, full).replaceAll("\\", "/")); + } + }; + walk(join(CLI_ROOT, "src")); + return out.sort(); +} + +/** One shard of a scope: its files only, no floor. The aggregate scores the scope. */ +function mainShard(scope, scopes, text, force) { + const { shard, of } = parseShard(text, scopes[scope]); + const dir = `reports/mutation/${scope}/shard-${shard}-of-${of}`; + mkdirSync(join(CLI_ROOT, dir), { recursive: true }); + pruneIncrementalFile(join(CLI_ROOT, dir, "incremental.json")); + const files = shardFiles(scopes[scope].mutate, sources(), shard, of); + writeFileSync(join(CLI_ROOT, dir, "manifest.json"), JSON.stringify({ scope, shard, of, files })); + const tempDirName = `.stryker-tmp-${scope}-${shard}`; + const base = JSON.parse(readFileSync(join(CLI_ROOT, "stryker.conf.json"), "utf8")); + const config = join(CLI_ROOT, dir, "stryker.conf.json"); + writeFileSync(config, JSON.stringify(shardConfig(base, { files, dir, tempDirName }))); + const args = ["run", config, ...(force ? ["--force"] : [])]; + const result = spawnSync(join(CLI_ROOT, "node_modules", ".bin", "stryker"), args, { + cwd: CLI_ROOT, + stdio: "inherit", + }); + rmSync(join(CLI_ROOT, tempDirName), { recursive: true, force: true }); + if (result.status !== 0) process.exit(result.status ?? 1); + const report = JSON.parse(readFileSync(join(CLI_ROOT, dir, "mutation.json"), "utf8")); + console.log( + `\nShard ${shard}/${of} of ${scope}: ${files.length} file(s), score ${scoreOf(report).toFixed(1)} (not gated)` + ); +} + +/** Scores a sharded scope once, over every shard's report in `from`, against its floor. */ +function mainAggregate(scope, scopes, from) { + const declared = scopes[scope]; + if (from === undefined || !existsSync(from)) { + console.error(`${scope}: no shard report was found${from === undefined ? "" : ` in ${from}`}`); + process.exit(1); + } + const shards = readdirSync(from) + .map((name) => join(from, name)) + .filter((dir) => existsSync(join(dir, "manifest.json"))) + .map((dir) => ({ + manifest: JSON.parse(readFileSync(join(dir, "manifest.json"), "utf8")), + report: JSON.parse(readFileSync(join(dir, "mutation.json"), "utf8")), + })) + .filter((shard) => shard.manifest.scope === scope); + const problem = aggregateVerdict({ + mutate: declared.mutate, + files: sources(), + of: declared.shards ?? 1, + shards, + }); + if (problem !== null) { + console.error(`${scope}: ${problem}`); + process.exit(1); + } + const merged = mergeReports(shards.map((shard) => shard.report)); + const score = scoreOf(merged); + console.log( + `\nReport: ${scope} over ${shards.length} shards (score ${score.toFixed(1)}, floor ${declared.break})` + ); + for (const survivor of survivorsOf(merged)) console.log(` survived: ${survivor}`); + const verdict = breakVerdict(score, declared); + if (verdict !== null) { + console.error(verdict); + process.exit(1); + } +} + function main() { const scopes = loadScopes(); const [scope, ...flags] = process.argv.slice(2); @@ -197,6 +310,10 @@ function main() { if (scope === undefined) usage("No scope given.", scopes); if (!Object.hasOwn(scopes, scope)) usage(`Unknown scope "${scope}".`, scopes); const force = flags.includes("--force"); + const shardAt = flags.indexOf("--shard"); + if (shardAt !== -1) return mainShard(scope, scopes, flags[shardAt + 1], force); + const aggregateAt = flags.indexOf("--aggregate"); + if (aggregateAt !== -1) return mainAggregate(scope, scopes, flags[aggregateAt + 1]); const scopeDir = join(REPORT_ROOT, scope); mkdirSync(scopeDir, { recursive: true }); diff --git a/cli/tests/architecture/helpers.ts b/cli/tests/architecture/helpers.ts index 12e3fbde0..b5f6fa380 100644 --- a/cli/tests/architecture/helpers.ts +++ b/cli/tests/architecture/helpers.ts @@ -4,6 +4,7 @@ */ import { existsSync, readdirSync, readFileSync, statSync } from "node:fs"; import { join, posix, relative, resolve } from "node:path"; +import { matchesGlob as scopeGlob } from "../../scripts/mutation-shards.mjs"; export const CLI_ROOT = resolve(import.meta.dirname, "..", ".."); export const SRC = join(CLI_ROOT, "src"); @@ -104,15 +105,7 @@ export function expectRatchet( * or a scope silently covers only its subdirectories — and `*` within one segment. */ export function matchesGlob(glob: string, path: string): boolean { - const pattern = glob - .split("/**/") - .map((segment) => segment.split("*").map(escapeGlobLiteral).join("[^/]*")) - .join("/(?:.*/)?"); - return new RegExp(`^${pattern}$`).test(path); -} - -function escapeGlobLiteral(literal: string): string { - return literal.replace(/[.+?^${}()|[\]\\]/g, "\\$&"); + return scopeGlob(glob, path); } // The context graph: single source of truth for which context may import which. It lives here diff --git a/cli/tests/architecture/mutation-shards.arch.test.ts b/cli/tests/architecture/mutation-shards.arch.test.ts new file mode 100644 index 000000000..23401b450 --- /dev/null +++ b/cli/tests/architecture/mutation-shards.arch.test.ts @@ -0,0 +1,184 @@ +import { describe, expect, it } from "vitest"; +import { legsOf } from "../../scripts/mutation-scopes-to-run.mjs"; +import { + aggregateVerdict, + inScope, + mergeReports, + partition, + scopeFiles, + shardFiles, +} from "../../scripts/mutation-shards.mjs"; +import { parseShard, scoreOf, shardConfig } from "../../scripts/run-mutation.mjs"; +import { read, sourceFiles } from "./helpers.js"; + +const scopes = (JSON.parse(read("mutation-scopes.json")) as { scopes: Record }) + .scopes; + +interface Scope { + readonly mutate: string | readonly string[]; + readonly break: number; + readonly shards?: number; +} + +const sharded = Object.entries(scopes).filter(([, scope]) => (scope.shards ?? 1) > 1); + +describe("splitting a scope into shards", () => { + it("declares shards only where a scope is worth splitting, never on the canary", () => { + expect(sharded.map(([name]) => name).sort()).toEqual([ + "framework", + "kernel", + "telemetry", + "tools", + ]); + expect(scopes.canary?.shards).toBeUndefined(); + }); + + it.each(sharded)("partitions %s: every file once, in exactly one shard", (_name, scope) => { + const all = scopeFiles(scope.mutate, sourceFiles()); + const of = scope.shards ?? 1; + const parts = Array.from({ length: of }, (_, i) => + shardFiles(scope.mutate, sourceFiles(), i + 1, of) + ); + expect(parts.flat().sort()).toEqual(all); + expect(new Set(parts.flat()).size).toBe(all.length); + for (const part of parts) expect(part.length).toBeGreaterThan(0); + }); + + it("balances the shards by weight, the same way on every run", () => { + const weights: Record = { + "src/a/h.ts": 9, + "src/a/m.ts": 5, + "src/a/n.ts": 4, + "src/a/l.ts": 1, + }; + const cut = () => + partition("src/a/**/*.ts", Object.keys(weights), 2, (path) => weights[path] ?? 0); + expect(cut()).toEqual([ + ["src/a/h.ts", "src/a/l.ts"], + ["src/a/m.ts", "src/a/n.ts"], + ]); + expect(cut()).toEqual(cut()); + }); + + it.each(sharded)( + "keeps the heaviest shard of %s within a quarter of its fair share", + (name, scope) => { + const of = scope.shards ?? 1; + const parts = partition(scope.mutate, sourceFiles(), of); + const weight = (files: string[]) => + files.reduce((sum, file) => sum + read(file).split("\n").length, 0); + const fair = weight(parts.flat()) / of; + expect(Math.max(...parts.map(weight)) / fair, name).toBeLessThan(1.25); + } + ); + + it("honours a ! glob, as the scope does", () => { + const mutate = ["src/t/**/*.ts", "!src/t/p/**/*.ts"]; + expect(inScope(mutate, "src/t/a.ts")).toBe(true); + expect(inScope(mutate, "src/t/p/b.ts")).toBe(false); + }); +}); + +describe("running the CI legs", () => { + it("gives a sharded scope one leg per shard, and any other scope one leg", () => { + const declared = { a: { mutate: "x", break: 1 }, b: { mutate: "y", break: 1, shards: 3 } }; + expect(legsOf(["a", "b"], declared)).toEqual([ + { scope: "a", shard: 1, of: 1 }, + { scope: "b", shard: 1, of: 3 }, + { scope: "b", shard: 2, of: 3 }, + { scope: "b", shard: 3, of: 3 }, + ]); + }); +}); + +const mutant = (status: string) => ({ status, mutatorName: "m", location: { start: { line: 1 } } }); +const report = (files: Record) => ({ + files: Object.fromEntries( + Object.entries(files).map(([name, statuses]) => [name, { mutants: statuses.map(mutant) }]) + ), +}); + +describe("scoring a sharded scope once, over every shard", () => { + const mutate = "src/a/**/*.ts"; + const files = ["src/a/x.ts", "src/a/y.ts", "src/a/z.ts", "src/a/w.ts"]; + const of = 2; + const weightOf = () => 1; + const shard = (i: number) => { + const assigned = shardFiles(mutate, files, i, of, weightOf); + return { + manifest: { shard: i, of, files: assigned }, + report: report(Object.fromEntries(assigned.map((file) => [file, ["Killed", "Survived"]]))), + }; + }; + + it("merges the shards into the report one run would have written", () => { + const merged = mergeReports([shard(1).report, shard(2).report]); + expect(Object.keys(merged.files).sort()).toEqual([...files].sort()); + expect(scoreOf(merged)).toBe(50); + }); + + it("passes when every shard is there and matches the partition", () => { + expect( + aggregateVerdict({ mutate, files, of, weightOf, shards: [shard(1), shard(2)] }) + ).toBeNull(); + }); + + it("fails when a shard is missing", () => { + expect(aggregateVerdict({ mutate, files, of, weightOf, shards: [shard(1)] })).toMatch( + /shard 2 of 2 is missing/ + ); + }); + + it("fails when a shard mutated other files than the partition gives it", () => { + const files2 = [...shard(2).manifest.files, ...shard(1).manifest.files]; + const wrong = { ...shard(2), manifest: { ...shard(2).manifest, files: files2 } }; + expect(aggregateVerdict({ mutate, files, of, weightOf, shards: [shard(1), wrong] })).toMatch( + /shard 2 of 2 mutated other files/ + ); + }); + + it("fails when the shards hold no mutant at all", () => { + const empty = [1, 2].map((i) => ({ ...shard(i), report: { files: {} } })); + expect(aggregateVerdict({ mutate, files, of, weightOf, shards: empty })).toMatch(/no mutant/); + }); +}); + +describe("running one shard", () => { + const base = JSON.parse(read("stryker.conf.json")) as Record; + + it("keeps the whole config and makes only its files, sandbox and reports its own", () => { + const config = shardConfig(base, { + files: ["src/kernel/a.ts"], + dir: "reports/mutation/kernel/shard-1-of-3", + tempDirName: ".stryker-tmp-kernel-1", + }); + const own = [ + "mutate", + "incremental", + "incrementalFile", + "tempDirName", + "htmlReporter", + "jsonReporter", + ]; + for (const [key, value] of Object.entries(base)) { + if (!own.includes(key)) expect(config[key], key).toEqual(value); + } + expect(config).toMatchObject({ + mutate: ["src/kernel/a.ts"], + incremental: true, + incrementalFile: "reports/mutation/kernel/shard-1-of-3/incremental.json", + tempDirName: ".stryker-tmp-kernel-1", + jsonReporter: { fileName: "reports/mutation/kernel/shard-1-of-3/mutation.json" }, + htmlReporter: { fileName: "reports/mutation/kernel/shard-1-of-3/report.html" }, + }); + }); + + it("takes only the cut the scope declares", () => { + expect(parseShard("2/3", { shards: 3 })).toEqual({ shard: 2, of: 3 }); + expect(() => parseShard("2/4", { shards: 3 })).toThrow(/does not fit/); + expect(() => parseShard("4/3", { shards: 3 })).toThrow(/does not fit/); + expect(() => parseShard("0/3", { shards: 3 })).toThrow(/does not fit/); + expect(() => parseShard("1/1", {})).not.toThrow(); + expect(() => parseShard("x", { shards: 3 })).toThrow(/takes i\/n/); + }); +}); diff --git a/scripts/__tests__/cli-ci-gate-covers-every-job.test.js b/scripts/__tests__/cli-ci-gate-covers-every-job.test.js index d67cee4ab..16701f98a 100644 --- a/scripts/__tests__/cli-ci-gate-covers-every-job.test.js +++ b/scripts/__tests__/cli-ci-gate-covers-every-job.test.js @@ -113,9 +113,12 @@ test("only a proven next snapshot tree skips the promotion or main mutation matr assert.match(promotion.run, /HEAD_SHA="\$snapshot_sha"/); assert.match(mutation.run, /steps\.promotion\.outputs\.trusted.*== "true"/); - assert.match(mutation.run, /scopes='\[\]'/); + assert.match(mutation.run, /legs='\[\]'/); + assert.match(mutation.run, /sharded='\[\]'/); assert.match(mutation.run, /else[\s\S]*mutation-scopes-to-run\.mjs/); - assert.equal(workflow.jobs["cli-mutation"].if, "needs.changes.outputs.mutation_scopes != '[]'"); + assert.equal(workflow.jobs["cli-mutation"].if, "needs.changes.outputs.mutation_legs != '[]'"); + // The scoring job runs whatever its shards did, so a crashed shard fails it, never skips it. + assert.match(workflow.jobs["cli-mutation-score"].if, /^always\(\) && needs\.changes\.outputs\.mutation_sharded != '\[\]'/); // These checks validate the current PR merge ref or main commit; reuse never turns them off. for (const name of [ From 671c3c8ad77a48be065b6e52fd825b4f9e260233 Mon Sep 17 00:00:00 2001 From: Baptiste LAFOURCADE Date: Mon, 12 Oct 2026 17:09:41 +0200 Subject: [PATCH 2/3] ci(cli): shard kernel in four, tools-codex and translate in two After the first split, the warm run's slowest legs were tools-codex (10.3 min), kernel (9.6) and translate. Their shares now run in parallel too. Kernel's heaviest file alone outweighs a quarter of the scope, so the balance check bounds a shard by a quarter over its fair share or by its heaviest file, whichever is larger. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01VWNxk63AGKkqE8HRqHLjGp --- cli/mutation-scopes.json | 8 +++++--- cli/tests/architecture/mutation-shards.arch.test.ts | 9 +++++++-- 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/cli/mutation-scopes.json b/cli/mutation-scopes.json index 17900ee5e..fae55210b 100644 --- a/cli/mutation-scopes.json +++ b/cli/mutation-scopes.json @@ -4,7 +4,7 @@ "kernel": { "mutate": "src/kernel/**/*.ts", "break": 95, - "shards": 3 + "shards": 4 }, "tools": { "mutate": [ @@ -26,7 +26,8 @@ }, "translate": { "mutate": "src/contexts/translate/**/*.ts", - "break": 85 + "break": 85, + "shards": 2 }, "distribution": { "mutate": "src/contexts/distribution/**/*.ts", @@ -51,7 +52,8 @@ }, "tools-codex": { "mutate": "src/contexts/tools/domain/profiles/codex/**/*.ts", - "break": 94 + "break": 94, + "shards": 2 }, "tools-copilot": { "mutate": "src/contexts/tools/domain/profiles/copilot/**/*.ts", diff --git a/cli/tests/architecture/mutation-shards.arch.test.ts b/cli/tests/architecture/mutation-shards.arch.test.ts index 23401b450..da803dc0f 100644 --- a/cli/tests/architecture/mutation-shards.arch.test.ts +++ b/cli/tests/architecture/mutation-shards.arch.test.ts @@ -29,6 +29,8 @@ describe("splitting a scope into shards", () => { "kernel", "telemetry", "tools", + "tools-codex", + "translate", ]); expect(scopes.canary?.shards).toBeUndefined(); }); @@ -61,14 +63,17 @@ describe("splitting a scope into shards", () => { }); it.each(sharded)( - "keeps the heaviest shard of %s within a quarter of its fair share", + "keeps the heaviest shard of %s within a quarter of its fair share, or its heaviest file", (name, scope) => { const of = scope.shards ?? 1; const parts = partition(scope.mutate, sourceFiles(), of); const weight = (files: string[]) => files.reduce((sum, file) => sum + read(file).split("\n").length, 0); const fair = weight(parts.flat()) / of; - expect(Math.max(...parts.map(weight)) / fair, name).toBeLessThan(1.25); + const heaviestFile = Math.max(...parts.flat().map((file) => weight([file]))); + expect(Math.max(...parts.map(weight)), name).toBeLessThanOrEqual( + Math.max(1.25 * fair, heaviestFile) + ); } ); From 98553183878bcdbd3b9157a91f0e791d012ae8a9 Mon Sep 17 00:00:00 2001 From: Baptiste LAFOURCADE Date: Mon, 12 Oct 2026 18:46:30 +0200 Subject: [PATCH 3/3] refactor(cli): one walk of src/ for the shards and the tests run-mutation.mjs walked src/ on its own, beside the architecture tests' sourceFiles(). Both now use the one in mutation-shards.mjs, so the CI cuts exactly the file list the partition tests check. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01VWNxk63AGKkqE8HRqHLjGp --- cli/scripts/mutation-shards.d.mts | 1 + cli/scripts/mutation-shards.mjs | 18 ++++++++++++++++-- cli/scripts/run-mutation.mjs | 22 ++++------------------ cli/tests/architecture/helpers.ts | 20 +++++++------------- 4 files changed, 28 insertions(+), 33 deletions(-) diff --git a/cli/scripts/mutation-shards.d.mts b/cli/scripts/mutation-shards.d.mts index 704723041..f36d2e72b 100644 --- a/cli/scripts/mutation-shards.d.mts +++ b/cli/scripts/mutation-shards.d.mts @@ -3,6 +3,7 @@ import type { MutationReport } from "./run-mutation.mjs"; type Mutate = string | readonly string[]; type WeightOf = (path: string) => number; +export function sourceFiles(): string[]; export function linesOf(path: string): number; export function matchesGlob(glob: string, path: string): boolean; export function inScope(mutate: Mutate, path: string): boolean; diff --git a/cli/scripts/mutation-shards.mjs b/cli/scripts/mutation-shards.mjs index 8bc1806a5..749208029 100644 --- a/cli/scripts/mutation-shards.mjs +++ b/cli/scripts/mutation-shards.mjs @@ -1,10 +1,24 @@ #!/usr/bin/env node -import { readFileSync } from "node:fs"; -import { dirname, join, resolve } from "node:path"; +import { readdirSync, readFileSync, statSync } from "node:fs"; +import { dirname, join, relative, resolve } from "node:path"; import { fileURLToPath } from "node:url"; const CLI_ROOT = resolve(dirname(fileURLToPath(import.meta.url)), ".."); +/** Every `.ts` file under `src/`, relative to `cli/` with `/`, sorted: what a scope is cut from. */ +export function sourceFiles() { + const out = []; + const walk = (dir) => { + for (const entry of readdirSync(dir)) { + const full = join(dir, entry); + if (statSync(full).isDirectory()) walk(full); + else if (entry.endsWith(".ts")) out.push(relative(CLI_ROOT, full).replaceAll("\\", "/")); + } + }; + walk(join(CLI_ROOT, "src")); + return out.sort(); +} + /** A file's weight in a shard: its line count, which the number of its mutants follows. */ export function linesOf(path) { return readFileSync(join(CLI_ROOT, path), "utf8").split("\n").length; diff --git a/cli/scripts/run-mutation.mjs b/cli/scripts/run-mutation.mjs index 11d50cdf4..ccb9496c9 100644 --- a/cli/scripts/run-mutation.mjs +++ b/cli/scripts/run-mutation.mjs @@ -7,12 +7,11 @@ import { readFileSync, renameSync, rmSync, - statSync, writeFileSync, } from "node:fs"; -import { dirname, join, relative, resolve } from "node:path"; +import { dirname, join, resolve } from "node:path"; import { fileURLToPath } from "node:url"; -import { aggregateVerdict, mergeReports, shardFiles } from "./mutation-shards.mjs"; +import { aggregateVerdict, mergeReports, shardFiles, sourceFiles } from "./mutation-shards.mjs"; const CLI_ROOT = resolve(dirname(fileURLToPath(import.meta.url)), ".."); const REPORT_ROOT = join(CLI_ROOT, "reports", "mutation"); @@ -227,26 +226,13 @@ function mainChanged(base = "origin/next") { for (const survivor of survivorsOf(report)) console.log(` survived: ${survivor}`); } -function sources() { - const out = []; - const walk = (dir) => { - for (const entry of readdirSync(dir)) { - const full = join(dir, entry); - if (statSync(full).isDirectory()) walk(full); - else if (entry.endsWith(".ts")) out.push(relative(CLI_ROOT, full).replaceAll("\\", "/")); - } - }; - walk(join(CLI_ROOT, "src")); - return out.sort(); -} - /** One shard of a scope: its files only, no floor. The aggregate scores the scope. */ function mainShard(scope, scopes, text, force) { const { shard, of } = parseShard(text, scopes[scope]); const dir = `reports/mutation/${scope}/shard-${shard}-of-${of}`; mkdirSync(join(CLI_ROOT, dir), { recursive: true }); pruneIncrementalFile(join(CLI_ROOT, dir, "incremental.json")); - const files = shardFiles(scopes[scope].mutate, sources(), shard, of); + const files = shardFiles(scopes[scope].mutate, sourceFiles(), shard, of); writeFileSync(join(CLI_ROOT, dir, "manifest.json"), JSON.stringify({ scope, shard, of, files })); const tempDirName = `.stryker-tmp-${scope}-${shard}`; const base = JSON.parse(readFileSync(join(CLI_ROOT, "stryker.conf.json"), "utf8")); @@ -282,7 +268,7 @@ function mainAggregate(scope, scopes, from) { .filter((shard) => shard.manifest.scope === scope); const problem = aggregateVerdict({ mutate: declared.mutate, - files: sources(), + files: sourceFiles(), of: declared.shards ?? 1, shards, }); diff --git a/cli/tests/architecture/helpers.ts b/cli/tests/architecture/helpers.ts index b5f6fa380..d13a8875a 100644 --- a/cli/tests/architecture/helpers.ts +++ b/cli/tests/architecture/helpers.ts @@ -2,9 +2,12 @@ * These tests read source as text and never import the code under test, so they stay fast * enough for a pre-commit hook and cannot be broken by runtime wiring. */ -import { existsSync, readdirSync, readFileSync, statSync } from "node:fs"; -import { join, posix, relative, resolve } from "node:path"; -import { matchesGlob as scopeGlob } from "../../scripts/mutation-shards.mjs"; +import { existsSync, readdirSync, readFileSync } from "node:fs"; +import { join, posix, resolve } from "node:path"; +import { + matchesGlob as scopeGlob, + sourceFiles as srcFiles, +} from "../../scripts/mutation-shards.mjs"; export const CLI_ROOT = resolve(import.meta.dirname, "..", ".."); export const SRC = join(CLI_ROOT, "src"); @@ -16,16 +19,7 @@ export function canonicalPath(path: string): string { } export function sourceFiles(): string[] { - const out: string[] = []; - const walk = (dir: string): void => { - for (const entry of readdirSync(dir)) { - const full = join(dir, entry); - if (statSync(full).isDirectory()) walk(full); - else if (entry.endsWith(".ts")) out.push(canonicalPath(relative(CLI_ROOT, full))); - } - }; - walk(SRC); - return out.sort(); + return srcFiles(); } export function read(relativePath: string): string {