diff --git a/.github/workflows/cli-ci.yml b/.github/workflows/cli-ci.yml index b7e2ca1bc..7c94687db 100644 --- a/.github/workflows/cli-ci.yml +++ b/.github/workflows/cli-ci.yml @@ -35,7 +35,8 @@ jobs: outputs: relevant: ${{ steps.filter.outputs.relevant }} trusted_promotion: ${{ steps.promotion.outputs.trusted }} - mutation_scopes: ${{ steps.mutation.outputs.scopes }} + mutation_legs: ${{ steps.mutation.outputs.legs }} + mutation_sharded: ${{ steps.mutation.outputs.sharded }} steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: @@ -183,7 +184,8 @@ jobs: run: | set -euo pipefail if [[ "${{ steps.promotion.outputs.trusted }}" == "true" ]]; then - scopes='[]' + legs='[]' + sharded='[]' else if [[ "${{ github.event_name }}" == "pull_request" ]]; then BASE="${{ github.event.pull_request.base.sha }}" @@ -199,10 +201,13 @@ jobs: changed="$(git diff --name-only "$BASE" "$HEAD")" all=false fi - scopes="$(ALL="$all" CHANGED="$changed" node cli/scripts/mutation-scopes-to-run.mjs)" + legs="$(ALL="$all" CHANGED="$changed" node cli/scripts/mutation-scopes-to-run.mjs)" + sharded="$(ALL="$all" CHANGED="$changed" OUTPUT=sharded node cli/scripts/mutation-scopes-to-run.mjs)" fi - echo "mutation scopes: $scopes" - echo "scopes=$scopes" >> "$GITHUB_OUTPUT" + echo "mutation legs: $legs" + echo "sharded scopes: $sharded" + echo "legs=$legs" >> "$GITHUB_OUTPUT" + echo "sharded=$sharded" >> "$GITHUB_OUTPUT" cli-typecheck: name: cli / Typecheck @@ -583,15 +588,21 @@ jobs: # nothing relevant. The one check the branch rulesets require — without it a pull request # merges with the real jobs red, because nothing names them. cli-mutation: - name: cli / Mutation (${{ matrix.scope }}) + # A heavy scope runs as several legs in parallel, each mutating its share of the files; the + # job below scores it once over every leg. Any other scope is one leg and gates itself. + name: cli / Mutation (${{ matrix.scope }}${{ matrix.of > 1 && format(' {0}/{1}', matrix.shard, matrix.of) || '' }}) needs: [changes] - if: needs.changes.outputs.mutation_scopes != '[]' + if: needs.changes.outputs.mutation_legs != '[]' runs-on: ubuntu-latest timeout-minutes: 90 strategy: fail-fast: false matrix: - scope: ${{ fromJSON(needs.changes.outputs.mutation_scopes) }} + include: ${{ fromJSON(needs.changes.outputs.mutation_legs) }} + env: + # `--` closes the name: `tools` must never restore `tools-codex`'s file. + LEG: ${{ matrix.of > 1 && format('{0}-shard-{1}-of-{2}', matrix.scope, matrix.shard, matrix.of) || matrix.scope }} + LEG_DIR: cli/reports/mutation/${{ matrix.of > 1 && format('{0}/shard-{1}-of-{2}', matrix.scope, matrix.shard, matrix.of) || matrix.scope }} steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - name: Install pnpm @@ -608,19 +619,51 @@ jobs: - run: cd cli && pnpm install --frozen-lockfile # The incremental file is what keeps a run to the mutants a change can move; the newest # one this branch or its base saved is restored, and this run's is saved whatever the score. - - name: Restore this scope's incremental file + - name: Restore this leg's incremental file uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 with: - path: cli/reports/mutation/${{ matrix.scope }}/incremental.json - key: mutation-${{ matrix.scope }}-${{ github.run_id }} - restore-keys: mutation-${{ matrix.scope }}- - - run: cd cli && node scripts/run-mutation.mjs "${{ matrix.scope }}" ${{ github.event_name == 'schedule' && '--force' || '' }} - - name: Save this scope's incremental file + path: ${{ env.LEG_DIR }}/incremental.json + key: mutation--${{ env.LEG }}--${{ github.run_id }} + restore-keys: mutation--${{ env.LEG }}-- + - run: cd cli && node scripts/run-mutation.mjs "${{ matrix.scope }}" ${{ matrix.of > 1 && format('--shard {0}/{1}', matrix.shard, matrix.of) || '' }} ${{ github.event_name == 'schedule' && '--force' || '' }} + - name: Save this leg's incremental file if: always() uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 with: - path: cli/reports/mutation/${{ matrix.scope }}/incremental.json - key: mutation-${{ matrix.scope }}-${{ github.run_id }} + path: ${{ env.LEG_DIR }}/incremental.json + key: mutation--${{ env.LEG }}--${{ github.run_id }} + - name: Hand this shard's report to the scoring job + if: matrix.of > 1 + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: mutation-${{ matrix.scope }}-shard-${{ matrix.shard }}-of-${{ matrix.of }} + path: | + ${{ env.LEG_DIR }}/manifest.json + ${{ env.LEG_DIR }}/mutation.json + if-no-files-found: error + retention-days: 1 + + cli-mutation-score: + name: cli / Mutation (${{ matrix.scope }}) + needs: [changes, cli-mutation] + # Always, so a crashed or cancelled shard is scored as missing instead of skipped. + if: always() && needs.changes.outputs.mutation_sharded != '[]' && needs.changes.outputs.mutation_sharded != '' + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + scope: ${{ fromJSON(needs.changes.outputs.mutation_sharded) }} + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 + with: + node-version: "22" + - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 + with: + pattern: mutation-${{ matrix.scope }}-shard-* + path: shards + - name: Score the scope over every shard, against its floor + run: node cli/scripts/run-mutation.mjs "${{ matrix.scope }}" --aggregate "$GITHUB_WORKSPACE/shards" gate: name: cli / gate @@ -639,6 +682,7 @@ jobs: - identifier-join - cli-jscpd - cli-mutation + - cli-mutation-score - kanban-checks - windows steps: @@ -660,6 +704,7 @@ jobs: "${{ needs.identifier-join.result }}" \ "${{ needs.cli-jscpd.result }}" \ "${{ needs.cli-mutation.result }}" \ + "${{ needs.cli-mutation-score.result }}" \ "${{ needs.kanban-checks.result }}" \ "${{ needs.windows.result }}"; do if [[ "$result" == "failure" || "$result" == "cancelled" ]]; then diff --git a/cli/mutation-scopes.json b/cli/mutation-scopes.json index 4c6938794..fae55210b 100644 --- a/cli/mutation-scopes.json +++ b/cli/mutation-scopes.json @@ -3,7 +3,8 @@ "scopes": { "kernel": { "mutate": "src/kernel/**/*.ts", - "break": 95 + "break": 95, + "shards": 4 }, "tools": { "mutate": [ @@ -15,15 +16,18 @@ "!src/contexts/tools/domain/profiles/kilo/**/*.ts", "!src/contexts/tools/domain/profiles/opencode/**/*.ts" ], - "break": 94 + "break": 94, + "shards": 4 }, "telemetry": { "mutate": "src/contexts/telemetry/**/*.ts", - "break": 93 + "break": 93, + "shards": 3 }, "translate": { "mutate": "src/contexts/translate/**/*.ts", - "break": 85 + "break": 85, + "shards": 2 }, "distribution": { "mutate": "src/contexts/distribution/**/*.ts", @@ -31,7 +35,8 @@ }, "framework": { "mutate": "src/contexts/framework/**/*.ts", - "break": 93 + "break": 93, + "shards": 4 }, "presentation": { "mutate": "src/presentation/**/*.ts", @@ -47,7 +52,8 @@ }, "tools-codex": { "mutate": "src/contexts/tools/domain/profiles/codex/**/*.ts", - "break": 94 + "break": 94, + "shards": 2 }, "tools-copilot": { "mutate": "src/contexts/tools/domain/profiles/copilot/**/*.ts", diff --git a/cli/scripts/mutation-scopes-to-run.d.mts b/cli/scripts/mutation-scopes-to-run.d.mts index 45eb1961e..a7bd03703 100644 --- a/cli/scripts/mutation-scopes-to-run.d.mts +++ b/cli/scripts/mutation-scopes-to-run.d.mts @@ -6,3 +6,7 @@ export function scopesToRun( scopes: Readonly>, options?: { readonly all?: boolean } ): string[]; +export function legsOf( + names: readonly string[], + scopes: Readonly> +): { scope: string; shard: number; of: number }[]; diff --git a/cli/scripts/mutation-scopes-to-run.mjs b/cli/scripts/mutation-scopes-to-run.mjs index e693bc684..2eb440cfa 100644 --- a/cli/scripts/mutation-scopes-to-run.mjs +++ b/cli/scripts/mutation-scopes-to-run.mjs @@ -12,6 +12,7 @@ export const HARNESS = [ "cli/vitest.mutation.config.ts", "cli/scripts/run-mutation.mjs", "cli/scripts/mutation-scopes-to-run.mjs", + "cli/scripts/mutation-shards.mjs", "cli/package.json", "cli/pnpm-lock.yaml", ".github/workflows/cli-ci.yml", @@ -36,10 +37,22 @@ export function scopesToRun(changed, scopes, { all = false } = {}) { .map(([name]) => name); } +/** The CI legs for `names`: one per shard of a sharded scope, one for any other. */ +export function legsOf(names, scopes) { + return names.flatMap((scope) => { + const of = scopes[scope].shards ?? 1; + return Array.from({ length: of }, (_, i) => ({ scope, shard: i + 1, of })); + }); +} + if (process.argv[1] !== undefined && resolve(process.argv[1]) === fileURLToPath(import.meta.url)) { const scopes = JSON.parse(readFileSync(join(CLI_ROOT, "mutation-scopes.json"), "utf8")).scopes; const changed = (process.env.CHANGED ?? "").split("\n").filter(Boolean); - process.stdout.write( - JSON.stringify(scopesToRun(changed, scopes, { all: process.env.ALL === "true" })) - ); + const names = scopesToRun(changed, scopes, { all: process.env.ALL === "true" }); + // `legs` feeds the mutation matrix; `sharded` the jobs that score a split scope once. + const output = + process.env.OUTPUT === "sharded" + ? names.filter((name) => (scopes[name].shards ?? 1) > 1) + : legsOf(names, scopes); + process.stdout.write(JSON.stringify(output)); } diff --git a/cli/scripts/mutation-shards.d.mts b/cli/scripts/mutation-shards.d.mts new file mode 100644 index 000000000..f36d2e72b --- /dev/null +++ b/cli/scripts/mutation-shards.d.mts @@ -0,0 +1,38 @@ +import type { MutationReport } from "./run-mutation.mjs"; + +type Mutate = string | readonly string[]; +type WeightOf = (path: string) => number; + +export function sourceFiles(): string[]; +export function linesOf(path: string): number; +export function matchesGlob(glob: string, path: string): boolean; +export function inScope(mutate: Mutate, path: string): boolean; +export function scopeFiles(mutate: Mutate, files: readonly string[]): string[]; +export function partition( + mutate: Mutate, + files: readonly string[], + of: number, + weightOf?: WeightOf +): string[][]; +export function shardFiles( + mutate: Mutate, + files: readonly string[], + shard: number, + of: number, + weightOf?: WeightOf +): string[]; +export function mergeReports(reports: readonly MutationReport[]): Required; +export function aggregateVerdict(input: { + readonly mutate: Mutate; + readonly files: readonly string[]; + readonly of: number; + readonly shards: readonly { + readonly manifest: { + readonly shard: number; + readonly of: number; + readonly files: readonly string[]; + }; + readonly report: MutationReport; + }[]; + readonly weightOf?: WeightOf; +}): string | null; diff --git a/cli/scripts/mutation-shards.mjs b/cli/scripts/mutation-shards.mjs new file mode 100644 index 000000000..749208029 --- /dev/null +++ b/cli/scripts/mutation-shards.mjs @@ -0,0 +1,99 @@ +#!/usr/bin/env node +import { readdirSync, readFileSync, statSync } from "node:fs"; +import { dirname, join, relative, resolve } from "node:path"; +import { fileURLToPath } from "node:url"; + +const CLI_ROOT = resolve(dirname(fileURLToPath(import.meta.url)), ".."); + +/** Every `.ts` file under `src/`, relative to `cli/` with `/`, sorted: what a scope is cut from. */ +export function sourceFiles() { + const out = []; + const walk = (dir) => { + for (const entry of readdirSync(dir)) { + const full = join(dir, entry); + if (statSync(full).isDirectory()) walk(full); + else if (entry.endsWith(".ts")) out.push(relative(CLI_ROOT, full).replaceAll("\\", "/")); + } + }; + walk(join(CLI_ROOT, "src")); + return out.sort(); +} + +/** A file's weight in a shard: its line count, which the number of its mutants follows. */ +export function linesOf(path) { + return readFileSync(join(CLI_ROOT, path), "utf8").split("\n").length; +} + +/** A `mutate` glob as `mutation-scopes.json` writes them: `**` crosses directories, `*` does not. */ +export function matchesGlob(glob, path) { + const pattern = glob + .split("/**/") + .map((segment) => segment.split("*").map(escapeGlobLiteral).join("[^/]*")) + .join("/(?:.*/)?"); + return new RegExp(`^${pattern}$`).test(path); +} + +function escapeGlobLiteral(literal) { + return literal.replace(/[.+?^${}()|[\]\\]/g, "\\$&"); +} + +/** Whether a scope's `mutate` takes `path`: one of its globs matches and no `!` glob does. */ +export function inScope(mutate, path) { + const globs = [mutate].flat(); + return ( + globs.some((glob) => !glob.startsWith("!") && matchesGlob(glob, path)) && + !globs.some((glob) => glob.startsWith("!") && matchesGlob(glob.slice(1), path)) + ); +} + +/** The scope's files among `files`, sorted. */ +export function scopeFiles(mutate, files) { + return files.filter((path) => inScope(mutate, path)).sort(); +} + +/** The scope's files cut into `of` shards of about the same weight: heaviest file first, each + * to the lightest shard so far, ties broken by path and by shard number. The same files at the + * same weights give the same shards on every machine. */ +export function partition(mutate, files, of, weightOf = linesOf) { + const weighted = scopeFiles(mutate, files) + .map((path) => ({ path, weight: weightOf(path) })) + .sort((a, b) => b.weight - a.weight || (a.path < b.path ? -1 : 1)); + const shards = Array.from({ length: of }, () => ({ weight: 0, files: [] })); + for (const { path, weight } of weighted) { + const lightest = shards.reduce((min, shard) => (shard.weight < min.weight ? shard : min)); + lightest.files.push(path); + lightest.weight += weight; + } + return shards.map((shard) => shard.files.sort()); +} + +/** The files shard `shard` (from 1) of `of` mutates. */ +export function shardFiles(mutate, files, shard, of, weightOf = linesOf) { + return partition(mutate, files, of, weightOf)[shard - 1]; +} + +/** The shards' reports as one: each file is in exactly one shard, so nothing overlaps. */ +export function mergeReports(reports) { + return { files: Object.assign({}, ...reports.map((report) => report.files ?? {})) }; +} + +/** Why a sharded scope cannot be scored, or null. Every shard must be there, each must have + * mutated exactly what the partition gives it at this commit, and something must have been + * mutated: a missing or drifted shard would lift the score by leaving mutants out. */ +export function aggregateVerdict({ mutate, files, of, shards, weightOf = linesOf }) { + for (let shard = 1; shard <= of; shard += 1) { + const found = shards.find((candidate) => candidate.manifest.shard === shard); + if (found === undefined) return `shard ${shard} of ${of} is missing`; + if (found.manifest.of !== of) + return `shard ${shard} was cut in ${found.manifest.of}, not ${of}`; + const expected = shardFiles(mutate, files, shard, of, weightOf); + const mutated = [...found.manifest.files].sort(); + if (JSON.stringify(mutated) !== JSON.stringify(expected)) { + return `shard ${shard} of ${of} mutated other files than the partition gives it`; + } + } + const mutants = Object.values(mergeReports(shards.map((s) => s.report)).files).flatMap( + (file) => file.mutants + ).length; + return mutants === 0 ? "the shards hold no mutant" : null; +} diff --git a/cli/scripts/run-mutation.d.mts b/cli/scripts/run-mutation.d.mts index 699f3a40f..2eaf44979 100644 --- a/cli/scripts/run-mutation.d.mts +++ b/cli/scripts/run-mutation.d.mts @@ -1,6 +1,8 @@ export interface MutationScope { readonly mutate: string | readonly string[]; readonly break: number; + /** CI legs the scope is split into; one when absent. */ + readonly shards?: number; } export interface MutationReport { @@ -36,3 +38,11 @@ export function canarySelectionVerdict(report: MutationReport): string | null; export function changedRanges(diff: string): string[]; export function changedArgs(ranges: readonly string[]): string[]; export function survivorsOf(report: MutationReport): string[]; +export function shardConfig( + base: Readonly>, + shard: { readonly files: readonly string[]; readonly dir: string; readonly tempDirName: string } +): Record; +export function parseShard( + text: string | undefined, + declared: { readonly shards?: number } +): { shard: number; of: number }; diff --git a/cli/scripts/run-mutation.mjs b/cli/scripts/run-mutation.mjs index 178e2ca77..ccb9496c9 100644 --- a/cli/scripts/run-mutation.mjs +++ b/cli/scripts/run-mutation.mjs @@ -1,8 +1,17 @@ #!/usr/bin/env node import { spawnSync } from "node:child_process"; -import { existsSync, mkdirSync, readFileSync, renameSync, rmSync, writeFileSync } from "node:fs"; +import { + existsSync, + mkdirSync, + readdirSync, + readFileSync, + renameSync, + rmSync, + writeFileSync, +} from "node:fs"; import { dirname, join, resolve } from "node:path"; import { fileURLToPath } from "node:url"; +import { aggregateVerdict, mergeReports, shardFiles, sourceFiles } from "./mutation-shards.mjs"; const CLI_ROOT = resolve(dirname(fileURLToPath(import.meta.url)), ".."); const REPORT_ROOT = join(CLI_ROOT, "reports", "mutation"); @@ -33,6 +42,33 @@ export function strykerArgs(scope, scopes, { force = false } = {}) { return args; } +/** The config one shard runs under: the whole `stryker.conf.json`, with only what it mutates, + * where it keeps its incremental file, its sandbox and its reports made its own, so two shards + * never share a file and each measures exactly what one run would have measured of its files. */ +export function shardConfig(base, { files, dir, tempDirName }) { + return { + ...base, + mutate: files, + incremental: true, + incrementalFile: `${dir}/incremental.json`, + tempDirName, + htmlReporter: { ...base.htmlReporter, fileName: `${dir}/report.html` }, + jsonReporter: { ...base.jsonReporter, fileName: `${dir}/mutation.json` }, + }; +} + +/** `i/n`, with `n` the shards the scope declares: a leg cut otherwise measures another split. */ +export function parseShard(text, declared) { + const match = /^(\d+)\/(\d+)$/.exec(text ?? ""); + const of = declared.shards ?? 1; + if (match === null) throw new Error(`--shard takes i/n, got "${text}"`); + const [shard, n] = [Number(match[1]), Number(match[2])]; + if (n !== of || shard < 1 || shard > of) { + throw new Error(`--shard ${text} does not fit the ${of} shard(s) the scope declares`); + } + return { shard, of }; +} + /** Stryker's own score: detected (killed, timed out) over detected plus undetected (survived, * uncovered); an ignored or errored mutant counts on neither side. No mutant scores zero. */ export function scoreOf(report) { @@ -145,7 +181,7 @@ function pruneIncrementalFile(path) { function usage(problem, scopes) { console.error( - `${problem}\n\nUsage: node scripts/run-mutation.mjs [--force]\n node scripts/run-mutation.mjs --changed []` + `${problem}\n\nUsage: node scripts/run-mutation.mjs [--force]\n node scripts/run-mutation.mjs --shard / [--force] (CI)\n node scripts/run-mutation.mjs --aggregate (CI)\n node scripts/run-mutation.mjs --changed []` ); console.error(`Scopes: ${Object.keys(scopes).join(", ")}`); process.exit(1); @@ -190,6 +226,69 @@ function mainChanged(base = "origin/next") { for (const survivor of survivorsOf(report)) console.log(` survived: ${survivor}`); } +/** One shard of a scope: its files only, no floor. The aggregate scores the scope. */ +function mainShard(scope, scopes, text, force) { + const { shard, of } = parseShard(text, scopes[scope]); + const dir = `reports/mutation/${scope}/shard-${shard}-of-${of}`; + mkdirSync(join(CLI_ROOT, dir), { recursive: true }); + pruneIncrementalFile(join(CLI_ROOT, dir, "incremental.json")); + const files = shardFiles(scopes[scope].mutate, sourceFiles(), shard, of); + writeFileSync(join(CLI_ROOT, dir, "manifest.json"), JSON.stringify({ scope, shard, of, files })); + const tempDirName = `.stryker-tmp-${scope}-${shard}`; + const base = JSON.parse(readFileSync(join(CLI_ROOT, "stryker.conf.json"), "utf8")); + const config = join(CLI_ROOT, dir, "stryker.conf.json"); + writeFileSync(config, JSON.stringify(shardConfig(base, { files, dir, tempDirName }))); + const args = ["run", config, ...(force ? ["--force"] : [])]; + const result = spawnSync(join(CLI_ROOT, "node_modules", ".bin", "stryker"), args, { + cwd: CLI_ROOT, + stdio: "inherit", + }); + rmSync(join(CLI_ROOT, tempDirName), { recursive: true, force: true }); + if (result.status !== 0) process.exit(result.status ?? 1); + const report = JSON.parse(readFileSync(join(CLI_ROOT, dir, "mutation.json"), "utf8")); + console.log( + `\nShard ${shard}/${of} of ${scope}: ${files.length} file(s), score ${scoreOf(report).toFixed(1)} (not gated)` + ); +} + +/** Scores a sharded scope once, over every shard's report in `from`, against its floor. */ +function mainAggregate(scope, scopes, from) { + const declared = scopes[scope]; + if (from === undefined || !existsSync(from)) { + console.error(`${scope}: no shard report was found${from === undefined ? "" : ` in ${from}`}`); + process.exit(1); + } + const shards = readdirSync(from) + .map((name) => join(from, name)) + .filter((dir) => existsSync(join(dir, "manifest.json"))) + .map((dir) => ({ + manifest: JSON.parse(readFileSync(join(dir, "manifest.json"), "utf8")), + report: JSON.parse(readFileSync(join(dir, "mutation.json"), "utf8")), + })) + .filter((shard) => shard.manifest.scope === scope); + const problem = aggregateVerdict({ + mutate: declared.mutate, + files: sourceFiles(), + of: declared.shards ?? 1, + shards, + }); + if (problem !== null) { + console.error(`${scope}: ${problem}`); + process.exit(1); + } + const merged = mergeReports(shards.map((shard) => shard.report)); + const score = scoreOf(merged); + console.log( + `\nReport: ${scope} over ${shards.length} shards (score ${score.toFixed(1)}, floor ${declared.break})` + ); + for (const survivor of survivorsOf(merged)) console.log(` survived: ${survivor}`); + const verdict = breakVerdict(score, declared); + if (verdict !== null) { + console.error(verdict); + process.exit(1); + } +} + function main() { const scopes = loadScopes(); const [scope, ...flags] = process.argv.slice(2); @@ -197,6 +296,10 @@ function main() { if (scope === undefined) usage("No scope given.", scopes); if (!Object.hasOwn(scopes, scope)) usage(`Unknown scope "${scope}".`, scopes); const force = flags.includes("--force"); + const shardAt = flags.indexOf("--shard"); + if (shardAt !== -1) return mainShard(scope, scopes, flags[shardAt + 1], force); + const aggregateAt = flags.indexOf("--aggregate"); + if (aggregateAt !== -1) return mainAggregate(scope, scopes, flags[aggregateAt + 1]); const scopeDir = join(REPORT_ROOT, scope); mkdirSync(scopeDir, { recursive: true }); diff --git a/cli/tests/architecture/helpers.ts b/cli/tests/architecture/helpers.ts index 12e3fbde0..d13a8875a 100644 --- a/cli/tests/architecture/helpers.ts +++ b/cli/tests/architecture/helpers.ts @@ -2,8 +2,12 @@ * These tests read source as text and never import the code under test, so they stay fast * enough for a pre-commit hook and cannot be broken by runtime wiring. */ -import { existsSync, readdirSync, readFileSync, statSync } from "node:fs"; -import { join, posix, relative, resolve } from "node:path"; +import { existsSync, readdirSync, readFileSync } from "node:fs"; +import { join, posix, resolve } from "node:path"; +import { + matchesGlob as scopeGlob, + sourceFiles as srcFiles, +} from "../../scripts/mutation-shards.mjs"; export const CLI_ROOT = resolve(import.meta.dirname, "..", ".."); export const SRC = join(CLI_ROOT, "src"); @@ -15,16 +19,7 @@ export function canonicalPath(path: string): string { } export function sourceFiles(): string[] { - const out: string[] = []; - const walk = (dir: string): void => { - for (const entry of readdirSync(dir)) { - const full = join(dir, entry); - if (statSync(full).isDirectory()) walk(full); - else if (entry.endsWith(".ts")) out.push(canonicalPath(relative(CLI_ROOT, full))); - } - }; - walk(SRC); - return out.sort(); + return srcFiles(); } export function read(relativePath: string): string { @@ -104,15 +99,7 @@ export function expectRatchet( * or a scope silently covers only its subdirectories — and `*` within one segment. */ export function matchesGlob(glob: string, path: string): boolean { - const pattern = glob - .split("/**/") - .map((segment) => segment.split("*").map(escapeGlobLiteral).join("[^/]*")) - .join("/(?:.*/)?"); - return new RegExp(`^${pattern}$`).test(path); -} - -function escapeGlobLiteral(literal: string): string { - return literal.replace(/[.+?^${}()|[\]\\]/g, "\\$&"); + return scopeGlob(glob, path); } // The context graph: single source of truth for which context may import which. It lives here diff --git a/cli/tests/architecture/mutation-shards.arch.test.ts b/cli/tests/architecture/mutation-shards.arch.test.ts new file mode 100644 index 000000000..da803dc0f --- /dev/null +++ b/cli/tests/architecture/mutation-shards.arch.test.ts @@ -0,0 +1,189 @@ +import { describe, expect, it } from "vitest"; +import { legsOf } from "../../scripts/mutation-scopes-to-run.mjs"; +import { + aggregateVerdict, + inScope, + mergeReports, + partition, + scopeFiles, + shardFiles, +} from "../../scripts/mutation-shards.mjs"; +import { parseShard, scoreOf, shardConfig } from "../../scripts/run-mutation.mjs"; +import { read, sourceFiles } from "./helpers.js"; + +const scopes = (JSON.parse(read("mutation-scopes.json")) as { scopes: Record }) + .scopes; + +interface Scope { + readonly mutate: string | readonly string[]; + readonly break: number; + readonly shards?: number; +} + +const sharded = Object.entries(scopes).filter(([, scope]) => (scope.shards ?? 1) > 1); + +describe("splitting a scope into shards", () => { + it("declares shards only where a scope is worth splitting, never on the canary", () => { + expect(sharded.map(([name]) => name).sort()).toEqual([ + "framework", + "kernel", + "telemetry", + "tools", + "tools-codex", + "translate", + ]); + expect(scopes.canary?.shards).toBeUndefined(); + }); + + it.each(sharded)("partitions %s: every file once, in exactly one shard", (_name, scope) => { + const all = scopeFiles(scope.mutate, sourceFiles()); + const of = scope.shards ?? 1; + const parts = Array.from({ length: of }, (_, i) => + shardFiles(scope.mutate, sourceFiles(), i + 1, of) + ); + expect(parts.flat().sort()).toEqual(all); + expect(new Set(parts.flat()).size).toBe(all.length); + for (const part of parts) expect(part.length).toBeGreaterThan(0); + }); + + it("balances the shards by weight, the same way on every run", () => { + const weights: Record = { + "src/a/h.ts": 9, + "src/a/m.ts": 5, + "src/a/n.ts": 4, + "src/a/l.ts": 1, + }; + const cut = () => + partition("src/a/**/*.ts", Object.keys(weights), 2, (path) => weights[path] ?? 0); + expect(cut()).toEqual([ + ["src/a/h.ts", "src/a/l.ts"], + ["src/a/m.ts", "src/a/n.ts"], + ]); + expect(cut()).toEqual(cut()); + }); + + it.each(sharded)( + "keeps the heaviest shard of %s within a quarter of its fair share, or its heaviest file", + (name, scope) => { + const of = scope.shards ?? 1; + const parts = partition(scope.mutate, sourceFiles(), of); + const weight = (files: string[]) => + files.reduce((sum, file) => sum + read(file).split("\n").length, 0); + const fair = weight(parts.flat()) / of; + const heaviestFile = Math.max(...parts.flat().map((file) => weight([file]))); + expect(Math.max(...parts.map(weight)), name).toBeLessThanOrEqual( + Math.max(1.25 * fair, heaviestFile) + ); + } + ); + + it("honours a ! glob, as the scope does", () => { + const mutate = ["src/t/**/*.ts", "!src/t/p/**/*.ts"]; + expect(inScope(mutate, "src/t/a.ts")).toBe(true); + expect(inScope(mutate, "src/t/p/b.ts")).toBe(false); + }); +}); + +describe("running the CI legs", () => { + it("gives a sharded scope one leg per shard, and any other scope one leg", () => { + const declared = { a: { mutate: "x", break: 1 }, b: { mutate: "y", break: 1, shards: 3 } }; + expect(legsOf(["a", "b"], declared)).toEqual([ + { scope: "a", shard: 1, of: 1 }, + { scope: "b", shard: 1, of: 3 }, + { scope: "b", shard: 2, of: 3 }, + { scope: "b", shard: 3, of: 3 }, + ]); + }); +}); + +const mutant = (status: string) => ({ status, mutatorName: "m", location: { start: { line: 1 } } }); +const report = (files: Record) => ({ + files: Object.fromEntries( + Object.entries(files).map(([name, statuses]) => [name, { mutants: statuses.map(mutant) }]) + ), +}); + +describe("scoring a sharded scope once, over every shard", () => { + const mutate = "src/a/**/*.ts"; + const files = ["src/a/x.ts", "src/a/y.ts", "src/a/z.ts", "src/a/w.ts"]; + const of = 2; + const weightOf = () => 1; + const shard = (i: number) => { + const assigned = shardFiles(mutate, files, i, of, weightOf); + return { + manifest: { shard: i, of, files: assigned }, + report: report(Object.fromEntries(assigned.map((file) => [file, ["Killed", "Survived"]]))), + }; + }; + + it("merges the shards into the report one run would have written", () => { + const merged = mergeReports([shard(1).report, shard(2).report]); + expect(Object.keys(merged.files).sort()).toEqual([...files].sort()); + expect(scoreOf(merged)).toBe(50); + }); + + it("passes when every shard is there and matches the partition", () => { + expect( + aggregateVerdict({ mutate, files, of, weightOf, shards: [shard(1), shard(2)] }) + ).toBeNull(); + }); + + it("fails when a shard is missing", () => { + expect(aggregateVerdict({ mutate, files, of, weightOf, shards: [shard(1)] })).toMatch( + /shard 2 of 2 is missing/ + ); + }); + + it("fails when a shard mutated other files than the partition gives it", () => { + const files2 = [...shard(2).manifest.files, ...shard(1).manifest.files]; + const wrong = { ...shard(2), manifest: { ...shard(2).manifest, files: files2 } }; + expect(aggregateVerdict({ mutate, files, of, weightOf, shards: [shard(1), wrong] })).toMatch( + /shard 2 of 2 mutated other files/ + ); + }); + + it("fails when the shards hold no mutant at all", () => { + const empty = [1, 2].map((i) => ({ ...shard(i), report: { files: {} } })); + expect(aggregateVerdict({ mutate, files, of, weightOf, shards: empty })).toMatch(/no mutant/); + }); +}); + +describe("running one shard", () => { + const base = JSON.parse(read("stryker.conf.json")) as Record; + + it("keeps the whole config and makes only its files, sandbox and reports its own", () => { + const config = shardConfig(base, { + files: ["src/kernel/a.ts"], + dir: "reports/mutation/kernel/shard-1-of-3", + tempDirName: ".stryker-tmp-kernel-1", + }); + const own = [ + "mutate", + "incremental", + "incrementalFile", + "tempDirName", + "htmlReporter", + "jsonReporter", + ]; + for (const [key, value] of Object.entries(base)) { + if (!own.includes(key)) expect(config[key], key).toEqual(value); + } + expect(config).toMatchObject({ + mutate: ["src/kernel/a.ts"], + incremental: true, + incrementalFile: "reports/mutation/kernel/shard-1-of-3/incremental.json", + tempDirName: ".stryker-tmp-kernel-1", + jsonReporter: { fileName: "reports/mutation/kernel/shard-1-of-3/mutation.json" }, + htmlReporter: { fileName: "reports/mutation/kernel/shard-1-of-3/report.html" }, + }); + }); + + it("takes only the cut the scope declares", () => { + expect(parseShard("2/3", { shards: 3 })).toEqual({ shard: 2, of: 3 }); + expect(() => parseShard("2/4", { shards: 3 })).toThrow(/does not fit/); + expect(() => parseShard("4/3", { shards: 3 })).toThrow(/does not fit/); + expect(() => parseShard("0/3", { shards: 3 })).toThrow(/does not fit/); + expect(() => parseShard("1/1", {})).not.toThrow(); + expect(() => parseShard("x", { shards: 3 })).toThrow(/takes i\/n/); + }); +}); diff --git a/scripts/__tests__/cli-ci-gate-covers-every-job.test.js b/scripts/__tests__/cli-ci-gate-covers-every-job.test.js index d67cee4ab..16701f98a 100644 --- a/scripts/__tests__/cli-ci-gate-covers-every-job.test.js +++ b/scripts/__tests__/cli-ci-gate-covers-every-job.test.js @@ -113,9 +113,12 @@ test("only a proven next snapshot tree skips the promotion or main mutation matr assert.match(promotion.run, /HEAD_SHA="\$snapshot_sha"/); assert.match(mutation.run, /steps\.promotion\.outputs\.trusted.*== "true"/); - assert.match(mutation.run, /scopes='\[\]'/); + assert.match(mutation.run, /legs='\[\]'/); + assert.match(mutation.run, /sharded='\[\]'/); assert.match(mutation.run, /else[\s\S]*mutation-scopes-to-run\.mjs/); - assert.equal(workflow.jobs["cli-mutation"].if, "needs.changes.outputs.mutation_scopes != '[]'"); + assert.equal(workflow.jobs["cli-mutation"].if, "needs.changes.outputs.mutation_legs != '[]'"); + // The scoring job runs whatever its shards did, so a crashed shard fails it, never skips it. + assert.match(workflow.jobs["cli-mutation-score"].if, /^always\(\) && needs\.changes\.outputs\.mutation_sharded != '\[\]'/); // These checks validate the current PR merge ref or main commit; reuse never turns them off. for (const name of [