ci(cli): split the four slowest mutation scopes into parallel shards

.github/workflows/cli-ci.yml

@@ -35,7 +35,8 @@ jobs:
     outputs:
       relevant: ${{ steps.filter.outputs.relevant }}
       trusted_promotion: ${{ steps.promotion.outputs.trusted }}
-      mutation_scopes: ${{ steps.mutation.outputs.scopes }}
+      mutation_legs: ${{ steps.mutation.outputs.legs }}
+      mutation_sharded: ${{ steps.mutation.outputs.sharded }}
     steps:
       - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
         with:
@@ -183,7 +184,8 @@ jobs:
         run: |
           set -euo pipefail
           if [[ "${{ steps.promotion.outputs.trusted }}" == "true" ]]; then
-            scopes='[]'
+            legs='[]'
+            sharded='[]'
           else
             if [[ "${{ github.event_name }}" == "pull_request" ]]; then
               BASE="${{ github.event.pull_request.base.sha }}"
@@ -199,10 +201,13 @@ jobs:
               changed="$(git diff --name-only "$BASE" "$HEAD")"
               all=false
             fi
-            scopes="$(ALL="$all" CHANGED="$changed" node cli/scripts/mutation-scopes-to-run.mjs)"
+            legs="$(ALL="$all" CHANGED="$changed" node cli/scripts/mutation-scopes-to-run.mjs)"
+            sharded="$(ALL="$all" CHANGED="$changed" OUTPUT=sharded node cli/scripts/mutation-scopes-to-run.mjs)"
           fi
-          echo "mutation scopes: $scopes"
-          echo "scopes=$scopes" >> "$GITHUB_OUTPUT"
+          echo "mutation legs: $legs"
+          echo "sharded scopes: $sharded"
+          echo "legs=$legs" >> "$GITHUB_OUTPUT"
+          echo "sharded=$sharded" >> "$GITHUB_OUTPUT"
 
   cli-typecheck:
     name: cli / Typecheck
@@ -583,15 +588,21 @@ jobs:
   # nothing relevant. The one check the branch rulesets require — without it a pull request
   # merges with the real jobs red, because nothing names them.
   cli-mutation:
-    name: cli / Mutation (${{ matrix.scope }})
+    # A heavy scope runs as several legs in parallel, each mutating its share of the files; the
+    # job below scores it once over every leg. Any other scope is one leg and gates itself.
+    name: cli / Mutation (${{ matrix.scope }}${{ matrix.of > 1 && format(' {0}/{1}', matrix.shard, matrix.of) || '' }})
     needs: [changes]
-    if: needs.changes.outputs.mutation_scopes != '[]'
+    if: needs.changes.outputs.mutation_legs != '[]'
     runs-on: ubuntu-latest
     timeout-minutes: 90
     strategy:
       fail-fast: false
       matrix:
-        scope: ${{ fromJSON(needs.changes.outputs.mutation_scopes) }}
+        include: ${{ fromJSON(needs.changes.outputs.mutation_legs) }}
+    env:
+      # `--` closes the name: `tools` must never restore `tools-codex`'s file.
+      LEG: ${{ matrix.of > 1 && format('{0}-shard-{1}-of-{2}', matrix.scope, matrix.shard, matrix.of) || matrix.scope }}
+      LEG_DIR: cli/reports/mutation/${{ matrix.of > 1 && format('{0}/shard-{1}-of-{2}', matrix.scope, matrix.shard, matrix.of) || matrix.scope }}
     steps:
       - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
       - name: Install pnpm
@@ -608,19 +619,51 @@ jobs:
       - run: cd cli && pnpm install --frozen-lockfile
       # The incremental file is what keeps a run to the mutants a change can move; the newest
       # one this branch or its base saved is restored, and this run's is saved whatever the score.
-      - name: Restore this scope's incremental file
+      - name: Restore this leg's incremental file
         uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
         with:
-          path: cli/reports/mutation/${{ matrix.scope }}/incremental.json
-          key: mutation-${{ matrix.scope }}-${{ github.run_id }}
-          restore-keys: mutation-${{ matrix.scope }}-
-      - run: cd cli && node scripts/run-mutation.mjs "${{ matrix.scope }}" ${{ github.event_name == 'schedule' && '--force' || '' }}
-      - name: Save this scope's incremental file
+          path: ${{ env.LEG_DIR }}/incremental.json
+          key: mutation--${{ env.LEG }}--${{ github.run_id }}
+          restore-keys: mutation--${{ env.LEG }}--
+      - run: cd cli && node scripts/run-mutation.mjs "${{ matrix.scope }}" ${{ matrix.of > 1 && format('--shard {0}/{1}', matrix.shard, matrix.of) || '' }} ${{ github.event_name == 'schedule' && '--force' || '' }}
+      - name: Save this leg's incremental file
         if: always()
         uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
         with:
-          path: cli/reports/mutation/${{ matrix.scope }}/incremental.json
-          key: mutation-${{ matrix.scope }}-${{ github.run_id }}
+          path: ${{ env.LEG_DIR }}/incremental.json
+          key: mutation--${{ env.LEG }}--${{ github.run_id }}
+      - name: Hand this shard's report to the scoring job
+        if: matrix.of > 1
+        uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
+        with:
+          name: mutation-${{ matrix.scope }}-shard-${{ matrix.shard }}-of-${{ matrix.of }}
+          path: |
+            ${{ env.LEG_DIR }}/manifest.json
+            ${{ env.LEG_DIR }}/mutation.json
+          if-no-files-found: error
+          retention-days: 1
+
+  cli-mutation-score:
+    name: cli / Mutation (${{ matrix.scope }})
+    needs: [changes, cli-mutation]
+    # Always, so a crashed or cancelled shard is scored as missing instead of skipped.
+    if: always() && needs.changes.outputs.mutation_sharded != '[]' && needs.changes.outputs.mutation_sharded != ''
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        scope: ${{ fromJSON(needs.changes.outputs.mutation_sharded) }}
+    steps:
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+      - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0
+        with:
+          node-version: "22"
+      - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2
+        with:
+          pattern: mutation-${{ matrix.scope }}-shard-*
+          path: shards
+      - name: Score the scope over every shard, against its floor
+        run: node cli/scripts/run-mutation.mjs "${{ matrix.scope }}" --aggregate "$GITHUB_WORKSPACE/shards"
 
   gate:
     name: cli / gate
@@ -639,6 +682,7 @@ jobs:
       - identifier-join
       - cli-jscpd
       - cli-mutation
+      - cli-mutation-score
       - kanban-checks
       - windows
     steps:
@@ -660,6 +704,7 @@ jobs:
             "${{ needs.identifier-join.result }}" \
             "${{ needs.cli-jscpd.result }}" \
             "${{ needs.cli-mutation.result }}" \
+            "${{ needs.cli-mutation-score.result }}" \
             "${{ needs.kanban-checks.result }}" \
             "${{ needs.windows.result }}"; do
             if [[ "$result" == "failure" || "$result" == "cancelled" ]]; then

cli/mutation-scopes.json

@@ -3,7 +3,8 @@
   "scopes": {
     "kernel": {
       "mutate": "src/kernel/**/*.ts",
-      "break": 95
+      "break": 95,
+      "shards": 4
     },
     "tools": {
       "mutate": [
@@ -15,23 +16,27 @@
         "!src/contexts/tools/domain/profiles/kilo/**/*.ts",
         "!src/contexts/tools/domain/profiles/opencode/**/*.ts"
       ],
-      "break": 94
+      "break": 94,
+      "shards": 4
     },
     "telemetry": {
       "mutate": "src/contexts/telemetry/**/*.ts",
-      "break": 93
+      "break": 93,
+      "shards": 3
     },
     "translate": {
       "mutate": "src/contexts/translate/**/*.ts",
-      "break": 85
+      "break": 85,
+      "shards": 2
     },
     "distribution": {
       "mutate": "src/contexts/distribution/**/*.ts",
       "break": 77
     },
     "framework": {
       "mutate": "src/contexts/framework/**/*.ts",
-      "break": 93
+      "break": 93,
+      "shards": 4
     },
     "presentation": {
       "mutate": "src/presentation/**/*.ts",
@@ -47,7 +52,8 @@
     },
     "tools-codex": {
       "mutate": "src/contexts/tools/domain/profiles/codex/**/*.ts",
-      "break": 94
+      "break": 94,
+      "shards": 2
     },
     "tools-copilot": {
       "mutate": "src/contexts/tools/domain/profiles/copilot/**/*.ts",

cli/scripts/mutation-scopes-to-run.d.mts

@@ -6,3 +6,7 @@ export function scopesToRun(
   scopes: Readonly<Record<string, MutationScope>>,
   options?: { readonly all?: boolean }
 ): string[];
+export function legsOf(
+  names: readonly string[],
+  scopes: Readonly<Record<string, MutationScope>>
+): { scope: string; shard: number; of: number }[];

cli/scripts/mutation-scopes-to-run.mjs

@@ -12,6 +12,7 @@ export const HARNESS = [
   "cli/vitest.mutation.config.ts",
   "cli/scripts/run-mutation.mjs",
   "cli/scripts/mutation-scopes-to-run.mjs",
+  "cli/scripts/mutation-shards.mjs",
   "cli/package.json",
   "cli/pnpm-lock.yaml",
   ".github/workflows/cli-ci.yml",
@@ -36,10 +37,22 @@ export function scopesToRun(changed, scopes, { all = false } = {}) {
     .map(([name]) => name);
 }
 
+/** The CI legs for `names`: one per shard of a sharded scope, one for any other. */
+export function legsOf(names, scopes) {
+  return names.flatMap((scope) => {
+    const of = scopes[scope].shards ?? 1;
+    return Array.from({ length: of }, (_, i) => ({ scope, shard: i + 1, of }));
+  });
+}
+
 if (process.argv[1] !== undefined && resolve(process.argv[1]) === fileURLToPath(import.meta.url)) {
   const scopes = JSON.parse(readFileSync(join(CLI_ROOT, "mutation-scopes.json"), "utf8")).scopes;
   const changed = (process.env.CHANGED ?? "").split("\n").filter(Boolean);
-  process.stdout.write(
-    JSON.stringify(scopesToRun(changed, scopes, { all: process.env.ALL === "true" }))
-  );
+  const names = scopesToRun(changed, scopes, { all: process.env.ALL === "true" });
+  // `legs` feeds the mutation matrix; `sharded` the jobs that score a split scope once.
+  const output =
+    process.env.OUTPUT === "sharded"
+      ? names.filter((name) => (scopes[name].shards ?? 1) > 1)
+      : legsOf(names, scopes);
+  process.stdout.write(JSON.stringify(output));
 }

cli/scripts/mutation-shards.d.mts

@@ -0,0 +1,38 @@
+import type { MutationReport } from "./run-mutation.mjs";
+
+type Mutate = string | readonly string[];
+type WeightOf = (path: string) => number;
+
+export function sourceFiles(): string[];
+export function linesOf(path: string): number;
+export function matchesGlob(glob: string, path: string): boolean;
+export function inScope(mutate: Mutate, path: string): boolean;
+export function scopeFiles(mutate: Mutate, files: readonly string[]): string[];
+export function partition(
+  mutate: Mutate,
+  files: readonly string[],
+  of: number,
+  weightOf?: WeightOf
+): string[][];
+export function shardFiles(
+  mutate: Mutate,
+  files: readonly string[],
+  shard: number,
+  of: number,
+  weightOf?: WeightOf
+): string[];
+export function mergeReports(reports: readonly MutationReport[]): Required<MutationReport>;
+export function aggregateVerdict(input: {
+  readonly mutate: Mutate;
+  readonly files: readonly string[];
+  readonly of: number;
+  readonly shards: readonly {
+    readonly manifest: {
+      readonly shard: number;
+      readonly of: number;
+      readonly files: readonly string[];
+    };
+    readonly report: MutationReport;
+  }[];
+  readonly weightOf?: WeightOf;
+}): string | null;

cli/scripts/mutation-shards.mjs

@@ -0,0 +1,99 @@
+#!/usr/bin/env node
+import { readdirSync, readFileSync, statSync } from "node:fs";
+import { dirname, join, relative, resolve } from "node:path";
+import { fileURLToPath } from "node:url";
+
+const CLI_ROOT = resolve(dirname(fileURLToPath(import.meta.url)), "..");
+
+/** Every `.ts` file under `src/`, relative to `cli/` with `/`, sorted: what a scope is cut from. */
+export function sourceFiles() {
+  const out = [];
+  const walk = (dir) => {
+    for (const entry of readdirSync(dir)) {
+      const full = join(dir, entry);
+      if (statSync(full).isDirectory()) walk(full);
+      else if (entry.endsWith(".ts")) out.push(relative(CLI_ROOT, full).replaceAll("\\", "/"));
+    }
+  };
+  walk(join(CLI_ROOT, "src"));
+  return out.sort();
+}
+
+/** A file's weight in a shard: its line count, which the number of its mutants follows. */
+export function linesOf(path) {
+  return readFileSync(join(CLI_ROOT, path), "utf8").split("\n").length;
+}
+
+/** A `mutate` glob as `mutation-scopes.json` writes them: `**` crosses directories, `*` does not. */
+export function matchesGlob(glob, path) {
+  const pattern = glob
+    .split("/**/")
+    .map((segment) => segment.split("*").map(escapeGlobLiteral).join("[^/]*"))
+    .join("/(?:.*/)?");
+  return new RegExp(`^${pattern}$`).test(path);
+}
+
+function escapeGlobLiteral(literal) {
+  return literal.replace(/[.+?^${}()|[\]\\]/g, "\\$&");
+}
+
+/** Whether a scope's `mutate` takes `path`: one of its globs matches and no `!` glob does. */
+export function inScope(mutate, path) {
+  const globs = [mutate].flat();
+  return (
+    globs.some((glob) => !glob.startsWith("!") && matchesGlob(glob, path)) &&
+    !globs.some((glob) => glob.startsWith("!") && matchesGlob(glob.slice(1), path))
+  );
+}
+
+/** The scope's files among `files`, sorted. */
+export function scopeFiles(mutate, files) {
+  return files.filter((path) => inScope(mutate, path)).sort();
+}
+
+/** The scope's files cut into `of` shards of about the same weight: heaviest file first, each
+ * to the lightest shard so far, ties broken by path and by shard number. The same files at the
+ * same weights give the same shards on every machine. */
+export function partition(mutate, files, of, weightOf = linesOf) {
+  const weighted = scopeFiles(mutate, files)
+    .map((path) => ({ path, weight: weightOf(path) }))
+    .sort((a, b) => b.weight - a.weight || (a.path < b.path ? -1 : 1));
+  const shards = Array.from({ length: of }, () => ({ weight: 0, files: [] }));
+  for (const { path, weight } of weighted) {
+    const lightest = shards.reduce((min, shard) => (shard.weight < min.weight ? shard : min));
+    lightest.files.push(path);
+    lightest.weight += weight;
+  }
+  return shards.map((shard) => shard.files.sort());
+}
+
+/** The files shard `shard` (from 1) of `of` mutates. */
+export function shardFiles(mutate, files, shard, of, weightOf = linesOf) {
+  return partition(mutate, files, of, weightOf)[shard - 1];
+}
+
+/** The shards' reports as one: each file is in exactly one shard, so nothing overlaps. */
+export function mergeReports(reports) {
+  return { files: Object.assign({}, ...reports.map((report) => report.files ?? {})) };
+}
+
+/** Why a sharded scope cannot be scored, or null. Every shard must be there, each must have
+ * mutated exactly what the partition gives it at this commit, and something must have been
+ * mutated: a missing or drifted shard would lift the score by leaving mutants out. */
+export function aggregateVerdict({ mutate, files, of, shards, weightOf = linesOf }) {
+  for (let shard = 1; shard <= of; shard += 1) {
+    const found = shards.find((candidate) => candidate.manifest.shard === shard);
+    if (found === undefined) return `shard ${shard} of ${of} is missing`;
+    if (found.manifest.of !== of)
+      return `shard ${shard} was cut in ${found.manifest.of}, not ${of}`;
+    const expected = shardFiles(mutate, files, shard, of, weightOf);
+    const mutated = [...found.manifest.files].sort();
+    if (JSON.stringify(mutated) !== JSON.stringify(expected)) {
+      return `shard ${shard} of ${of} mutated other files than the partition gives it`;
+    }
+  }
+  const mutants = Object.values(mergeReports(shards.map((s) => s.report)).files).flatMap(
+    (file) => file.mutants
+  ).length;
+  return mutants === 0 ? "the shards hold no mutant" : null;
+}

cli/scripts/run-mutation.d.mts

@@ -1,6 +1,8 @@
 export interface MutationScope {
   readonly mutate: string | readonly string[];
   readonly break: number;
+  /** CI legs the scope is split into; one when absent. */
+  readonly shards?: number;
 }
 
 export interface MutationReport {
@@ -36,3 +38,11 @@ export function canarySelectionVerdict(report: MutationReport): string | null;
 export function changedRanges(diff: string): string[];
 export function changedArgs(ranges: readonly string[]): string[];
 export function survivorsOf(report: MutationReport): string[];
+export function shardConfig(
+  base: Readonly<Record<string, unknown>>,
+  shard: { readonly files: readonly string[]; readonly dir: string; readonly tempDirName: string }
+): Record<string, unknown>;
+export function parseShard(
+  text: string | undefined,
+  declared: { readonly shards?: number }
+): { shard: number; of: number };