v0.2.0 perf: speed up CI with bounded test shards and cleanup

.github/workflows/ci.yml

@@ -31,6 +31,7 @@ jobs:
       packages: ${{ steps.classify.outputs.packages }}
       test-packages: ${{ steps.classify.outputs.test-packages }}
       typecheck-packages: ${{ steps.classify.outputs.typecheck-packages }}
+      unit-matrix: ${{ steps.test-plan.outputs.unit-matrix }}
       daemon: ${{ steps.classify.outputs.daemon }}
       web: ${{ steps.classify.outputs.web }}
     steps:
@@ -86,6 +87,37 @@ jobs:
           for (const [key, value] of Object.entries(outputs)) console.log(`${key}=${value}`);
           NODE
 
+      - name: Restore timing hints once for the entire run
+        if: steps.policy.outputs.execution-tier == 'complete' && steps.classify.outputs.mode != 'docs'
+        uses: actions/cache/restore@0057852bfaa89a56745cba8c7296529d2fc39830
+        with:
+          path: .codex-artifacts/ci/timings.json
+          key: ci-node-timings-v1-${{ github.run_id }}-${{ github.run_attempt }}
+          restore-keys: ci-node-timings-v1-
+
+      - name: Plan complete core and daemon file coverage
+        id: test-plan
+        env:
+          SELECTED_TEST_PACKAGES: ${{ steps.classify.outputs.test-packages }}
+        shell: bash
+        run: |
+          set -euo pipefail
+          unit_matrix="$(node scripts/ci/test-shard-plan.mjs \
+            --packages="$SELECTED_TEST_PACKAGES" \
+            --timings=.codex-artifacts/ci/timings.json \
+            --output=.codex-artifacts/ci/test-plan.json)"
+          printf 'unit-matrix=%s\n' "$unit_matrix" >> "$GITHUB_OUTPUT"
+
+      - name: Publish the immutable test plan for every shard
+        if: steps.policy.outputs.execution-tier == 'complete' && steps.classify.outputs.mode != 'docs'
+        uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9
+        with:
+          name: ci-test-plan
+          include-hidden-files: true
+          if-no-files-found: error
+          retention-days: 7
+          path: .codex-artifacts/ci/test-plan.json
+
   common-fast:
     name: Common checks (fast lane)
     needs: scope
@@ -112,18 +144,26 @@ jobs:
       - name: Check scanner boundaries, execution policy, classification, and platform CI gates
         run: node --test scripts/check-public-repo-safety.test.mjs scripts/ci/*.test.mjs scripts/native-execution-engines/verify-codex.test.mjs
 
+      # Final file scope cannot exclude secrets introduced and later deleted.
       - name: Scan Git history for secrets
-        if: needs.scope.outputs.execution-tier == 'complete' && needs.scope.outputs.mode != 'docs'
-        run: >-
-          docker run --rm
-          -v "$GITHUB_WORKSPACE:/repo:ro"
-          zricethezav/gitleaks@sha256:c00b6bd0aeb3071cbcb79009cb16a60dd9e0a7c60e2be9ab65d25e6bc8abbb7f
-          git --no-banner --no-color --redact=100
-          --log-opts=HEAD
-          --config /repo/.gitleaks.toml
-          --gitleaks-ignore-path /repo/.gitleaksignore
-          --timeout 600
-          /repo
+        if: needs.scope.outputs.execution-tier == 'complete'
+        env:
+          BASE_SHA: ${{ github.event.pull_request.base.sha }}
+        shell: bash
+        run: |
+          set -euo pipefail
+          scan_plan="$(node scripts/ci/secret-scan.mjs --event="$GITHUB_EVENT_NAME" --base="$BASE_SHA" --head=HEAD)"
+          echo "Secret scan plan: $scan_plan"
+          log_opts="$(SCAN_PLAN="$scan_plan" node --input-type=module -e 'console.log(JSON.parse(process.env.SCAN_PLAN).logOpts)')"
+          docker run --rm \
+            -v "$GITHUB_WORKSPACE:/repo:ro" \
+            zricethezav/gitleaks@sha256:c00b6bd0aeb3071cbcb79009cb16a60dd9e0a7c60e2be9ab65d25e6bc8abbb7f \
+            git --no-banner --no-color --redact=100 \
+            --log-opts="$log_opts" \
+            --config /repo/.gitleaks.toml \
+            --gitleaks-ignore-path /repo/.gitleaksignore \
+            --timeout 600 \
+            /repo
 
       - name: Check pull request whitespace
         if: github.event_name == 'pull_request'
@@ -310,11 +350,14 @@ jobs:
         run: node scripts/ci/run-packages.mjs --task=typecheck:ci --packages="$SELECTED_PACKAGES"
 
   unit-tests:
-    name: Unit tests and Web checks
+    name: Unit tests and Web checks (${{ matrix.suite }} ${{ matrix.shard }})
     needs: [scope, build]
     if: ${{ !cancelled() && needs.scope.outputs.execution-tier == 'complete' && needs.scope.outputs.mode != 'docs' && needs.build.result == 'success' }}
     runs-on: ubuntu-latest
     timeout-minutes: 25
+    strategy:
+      fail-fast: false
+      matrix: ${{ fromJSON(needs.scope.outputs.unit-matrix) }}
     env:
       SELECTED_PACKAGES: ${{ needs.scope.outputs.test-packages }}
       WEB_SELECTED: ${{ needs.scope.outputs.web }}
@@ -350,15 +393,42 @@ jobs:
           name: ci-dist
           path: .
 
+      - name: Download the shared test plan
+        if: matrix.suite == 'core'
+        uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333
+        with:
+          name: ci-test-plan
+          path: .codex-artifacts/ci
+
       - name: Run selected unit tests without rebuilding dependencies
-        run: node scripts/ci/run-packages.mjs --task=test:ci --packages="$SELECTED_PACKAGES" --exclude='["@ling-stack/daemon"]'
+        if: matrix.suite == 'packages'
+        run: node scripts/ci/run-packages.mjs --task=test:ci --packages="$SELECTED_PACKAGES" --exclude='["@ling-stack/daemon","@ling-stack/core"]'
+
+      - name: Run bounded core shard
+        if: matrix.suite == 'core'
+        run: >-
+          node scripts/ci/node-test-shard.mjs --suite=core
+          --shard=${{ matrix.shard }} --shard-count=4 --concurrency=2
+          --shard-timeout-ms=1200000
+          --plan=.codex-artifacts/ci/test-plan.json
+          --report=.codex-artifacts/ci/reports/core-${{ matrix.shard }}.json
+
+      - name: Preserve core file timings and failure logs
+        if: always() && matrix.suite == 'core'
+        uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9
+        with:
+          name: ci-test-report-core-${{ matrix.shard }}
+          include-hidden-files: true
+          if-no-files-found: error
+          retention-days: 7
+          path: .codex-artifacts/ci/reports/**
 
       - name: Check Web architecture and UI rules
-        if: env.WEB_SELECTED == 'true'
+        if: matrix.suite == 'packages' && env.WEB_SELECTED == 'true'
         run: corepack pnpm --filter @ling-stack/web lint
 
       - name: Check translations
-        if: env.WEB_SELECTED == 'true'
+        if: matrix.suite == 'packages' && env.WEB_SELECTED == 'true'
         run: corepack pnpm --filter @ling-stack/web check:i18n
 
   daemon-tests:
@@ -368,9 +438,7 @@ jobs:
     # Full discovery is unchanged: original Darwin-only positive cases must
     # execute on the real Host, not skip or weaken the production platform gate.
     runs-on: macos-26
-    # Run 39274142315 reached 1,584 passing tests in shard 1 before the 25m
-    # job limit cancelled it while tests were still progressing. Include setup
-    # and room for the remaining files without changing discovery or isolation.
+    # File/shard watchdogs report failures before this final infrastructure cap.
     timeout-minutes: 40
     strategy:
       fail-fast: false
@@ -408,6 +476,12 @@ jobs:
           name: ci-dist
           path: .
 
+      - name: Download the shared test plan
+        uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333
+        with:
+          name: ci-test-plan
+          path: .codex-artifacts/ci
+
       # ci-dist comes from Ubuntu; native codecs must be rebuilt on this macOS host.
       - name: Build the host image codec
         env:
@@ -417,10 +491,22 @@ jobs:
           corepack pnpm --filter @ling-stack/daemon build:image-sanitizer
 
       - name: Run isolated daemon shard
-        # Each worker retains serial file execution and its own home/workspace.
-        # Short native lifecycle fixtures retain exclusive execution first.
-        # Reuse the same four macOS jobs instead of adding more queued runners.
-        run: node scripts/ci/daemon-test-shard.mjs --shard=${{ matrix.shard }} --shard-count=4 --concurrency=2
+        # Every file owns its home/workspace and process group. Short native
+        # lifecycle fixtures still finish exclusively before parallel workers.
+        run: >-
+          node scripts/ci/daemon-test-shard.mjs --shard=${{ matrix.shard }} --shard-count=4 --concurrency=2
+          --plan=.codex-artifacts/ci/test-plan.json
+          --report=.codex-artifacts/ci/reports/daemon-${{ matrix.shard }}.json
+
+      - name: Preserve daemon file timings and failure logs
+        if: always()
+        uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9
+        with:
+          name: ci-test-report-daemon-${{ matrix.shard }}
+          include-hidden-files: true
+          if-no-files-found: error
+          retention-days: 7
+          path: .codex-artifacts/ci/reports/**
 
       - name: Collect fixed OS bootstrap source for review only
         if: always()
@@ -615,3 +701,34 @@ jobs:
         run: |
           set -euo pipefail
           node scripts/ci/execution-policy.mjs --check-quality
+
+      - name: Download the shared test plan for coverage verification
+        if: needs.scope.outputs.execution-tier == 'complete' && (needs.scope.outputs.daemon == 'true' || contains(needs.scope.outputs.test-packages, '"@ling-stack/core"'))
+        uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333
+        with:
+          name: ci-test-plan
+          path: .codex-artifacts/ci
+
+      - name: Download all successful shard reports
+        if: needs.scope.outputs.execution-tier == 'complete' && (needs.scope.outputs.daemon == 'true' || contains(needs.scope.outputs.test-packages, '"@ling-stack/core"'))
+        uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333
+        with:
+          pattern: ci-test-report-*
+          merge-multiple: true
+          path: .codex-artifacts/ci/reports
+
+      - name: Verify every planned file passed exactly once and collect timings
+        if: needs.scope.outputs.execution-tier == 'complete' && (needs.scope.outputs.daemon == 'true' || contains(needs.scope.outputs.test-packages, '"@ling-stack/core"'))
+        run: >-
+          node scripts/ci/test-shard-plan.mjs
+          --plan=.codex-artifacts/ci/test-plan.json --reports=.codex-artifacts/ci/reports
+          --output=.codex-artifacts/ci/timings.json
+
+      # Only validated default-branch runs publish hints. PRs consume them but
+      # cannot replace the baseline; all shards use the same uploaded plan.
+      - name: Save trusted default-branch test timings
+        if: needs.scope.outputs.execution-tier == 'complete' && needs.scope.outputs.mode != 'docs' && github.ref == format('refs/heads/{0}', github.event.repository.default_branch)
+        uses: actions/cache/save@0057852bfaa89a56745cba8c7296529d2fc39830
+        with:
+          path: .codex-artifacts/ci/timings.json
+          key: ci-node-timings-v1-${{ github.run_id }}-${{ github.run_attempt }}

packages/daemon/test/channel-workspace.test.mjs

@@ -4,7 +4,6 @@ import { createHash } from "node:crypto";
 import { spawnSync } from "node:child_process";
 import { readFileSync, realpathSync } from "node:fs";
 import { access, mkdir, mkdtemp, readFile, rm, writeFile } from "node:fs/promises";
-import { createServer } from "node:http";
 import { createRequire } from "node:module";
 import { tmpdir } from "node:os";
 import { resolve } from "node:path";
@@ -49,6 +48,7 @@ import { commandOperationCandidate, fileOperationCandidate } from "./support/cha
 import { messageText, startRealPiChannelFixture, systemText } from "./support/channel-operation-real-pi-fixture.mjs";
 import { prepareFixtureTaskInput } from "./support/prepared-input-fixture.mjs";
 import { ToolFreePiRunner } from "./support/tool-free-pi-runner.mjs";
+import { startMockOpenAICompatibleServer } from "./support/mock-openai-compatible-server.mjs";
 
 const require = createRequire(import.meta.url);
 const ASYNC_SETTLE_TIMEOUT_MS = 10_000;
@@ -2442,9 +2442,9 @@ test("materialized image attachment lets Ling answer without materialization cla
   }
 });
 
-test("provider-backed image materialization works without injected summarizer", async () => {
+test("provider-backed image materialization works without injected summarizer", async (t) => {
   const cwd = await mkdtemp(resolve(tmpdir(), "ling-stack-provider-backed-image-materialization-"));
-  const relay = await startMockOpenAICompatibleServer({
+  const relay = await startMockOpenAICompatibleServer(t, {
     responseContent: ({ body }) => {
       const content = body?.messages?.[0]?.content;
       if (Array.isArray(content) && content.some((part) => part?.type === "image_url")) {
@@ -2521,7 +2521,6 @@ test("provider-backed image materialization works without injected summarizer",
     assert.match(imagePart.image_url.url, /^data:image\/png;base64,/);
   } finally {
     await daemon.close();
-    await relay.close();
     await rm(cwd, { recursive: true, force: true });
   }
 });
@@ -11754,10 +11753,10 @@ test("AI teammate private workspace endpoints persist roster, setup, private mes
   }
 });
 
-test("AI teammate private reply waits for startup provider refresh instead of using stale timeout", async () => {
+test("AI teammate private reply waits for startup provider refresh instead of using stale timeout", async (t) => {
   const cwd = await mkdtemp(resolve(tmpdir(), "ling-stack-direct-provider-startup-refresh-"));
   await ensureGitRepo(cwd);
-  const relay = await startMockOpenAICompatibleServer();
+  const relay = await startMockOpenAICompatibleServer(t);
   const runner = new RecordingRunner();
   let daemon = await startDaemonForTest(runner);
   try {
@@ -11806,7 +11805,6 @@ test("AI teammate private reply waits for startup provider refresh instead of us
     assert.equal(refreshed.provider.lastCheck.code, "ready");
   } finally {
     await daemon.close();
-    await relay.close();
     await rm(cwd, { recursive: true, force: true });
   }
 });
@@ -16022,43 +16020,6 @@ function simulateLegacyChannelWithoutModel(cwd, channelId) {
   }
 }
 
-async function startMockOpenAICompatibleServer(options = {}) {
-  const requests = [];
-  const server = createServer(async (request, response) => {
-    const chunks = [];
-    for await (const chunk of request) {
-      chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk));
-    }
-    const bodyText = Buffer.concat(chunks).toString("utf8");
-    const body = bodyText ? JSON.parse(bodyText) : {};
-    requests.push({
-      method: request.method,
-      path: request.url,
-      authorization: request.headers.authorization,
-      body,
-    });
-    const responseContent = typeof options.responseContent === "function"
-      ? options.responseContent({ request, body })
-      : options.responseContent ?? "OK";
-    response.writeHead(200, { "content-type": "application/json" });
-    response.end(JSON.stringify({
-      id: "chatcmpl-startup-refresh",
-      object: "chat.completion",
-      choices: [{ index: 0, message: { role: "assistant", content: responseContent }, finish_reason: "stop" }],
-    }));
-  });
-  await new Promise((resolveListen) => server.listen(0, "127.0.0.1", resolveListen));
-  const address = server.address();
-  assert.ok(address && typeof address === "object");
-  return {
-    url: `http://127.0.0.1:${address.port}`,
-    requests,
-    close: () => new Promise((resolveClose, rejectClose) => {
-      server.close((error) => error ? rejectClose(error) : resolveClose());
-    }),
-  };
-}
-
 async function postTaskRunAndReadReadyRunId(baseUrl, cwd, taskId, body) {
   const response = await fetch(`${baseUrl}/tasks/${encodeURIComponent(taskId)}/run?cwd=${encodeURIComponent(cwd)}`, {
     method: "POST",

packages/daemon/test/mock-openai-compatible-server.test.mjs

@@ -0,0 +1,42 @@
+import assert from "node:assert/strict";
+import { spawnSync } from "node:child_process";
+import { mkdtemp, rm, writeFile } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import test from "node:test";
+
+test("mock relays close after startup and shutdown failures without hiding failed tests", async t => {
+  const directory = await mkdtemp(join(tmpdir(), "ling-mock-relay-cleanup-"));
+  t.after(() => rm(directory, { recursive: true, force: true }));
+  const fixture = join(directory, "failures.mjs");
+  const helper = new URL("./support/mock-openai-compatible-server.mjs", import.meta.url).href;
+  await writeFile(fixture, `
+    import assert from "node:assert/strict";
+    import test from "node:test";
+    import { startMockOpenAICompatibleServer } from ${JSON.stringify(helper)};
+    const urls = [];
+    test("startup failure", async t => {
+      urls.push((await startMockOpenAICompatibleServer(t)).url);
+      throw new Error("runtime_already_running");
+    });
+    test("shutdown failure", async t => {
+      urls.push((await startMockOpenAICompatibleServer(t)).url);
+      try { assert.ok(true); }
+      finally { throw new Error("harness_shutdown_incomplete_stores_retained"); }
+    });
+    test("all relays closed", async () => {
+      assert.equal(urls.length, 2);
+      for (const url of urls) await assert.rejects(fetch(url), error => error.cause?.code === "ECONNREFUSED");
+    });
+  `);
+  const env = { ...process.env };
+  delete env.NODE_TEST_CONTEXT;
+  // Run node:test in-process so the watchdog also owns any leaked HTTP listener.
+  const result = spawnSync(process.execPath, [fixture], { env, encoding: "utf8", timeout: 5000, killSignal: "SIGKILL" });
+  assert.ifError(result.error);
+  assert.equal(result.status, 1, result.stdout + result.stderr);
+  assert.match(result.stdout, /runtime_already_running/u);
+  assert.match(result.stdout, /harness_shutdown_incomplete_stores_retained/u);
+  assert.match(result.stdout, /ok 3 - all relays closed/u);
+  assert.match(result.stdout, /# pass 1\n# fail 2/u);
+});