From 19b01d6a64e70f91f644f4063dfa61955eeaddc4 Mon Sep 17 00:00:00 2001 From: tianxiaochannel-oss88 Date: Mon, 2 Nov 2026 01:19:42 +0800 Subject: [PATCH 1/2] perf(ci): bound test execution and balance shards Isolate test files, enforce bounded execution and cleanup, and collect exact-once shard reports with trusted timing hints. Close mock HTTP relays on startup and shutdown failures. Scan all PR commits for secrets with safe full-history fallback while preserving mandatory platform gates. --- .github/workflows/ci.yml | 161 +++++++-- .../daemon/test/channel-workspace.test.mjs | 49 +-- .../mock-openai-compatible-server.test.mjs | 42 +++ .../support/mock-openai-compatible-server.mjs | 37 ++ scripts/ci/README.md | 82 +++++ scripts/ci/daemon-test-shard.mjs | 189 +--------- scripts/ci/daemon-test-shard.test.mjs | 29 ++ scripts/ci/execution-workflow.test.mjs | 4 +- scripts/ci/node-test-shard.mjs | 247 +++++++++++++ scripts/ci/node-test-shard.test.mjs | 182 ++++++++++ scripts/ci/platform-lanes.test.mjs | 2 +- scripts/ci/runtime-workflow.test.mjs | 69 ++++ scripts/ci/secret-scan.mjs | 113 ++++++ scripts/ci/secret-scan.test.mjs | 341 ++++++++++++++++++ scripts/ci/test-processes.mjs | 71 ++++ scripts/ci/test-processes.test.mjs | 71 ++++ scripts/ci/test-shard-plan.mjs | 149 ++++++++ scripts/ci/test-shard-plan.test.mjs | 122 +++++++ 18 files changed, 1717 insertions(+), 243 deletions(-) create mode 100644 packages/daemon/test/mock-openai-compatible-server.test.mjs create mode 100644 packages/daemon/test/support/mock-openai-compatible-server.mjs create mode 100644 scripts/ci/README.md create mode 100644 scripts/ci/node-test-shard.mjs create mode 100644 scripts/ci/node-test-shard.test.mjs create mode 100644 scripts/ci/runtime-workflow.test.mjs create mode 100644 scripts/ci/secret-scan.mjs create mode 100644 scripts/ci/secret-scan.test.mjs create mode 100644 scripts/ci/test-processes.mjs create mode 100644 scripts/ci/test-processes.test.mjs create mode 100644 scripts/ci/test-shard-plan.mjs create mode 100644 scripts/ci/test-shard-plan.test.mjs diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4dda64c93..22dc140d4 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -31,6 +31,7 @@ jobs: packages: ${{ steps.classify.outputs.packages }} test-packages: ${{ steps.classify.outputs.test-packages }} typecheck-packages: ${{ steps.classify.outputs.typecheck-packages }} + unit-matrix: ${{ steps.test-plan.outputs.unit-matrix }} daemon: ${{ steps.classify.outputs.daemon }} web: ${{ steps.classify.outputs.web }} steps: @@ -86,6 +87,37 @@ jobs: for (const [key, value] of Object.entries(outputs)) console.log(`${key}=${value}`); NODE + - name: Restore timing hints once for the entire run + if: steps.policy.outputs.execution-tier == 'complete' && steps.classify.outputs.mode != 'docs' + uses: actions/cache/restore@0057852bfaa89a56745cba8c7296529d2fc39830 + with: + path: .codex-artifacts/ci/timings.json + key: ci-node-timings-v1-${{ github.run_id }}-${{ github.run_attempt }} + restore-keys: ci-node-timings-v1- + + - name: Plan complete core and daemon file coverage + id: test-plan + env: + SELECTED_TEST_PACKAGES: ${{ steps.classify.outputs.test-packages }} + shell: bash + run: | + set -euo pipefail + unit_matrix="$(node scripts/ci/test-shard-plan.mjs \ + --packages="$SELECTED_TEST_PACKAGES" \ + --timings=.codex-artifacts/ci/timings.json \ + --output=.codex-artifacts/ci/test-plan.json)" + printf 'unit-matrix=%s\n' "$unit_matrix" >> "$GITHUB_OUTPUT" + + - name: Publish the immutable test plan for every shard + if: steps.policy.outputs.execution-tier == 'complete' && steps.classify.outputs.mode != 'docs' + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 + with: + name: ci-test-plan + include-hidden-files: true + if-no-files-found: error + retention-days: 7 + path: .codex-artifacts/ci/test-plan.json + common-fast: name: Common checks (fast lane) needs: scope @@ -112,18 +144,26 @@ jobs: - name: Check scanner boundaries, execution policy, classification, and platform CI gates run: node --test scripts/check-public-repo-safety.test.mjs scripts/ci/*.test.mjs scripts/native-execution-engines/verify-codex.test.mjs + # Final file scope cannot exclude secrets introduced and later deleted. - name: Scan Git history for secrets - if: needs.scope.outputs.execution-tier == 'complete' && needs.scope.outputs.mode != 'docs' - run: >- - docker run --rm - -v "$GITHUB_WORKSPACE:/repo:ro" - zricethezav/gitleaks@sha256:c00b6bd0aeb3071cbcb79009cb16a60dd9e0a7c60e2be9ab65d25e6bc8abbb7f - git --no-banner --no-color --redact=100 - --log-opts=HEAD - --config /repo/.gitleaks.toml - --gitleaks-ignore-path /repo/.gitleaksignore - --timeout 600 - /repo + if: needs.scope.outputs.execution-tier == 'complete' + env: + BASE_SHA: ${{ github.event.pull_request.base.sha }} + shell: bash + run: | + set -euo pipefail + scan_plan="$(node scripts/ci/secret-scan.mjs --event="$GITHUB_EVENT_NAME" --base="$BASE_SHA" --head=HEAD)" + echo "Secret scan plan: $scan_plan" + log_opts="$(SCAN_PLAN="$scan_plan" node --input-type=module -e 'console.log(JSON.parse(process.env.SCAN_PLAN).logOpts)')" + docker run --rm \ + -v "$GITHUB_WORKSPACE:/repo:ro" \ + zricethezav/gitleaks@sha256:c00b6bd0aeb3071cbcb79009cb16a60dd9e0a7c60e2be9ab65d25e6bc8abbb7f \ + git --no-banner --no-color --redact=100 \ + --log-opts="$log_opts" \ + --config /repo/.gitleaks.toml \ + --gitleaks-ignore-path /repo/.gitleaksignore \ + --timeout 600 \ + /repo - name: Check pull request whitespace if: github.event_name == 'pull_request' @@ -310,11 +350,14 @@ jobs: run: node scripts/ci/run-packages.mjs --task=typecheck:ci --packages="$SELECTED_PACKAGES" unit-tests: - name: Unit tests and Web checks + name: Unit tests and Web checks (${{ matrix.suite }} ${{ matrix.shard }}) needs: [scope, build] if: ${{ !cancelled() && needs.scope.outputs.execution-tier == 'complete' && needs.scope.outputs.mode != 'docs' && needs.build.result == 'success' }} runs-on: ubuntu-latest timeout-minutes: 25 + strategy: + fail-fast: false + matrix: ${{ fromJSON(needs.scope.outputs.unit-matrix) }} env: SELECTED_PACKAGES: ${{ needs.scope.outputs.test-packages }} WEB_SELECTED: ${{ needs.scope.outputs.web }} @@ -350,15 +393,42 @@ jobs: name: ci-dist path: . + - name: Download the shared test plan + if: matrix.suite == 'core' + uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 + with: + name: ci-test-plan + path: .codex-artifacts/ci + - name: Run selected unit tests without rebuilding dependencies - run: node scripts/ci/run-packages.mjs --task=test:ci --packages="$SELECTED_PACKAGES" --exclude='["@ling-stack/daemon"]' + if: matrix.suite == 'packages' + run: node scripts/ci/run-packages.mjs --task=test:ci --packages="$SELECTED_PACKAGES" --exclude='["@ling-stack/daemon","@ling-stack/core"]' + + - name: Run bounded core shard + if: matrix.suite == 'core' + run: >- + node scripts/ci/node-test-shard.mjs --suite=core + --shard=${{ matrix.shard }} --shard-count=4 --concurrency=2 + --shard-timeout-ms=1200000 + --plan=.codex-artifacts/ci/test-plan.json + --report=.codex-artifacts/ci/reports/core-${{ matrix.shard }}.json + + - name: Preserve core file timings and failure logs + if: always() && matrix.suite == 'core' + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 + with: + name: ci-test-report-core-${{ matrix.shard }} + include-hidden-files: true + if-no-files-found: error + retention-days: 7 + path: .codex-artifacts/ci/reports/** - name: Check Web architecture and UI rules - if: env.WEB_SELECTED == 'true' + if: matrix.suite == 'packages' && env.WEB_SELECTED == 'true' run: corepack pnpm --filter @ling-stack/web lint - name: Check translations - if: env.WEB_SELECTED == 'true' + if: matrix.suite == 'packages' && env.WEB_SELECTED == 'true' run: corepack pnpm --filter @ling-stack/web check:i18n daemon-tests: @@ -368,9 +438,7 @@ jobs: # Full discovery is unchanged: original Darwin-only positive cases must # execute on the real Host, not skip or weaken the production platform gate. runs-on: macos-26 - # Run 39274142315 reached 1,584 passing tests in shard 1 before the 25m - # job limit cancelled it while tests were still progressing. Include setup - # and room for the remaining files without changing discovery or isolation. + # File/shard watchdogs report failures before this final infrastructure cap. timeout-minutes: 40 strategy: fail-fast: false @@ -408,6 +476,12 @@ jobs: name: ci-dist path: . + - name: Download the shared test plan + uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 + with: + name: ci-test-plan + path: .codex-artifacts/ci + # ci-dist comes from Ubuntu; native codecs must be rebuilt on this macOS host. - name: Build the host image codec env: @@ -417,10 +491,22 @@ jobs: corepack pnpm --filter @ling-stack/daemon build:image-sanitizer - name: Run isolated daemon shard - # Each worker retains serial file execution and its own home/workspace. - # Short native lifecycle fixtures retain exclusive execution first. - # Reuse the same four macOS jobs instead of adding more queued runners. - run: node scripts/ci/daemon-test-shard.mjs --shard=${{ matrix.shard }} --shard-count=4 --concurrency=2 + # Every file owns its home/workspace and process group. Short native + # lifecycle fixtures still finish exclusively before parallel workers. + run: >- + node scripts/ci/daemon-test-shard.mjs --shard=${{ matrix.shard }} --shard-count=4 --concurrency=2 + --plan=.codex-artifacts/ci/test-plan.json + --report=.codex-artifacts/ci/reports/daemon-${{ matrix.shard }}.json + + - name: Preserve daemon file timings and failure logs + if: always() + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 + with: + name: ci-test-report-daemon-${{ matrix.shard }} + include-hidden-files: true + if-no-files-found: error + retention-days: 7 + path: .codex-artifacts/ci/reports/** - name: Collect fixed OS bootstrap source for review only if: always() @@ -615,3 +701,34 @@ jobs: run: | set -euo pipefail node scripts/ci/execution-policy.mjs --check-quality + + - name: Download the shared test plan for coverage verification + if: needs.scope.outputs.execution-tier == 'complete' && (needs.scope.outputs.daemon == 'true' || contains(needs.scope.outputs.test-packages, '"@ling-stack/core"')) + uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 + with: + name: ci-test-plan + path: .codex-artifacts/ci + + - name: Download all successful shard reports + if: needs.scope.outputs.execution-tier == 'complete' && (needs.scope.outputs.daemon == 'true' || contains(needs.scope.outputs.test-packages, '"@ling-stack/core"')) + uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 + with: + pattern: ci-test-report-* + merge-multiple: true + path: .codex-artifacts/ci/reports + + - name: Verify every planned file passed exactly once and collect timings + if: needs.scope.outputs.execution-tier == 'complete' && (needs.scope.outputs.daemon == 'true' || contains(needs.scope.outputs.test-packages, '"@ling-stack/core"')) + run: >- + node scripts/ci/test-shard-plan.mjs + --plan=.codex-artifacts/ci/test-plan.json --reports=.codex-artifacts/ci/reports + --output=.codex-artifacts/ci/timings.json + + # Only validated default-branch runs publish hints. PRs consume them but + # cannot replace the baseline; all shards use the same uploaded plan. + - name: Save trusted default-branch test timings + if: needs.scope.outputs.execution-tier == 'complete' && needs.scope.outputs.mode != 'docs' && github.ref == format('refs/heads/{0}', github.event.repository.default_branch) + uses: actions/cache/save@0057852bfaa89a56745cba8c7296529d2fc39830 + with: + path: .codex-artifacts/ci/timings.json + key: ci-node-timings-v1-${{ github.run_id }}-${{ github.run_attempt }} diff --git a/packages/daemon/test/channel-workspace.test.mjs b/packages/daemon/test/channel-workspace.test.mjs index ddc421a42..52c30e818 100644 --- a/packages/daemon/test/channel-workspace.test.mjs +++ b/packages/daemon/test/channel-workspace.test.mjs @@ -4,7 +4,6 @@ import { createHash } from "node:crypto"; import { spawnSync } from "node:child_process"; import { readFileSync, realpathSync } from "node:fs"; import { access, mkdir, mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; -import { createServer } from "node:http"; import { createRequire } from "node:module"; import { tmpdir } from "node:os"; import { resolve } from "node:path"; @@ -49,6 +48,7 @@ import { commandOperationCandidate, fileOperationCandidate } from "./support/cha import { messageText, startRealPiChannelFixture, systemText } from "./support/channel-operation-real-pi-fixture.mjs"; import { prepareFixtureTaskInput } from "./support/prepared-input-fixture.mjs"; import { ToolFreePiRunner } from "./support/tool-free-pi-runner.mjs"; +import { startMockOpenAICompatibleServer } from "./support/mock-openai-compatible-server.mjs"; const require = createRequire(import.meta.url); const ASYNC_SETTLE_TIMEOUT_MS = 10_000; @@ -2442,9 +2442,9 @@ test("materialized image attachment lets Ling answer without materialization cla } }); -test("provider-backed image materialization works without injected summarizer", async () => { +test("provider-backed image materialization works without injected summarizer", async (t) => { const cwd = await mkdtemp(resolve(tmpdir(), "ling-stack-provider-backed-image-materialization-")); - const relay = await startMockOpenAICompatibleServer({ + const relay = await startMockOpenAICompatibleServer(t, { responseContent: ({ body }) => { const content = body?.messages?.[0]?.content; if (Array.isArray(content) && content.some((part) => part?.type === "image_url")) { @@ -2521,7 +2521,6 @@ test("provider-backed image materialization works without injected summarizer", assert.match(imagePart.image_url.url, /^data:image\/png;base64,/); } finally { await daemon.close(); - await relay.close(); await rm(cwd, { recursive: true, force: true }); } }); @@ -11754,10 +11753,10 @@ test("AI teammate private workspace endpoints persist roster, setup, private mes } }); -test("AI teammate private reply waits for startup provider refresh instead of using stale timeout", async () => { +test("AI teammate private reply waits for startup provider refresh instead of using stale timeout", async (t) => { const cwd = await mkdtemp(resolve(tmpdir(), "ling-stack-direct-provider-startup-refresh-")); await ensureGitRepo(cwd); - const relay = await startMockOpenAICompatibleServer(); + const relay = await startMockOpenAICompatibleServer(t); const runner = new RecordingRunner(); let daemon = await startDaemonForTest(runner); try { @@ -11806,7 +11805,6 @@ test("AI teammate private reply waits for startup provider refresh instead of us assert.equal(refreshed.provider.lastCheck.code, "ready"); } finally { await daemon.close(); - await relay.close(); await rm(cwd, { recursive: true, force: true }); } }); @@ -16022,43 +16020,6 @@ function simulateLegacyChannelWithoutModel(cwd, channelId) { } } -async function startMockOpenAICompatibleServer(options = {}) { - const requests = []; - const server = createServer(async (request, response) => { - const chunks = []; - for await (const chunk of request) { - chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk)); - } - const bodyText = Buffer.concat(chunks).toString("utf8"); - const body = bodyText ? JSON.parse(bodyText) : {}; - requests.push({ - method: request.method, - path: request.url, - authorization: request.headers.authorization, - body, - }); - const responseContent = typeof options.responseContent === "function" - ? options.responseContent({ request, body }) - : options.responseContent ?? "OK"; - response.writeHead(200, { "content-type": "application/json" }); - response.end(JSON.stringify({ - id: "chatcmpl-startup-refresh", - object: "chat.completion", - choices: [{ index: 0, message: { role: "assistant", content: responseContent }, finish_reason: "stop" }], - })); - }); - await new Promise((resolveListen) => server.listen(0, "127.0.0.1", resolveListen)); - const address = server.address(); - assert.ok(address && typeof address === "object"); - return { - url: `http://127.0.0.1:${address.port}`, - requests, - close: () => new Promise((resolveClose, rejectClose) => { - server.close((error) => error ? rejectClose(error) : resolveClose()); - }), - }; -} - async function postTaskRunAndReadReadyRunId(baseUrl, cwd, taskId, body) { const response = await fetch(`${baseUrl}/tasks/${encodeURIComponent(taskId)}/run?cwd=${encodeURIComponent(cwd)}`, { method: "POST", diff --git a/packages/daemon/test/mock-openai-compatible-server.test.mjs b/packages/daemon/test/mock-openai-compatible-server.test.mjs new file mode 100644 index 000000000..1de39b002 --- /dev/null +++ b/packages/daemon/test/mock-openai-compatible-server.test.mjs @@ -0,0 +1,42 @@ +import assert from "node:assert/strict"; +import { spawnSync } from "node:child_process"; +import { mkdtemp, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import test from "node:test"; + +test("mock relays close after startup and shutdown failures without hiding failed tests", async t => { + const directory = await mkdtemp(join(tmpdir(), "ling-mock-relay-cleanup-")); + t.after(() => rm(directory, { recursive: true, force: true })); + const fixture = join(directory, "failures.mjs"); + const helper = new URL("./support/mock-openai-compatible-server.mjs", import.meta.url).href; + await writeFile(fixture, ` + import assert from "node:assert/strict"; + import test from "node:test"; + import { startMockOpenAICompatibleServer } from ${JSON.stringify(helper)}; + const urls = []; + test("startup failure", async t => { + urls.push((await startMockOpenAICompatibleServer(t)).url); + throw new Error("runtime_already_running"); + }); + test("shutdown failure", async t => { + urls.push((await startMockOpenAICompatibleServer(t)).url); + try { assert.ok(true); } + finally { throw new Error("harness_shutdown_incomplete_stores_retained"); } + }); + test("all relays closed", async () => { + assert.equal(urls.length, 2); + for (const url of urls) await assert.rejects(fetch(url), error => error.cause?.code === "ECONNREFUSED"); + }); + `); + const env = { ...process.env }; + delete env.NODE_TEST_CONTEXT; + // Run node:test in-process so the watchdog also owns any leaked HTTP listener. + const result = spawnSync(process.execPath, [fixture], { env, encoding: "utf8", timeout: 5000, killSignal: "SIGKILL" }); + assert.ifError(result.error); + assert.equal(result.status, 1, result.stdout + result.stderr); + assert.match(result.stdout, /runtime_already_running/u); + assert.match(result.stdout, /harness_shutdown_incomplete_stores_retained/u); + assert.match(result.stdout, /ok 3 - all relays closed/u); + assert.match(result.stdout, /# pass 1\n# fail 2/u); +}); diff --git a/packages/daemon/test/support/mock-openai-compatible-server.mjs b/packages/daemon/test/support/mock-openai-compatible-server.mjs new file mode 100644 index 000000000..4cbad1c81 --- /dev/null +++ b/packages/daemon/test/support/mock-openai-compatible-server.mjs @@ -0,0 +1,37 @@ +import assert from "node:assert/strict"; +import { createServer } from "node:http"; + +export async function startMockOpenAICompatibleServer(t, options = {}) { + const requests = []; + const server = createServer(async (request, response) => { + const chunks = []; + for await (const chunk of request) { + chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk)); + } + const bodyText = Buffer.concat(chunks).toString("utf8"); + const body = bodyText ? JSON.parse(bodyText) : {}; + requests.push({ + method: request.method, + path: request.url, + authorization: request.headers.authorization, + body, + }); + const responseContent = typeof options.responseContent === "function" + ? options.responseContent({ request, body }) + : options.responseContent ?? "OK"; + response.writeHead(200, { "content-type": "application/json" }); + response.end(JSON.stringify({ + id: "chatcmpl-startup-refresh", + object: "chat.completion", + choices: [{ index: 0, message: { role: "assistant", content: responseContent }, finish_reason: "stop" }], + })); + }); + // Register ownership before any caller can fail during daemon startup/stop. + t.after(() => new Promise((resolveClose, rejectClose) => { + server.close((error) => error ? rejectClose(error) : resolveClose()); + })); + await new Promise((resolveListen) => server.listen(0, "127.0.0.1", resolveListen)); + const address = server.address(); + assert.ok(address && typeof address === "object"); + return { url: `http://127.0.0.1:${address.port}`, requests }; +} diff --git a/scripts/ci/README.md b/scripts/ci/README.md new file mode 100644 index 000000000..b3c14353f --- /dev/null +++ b/scripts/ci/README.md @@ -0,0 +1,82 @@ +# CI Test Execution + +The existing quality check remains the required merge gate. Ready PRs run the +selected package graph; Draft policy, dependency checks, Web checks and native +platform acceptance are unchanged. + +## Test Lanes + +- Core: four Linux shards, two serial file workers per shard. These jobs are + included in the existing `unit-tests` matrix only when core is selected. +- Other packages and Web checks: one Linux matrix entry, excluding core and + daemon because their complete discovery runs in their own entries. +- Daemon: four macOS shards, two serial file workers per shard. The native Kimi + lifecycle files run exclusively before other files on their assigned runner. +- Linux restricted-command contracts and real macOS native smoke remain separate + mandatory lanes. No daemon files have been moved to Linux or removed. + +Every file gets a private runtime home/workspace and a POSIX process group. +An assertion failure does not suppress later files. A file exceeding five +minutes fails and its group is terminated; it is not retried or forced green. +SIGTERM gets up to five seconds before SIGKILL. Cancellation stops new work. +The core queue has a 20-minute deadline and daemon has a 30-minute deadline, +leaving setup and artifact upload time within their unchanged job caps. +Existing per-test assertion and timeout contracts still apply. + +Detached/reparented leak cleanup uses per-file ownership tags and sampled process +ancestry in addition to process groups. This is CI leak cleanup, not a security +sandbox: processes that deliberately clear their ownership tags and escape +ancestry sampling are outside its guarantees. + +## Plans And Evidence + +Scope restores timing hints once and uploads `ci-test-plan`. Every shard uses +that same plan and verifies it against fresh discovery. Unknown/new files fall +back to source-size/category weights; invalid caches never reduce coverage. +Measured timings influence both outer shards and inner worker placement, not +test deadlines or platform selection. + +`ci-test-report-core-*` and `ci-test-report-daemon-*` contain JSON results and +per-file stdout/stderr logs. The JSON is updated before each file starts and +after it finishes, so interrupted runs retain the last active file. Timeouts, +cancellations, launch errors and log/cleanup failures fail the job. The aggregate +requires every planned file to have passed exactly once before saving new timing +hints. Only successful default-branch runs publish timing caches. + +Generate a local plan without running tests: + +```sh +node scripts/ci/test-shard-plan.mjs \ + --packages='["@ling-stack/core","@ling-stack/daemon"]' \ + --output=.codex-artifacts/ci/test-plan.json +``` + +On the suite's required platform, run a shard with that immutable plan: + +```sh +node scripts/ci/daemon-test-shard.mjs --shard=0 --shard-count=4 --concurrency=2 \ + --plan=.codex-artifacts/ci/test-plan.json \ + --report=.codex-artifacts/ci/reports/daemon-0.json +``` + +## Secret Scans + +Complete-tier PRs scan every commit after the unique merge-base, including merged +branches and merge-resolution patches. Docs-only scope and the final file diff +never disable these history scans: an introduced-then-deleted secret is still +scanned. Draft preview runs continue to defer the scan; a Draft promoted to +complete by `full-ci` runs it. + +Missing/ambiguous history or changes to scanner configuration/ignore policy +select full history. Scheduled/manual runs scan full reachable history. Checkout +must retain complete history; the scanner cannot reconstruct a shallow clone. + +## Validation + +```sh +node --test scripts/ci/*.test.mjs +``` + +Local fixture and package checks are not evidence of Linux/macOS hosted-runner +latency. Compare queue, setup, test-file and total durations after the next +authorized complete CI run before claiming a wall-clock improvement. diff --git a/scripts/ci/daemon-test-shard.mjs b/scripts/ci/daemon-test-shard.mjs index dd3d02a06..ec6ee3abb 100644 --- a/scripts/ci/daemon-test-shard.mjs +++ b/scripts/ci/daemon-test-shard.mjs @@ -1,181 +1,22 @@ -import { spawn } from "node:child_process"; -import { mkdir, mkdtemp, readdir, rm } from "node:fs/promises"; -import { statSync } from "node:fs"; -import { tmpdir } from "node:os"; -import { dirname, join, resolve } from "node:path"; -import { fileURLToPath, pathToFileURL } from "node:url"; +import { readdir } from "node:fs/promises"; +import { join, resolve } from "node:path"; +import { pathToFileURL } from "node:url"; +import { createTestIsolation, runNodeTestFiles, runTestShard } from "./node-test-shard.mjs"; +import { testSuites } from "./test-shard-plan.mjs"; -const packageRoot = resolve(dirname(fileURLToPath(import.meta.url)), "../../packages/daemon"); +export { assignTestShards, estimateTestWeight } from "./test-shard-plan.mjs"; +export { createTestIsolation as createDaemonShardIsolation }; -// These real native lifecycle fixtures have short observation windows. Keep -// their original exclusive execution before starting other test workers. -const exclusiveTestFiles = new Set([ - "test/acp-kimi-host-read.test.mjs", - "test/acp-kimi-host-terminal.test.mjs", -]); - -/** - * Return daemon test files in a stable order. The caller passes these paths to - * Node's test runner, so discovery stays independent from shell globbing. - * @param {string} testDirectory - * @returns {Promise} - */ -export async function discoverDaemonTestFiles(testDirectory = join(packageRoot, "test")) { - return (await readdir(testDirectory)) - .filter((file) => file.endsWith(".test.mjs") && file !== "dependency-security.test.mjs") - .sort() - .map((file) => `test/${file}`); -} - -/** - * Estimate relative test cost using stable filename categories. This is only - * a first-pass balancer; a future timing artifact can replace these weights - * without changing the shard contract. - * @param {string} file - * @returns {number} - */ -export function estimateTestWeight(file) { - const keywordWeight = /(?:integration|computer-use|remote|channel|workflow|workspace|media|mcp|runner|runtime)/iu.test(file) - ? 3 - : /(?:storage|conversation|startup|provider|terminal|skill|cua|office)/iu.test(file) - ? 2 - : 1; - const absoluteFile = file.startsWith("test/") ? resolve(packageRoot, file) : file; - try { - // Source size tracks the number of test cases and setup work better than - // filename categories alone, while the keyword floor covers small costly - // integration fixtures. - return Math.max(keywordWeight, Math.ceil(statSync(absoluteFile).size / 1024)); - } catch { - return keywordWeight; - } -} - -/** - * Greedily assign the heaviest tests first to the currently lightest bucket. - * The tie-breaker is shard index, making assignments reproducible across runs. - * @param {string[]} files - * @param {number} shardCount - * @returns {{files: string[], weight: number}[]} - */ -export function assignTestShards(files, shardCount) { - if (!Number.isInteger(shardCount) || shardCount < 1) { - throw new RangeError("shardCount must be a positive integer"); - } - const buckets = Array.from({ length: shardCount }, () => ({ files: [], weight: 0 })); - const weightedFiles = [...new Set(files)] - .map((file) => ({ file, weight: estimateTestWeight(file) })) - .sort((left, right) => right.weight - left.weight || left.file.localeCompare(right.file)); - - for (const item of weightedFiles) { - const target = buckets.reduce((best, bucket, index) => ( - bucket.weight < best.weight || (bucket.weight === best.weight && index < best.index) - ? { ...bucket, index } - : best - ), { ...buckets[0], index: 0 }); - buckets[target.index].files.push(item.file); - buckets[target.index].weight += item.weight; - } - - return buckets.map((bucket) => ({ ...bucket, files: bucket.files.sort() })); -} - -export async function createDaemonShardIsolation(shard, temporaryDirectory = tmpdir()) { - const root = await mkdtemp(join(temporaryDirectory, `ling-stack-daemon-shard-${shard}-`)); - const home = join(root, "home"); - const workspace = join(root, "workspace"); - await Promise.all([mkdir(home), mkdir(workspace)]); - return { root, home, workspace }; -} - -/** @param {string[]} argv */ -function parseArgs(argv) { - const parsed = {}; - for (let index = 0; index < argv.length; index += 1) { - const value = argv[index]; - if (!value.startsWith("--")) continue; - const [key, inlineValue] = value.slice(2).split("=", 2); - parsed[key] = inlineValue ?? argv[++index]; - } - return parsed; -} - -async function runWorker(files, { shard, worker, cwd, temporaryDirectory, stdio }) { - const isolation = await createDaemonShardIsolation(`${shard}-worker-${worker}`, temporaryDirectory); - const started = performance.now(); - try { - const env = { - ...process.env, - LING_STACK_HOME: isolation.home, - LING_STACK_DEFAULT_WORKSPACE_CWD: isolation.workspace, - }; - // This is a new test coordinator even when invoked by a runner contract test. - delete env.NODE_TEST_CONTEXT; - const status = await new Promise((resolveStatus, reject) => { - const child = spawn(process.execPath, ["--test", "--test-concurrency=1", ...files], { - cwd, - env, - stdio, - }); - child.once("error", reject); - child.once("close", (code) => resolveStatus(code ?? 1)); - }); - console.log(`Daemon shard ${shard}, worker ${worker}: ${files.length} files, exit ${status}, ${((performance.now() - started) / 1000).toFixed(1)}s.`); - return status; - } finally { - await rm(isolation.root, { recursive: true, force: true }); - } -} - -/** Keep files serial within each isolated worker, without adding hosted jobs. */ -export async function runDaemonTestFiles(files, { - concurrency = 1, shard = 0, cwd = packageRoot, temporaryDirectory = tmpdir(), stdio = "inherit", -} = {}) { - if (!Number.isInteger(concurrency) || concurrency < 1) { - throw new RangeError("concurrency must be a positive integer"); - } - const uniqueFiles = [...new Set(files)]; - if (uniqueFiles.length === 0) return 0; - const exclusive = concurrency > 1 ? uniqueFiles.filter(file => exclusiveTestFiles.has(file)).sort() : []; - const parallel = uniqueFiles.filter(file => !exclusive.includes(file)); - const results = []; - if (exclusive.length > 0) { - results.push(...await Promise.allSettled([runWorker(exclusive, { - shard, worker: "exclusive", cwd, temporaryDirectory, stdio, - })])); - } - const workers = parallel.length > 0 - ? assignTestShards(parallel, Math.min(concurrency, parallel.length)) : []; - // Wait for all workers even after a failure so their evidence and cleanup - // complete before this process reports the aggregate result. - results.push(...await Promise.allSettled(workers.map((bucket, worker) => runWorker(bucket.files, { - shard, worker, cwd, temporaryDirectory, stdio, - })))); - const error = results.find(result => result.status === "rejected"); - if (error) throw error.reason; - return results.some(result => result.value !== 0) ? 1 : 0; +export async function discoverDaemonTestFiles(testDirectory = join(testSuites.daemon.cwd, "test")) { + return (await readdir(testDirectory)).filter(file => file.endsWith(".test.mjs") && !testSuites.daemon.excluded.includes(file)) + .sort().map(file => `test/${file}`); } -async function runShard() { - const args = parseArgs(process.argv.slice(2)); - const shard = Number(args.shard ?? 0); - const shardCount = Number(args["shard-count"] ?? 4); - if (!Number.isInteger(shard) || !Number.isInteger(shardCount) || shard < 0 || shard >= shardCount) { - throw new RangeError(`expected --shard in [0, ${shardCount - 1}]`); - } - - const files = await discoverDaemonTestFiles(args["test-directory"] ?? join(packageRoot, "test")); - const bucket = assignTestShards(files, shardCount)[shard]; - const concurrency = Number(args.concurrency ?? 1); - console.log(`Daemon shard ${shard + 1}/${shardCount}: ${bucket.files.length} files, weight ${bucket.weight}, concurrency ${concurrency}.`); - return runDaemonTestFiles(bucket.files, { shard, concurrency }); +export function runDaemonTestFiles(files, options) { + return runNodeTestFiles(files, { ...testSuites.daemon, suite: "daemon", ...options }); } -if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) { - try { - process.exitCode = await runShard(); - } catch (error) { - console.error(error instanceof Error ? error.message : error); - process.exitCode = 2; - } +if (process.argv[1] && import.meta.url === pathToFileURL(resolve(process.argv[1])).href) { + try { process.exitCode = await runTestShard("daemon", process.argv.slice(2)); } + catch (error) { console.error(error); process.exitCode = 2; } } diff --git a/scripts/ci/daemon-test-shard.test.mjs b/scripts/ci/daemon-test-shard.test.mjs index 2f6c694c3..7ee045080 100644 --- a/scripts/ci/daemon-test-shard.test.mjs +++ b/scripts/ci/daemon-test-shard.test.mjs @@ -1,8 +1,10 @@ import assert from "node:assert/strict"; +import { spawn } from "node:child_process"; import { mkdir, mkdtemp, readFile, readdir, rm, stat, writeFile } from "node:fs/promises"; import { tmpdir } from "node:os"; import { isAbsolute, join, resolve } from "node:path"; import test from "node:test"; +import { pathToFileURL } from "node:url"; import { assignTestShards, createDaemonShardIsolation, @@ -26,6 +28,33 @@ async function runnerFixture(t, sources, names = sources.map((_, index) => `${in options: { cwd: root, temporaryDirectory: join(root, "isolation"), stdio: "ignore" } }; } +test("bounded runner fails a leaked file and still completes its next file", { skip: process.platform === "win32" }, async (t) => { + const fixture = await runnerFixture(t, [ + "import test from 'node:test'; test('passes but leaks a handle', () => { setInterval(() => {}, 1000); });", + "import { writeFileSync } from 'node:fs'; import test from 'node:test'; test('next file', () => writeFileSync('next-ran', 'yes'));", + ]); + const runner = pathToFileURL(resolve("scripts/ci/daemon-test-shard.mjs")).href; + const source = `import { runDaemonTestFiles } from ${JSON.stringify(runner)}; + process.exitCode = await runDaemonTestFiles(${JSON.stringify(fixture.files)}, + ${JSON.stringify({ ...fixture.options, fileTimeoutMs: 800, terminationGraceMs: 100 })});`; + const child = spawn(process.execPath, ["--input-type=module", "-e", source], { + detached: true, stdio: "ignore", env: { ...process.env, NODE_TEST_CONTEXT: "" }, + }); + let watchdogExpired = false; + const watchdog = setTimeout(() => { + watchdogExpired = true; + try { process.kill(-child.pid, "SIGKILL"); } catch (error) { if (error.code !== "ESRCH") throw error; } + }, 5000); + const status = await new Promise((resolveStatus, reject) => { + child.once("error", reject); + child.once("close", resolveStatus); + }).finally(() => clearTimeout(watchdog)); + assert.equal(watchdogExpired, false, "file deadline must finish before the outer safety watchdog"); + assert.equal(status, 1, "a leaked handle must fail, never force a green result"); + assert.equal(await readFile(join(fixture.root, "next-ran"), "utf8"), "yes"); + assert.deepEqual(await readdir(fixture.options.temporaryDirectory), []); +}); + test("daemon workers overlap while retaining separate homes and workspaces", async (t) => { const fixture = await runnerFixture(t, [0, 1].map(index => ` import assert from 'node:assert/strict'; diff --git a/scripts/ci/execution-workflow.test.mjs b/scripts/ci/execution-workflow.test.mjs index 7fe45af45..327d23842 100644 --- a/scripts/ci/execution-workflow.test.mjs +++ b/scripts/ci/execution-workflow.test.mjs @@ -19,7 +19,7 @@ function jobSource(id) { /** Execute the actual workflow aggregation command with simulated job outcomes. */ function runQuality(overrides = {}) { - const command = jobSource("quality").split(" run: |\n")[1]; + const command = /^ {8}run: \|\n((?: {10}[^\n]*(?:\n|$)|\n)*)/mu.exec(jobSource("quality"))?.[1]; assert.ok(command, "quality must retain an executable aggregation gate"); return spawnSync("bash", ["-c", command.replace(/^ {10}/gmu, "")], { cwd: root, @@ -53,7 +53,7 @@ test("every expensive job requires complete execution and retains its scope sele } const common = jobSource("common-fast"); assert.match(common, /run: node --test scripts\/check-public-repo-safety\.test\.mjs scripts\/ci\/\*\.test\.mjs/u); - assert.match(common, /name: Scan Git history for secrets\n if: needs\.scope\.outputs\.execution-tier == 'complete' && needs\.scope\.outputs\.mode != 'docs'/u); + assert.match(common, /name: Scan Git history for secrets\n if: needs\.scope\.outputs\.execution-tier == 'complete'\n/u); const dependencyPolicy = jobSource("dependency-policy"); assert.match(dependencyPolicy, /name: Check production dependency update policy\n run: node --test scripts\/production-dependency-policy\.test\.mjs/u); }); diff --git a/scripts/ci/node-test-shard.mjs b/scripts/ci/node-test-shard.mjs new file mode 100644 index 000000000..c38b7cd07 --- /dev/null +++ b/scripts/ci/node-test-shard.mjs @@ -0,0 +1,247 @@ +import { spawn } from "node:child_process"; +import { randomUUID } from "node:crypto"; +import { createWriteStream } from "node:fs"; +import { mkdir, mkdtemp, readFile, rename, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { basename, dirname, join, resolve } from "node:path"; +import { pathToFileURL } from "node:url"; +import { finished } from "node:stream/promises"; +import { assignTestShards, assertSuiteCoverage, discoverTestFiles, parseArguments, testSuites } from "./test-shard-plan.mjs"; +import { trackTestProcesses } from "./test-processes.mjs"; + +export const defaultFileTimeoutMs = 300_000; +export const defaultShardTimeoutMs = 1_800_000; + +export async function createTestIsolation(label, temporaryDirectory = tmpdir()) { + const root = await mkdtemp(join(temporaryDirectory, `ling-stack-test-${label}-`)); + const home = join(root, "home"); + const workspace = join(root, "workspace"); + try { await Promise.all([mkdir(home), mkdir(workspace)]); } + catch (error) { await rm(root, { recursive: true, force: true }); throw error; } + return { root, home, workspace }; +} + +/** Each file owns an isolated runtime and bounded process cleanup. */ +async function runFile(file, { cwd, shard, worker, suite, temporaryDirectory, stdio, signal, + fileTimeoutMs, terminationGraceMs, reportPath }) { + const isolation = await createTestIsolation(`${suite}-${shard}-${worker}`, temporaryDirectory); + const started = performance.now(); + let result; + let log; + try { + if (reportPath) { + const logPath = join(dirname(reportPath), "logs", `${suite}-${shard}`, `${basename(file)}.log`); + await mkdir(dirname(logPath), { recursive: true }); + log = createWriteStream(logPath); + await new Promise((done, reject) => { log.once("open", done); log.once("error", reject); }); + } + if (signal.aborted) return { file, worker, status: "cancelled", exitCode: null, signal: null, + durationMs: Math.max(1, performance.now() - started) }; + const token = randomUUID(); + const env = { ...process.env, LING_STACK_HOME: isolation.home, LING_STACK_DEFAULT_WORKSPACE_CWD: isolation.workspace, + LING_STACK_CI_TEST_OWNER: token }; + delete env.NODE_TEST_CONTEXT; + const child = spawn(process.execPath, ["--test", "--test-concurrency=1", file], { + cwd, env, detached: true, stdio: ["ignore", "pipe", "pipe"], + }); + let stopped; + let escalation; + let logError; + let cleanupError; + const processes = trackTestProcesses(child.pid, token); + const terminate = signal => { + // Observe ancestry before terminating parents that may orphan a child. + try { processes.signal(signal); } + catch (error) { + cleanupError ??= error; + try { child.kill(signal); } catch (error) { cleanupError ??= error; } + } + }; + const stop = reason => { + stopped ??= reason; + terminate("SIGTERM"); + escalation ??= setTimeout(() => terminate("SIGKILL"), terminationGraceMs); + }; + const observe = setInterval(() => { + try { processes.live(); } + catch (error) { cleanupError ??= error; stop("cleanup_error"); } + }, 200); + const onAbort = () => stop("cancelled"); + const onLogError = error => { logError = error; stop("log_error"); }; + log?.on("error", onLogError); + for (const [stream, destination] of [[child.stdout, process.stdout], [child.stderr, process.stderr]]) { + if (log) stream.pipe(log, { end: false }); + if (stdio === "inherit") stream.pipe(destination, { end: false }); + if (!log && stdio !== "inherit") stream.resume(); + } + const exited = new Promise((done, reject) => { + child.once("error", reject); + child.once("exit", (code, exitSignal) => done({ code, signal: exitSignal })); + }); + const closed = new Promise(done => child.once("close", done)); + const deadline = setTimeout(() => stop("timed_out"), fileTimeoutMs); + signal.addEventListener("abort", onAbort, { once: true }); + if (signal.aborted) onAbort(); + let teardownTimer; + try { + const exit = await exited; + clearTimeout(deadline); + // Even a successful coordinator can leave a grandchild alive. Reap its + // group before deleting the private workspace or moving to another file. + if (processes.live(true).length) stop("leaked_processes"); + const teardown = new Promise(done => { + teardownTimer = setTimeout(() => { + stop("teardown_timeout"); + terminate("SIGKILL"); + child.stdout.destroy(); + child.stderr.destroy(); + done(); + }, terminationGraceMs + 1000); + }); + await Promise.race([closed, teardown]); + if (!(await processes.waitForExit(terminationGraceMs))) { + terminate("SIGKILL"); + if (!(await processes.waitForExit(1000))) throw new Error("CI-owned test processes survived SIGKILL"); + } + if (logError) throw logError; + if (cleanupError) throw cleanupError; + result = { file, worker, status: stopped ?? (exit.code === 0 ? "passed" : "failed"), + exitCode: exit.code, signal: exit.signal, durationMs: Math.max(1, performance.now() - started) }; + } finally { + clearTimeout(deadline); + clearTimeout(escalation); + clearTimeout(teardownTimer); + clearInterval(observe); + signal.removeEventListener("abort", onAbort); + try { + terminate("SIGKILL"); + if (!(await processes.waitForExit(1000))) throw new Error("CI-owned test processes survived final cleanup"); + if (cleanupError) throw cleanupError; + } finally { child.stdout.destroy(); child.stderr.destroy(); } + } + } finally { + try { + if (log) { log.end(); await finished(log); } + } finally { await rm(isolation.root, { recursive: true, force: true }); } + } + return result; +} + +export async function runNodeTestFiles(files, { + concurrency = 1, shard = 0, shardCount = 1, suite = "tests", cwd = process.cwd(), exclusive = [], timings = {}, temporaryDirectory = tmpdir(), stdio = "inherit", + fileTimeoutMs = defaultFileTimeoutMs, shardTimeoutMs = defaultShardTimeoutMs, terminationGraceMs = 5000, + reportPath, signal: parentSignal, +} = {}) { + for (const [name, value] of Object.entries({ concurrency, fileTimeoutMs, shardTimeoutMs, terminationGraceMs })) { + if (!Number.isSafeInteger(value) || value < 1) throw new RangeError(`${name} must be a positive integer`); + } + if (process.platform === "win32") throw new Error("This CI runner requires POSIX process groups (Linux or macOS)"); + const selected = [...new Set(files)]; + const records = new Map(selected.map(file => [file, { file, status: "pending" }])); + const controller = new AbortController(); + const relayAbort = () => controller.abort(parentSignal.reason); + parentSignal?.addEventListener("abort", relayAbort, { once: true }); + if (parentSignal?.aborted) relayAbort(); + const shardDeadline = setTimeout(() => controller.abort(new Error("shard_deadline_exceeded")), shardTimeoutMs); + let complete = false; + let writes = Promise.resolve(); + const persist = () => { + if (!reportPath) return Promise.resolve(); + const snapshot = JSON.stringify({ version: 1, suite, shard, shardCount, platform: process.platform, + complete, fileTimeoutMs, shardTimeoutMs, results: selected.map(file => records.get(file)) }, null, 2); + writes = writes.then(async () => { + await mkdir(dirname(reportPath), { recursive: true }); + await writeFile(`${reportPath}.tmp`, `${snapshot}\n`); + await rename(`${reportPath}.tmp`, reportPath); + }); + return writes; + }; + const errors = []; + const worker = async (bucket, index) => { + for (const file of bucket) { + if (controller.signal.aborted) break; + console.log(`[${suite} shard ${shard}, worker ${index}] START ${file} (deadline ${fileTimeoutMs}ms)`); + records.set(file, { file, worker: index, status: "running" }); + await persist(); + try { + const result = await runFile(file, { cwd, shard, worker: index, suite, temporaryDirectory, stdio, + signal: controller.signal, fileTimeoutMs, terminationGraceMs, reportPath }); + records.set(file, result); + console.log(`[${suite} shard ${shard}, worker ${index}] END ${file}: ${result.status}, ${(result.durationMs / 1000).toFixed(2)}s`); + } catch (error) { + errors.push(error); + records.set(file, { file, worker: index, status: "error", error: error.message }); + console.error(`[${suite} shard ${shard}, worker ${index}] ERROR ${file}: ${error.message}`); + } + await persist(); + } + }; + try { + await persist(); + const exclusiveFiles = selected.filter(file => exclusive.includes(file)).sort(); + await worker(exclusiveFiles, "exclusive"); + const parallel = selected.filter(file => !exclusive.includes(file)); + const buckets = parallel.length ? assignTestShards(parallel, Math.min(concurrency, parallel.length), { cwd, timings }) : []; + const outcomes = await Promise.allSettled(buckets.map((bucket, index) => worker(bucket.files, index))); + for (const outcome of outcomes) if (outcome.status === "rejected") errors.push(outcome.reason); + complete = !controller.signal.aborted && [...records.values()].every(result => result.status !== "pending" && result.status !== "running"); + } finally { + clearTimeout(shardDeadline); + parentSignal?.removeEventListener("abort", relayAbort); + for (const [file, record] of records) { + if (record.status === "pending" || record.status === "running") records.set(file, { ...record, status: "cancelled" }); + } + await persist(); + } + if (errors.length) throw errors[0]; + if (controller.signal.aborted) console.error(`[${suite} shard ${shard}] Cancelled: ${controller.signal.reason}`); + return complete && [...records.values()].every(result => result.status === "passed") ? 0 : 1; +} + +export async function runTestShard(suiteName, argv) { + const args = parseArguments(argv, ["--shard", "--shard-count", "--concurrency", "--plan", "--report", "--file-timeout-ms", "--shard-timeout-ms"]); + const config = testSuites[suiteName]; + if (!config) throw new Error(`Unknown suite: ${suiteName}`); + const shard = Number(args["--shard"] ?? 0); + const shardCount = Number(args["--shard-count"] ?? config.shardCount); + if (!Number.isInteger(shard) || !Number.isInteger(shardCount) || shardCount < 1 || shard < 0 || shard >= shardCount) { + throw new RangeError(`expected --shard in [0, ${shardCount - 1}]`); + } + const discovered = await discoverTestFiles(config.cwd, config.excluded); + let shards = assignTestShards(discovered, shardCount, { cwd: config.cwd }); + let timings = {}; + if (args["--plan"]) { + const plan = JSON.parse(await readFile(args["--plan"], "utf8")); + const suite = plan.suites?.[suiteName]; + if (plan.version !== 1 || suite?.shardCount !== shardCount || suite.platform !== process.platform) throw new Error("Invalid test plan for this suite/platform"); + assertSuiteCoverage(suite, discovered); + shards = suite.shards; + timings = suite.timings ?? {}; + } + const files = shards[shard].files; + console.log(`${suiteName} shard ${shard + 1}/${shardCount}: ${files.length} files, weight ${shards[shard].weight.toFixed(2)}.`); + const controller = new AbortController(); + const onInt = () => controller.abort(new Error("SIGINT")); + const onTerm = () => controller.abort(new Error("SIGTERM")); + process.once("SIGINT", onInt); + process.once("SIGTERM", onTerm); + try { + return await runNodeTestFiles(files, { ...config, suite: suiteName, shard, shardCount, timings, + concurrency: Number(args["--concurrency"] ?? config.concurrency), reportPath: args["--report"], signal: controller.signal, + fileTimeoutMs: Number(args["--file-timeout-ms"] ?? defaultFileTimeoutMs), + shardTimeoutMs: Number(args["--shard-timeout-ms"] ?? defaultShardTimeoutMs) }); + } finally { + process.removeListener("SIGINT", onInt); + process.removeListener("SIGTERM", onTerm); + } +} + +if (process.argv[1] && import.meta.url === pathToFileURL(resolve(process.argv[1])).href) { + try { + const args = process.argv.slice(2); + const index = args.findIndex(value => value.startsWith("--suite=")); + if (index < 0) throw new Error("--suite=core or --suite=daemon is required"); + const [suite] = args.splice(index, 1); + process.exitCode = await runTestShard(suite.slice("--suite=".length), args); + } catch (error) { console.error(error); process.exitCode = 2; } +} diff --git a/scripts/ci/node-test-shard.test.mjs b/scripts/ci/node-test-shard.test.mjs new file mode 100644 index 000000000..d573d2e90 --- /dev/null +++ b/scripts/ci/node-test-shard.test.mjs @@ -0,0 +1,182 @@ +import assert from "node:assert/strict"; +import { spawn } from "node:child_process"; +import { mkdir, mkdtemp, readFile, readdir, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { setTimeout as delay } from "node:timers/promises"; +import test from "node:test"; +import { runNodeTestFiles } from "./node-test-shard.mjs"; + +async function fixture(t, sources) { + const cwd = await mkdtemp(join(tmpdir(), "ling-node-runner-")); + t.after(() => rm(cwd, { recursive: true, force: true })); + await Promise.all([mkdir(join(cwd, "test")), mkdir(join(cwd, "isolation"))]); + await Promise.all(sources.map((source, index) => writeFile(join(cwd, `test/${index}.test.mjs`), source))); + const reportPath = join(cwd, "reports", "tests-0.json"); + return { cwd, reportPath, files: sources.map((_, index) => `test/${index}.test.mjs`), + options: { cwd, reportPath, stdio: "ignore", temporaryDirectory: join(cwd, "isolation"), terminationGraceMs: 100 } }; +} + +async function waitFor(check) { + const deadline = AbortSignal.timeout(5000); + for (;;) { + const result = await check(); + if (result) return result; + await delay(10, undefined, { signal: deadline }); + } +} + +test("reports are durable while a file is running, and preserve stdout/stderr after completion", async t => { + const f = await fixture(t, [` + import { writeFile, access } from 'node:fs/promises'; + import { setTimeout } from 'node:timers/promises'; + import test from 'node:test'; + test('wait for release', { timeout: 10000 }, async () => { + console.log('fixture stdout'); console.error('fixture stderr'); + await writeFile('ready', process.env.LING_STACK_HOME); + for (;;) { try { await access('release'); break; } catch {} await setTimeout(10); } + }); + `]); + const run = runNodeTestFiles(f.files, f.options); + await waitFor(async () => readFile(join(f.cwd, "ready"), "utf8").catch(() => false)); + const running = JSON.parse(await readFile(f.reportPath, "utf8")); + assert.equal(running.complete, false); + assert.equal(running.results[0].status, "running"); + await writeFile(join(f.cwd, "release"), "done"); + assert.equal(await run, 0); + const report = JSON.parse(await readFile(f.reportPath, "utf8")); + assert.equal(report.complete, true); + assert.equal(report.results[0].status, "passed"); + assert.ok(report.results[0].durationMs > 0); + const log = await readFile(join(f.cwd, "reports/logs/tests-0/0.test.mjs.log"), "utf8"); + assert.match(log, /fixture stdout/u); + assert.match(log, /fixture stderr/u); + assert.deepEqual(await readdir(f.options.temporaryDirectory), []); +}); + +test("serial files never reuse a previous file's runtime home or workspace", async t => { + const f = await fixture(t, [0, 1].map(index => ` + import { writeFileSync } from 'node:fs'; import test from 'node:test'; + test('home', () => writeFileSync('${index}.json', JSON.stringify({ + home: process.env.LING_STACK_HOME, workspace: process.env.LING_STACK_DEFAULT_WORKSPACE_CWD }))); + `)); + assert.equal(await runNodeTestFiles(f.files, f.options), 0); + const records = await Promise.all([0, 1].map(async index => JSON.parse(await readFile(join(f.cwd, `${index}.json`), "utf8")))); + assert.notEqual(records[0].home, records[1].home); + assert.notEqual(records[0].workspace, records[1].workspace); +}); + +test("file deadlines kill SIGTERM-resistant descendants, fail the file and continue", async t => { + const f = await fixture(t, [ + `import { spawn } from 'node:child_process'; import { writeFileSync } from 'node:fs'; import test from 'node:test'; + test('leaked child', () => { + const child = spawn(process.execPath, ['-e', "process.on('SIGTERM', () => {}); setInterval(() => {}, 1000)"], { stdio: 'ignore' }); + writeFileSync('child-pid', String(child.pid)); + setInterval(() => {}, 1000); + });`, + "import test from 'node:test'; test('next file still runs', () => {});", + ]); + assert.equal(await runNodeTestFiles(f.files, { ...f.options, fileTimeoutMs: 800 }), 1); + const report = JSON.parse(await readFile(f.reportPath, "utf8")); + assert.equal(report.complete, true); + assert.deepEqual(report.results.map(result => result.status), ["timed_out", "passed"]); + const pid = Number(await readFile(join(f.cwd, "child-pid"), "utf8")); + await waitFor(async () => { + try { process.kill(pid, 0); return false; } + catch (error) { assert.equal(error.code, "ESRCH"); return true; } + }); + assert.deepEqual(await readdir(f.options.temporaryDirectory), []); +}); + +for (const timeout of [false, true]) test(`detached descendants cannot survive ${timeout ? "a file timeout" : "successful coordinator exit"}`, async t => { + const f = await fixture(t, [ + `import { spawn } from 'node:child_process'; import { writeFileSync } from 'node:fs'; import test from 'node:test'; + test('detached leak', () => { + const child = spawn(process.execPath, ['-e', "process.on('SIGTERM', () => {}); setInterval(() => {}, 1000)"], + { detached: true, stdio: 'ignore' }); + writeFileSync('detached-pid', String(child.pid)); child.unref(); + ${timeout ? "setInterval(() => {}, 1000);" : ""} + });`, + "import test from 'node:test'; test('next file still runs', () => {});", + ]); + let pid; + t.after(() => { + if (pid) { try { process.kill(pid, "SIGKILL"); } catch (error) { if (error.code !== "ESRCH") throw error; } } + }); + let code; + try { code = await runNodeTestFiles(f.files, { ...f.options, fileTimeoutMs: 800 }); } + finally { pid = Number(await readFile(join(f.cwd, "detached-pid"), "utf8").catch(() => 0)); } + assert.equal(code, 1); + const report = JSON.parse(await readFile(f.reportPath, "utf8")); + assert.deepEqual(report.results.map(result => result.status), [timeout ? "timed_out" : "leaked_processes", "passed"]); + await waitFor(async () => { + try { process.kill(pid, 0); return false; } + catch (error) { assert.equal(error.code, "ESRCH"); return true; } + }); +}); + +test("cancellation reaps a detached child with a cleared environment without touching unrelated processes", async t => { + const f = await fixture(t, [ + `import { spawn } from 'node:child_process'; import { writeFileSync } from 'node:fs'; + const child = spawn(process.execPath, ['-e', "setInterval(() => {}, 1000)"], + { detached: true, stdio: 'ignore', env: {} }); + writeFileSync('detached-pid', String(child.pid)); child.unref(); setInterval(() => {}, 1000);`, + ]); + const unrelated = spawn(process.execPath, ["-e", "setInterval(() => {}, 1000)"], { detached: true, stdio: "ignore" }); + const unrelatedClosed = new Promise(done => unrelated.once("close", done)); + t.after(async () => { + unrelated.kill("SIGKILL"); + await unrelatedClosed; + }); + const controller = new AbortController(); + const run = runNodeTestFiles(f.files, { ...f.options, signal: controller.signal }); + const pid = Number(await waitFor(() => readFile(join(f.cwd, "detached-pid"), "utf8").catch(() => false))); + t.after(() => { try { process.kill(pid, "SIGKILL"); } catch (error) { if (error.code !== "ESRCH") throw error; } }); + controller.abort("fixture cancelled"); + assert.equal(await run, 1); + assert.equal(process.kill(unrelated.pid, 0), true); + await waitFor(async () => { + try { process.kill(pid, 0); return false; } + catch (error) { assert.equal(error.code, "ESRCH"); return true; } + }); +}); + +test("cancellation stops active files, does not launch queued files, and retains incomplete evidence", async t => { + const f = await fixture(t, [ + "import { writeFileSync } from 'node:fs'; writeFileSync('ready', 'yes'); setInterval(() => {}, 1000);", + "throw new Error('cancelled queue must not launch');", + ]); + const controller = new AbortController(); + const run = runNodeTestFiles(f.files, { ...f.options, signal: controller.signal }); + await waitFor(async () => readFile(join(f.cwd, "ready"), "utf8").catch(() => false)); + controller.abort(new Error("fixture cancelled")); + assert.equal(await run, 1); + const report = JSON.parse(await readFile(f.reportPath, "utf8")); + assert.equal(report.complete, false); + assert.deepEqual(report.results.map(result => result.status), ["cancelled", "cancelled"]); + assert.deepEqual(await readdir(f.options.temporaryDirectory), []); +}); + +test("the shard deadline bounds the entire queue independently of file deadlines", async t => { + const f = await fixture(t, ["setInterval(() => {}, 1000);", "throw new Error('not reached');"]); + assert.equal(await runNodeTestFiles(f.files, { ...f.options, fileTimeoutMs: 10000, shardTimeoutMs: 500 }), 1); + const report = JSON.parse(await readFile(f.reportPath, "utf8")); + assert.equal(report.complete, false); + assert.deepEqual(report.results.map(result => result.status), ["cancelled", "cancelled"]); +}); + +test("an already cancelled shard cannot become green, including an empty assignment", async t => { + const f = await fixture(t, ["throw new Error('not reached');"]); + const signal = AbortSignal.abort("before launch"); + for (const files of [[], f.files]) assert.equal(await runNodeTestFiles(files, { ...f.options, signal }), 1); + assert.deepEqual(await readdir(f.options.temporaryDirectory), []); +}); + +test("log creation failures still remove the private isolation directory", async t => { + const f = await fixture(t, ["import 'node:test';"]); + await mkdir(join(f.cwd, "reports/logs/tests-0/0.test.mjs.log"), { recursive: true }); + await assert.rejects(runNodeTestFiles(f.files, f.options), { code: "EISDIR" }); + assert.deepEqual(await readdir(f.options.temporaryDirectory), []); + const report = JSON.parse(await readFile(f.reportPath, "utf8")); + assert.equal(report.results[0].status, "error"); +}); diff --git a/scripts/ci/platform-lanes.test.mjs b/scripts/ci/platform-lanes.test.mjs index 056e57913..08ccfc80d 100644 --- a/scripts/ci/platform-lanes.test.mjs +++ b/scripts/ci/platform-lanes.test.mjs @@ -17,7 +17,7 @@ function jobSource(id) { } function runQuality(overrides = {}) { - const run = / run: (?:(\|)\n([\s\S]+)|([^\n]+))/u.exec(jobSource("quality")); + const run = /^ {8}run: (?:(\|)\n((?: {10}[^\n]*(?:\n|$)|\n)*)|([^\n]+))/mu.exec(jobSource("quality")); const script = run?.[1] ? run[2].replace(/^ {10}/gmu, "") : run?.[3]; assert.ok(script, "quality must retain an executable aggregation gate"); return spawnSync("bash", ["-c", script], { diff --git a/scripts/ci/runtime-workflow.test.mjs b/scripts/ci/runtime-workflow.test.mjs new file mode 100644 index 000000000..27615fa63 --- /dev/null +++ b/scripts/ci/runtime-workflow.test.mjs @@ -0,0 +1,69 @@ +import assert from "node:assert/strict"; +import { readFileSync } from "node:fs"; +import test from "node:test"; + +const workflow = readFileSync(new URL("../../.github/workflows/ci.yml", import.meta.url), "utf8"); +function job(id) { + const match = new RegExp(`^ ${id}:\\n([\\s\\S]*?)(?=^ [a-z][a-z-]*:|$(?![\\s\\S]))`, "mu").exec(workflow); + assert.ok(match, id); + return match[1]; +} + +test("all core matrix jobs are part of the existing mandatory unit gate", () => { + const source = job("unit-tests"); + assert.match(source, /matrix: \$\{\{ fromJSON\(needs\.scope\.outputs\.unit-matrix\) \}\}/u); + assert.match(source, /fail-fast: false/u); + assert.match(source, /node-test-shard\.mjs --suite=core/u); + assert.match(source, /--shard=\$\{\{ matrix\.shard \}\} --shard-count=4 --concurrency=2/u); + assert.match(source, /--shard-timeout-ms=1200000/u); + assert.match(source, /--exclude='\["@ling-stack\/daemon","@ling-stack\/core"\]'/u); + assert.match(source, /if: matrix\.suite == 'packages' && env\.WEB_SELECTED == 'true'/u); + assert.match(job("quality"), /UNIT_RESULT: \$\{\{ needs\.unit-tests\.result \}\}/u); + assert.doesNotMatch(source, /continue-on-error|--test-skip-pattern|--test-name-pattern/u); +}); + +test("timings are restored once and every shard consumes one immutable plan", () => { + const scope = job("scope"); + assert.match(scope, /actions\/cache\/restore@[a-f0-9]{40}/u); + assert.match(scope, /unit-matrix: \$\{\{ steps\.test-plan\.outputs\.unit-matrix \}\}/u); + assert.match(scope, /test-shard-plan\.mjs/u); + assert.match(scope, /name: ci-test-plan\n include-hidden-files: true/u); + for (const name of ["unit-tests", "daemon-tests"]) { + const source = job(name); + assert.match(source, /name: ci-test-plan/u); + assert.match(source, /--plan=\.codex-artifacts\/ci\/test-plan\.json/u); + assert.doesNotMatch(source, /actions\/cache\/restore/u); + assert.match(source, /name: ci-test-report-(?:core|daemon)-\$\{\{ matrix\.shard \}\}\n include-hidden-files: true/u); + assert.match(source, /if-no-files-found: error/u); + } +}); + +test("the aggregate verifies reports before caching only default-branch successful timings", () => { + const quality = job("quality"); + const policy = quality.indexOf("node scripts/ci/execution-policy.mjs --check-quality"); + const coverage = quality.indexOf("node scripts/ci/test-shard-plan.mjs"); + const cache = quality.indexOf("uses: actions/cache/save@"); + assert.ok(policy >= 0 && coverage > policy && cache > coverage); + assert.match(quality, /pattern: ci-test-report-\*\n merge-multiple: true/u); + assert.match(quality, /--plan=\.codex-artifacts\/ci\/test-plan\.json --reports=\.codex-artifacts\/ci\/reports/u); + assert.match(quality, /github\.ref == format\('refs\/heads\/\{0\}', github\.event\.repository\.default_branch\)/u); + assert.doesNotMatch(quality, /continue-on-error/u); +}); + +test("complete PR history scans ignore docs-only final diffs while Draft previews stay deferred", () => { + const common = job("common-fast"); + assert.doesNotMatch(common, /^ if:/mu, "the common job must not bypass history scans based on file scope"); + assert.doesNotMatch(workflow, /^\s+paths(?:-ignore)?:/mu, "docs-only final diffs must still trigger validation"); + assert.match(common, /^ - name: Scan Git history for secrets\n if: needs\.scope\.outputs\.execution-tier == 'complete'$/mu); +}); + +test("secret scans use commit history planning and retain full checkout and pinned scanner", () => { + const common = job("common-fast"); + assert.match(common, /fetch-depth: 0/u); + assert.match(common, /secret-scan\.mjs --event="\$GITHUB_EVENT_NAME" --base="\$BASE_SHA" --head=HEAD/u); + assert.match(common, /--log-opts="\$log_opts"/u); + assert.match(common, /gitleaks@sha256:[a-f0-9]{64}/u); + assert.match(common, /--redact=100/u); + assert.match(common, /--gitleaks-ignore-path \/repo\/\.gitleaksignore/u); + assert.doesNotMatch(common, /--log-opts=HEAD/u); +}); diff --git a/scripts/ci/secret-scan.mjs b/scripts/ci/secret-scan.mjs new file mode 100644 index 000000000..45e901e35 --- /dev/null +++ b/scripts/ci/secret-scan.mjs @@ -0,0 +1,113 @@ +import { execFileSync } from "node:child_process"; +import { resolve } from "node:path"; +import { pathToFileURL } from "node:url"; +import { parseArgs } from "node:util"; + +const HISTORY_LOG_OPTIONS = ["--full-history", "--diff-merges=first-parent"]; +const COMMIT_ID = /^(?:[a-f0-9]{40}|[a-f0-9]{64})$/u; +// These are the policy files explicitly passed to Gitleaks by CI. +const SCANNER_POLICY_PATHS = [":(top,literal).gitleaks.toml", ":(top,literal).gitleaksignore"]; + +function git(cwd, args) { + return execFileSync("git", ["-c", "color.ui=false", ...args], { + cwd, + encoding: "utf8", + stdio: ["ignore", "pipe", "pipe"], + env: { ...process.env, GIT_OPTIONAL_LOCKS: "0" }, + timeout: 30_000, + }).trim(); +} + +function resolveCommit(cwd, revision) { + if (typeof revision !== "string" || revision.length === 0 || revision.startsWith("-") || /[\s\u0000-\u001f\u007f]/u.test(revision)) { + throw new Error("Expected a single non-option Git revision."); + } + const commit = git(cwd, ["rev-parse", "--verify", "--end-of-options", `${revision}^{commit}`]); + if (!COMMIT_ID.test(commit)) throw new Error("Git revision did not resolve to one commit ID."); + return commit; +} + +/** + * Plan Gitleaks' git-log arguments, never a working-tree or endpoint file diff. + * Full mode means all history reachable from the validated head, like CI's + * previous HEAD scan; it cannot restore history missing from a shallow clone. + * Invalid heads throw even on full-scan events. Uncertain baselines fall back. + * @param {{eventName?: string, base?: string, head?: string, cwd?: string}} input + * @returns {{mode: "range"|"full", logOpts: string, reason: string}} + */ +export function planSecretScan({ eventName, base, head = "HEAD", cwd = process.cwd() } = {}) { + let headCommit; + try { + headCommit = resolveCommit(cwd, head); + } catch (error) { + throw new Error("Secret scan head is invalid or cannot be resolved to a commit.", { cause: error }); + } + + const full = (reason) => ({ + mode: "full", + logOpts: [...HISTORY_LOG_OPTIONS, headCommit].join(" "), + reason, + }); + if (eventName !== "pull_request") return full("Non-PR event requires full history."); + if (base === undefined || base === null || base === "") return full("PR base is missing."); + + let baseCommit; + try { + baseCommit = resolveCommit(cwd, base); + } catch { + return full("PR base is invalid or unavailable."); + } + + try { + if (git(cwd, ["rev-parse", "--is-shallow-repository"]) !== "false") { + return full("Repository is shallow or its history completeness is unknown."); + } + } catch { + return full("Repository history completeness could not be determined."); + } + + let baselines; + try { + baselines = git(cwd, ["merge-base", "--all", baseCommit, headCommit]).split(/\r?\n/u); + } catch { + return full("Safe merge-base is unavailable."); + } + if (baselines.length !== 1 || !COMMIT_ID.test(baselines[0])) { + return full("Safe merge-base is missing or ambiguous."); + } + + const range = `${baselines[0]}..${headCommit}`; + try { + // Inspect every commit, including reverted policy edits and merge-only edits. + // An endpoint diff would miss these and could skip newly unignored history. + const policyChange = git(cwd, [ + "log", "--format=%H", "--max-count=1", ...HISTORY_LOG_OPTIONS, "--no-patch", range, + "--", ...SCANNER_POLICY_PATHS, + ]); + if (policyChange) return full("Gitleaks configuration or ignore policy changed in PR history."); + } catch { + return full("Gitleaks policy history could not be inspected safely."); + } + + return { + mode: "range", + // Diff merge commits against their first parent without restricting traversal + // to the first-parent chain: merged branches may contain removed secrets. + logOpts: [...HISTORY_LOG_OPTIONS, range].join(" "), + reason: "All PR commits after the unique safe merge-base, including merge resolutions.", + }; +} + +if (process.argv[1] && import.meta.url === pathToFileURL(resolve(process.argv[1])).href) { + try { + const { values } = parseArgs({ + options: { event: { type: "string" }, base: { type: "string" }, head: { type: "string" } }, + allowPositionals: false, + strict: true, + }); + console.log(JSON.stringify(planSecretScan({ eventName: values.event, base: values.base, head: values.head }))); + } catch (error) { + console.error(error instanceof Error ? error.message : "Secret scan planning failed."); + process.exitCode = 1; + } +} diff --git a/scripts/ci/secret-scan.test.mjs b/scripts/ci/secret-scan.test.mjs new file mode 100644 index 000000000..c56d8534c --- /dev/null +++ b/scripts/ci/secret-scan.test.mjs @@ -0,0 +1,341 @@ +import assert from "node:assert/strict"; +import { execFileSync, spawnSync } from "node:child_process"; +import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { devNull, tmpdir } from "node:os"; +import { dirname, join } from "node:path"; +import test from "node:test"; +import { fileURLToPath, pathToFileURL } from "node:url"; +import { planSecretScan } from "./secret-scan.mjs"; + +const scriptPath = fileURLToPath(new URL("./secret-scan.mjs", import.meta.url)); +const fixtureEnv = { + ...Object.fromEntries(Object.entries(process.env).filter(([name]) => !name.startsWith("GIT_"))), + GIT_CONFIG_NOSYSTEM: "1", + GIT_CONFIG_GLOBAL: devNull, + GIT_AUTHOR_DATE: "2020-01-01T00:00:00Z", + GIT_COMMITTER_DATE: "2020-01-01T00:00:00Z", +}; +const policyFiles = { + ".gitleaks.toml": "[extend]\nuseDefault = true\n", + ".gitleaksignore": "# reviewed fixture fingerprints\n", +}; + +function git(cwd, ...args) { + return execFileSync("git", [ + "-c", "user.name=Secret scan fixture", "-c", "user.email=secret-scan-fixture@example.invalid", + "-c", "commit.gpgsign=false", "-c", "core.hooksPath=" + devNull, + "-c", "core.autocrlf=false", "-c", "color.ui=false", ...args, + ], { cwd, env: fixtureEnv, encoding: "utf8", stdio: ["ignore", "pipe", "pipe"] }).trim(); +} + +function repository(t) { + const cwd = mkdtempSync(join(tmpdir(), "ling-secret-scan-")); + t.after(() => rmSync(cwd, { recursive: true, force: true })); + git(cwd, "init", "--quiet", "--initial-branch=main", "--template="); + function commit(files, message) { + for (const [path, content] of Object.entries(files)) { + const target = join(cwd, path); + if (content === null) { + rmSync(target); + } else { + mkdirSync(dirname(target), { recursive: true }); + writeFileSync(target, content); + } + } + git(cwd, "add", "--all"); + git(cwd, "commit", "--quiet", "-m", message); + return git(cwd, "rev-parse", "HEAD"); + } + const root = commit({ ...policyFiles, "baseline.txt": "HISTORICAL_FIXTURE_MARKER\n", "conflict.txt": "shared\n" }, "baseline"); + return { cwd, root, commit }; +} + +function plan(repo, base = repo.root, extra = {}) { + return planSecretScan({ cwd: repo.cwd, eventName: "pull_request", base, ...extra }); +} + +function log(repo, scan) { + // Match Gitleaks' git log -p -U0 invocation and space-splitting of logOpts. + return git(repo.cwd, "log", "-p", "-U0", ...scan.logOpts.split(" ")); +} + +function commits(repo, scan) { + const output = git(repo.cwd, "log", "--format=%H", ...scan.logOpts.split(" "), "--no-patch"); + return output ? output.split(/\r?\n/u).sort() : []; +} + +function assertFull(repo, scan, reason, head = "HEAD") { + assert.equal(scan.mode, "full"); + assert.match(scan.reason, reason); + assert.equal(scan.logOpts, `--full-history --diff-merges=first-parent ${git(repo.cwd, "rev-parse", head)}`); +} + +test("PR scans both commits of an introduced-then-deleted secret with an empty endpoint diff", (t) => { + const repo = repository(t); + const introduced = repo.commit({ "transient.txt": "TRANSIENT_SECRET_MARKER\n" }, "introduce marker"); + const removed = repo.commit({ "transient.txt": null }, "remove marker"); + assert.equal(git(repo.cwd, "diff", repo.root, removed), ""); + + const scan = plan(repo); + assert.equal(scan.mode, "range"); + assert.equal(scan.logOpts, `--full-history --diff-merges=first-parent ${repo.root}..${removed}`); + assert.deepEqual(commits(repo, scan), [introduced, removed].sort()); + assert.match(log(repo, scan), /^\+TRANSIENT_SECRET_MARKER$/mu); + assert.doesNotMatch(log(repo, scan), /HISTORICAL_FIXTURE_MARKER/u); +}); + +test("divergent base and head use their merge-base, not either branch's endpoint tree", (t) => { + const repo = repository(t); + git(repo.cwd, "branch", "feature"); + const base = repo.commit({ "base-only.txt": "BASE_ONLY_MARKER\n" }, "base advanced"); + git(repo.cwd, "checkout", "--quiet", "feature"); + const head = repo.commit({ "feature.txt": "FEATURE_MARKER\n" }, "feature advanced"); + + const scan = plan(repo, "refs/heads/main"); + assert.equal(scan.logOpts, `--full-history --diff-merges=first-parent ${repo.root}..${head}`); + assert.deepEqual(commits(repo, scan), [head]); + assert.ok(!commits(repo, scan).includes(base)); + assert.match(log(repo, scan), /^\+FEATURE_MARKER$/mu); + assert.doesNotMatch(log(repo, scan), /BASE_ONLY_MARKER/u); +}); + +test("a tree-same merge retains second-parent commits containing a removed secret", (t) => { + const repo = repository(t); + git(repo.cwd, "branch", "side"); + const main = repo.commit({ "main.txt": "main change\n" }, "main change"); + git(repo.cwd, "checkout", "--quiet", "side"); + const introduced = repo.commit({ "side.txt": "SECOND_PARENT_SECRET_MARKER\n" }, "side introduction"); + const removed = repo.commit({ "side.txt": null }, "side removal"); + git(repo.cwd, "checkout", "--quiet", "main"); + git(repo.cwd, "merge", "--quiet", "--no-ff", "side", "-m", "merge side"); + const merge = git(repo.cwd, "rev-parse", "HEAD"); + assert.equal(git(repo.cwd, "diff", main, merge), ""); + + const scan = plan(repo); + assert.equal(scan.mode, "range"); + assert.deepEqual(commits(repo, scan), [main, introduced, removed, merge].sort()); + assert.match(log(repo, scan), /^\+SECOND_PARENT_SECRET_MARKER$/mu); + assert.ok(!scan.logOpts.split(" ").includes("--first-parent")); +}); + +test("real conflict-resolution additions survive a later revert and are present in the planned patch stream", (t) => { + const repo = repository(t); + git(repo.cwd, "branch", "side"); + const main = repo.commit({ "conflict.txt": "main version\n" }, "main conflict"); + git(repo.cwd, "checkout", "--quiet", "side"); + const side = repo.commit({ "conflict.txt": "side version\n" }, "side conflict"); + git(repo.cwd, "checkout", "--quiet", "main"); + assert.throws(() => git(repo.cwd, "merge", "--no-ff", "side", "-m", "conflicting merge"), (error) => error.status === 1); + assert.match(readFileSync(join(repo.cwd, "conflict.txt"), "utf8"), /^<<<<<<< /mu); + const merge = repo.commit({ "conflict.txt": "MERGE_RESOLUTION_SECRET_MARKER\n" }, "resolve conflict"); + const reverted = repo.commit({ "conflict.txt": "shared\n" }, "restore original content"); + assert.equal(git(repo.cwd, "rev-parse", `${merge}^1`), main); + assert.equal(git(repo.cwd, "rev-parse", `${merge}^2`), side); + assert.equal(git(repo.cwd, "diff", repo.root, reverted), ""); + assert.doesNotMatch(git(repo.cwd, "log", "-p", "-U0", "--diff-merges=off", `${repo.root}..${reverted}`), /^\+MERGE_RESOLUTION_SECRET_MARKER$/mu); + + const scan = plan(repo); + assert.equal(scan.mode, "range"); + assert.deepEqual(commits(repo, scan), [main, side, merge, reverted].sort()); + assert.match(log(repo, scan), /^\+MERGE_RESOLUTION_SECRET_MARKER$/mu); +}); + +test("GitHub-style synthetic merge heads include the PR branch, not already-baselined base commits", (t) => { + const repo = repository(t); + git(repo.cwd, "branch", "feature"); + const base = repo.commit({ "base.txt": "BASE_ADVANCE_MARKER\n" }, "base advanced"); + git(repo.cwd, "checkout", "--quiet", "feature"); + const introduced = repo.commit({ "feature.txt": "PR_BRANCH_SECRET_MARKER\n" }, "feature introduction"); + const removed = repo.commit({ "feature.txt": null }, "feature removal"); + git(repo.cwd, "checkout", "--quiet", "main"); + git(repo.cwd, "merge", "--quiet", "--no-ff", "feature", "-m", "synthetic PR merge"); + const merge = git(repo.cwd, "rev-parse", "HEAD"); + + const scan = plan(repo, base); + assert.equal(scan.logOpts, `--full-history --diff-merges=first-parent ${base}..${merge}`); + assert.deepEqual(commits(repo, scan), [introduced, removed, merge].sort()); + assert.match(log(repo, scan), /^\+PR_BRANCH_SECRET_MARKER$/mu); + assert.doesNotMatch(log(repo, scan), /BASE_ADVANCE_MARKER/u); +}); + +test("valid identical base and head produce an empty, explicitly bounded PR range", (t) => { + const repo = repository(t); + const scan = plan(repo); + assert.equal(scan.mode, "range"); + assert.equal(scan.logOpts, `--full-history --diff-merges=first-parent ${repo.root}..${repo.root}`); + assert.deepEqual(commits(repo, scan), []); +}); + +test("schedule, manual, missing, and unknown events scan full reachable head history", (t) => { + const repo = repository(t); + const head = repo.commit({ "new.txt": "new content\n" }, "new commit"); + for (const eventName of ["schedule", "workflow_dispatch", "push", "pull_request_target", "", undefined]) { + const scan = plan(repo, repo.root, { eventName }); + assertFull(repo, scan, /Non-PR/u); + assert.deepEqual(commits(repo, scan), [repo.root, head].sort()); + assert.match(log(repo, scan), /^\+HISTORICAL_FIXTURE_MARKER$/mu); + } +}); + +test("missing, unknown, malformed, option-like, and non-commit bases conservatively scan full history", (t) => { + const repo = repository(t); + const blob = git(repo.cwd, "rev-parse", "HEAD:baseline.txt"); + for (const base of [undefined, null, "", "unknown-base", "0".repeat(40), "--all", "--not", "HEAD --all", "HEAD\n--all", "HEAD\0", "^HEAD", "HEAD..main", blob, 123]) { + const scan = planSecretScan({ cwd: repo.cwd, eventName: "pull_request", base }); + assertFull(repo, scan, /base/u); + assert.deepEqual(commits(repo, scan), [repo.root]); + } +}); + +test("unrelated histories cannot supply a safe merge-base", (t) => { + const repo = repository(t); + const tree = git(repo.cwd, "rev-parse", "HEAD^{tree}"); + const unrelated = git(repo.cwd, "commit-tree", tree, "-m", "unrelated root"); + const scan = plan(repo, repo.root, { head: unrelated }); + assertFull(repo, scan, /merge-base.*unavailable/u, unrelated); + assert.deepEqual(commits(repo, scan), [unrelated]); +}); + +test("criss-cross merges with multiple best merge-bases fall back to full history", (t) => { + const repo = repository(t); + const tree = git(repo.cwd, "rev-parse", "HEAD^{tree}"); + const left = git(repo.cwd, "commit-tree", tree, "-p", repo.root, "-m", "left"); + const right = git(repo.cwd, "commit-tree", tree, "-p", repo.root, "-m", "right"); + const base = git(repo.cwd, "commit-tree", tree, "-p", left, "-p", right, "-m", "left merge"); + const head = git(repo.cwd, "commit-tree", tree, "-p", right, "-p", left, "-m", "right merge"); + assert.deepEqual(git(repo.cwd, "merge-base", "--all", base, head).split(/\r?\n/u).sort(), [left, right].sort()); + + const scan = plan(repo, base, { head }); + assertFull(repo, scan, /ambiguous/u, head); + assert.deepEqual(commits(repo, scan), [repo.root, left, right, head].sort()); +}); + +test("a shallow clone falls back even when both commits and a merge-base are available", (t) => { + const repo = repository(t); + const base = repo.commit({ "one.txt": "one\n" }, "one"); + repo.commit({ "two.txt": "two\n" }, "two"); + const clone = join(repo.cwd, "shallow-clone"); + git(repo.cwd, "clone", "--quiet", "--depth=2", pathToFileURL(repo.cwd).href, clone); + assert.equal(git(clone, "rev-parse", "--is-shallow-repository"), "true"); + assert.equal(git(clone, "merge-base", base, "HEAD"), base); + + const scan = planSecretScan({ cwd: clone, eventName: "pull_request", base }); + assertFull({ cwd: clone }, scan, /shallow/u); +}); + +test("missing history objects never produce a narrow plan", (t) => { + const repo = repository(t); + const base = repo.commit({ "new.txt": "new\n" }, "new"); + repo.commit({ "later.txt": "later\n" }, "later"); + rmSync(join(repo.cwd, ".git", "objects", repo.root.slice(0, 2), repo.root.slice(2))); + assertFull(repo, plan(repo, base), /merge-base.*unavailable/u); +}); + +test("unreadable policy history falls back when commit resolution and merge-base still succeed", (t) => { + const repo = repository(t); + const head = repo.commit({ "new.txt": "new\n" }, "new"); + const tree = git(repo.cwd, "rev-parse", "HEAD^{tree}"); + rmSync(join(repo.cwd, ".git", "objects", tree.slice(0, 2), tree.slice(2))); + assert.equal(git(repo.cwd, "merge-base", repo.root, head), repo.root); + assertFull(repo, plan(repo), /policy history/u); +}); + +test("policy modifications, reverts, deletions, and additions require a full scan", async (t) => { + for (const [path, original] of Object.entries(policyFiles)) { + await t.test(path, (t) => { + const repo = repository(t); + repo.commit({ [path]: original + "# policy change\n" }, "edit policy"); + assertFull(repo, plan(repo), /policy changed/u); + const restored = repo.commit({ [path]: original }, "restore policy"); + assert.equal(git(repo.cwd, "diff", repo.root, restored), ""); + assertFull(repo, plan(repo), /policy changed/u); + const deleted = repo.commit({ [path]: null }, "delete policy"); + assertFull(repo, plan(repo, restored), /policy changed/u); + repo.commit({ [path]: original }, "add policy"); + assertFull(repo, plan(repo, deleted), /policy changed/u); + }); + } +}); + +test("policy edits introduced only by a merge commit trigger a full scan", (t) => { + const repo = repository(t); + git(repo.cwd, "branch", "side"); + repo.commit({ "main.txt": "main\n" }, "main"); + git(repo.cwd, "checkout", "--quiet", "side"); + repo.commit({ "side.txt": "side\n" }, "side"); + git(repo.cwd, "checkout", "--quiet", "main"); + git(repo.cwd, "merge", "--quiet", "--no-ff", "--no-commit", "side"); + const merge = repo.commit({ ".gitleaksignore": "# merge-only policy change\n" }, "merge policy edit"); + assert.equal(git(repo.cwd, "log", "--format=%H", "--no-merges", `${repo.root}..${merge}`, "--", ".gitleaksignore"), ""); + assertFull(repo, plan(repo), /policy changed/u); +}); + +test("root policy paths remain guarded when cwd is a repository subdirectory", (t) => { + const repo = repository(t); + repo.commit({ ".gitleaksignore": "# changed\n", "nested/file.txt": "nested\n" }, "policy and nested file"); + assertFull(repo, plan(repo, repo.root, { cwd: join(repo.cwd, "nested") }), /policy changed/u); +}); + +test("invalid heads throw for PR and non-PR events rather than returning a successful full plan", (t) => { + const repo = repository(t); + const blob = git(repo.cwd, "rev-parse", "HEAD:baseline.txt"); + for (const eventName of ["pull_request", "schedule", "workflow_dispatch"]) { + for (const head of ["", null, 123, "unknown-head", "0".repeat(40), "--all", "--not", "HEAD --all", "HEAD\n--all", "HEAD\0", "^HEAD", "HEAD..main", blob]) { + assert.throws(() => plan(repo, repo.root, { eventName, head }), /head.*invalid/u); + } + } + git(repo.cwd, "symbolic-ref", "HEAD", "refs/heads/missing"); + assert.throws(() => plan(repo), /head.*invalid/u); + assert.throws(() => plan(repo, undefined, { eventName: "schedule" }), /head.*invalid/u); +}); + +test("annotated tags and ordinary refs are resolved to immutable commit IDs before building logOpts", (t) => { + const repo = repository(t); + git(repo.cwd, "tag", "--no-sign", "-a", "baseline-tag", "-m", "baseline tag"); + const head = repo.commit({ "head.txt": "head\n" }, "head"); + const scan = plan(repo, "baseline-tag", { head: "refs/heads/main" }); + assert.equal(scan.logOpts, `--full-history --diff-merges=first-parent ${repo.root}..${head}`); + assert.deepEqual(commits(repo, scan), [head]); +}); + +test("CLI emits exactly one JSON plan and supports the documented equals-style options", (t) => { + const repo = repository(t); + repo.commit({ "new.txt": "new\n" }, "new"); + const result = spawnSync(process.execPath, [scriptPath, "--event=pull_request", `--base=${repo.root}`, "--head=HEAD"], { + cwd: repo.cwd, env: fixtureEnv, encoding: "utf8", + }); + assert.equal(result.status, 0, result.stderr); + assert.equal(result.stderr, ""); + assert.equal(result.stdout.trim().split("\n").length, 1); + assert.deepEqual(JSON.parse(result.stdout), plan(repo)); + + const full = spawnSync(process.execPath, [scriptPath, "--event=workflow_dispatch"], { cwd: repo.cwd, env: fixtureEnv, encoding: "utf8" }); + assert.equal(full.status, 0, full.stderr); + assertFull(repo, JSON.parse(full.stdout), /Non-PR/u); +}); + +test("CLI rejects malformed arguments and invalid heads without emitting a green JSON plan", (t) => { + const repo = repository(t); + const injectedFile = join(repo.cwd, "injected-output"); + for (const args of [ + ["--event=pull_request", "--head=unknown-head"], + ["--event=schedule", "--head="], + ["--event=workflow_dispatch", "--head=--all"], + [`--head=--output=${injectedFile}`], + [`--head=HEAD;touch ${injectedFile}`], + ["--unknown=option"], ["--event"], ["unexpected-positional"], + ]) { + const result = spawnSync(process.execPath, [scriptPath, ...args], { cwd: repo.cwd, env: fixtureEnv, encoding: "utf8" }); + assert.notEqual(result.status, 0, args.join(" ")); + assert.equal(result.stdout, ""); + assert.notEqual(result.stderr, ""); + } + assert.equal(existsSync(injectedFile), false); + + const fallback = spawnSync(process.execPath, [scriptPath, "--event=pull_request", `--base=--output=${injectedFile}`], { + cwd: repo.cwd, env: fixtureEnv, encoding: "utf8", + }); + assert.equal(fallback.status, 0, fallback.stderr); + assertFull(repo, JSON.parse(fallback.stdout), /base.*invalid/u); + assert.equal(existsSync(injectedFile), false); +}); diff --git a/scripts/ci/test-processes.mjs b/scripts/ci/test-processes.mjs new file mode 100644 index 000000000..9061fda0a --- /dev/null +++ b/scripts/ci/test-processes.mjs @@ -0,0 +1,71 @@ +import { execFileSync } from "node:child_process"; +import { setTimeout as delay } from "node:timers/promises"; + +function ps(args) { + try { + return execFileSync("ps", args, { encoding: "utf8", timeout: 5000, maxBuffer: 32 * 1024 * 1024, + env: { ...process.env, LC_ALL: "C" }, stdio: ["ignore", "pipe", "pipe"] }); + } catch { + // ps environment output can contain credentials. Never attach it to errors. + throw new Error("Unable to inspect CI-owned test processes"); + } +} + +function processTable() { + return ps(["-axo", "pid=,ppid=,pgid=,stat=,lstart="]).trim().split(/\n/u).flatMap(line => { + const match = /^\s*(\d+)\s+(\d+)\s+(\d+)\s+(\S+)\s+(.+)$/u.exec(line); + return match ? [{ pid: Number(match[1]), parent: Number(match[2]), group: Number(match[3]), state: match[4], started: match[5] }] : []; + }); +} + +/** Track observed ancestry plus inherited ownership tags across setsid/reparenting. + * This is test cleanup, not a sandbox for processes deliberately hiding ownership. */ +export function trackTestProcesses(rootPid, token) { + const owned = new Map(); + const marker = `LING_STACK_CI_TEST_OWNER=${token}`; + function live(includeTagged = false) { + if (!rootPid) return []; + const table = processTable(); + const current = new Map(table.map(entry => [entry.pid, entry])); + for (const [pid, started] of owned) if (current.get(pid)?.started !== started) owned.delete(pid); + const tagged = new Set(); + if (includeTagged || owned.size === 0) { + for (const line of ps(["eww", "-U", String(process.getuid()), "-o", "pid=,command="]).split(/\n/u)) { + const match = /^\s*(\d+)\s+(.+)$/u.exec(line); + if (match && match[2].split(/\s+/u).includes(marker)) tagged.add(Number(match[1])); + } + } + for (const entry of table) if (tagged.has(entry.pid)) owned.set(entry.pid, entry.started); + // A numeric PGID alone is not ownership: require a still-identified member + // before adopting its group, including when the original leader has exited. + const originalGroup = table.some(entry => entry.group === rootPid && owned.get(entry.pid) === entry.started); + if (originalGroup) for (const entry of table) if (entry.group === rootPid) owned.set(entry.pid, entry.started); + let changed; + do { + changed = false; + for (const entry of table) { + if (!owned.has(entry.pid) && owned.has(entry.parent)) { owned.set(entry.pid, entry.started); changed = true; } + } + } while (changed); + return table.filter(entry => owned.get(entry.pid) === entry.started && !entry.state.startsWith("Z")); + } + return { + live, + signal(signal) { + for (const entry of live(true).reverse()) { + try { process.kill(entry.pid, signal); } + catch (error) { + if (error.code !== "ESRCH" && !(error.code === "EPERM" && !live().some(value => value.pid === entry.pid))) throw error; + } + } + }, + async waitForExit(timeoutMs) { + const deadline = performance.now() + timeoutMs; + while (live(true).length) { + if (performance.now() >= deadline) return false; + await delay(25); + } + return true; + }, + }; +} diff --git a/scripts/ci/test-processes.test.mjs b/scripts/ci/test-processes.test.mjs new file mode 100644 index 000000000..c1afb55d1 --- /dev/null +++ b/scripts/ci/test-processes.test.mjs @@ -0,0 +1,71 @@ +import assert from "node:assert/strict"; +import childProcess from "node:child_process"; +import { syncBuiltinESMExports } from "node:module"; +import test from "node:test"; +import { trackTestProcesses } from "./test-processes.mjs"; + +function fixture(t, initial) { + let table = initial; + let tagged = new Set([101]); + const signals = []; + t.mock.method(childProcess, "execFileSync", (_command, args) => args[0] === "-axo" + ? table.map(entry => `${entry.pid} ${entry.parent ?? 1} ${entry.group} S ${entry.started}`).join("\n") + : [...tagged].map(pid => `${pid} node LING_STACK_CI_TEST_OWNER=fixture-owner`).join("\n")); + t.mock.method(process, "kill", (pid, signal) => { signals.push({ pid, signal }); return true; }); + syncBuiltinESMExports(); + t.after(() => { t.mock.restoreAll(); syncBuiltinESMExports(); }); + return { tracker: trackTestProcesses(101, "fixture-owner"), signals, + update(next, tags = []) { table = next; tagged = new Set(tags); } }; +} + +test("a recycled orphan process group is not evidence of test ownership", t => { + const f = fixture(t, [{ pid: 101, group: 101, started: "original" }]); + assert.deepEqual(f.tracker.live(true).map(entry => entry.pid), [101]); + f.update([{ pid: 202, group: 101, started: "unrelated" }]); + f.tracker.signal("SIGKILL"); + assert.deepEqual(f.signals, []); +}); + +test("a replacement root PID does not inherit the previous root's ownership", t => { + const f = fixture(t, [{ pid: 101, group: 101, started: "original" }]); + f.tracker.live(true); + f.update([{ pid: 101, group: 101, started: "replacement" }, { pid: 202, group: 101, started: "replacement-child" }]); + f.tracker.signal("SIGKILL"); + assert.deepEqual(f.signals, []); +}); + +test("a root PID seen only after replacement cannot prove ownership without its tag", t => { + const f = fixture(t, []); + f.update([{ pid: 101, group: 101, started: "unrelated" }]); + f.tracker.signal("SIGKILL"); + assert.deepEqual(f.signals, []); +}); + +test("observed detached descendants retain ownership after reparenting with a cleared environment", t => { + const f = fixture(t, [{ pid: 101, group: 101, started: "original" }, { pid: 202, parent: 101, group: 202, started: "child" }]); + assert.deepEqual(f.tracker.live(true).map(entry => entry.pid), [101, 202]); + f.update([{ pid: 202, group: 202, started: "child" }, { pid: 303, group: 303, started: "spectator" }]); + f.tracker.signal("SIGKILL"); + assert.deepEqual(f.signals, [{ pid: 202, signal: "SIGKILL" }]); +}); + +test("an inherited ownership tag finds detached descendants even after an unobserved root exit", t => { + const f = fixture(t, []); + f.update([{ pid: 202, group: 202, started: "child" }], [202]); + f.tracker.signal("SIGTERM"); + assert.deepEqual(f.signals, [{ pid: 202, signal: "SIGTERM" }]); +}); + +test("inspection errors do not expose process environments", t => { + fixture(t, []); + t.mock.method(childProcess, "execFileSync", () => { + throw Object.assign(new Error("private-process-environment"), { stdout: "private-process-environment", stderr: "private-process-environment" }); + }); + syncBuiltinESMExports(); + assert.throws(() => trackTestProcesses(101, "fixture-owner").live(true), error => { + assert.equal(error.message, "Unable to inspect CI-owned test processes"); + assert.equal(error.cause, undefined); + assert.equal(error.stdout, undefined); + return true; + }); +}); diff --git a/scripts/ci/test-shard-plan.mjs b/scripts/ci/test-shard-plan.mjs new file mode 100644 index 000000000..df7688d05 --- /dev/null +++ b/scripts/ci/test-shard-plan.mjs @@ -0,0 +1,149 @@ +import { readFile, readdir, mkdir, writeFile } from "node:fs/promises"; +import { statSync } from "node:fs"; +import { dirname, join, resolve } from "node:path"; +import { fileURLToPath, pathToFileURL } from "node:url"; + +const root = resolve(dirname(fileURLToPath(import.meta.url)), "../.."); +export const testSuites = { + core: { packageName: "@ling-stack/core", cwd: join(root, "packages/core"), platform: "linux", shardCount: 4, concurrency: 2, excluded: [], exclusive: [] }, + daemon: { packageName: "@ling-stack/daemon", cwd: join(root, "packages/daemon"), platform: "darwin", shardCount: 4, concurrency: 2, + excluded: ["dependency-security.test.mjs"], + exclusive: ["test/acp-kimi-host-read.test.mjs", "test/acp-kimi-host-terminal.test.mjs"] }, +}; + +export async function discoverTestFiles(cwd, excluded = []) { + return (await readdir(join(cwd, "test"))).filter(file => file.endsWith(".test.mjs") && !excluded.includes(file)) + .sort().map(file => `test/${file}`); +} + +/** Timing hints only change placement, never discovery, coverage or deadlines. */ +export function estimateTestWeight(file, { cwd = testSuites.daemon.cwd, timings = {} } = {}) { + const measured = timings[file]; + if (Number.isFinite(measured) && measured > 0 && measured <= 3_600_000) return measured / 1000; + const floor = /(?:integration|computer-use|remote|channel|workflow|workspace|media|mcp|runner|runtime)/iu.test(file) + ? 3 : /(?:storage|conversation|startup|provider|terminal|skill|cua|office)/iu.test(file) ? 2 : 1; + try { return Math.max(floor, Math.ceil(statSync(resolve(cwd, file)).size / 1024)); } + catch (error) { if (error.code !== "ENOENT") throw error; return floor; } +} + +export function assignTestShards(files, shardCount, options) { + if (!Number.isInteger(shardCount) || shardCount < 1) throw new RangeError("shardCount must be a positive integer"); + const buckets = Array.from({ length: shardCount }, () => ({ files: [], weight: 0 })); + const weighted = [...new Set(files)].map(file => ({ file, weight: estimateTestWeight(file, options) })) + .sort((a, b) => b.weight - a.weight || a.file.localeCompare(b.file)); + for (const item of weighted) { + let target = 0; + for (let index = 1; index < buckets.length; index += 1) { + if (buckets[index].weight < buckets[target].weight) target = index; + } + buckets[target].files.push(item.file); + buckets[target].weight += item.weight; + } + return buckets.map(bucket => ({ ...bucket, files: bucket.files.sort() })); +} + +export async function loadTimingHints(path) { + try { + const hints = JSON.parse(await readFile(path, "utf8")); + if (hints.version !== 1 || !hints.suites || typeof hints.suites !== "object") throw new Error("unsupported timing schema"); + return hints; + } catch (error) { + if (error.code !== "ENOENT") console.warn(`Ignoring unusable CI timing hints: ${error.message}`); + return { version: 1, suites: {} }; + } +} + +export async function createTestPlan(packages, { timings = { suites: {} } } = {}) { + if (!Array.isArray(packages) || packages.some(name => typeof name !== "string")) throw new TypeError("packages must be a string array"); + const suites = {}; + for (const [name, config] of Object.entries(testSuites)) { + if (!packages.includes(config.packageName)) continue; + const files = await discoverTestFiles(config.cwd, config.excluded); + if (files.length === 0) throw new Error(`No tests discovered for ${name}`); + const hints = timings.suites?.[name]; + const durations = hints?.platform === config.platform ? hints.durationsMs : {}; + suites[name] = { platform: config.platform, files, shardCount: config.shardCount, concurrency: config.concurrency, timings: durations ?? {}, + shards: assignTestShards(files, config.shardCount, { cwd: config.cwd, timings: durations ?? {} }) }; + } + return { version: 1, suites, unitMatrix: { include: [ + { suite: "packages", shard: 0 }, + ...Array.from({ length: suites.core?.shardCount ?? 0 }, (_, shard) => ({ suite: "core", shard })), + ] } }; +} + +export function assertSuiteCoverage(suite, discovered) { + const assigned = suite?.shards?.flatMap(bucket => bucket.files); + const expected = [...discovered].sort(); + if (!Array.isArray(assigned) || suite.shards.length !== suite.shardCount + || JSON.stringify([...assigned].sort()) !== JSON.stringify(expected) + || JSON.stringify(suite.files) !== JSON.stringify(expected) + || new Set(assigned).size !== assigned.length) { + throw new Error("test_plan_coverage_mismatch: every discovered file must be assigned exactly once"); + } +} + +/** The aggregate gate checks reports against the immutable plan before saving hints. */ +export function mergeTestReports(plan, reports) { + if (plan.version !== 1) throw new Error("Unsupported test plan version"); + const expected = new Map(); + for (const [name, suite] of Object.entries(plan.suites)) { + assertSuiteCoverage(suite, suite.files); + suite.shards.forEach((bucket, shard) => expected.set(`${name}-${shard}`, { name, suite, bucket, shard })); + } + const hints = { version: 1, suites: {} }; + for (const report of reports) { + const key = `${report.suite}-${report.shard}`; + const entry = expected.get(key); + if (!entry || report.version !== 1 || report.complete !== true || report.shardCount !== entry.suite.shardCount + || report.platform !== entry.suite.platform) throw new Error(`Invalid, duplicate or incomplete test report: ${key}`); + const files = report.results?.map(result => result.file).sort(); + if (JSON.stringify(files) !== JSON.stringify(entry.bucket.files)) throw new Error(`test_report_coverage_mismatch: ${key}`); + for (const result of report.results) { + if (result.status !== "passed" || !Number.isFinite(result.durationMs) || result.durationMs <= 0) { + throw new Error(`Unsuccessful test report: ${key} ${result.file} ${result.status}`); + } + } + hints.suites[entry.name] ??= { platform: entry.suite.platform, durationsMs: {} }; + Object.assign(hints.suites[entry.name].durationsMs, Object.fromEntries(report.results.map(result => [result.file, result.durationMs]))); + expected.delete(key); + } + if (expected.size > 0) throw new Error(`Missing test reports: ${[...expected.keys()].join(", ")}`); + return hints; +} + +export function parseArguments(argv, allowed) { + const args = {}; + for (let index = 0; index < argv.length; index += 1) { + const [key, ...rest] = argv[index].split("="); + if (!allowed.includes(key) || Object.hasOwn(args, key)) throw new Error(`Unknown or duplicate argument: ${key}`); + const value = rest.length ? rest.join("=") : argv[++index]; + if (!value || value.startsWith("--")) throw new Error(`Missing value for ${key}`); + args[key] = value; + } + return args; +} + +async function main() { + const args = parseArguments(process.argv.slice(2), ["--packages", "--timings", "--output", "--plan", "--reports"]); + let result; + if (args["--reports"]) { + const plan = JSON.parse(await readFile(args["--plan"], "utf8")); + const directory = args["--reports"]; + const files = (await readdir(directory)).filter(file => file.endsWith(".json")).sort(); + const reports = await Promise.all(files.map(async file => JSON.parse(await readFile(join(directory, file), "utf8")))); + result = mergeTestReports(plan, reports); + console.log(`Verified complete test coverage for ${Object.keys(plan.suites).join(", ")}.`); + } else { + result = await createTestPlan(JSON.parse(args["--packages"]), { + timings: args["--timings"] ? await loadTimingHints(args["--timings"]) : undefined, + }); + console.log(JSON.stringify(result.unitMatrix)); + } + if (!args["--output"]) throw new Error("--output is required"); + await mkdir(dirname(resolve(args["--output"])), { recursive: true }); + await writeFile(args["--output"], `${JSON.stringify(result, null, 2)}\n`); +} + +if (process.argv[1] && import.meta.url === pathToFileURL(resolve(process.argv[1])).href) { + try { await main(); } catch (error) { console.error(error); process.exitCode = 1; } +} diff --git a/scripts/ci/test-shard-plan.test.mjs b/scripts/ci/test-shard-plan.test.mjs new file mode 100644 index 000000000..4576e036d --- /dev/null +++ b/scripts/ci/test-shard-plan.test.mjs @@ -0,0 +1,122 @@ +import assert from "node:assert/strict"; +import { execFileSync } from "node:child_process"; +import { mkdir, mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { fileURLToPath } from "node:url"; +import test from "node:test"; +import { assertSuiteCoverage, assignTestShards, createTestPlan, discoverTestFiles, estimateTestWeight, + loadTimingHints, mergeTestReports, testSuites } from "./test-shard-plan.mjs"; + +test("core and daemon plans preserve full discovery exactly once with dynamic core jobs", async () => { + for (const packages of [[], ["@ling-stack/core"], ["@ling-stack/daemon"], ["@ling-stack/core", "@ling-stack/daemon"]]) { + const plan = await createTestPlan(packages); + assert.deepEqual(Object.keys(plan.suites), Object.keys(testSuites).filter(name => packages.includes(testSuites[name].packageName))); + assert.equal(plan.unitMatrix.include.length, packages.includes("@ling-stack/core") ? 5 : 1); + assert.deepEqual(plan.unitMatrix.include[0], { suite: "packages", shard: 0 }); + for (const [name, suite] of Object.entries(plan.suites)) { + const discovered = await discoverTestFiles(testSuites[name].cwd, testSuites[name].excluded); + assert.ok(discovered.length > 100); + assert.doesNotThrow(() => assertSuiteCoverage(suite, discovered)); + assert.equal(suite.shardCount, 4); + } + if (plan.suites.daemon) assert.ok(!plan.suites.daemon.files.includes("test/dependency-security.test.mjs")); + } +}); + +test("measured durations balance placement deterministically without changing coverage", () => { + const files = ["test/a.test.mjs", "test/b.test.mjs", "test/c.test.mjs", "test/new.test.mjs"]; + const options = { timings: { [files[0]]: 100_000, [files[1]]: 90_000, [files[2]]: 10_000 } }; + const plan = assignTestShards(files, 2, options); + assert.deepEqual(plan, assignTestShards([...files].reverse(), 2, options)); + assert.deepEqual(plan.map(bucket => bucket.files), [[files[0], files[3]], [files[1], files[2]]]); + assert.equal(estimateTestWeight(files[0], options), 100); + assert.ok(estimateTestWeight(files[3], options) > 0); + for (const duration of [0, -1, NaN, Infinity, "100", 3_600_001]) { + assert.equal(estimateTestWeight(files[3], { timings: { [files[3]]: duration } }), 1); + } +}); + +test("wrong-platform timings cannot influence a suite's plan", async () => { + const base = await createTestPlan(["@ling-stack/core"]); + const wrongPlatform = await createTestPlan(["@ling-stack/core"], { timings: { + suites: { core: { platform: "darwin", durationsMs: Object.fromEntries(base.suites.core.files.map(file => [file, 1])) } }, + } }); + assert.deepEqual(wrongPlatform, base); +}); + +test("unavailable and corrupt timing caches fall back to complete plans", async t => { + const root = await mkdtemp(join(tmpdir(), "ling-ci-timings-")); + t.after(() => rm(root, { recursive: true, force: true })); + const path = join(root, "timings.json"); + assert.deepEqual(await loadTimingHints(path), { version: 1, suites: {} }); + for (const invalid of ["{", "null", '{"version":2,"suites":{}}']) { + await writeFile(path, invalid); + assert.deepEqual(await loadTimingHints(path), { version: 1, suites: {} }); + } + await writeFile(path, '{"version":1,"suites":{}}'); + assert.deepEqual(await loadTimingHints(path), { version: 1, suites: {} }); +}); + +test("coverage rejects omissions, duplicate files, changed discovery and wrong shard counts", () => { + const files = ["test/a.test.mjs", "test/b.test.mjs"]; + const valid = { files, shardCount: 2, shards: assignTestShards(files, 2) }; + assert.doesNotThrow(() => assertSuiteCoverage(valid, files)); + for (const suite of [null, {}, { ...valid, files: [files[0]] }, { ...valid, shardCount: 3 }, + { ...valid, shards: [{ files }, { files: [files[0]] }] }, { ...valid, shards: [{ files: [] }, { files: [] }] }]) { + assert.throws(() => assertSuiteCoverage(suite, files), /coverage_mismatch/u); + } + assert.throws(() => assertSuiteCoverage(valid, [...files, "test/new.test.mjs"]), /coverage_mismatch/u); +}); + +async function reportFixture() { + const plan = await createTestPlan(["@ling-stack/core", "@ling-stack/daemon"]); + const reports = Object.entries(plan.suites).flatMap(([suite, entry]) => entry.shards.map((bucket, shard) => ({ + version: 1, suite, shard, shardCount: entry.shardCount, platform: entry.platform, complete: true, + results: bucket.files.map((file, index) => ({ file, status: "passed", durationMs: index + 1 })), + }))); + return { plan, reports }; +} + +test("only full, successful, same-platform shard reports become timing hints", async () => { + const { plan, reports } = await reportFixture(); + const hints = mergeTestReports(plan, reports); + for (const [name, suite] of Object.entries(plan.suites)) { + assert.deepEqual(Object.keys(hints.suites[name].durationsMs).sort(), suite.files); + assert.equal(hints.suites[name].platform, suite.platform); + } + assert.deepEqual(mergeTestReports(plan, [...reports].reverse()), hints); + assert.throws(() => mergeTestReports(plan, reports.slice(1)), /Missing test reports/u); + assert.throws(() => mergeTestReports(plan, [...reports, reports[0]]), /duplicate/u); + for (const patch of [{ complete: false }, { version: 2 }, { suite: "other" }, { shardCount: 99 }, { platform: "win32" }, { results: [] }]) { + assert.throws(() => mergeTestReports(plan, [{ ...reports[0], ...patch }, ...reports.slice(1)])); + } + for (const status of ["failed", "timed_out", "cancelled", "pending", "error", "leaked_processes"]) { + const broken = structuredClone(reports); + broken[0].results[0].status = status; + assert.throws(() => mergeTestReports(plan, broken), /Unsuccessful test report/u); + } + const duplicate = structuredClone(reports); + duplicate[0].results.push(duplicate[0].results[0]); + assert.throws(() => mergeTestReports(plan, duplicate), /coverage_mismatch/u); +}); + +test("the plan CLI publishes a parseable matrix and the aggregate CLI verifies downloaded report paths", async t => { + const root = await mkdtemp(join(tmpdir(), "ling-ci-plan-cli-")); + t.after(() => rm(root, { recursive: true, force: true })); + const script = fileURLToPath(new URL("./test-shard-plan.mjs", import.meta.url)); + const path = join(root, "test-plan.json"); + const matrix = JSON.parse(execFileSync(process.execPath, [script, '--packages=["@ling-stack/core","@ling-stack/daemon"]', + `--output=${path}`], { encoding: "utf8" })); + const plan = JSON.parse(await readFile(path, "utf8")); + assert.deepEqual(matrix, plan.unitMatrix); + const fixture = await reportFixture(); + assert.deepEqual(plan, fixture.plan); + const reports = join(root, "reports"); + await mkdir(join(reports, "logs"), { recursive: true }); + await Promise.all(fixture.reports.map(report => writeFile(join(reports, `${report.suite}-${report.shard}.json`), JSON.stringify(report)))); + const output = join(root, "timings.json"); + assert.match(execFileSync(process.execPath, [script, `--plan=${path}`, `--reports=${reports}`, `--output=${output}`], + { encoding: "utf8" }), /Verified complete test coverage/u); + assert.deepEqual(JSON.parse(await readFile(output, "utf8")), mergeTestReports(plan, fixture.reports)); +}); From dd1212c2bb215487c6fa6928e42cedab266620a6 Mon Sep 17 00:00:00 2001 From: tianxiaochannel-oss88 Date: Mon, 2 Nov 2026 01:59:27 +0800 Subject: [PATCH 2/2] fix(ci): tighten detached process tracking --- scripts/ci/node-test-shard.mjs | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/scripts/ci/node-test-shard.mjs b/scripts/ci/node-test-shard.mjs index c38b7cd07..5ed54df8b 100644 --- a/scripts/ci/node-test-shard.mjs +++ b/scripts/ci/node-test-shard.mjs @@ -49,6 +49,9 @@ async function runFile(file, { cwd, shard, worker, suite, temporaryDirectory, st let logError; let cleanupError; const processes = trackTestProcesses(child.pid, token); + // Capture descendants before a fast coordinator can exit and reparent them. + try { processes.live(true); } + catch (error) { cleanupError = error; } const terminate = signal => { // Observe ancestry before terminating parents that may orphan a child. try { processes.signal(signal); } @@ -65,7 +68,7 @@ async function runFile(file, { cwd, shard, worker, suite, temporaryDirectory, st const observe = setInterval(() => { try { processes.live(); } catch (error) { cleanupError ??= error; stop("cleanup_error"); } - }, 200); + }, 25); const onAbort = () => stop("cancelled"); const onLogError = error => { logError = error; stop("log_error"); }; log?.on("error", onLogError); @@ -79,6 +82,12 @@ async function runFile(file, { cwd, shard, worker, suite, temporaryDirectory, st child.once("exit", (code, exitSignal) => done({ code, signal: exitSignal })); }); const closed = new Promise(done => child.once("close", done)); + child.once("exit", () => { + // The exit event is the last reliable boundary before the coordinator is + // reaped; take one final ownership snapshot before teardown starts. + try { processes.live(true); } + catch (error) { cleanupError ??= error; stop("cleanup_error"); } + }); const deadline = setTimeout(() => stop("timed_out"), fileTimeoutMs); signal.addEventListener("abort", onAbort, { once: true }); if (signal.aborted) onAbort();