ci: add isolated non-blocking timing telemetry

.github/workflows/ci.yml

@@ -732,3 +732,103 @@ jobs:
         with:
           path: .codex-artifacts/ci/timings.json
           key: ci-node-timings-v1-${{ github.run_id }}-${{ github.run_attempt }}
+
+  timing-telemetry:
+    name: CI timing telemetry
+    needs: [scope, quality]
+    if: always()
+    runs-on: ubuntu-latest
+    timeout-minutes: 3
+    continue-on-error: true
+    permissions:
+      actions: read
+      contents: read
+    steps:
+      - name: Check out timing telemetry without retained credentials
+        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1
+        with:
+          fetch-depth: 1
+          persist-credentials: false
+
+      - name: Set up Node.js
+        uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1
+        with:
+          node-version: 22.22.1
+
+      - name: Download available shard reports including failures
+        if: always() && needs.scope.outputs.execution-tier == 'complete' && (needs.scope.outputs.daemon == 'true' || contains(needs.scope.outputs.test-packages, '"@ling-stack/core"'))
+        continue-on-error: true
+        uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333
+        with:
+          pattern: ci-test-report-*
+          merge-multiple: true
+          path: .codex-artifacts/ci/reports
+
+      - name: Collect workflow timing metadata
+        if: always()
+        continue-on-error: true
+        shell: bash
+        env:
+          GH_TOKEN: ${{ github.token }}
+        run: |
+          set -u
+          set +e
+          mkdir -p .codex-artifacts/ci
+          run_status=0
+          gh api "repos/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}" > .codex-artifacts/ci/telemetry-run.json || run_status=$?
+          if [ "$run_status" -ne 0 ]; then
+            printf '{"telemetry_unavailable":true,"reason":"workflow run API request failed"}\n' > .codex-artifacts/ci/telemetry-run.json
+          fi
+          jobs_status=0
+          gh api --paginate --slurp "repos/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}/attempts/${GITHUB_RUN_ATTEMPT}/jobs?per_page=100" > .codex-artifacts/ci/telemetry-jobs-pages.json || jobs_status=$?
+          if [ "$jobs_status" -ne 0 ]; then
+            printf '{"telemetry_unavailable":true,"reason":"workflow jobs API request failed"}\n' > .codex-artifacts/ci/telemetry-jobs-pages.json
+          else
+            node --input-type=module <<'NODE'
+          import { readFile, writeFile } from "node:fs/promises";
+          const pages = JSON.parse(await readFile(".codex-artifacts/ci/telemetry-jobs-pages.json", "utf8"));
+          if (!Array.isArray(pages) || pages.some(page => !Array.isArray(page?.jobs))) throw new Error("Invalid jobs API pages");
+          const jobs = pages.flatMap(page => page.jobs);
+          await writeFile(".codex-artifacts/ci/telemetry-jobs.json", `${JSON.stringify({ jobs }, null, 2)}\n`);
+          NODE
+          fi
+          if [ ! -f .codex-artifacts/ci/telemetry-jobs.json ]; then
+            printf '{"jobs":[],"telemetry_unavailable":true}\n' > .codex-artifacts/ci/telemetry-jobs.json
+          fi
+          exit 0
+
+      - name: Render CI timing telemetry
+        if: always()
+        continue-on-error: true
+        shell: bash
+        run: |
+          set +e
+          node scripts/ci/ci-telemetry.mjs \
+            --run=.codex-artifacts/ci/telemetry-run.json \
+            --jobs=.codex-artifacts/ci/telemetry-jobs.json \
+            --reports-dir=.codex-artifacts/ci/reports \
+            --output=.codex-artifacts/ci/ci-telemetry.json \
+            --summary=.codex-artifacts/ci/ci-telemetry.md
+          telemetry_status=$?
+          if [ "$telemetry_status" -ne 0 ]; then
+            printf '{"version":1,"telemetryUnavailable":true}\n' > .codex-artifacts/ci/ci-telemetry.json
+            printf '## CI timing telemetry\n\nTelemetry was unavailable; this did not affect the quality gate.\n' > .codex-artifacts/ci/ci-telemetry.md
+            echo "Warning: CI timing telemetry was unavailable" >&2
+          fi
+          cat .codex-artifacts/ci/ci-telemetry.md >> "$GITHUB_STEP_SUMMARY"
+          exit 0
+
+      - name: Preserve CI timing telemetry
+        if: always()
+        continue-on-error: true
+        uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9
+        with:
+          name: ci-telemetry
+          include-hidden-files: true
+          if-no-files-found: warn
+          retention-days: 14
+          path: |
+            .codex-artifacts/ci/ci-telemetry.json
+            .codex-artifacts/ci/ci-telemetry.md
+            .codex-artifacts/ci/telemetry-run.json
+            .codex-artifacts/ci/telemetry-jobs.json

docs/maintenance/ci.md

@@ -129,6 +129,18 @@ Linux 每日回归仅在同一提交、同一检查方案已有有效的完整
 
 每次运行分别记录排队时间、页面总时长、各任务运行时间、Linux／macOS 累计任务分钟、测试数量与原生跳过数量。累计任务分钟是性能观察，不能在没有账单证据时称为实际扣额；失败任务已运行的部分也会消耗资源。
 
+独立的 `timing-telemetry` 任务在 `quality` 汇总结束后，尽力保存
+`ci-telemetry` artifact（保留 14 天）并写入 GitHub Step Summary。质量门
+不依赖该任务；采集、下载、解析、上传或观测任务失败都不改变原有门禁。
+质量门失败时仍下载可用分片报告，记录本次运行尝试的 run/job/step 观测、
+逐文件结果、Core/Daemon 最大分片文件累计耗时、分片不均衡比和可用报告中的
+timeout/leaked-process 计数。job 启动延迟包含依赖等待，不能当作纯 runner
+排队时间；并发文件累计耗时不等于分片实际时长；截至已完成任务的经过时间
+不等于依赖图关键路径，且不包含仍在执行的任务。run 总时长标为采集时
+已观测值，报告缺失也不能据此断言全量测试没有超时或泄漏。先观察至少
+5～10 次完整 hosted run，再决定是否调整分片、worker 并发或增加 E2E；当前
+不据此承诺固定提速比例。
+
 缓存继续用于锁定依赖和同一次构建输出复用。不得复用与提交、平台或方案不匹配的测试成功。分片缩短等待不一定减少累计费用；不在测量前承诺节省比例，也不把公开仓库、自托管 runner 或增加支出上限当作本轮方案的前提。
 
 ## 实施顺序与验收

packages/daemon/test/channel-node-invocation-binding.test.mjs

@@ -7,6 +7,7 @@ import { workflowExactCommandBindingRejection } from "../dist/features/runs/appr
 import { channelOperationPlanFingerprintV1, isChannelIntentExecutionPlanV1, nodeFunctionGoalCommand, parseNodeFunctionInvocation } from "@ling-stack/contracts";
 import { executeUnifiedBashTool } from "../dist/features/runs/unified-exec.js";
 import { createUnifiedExecFixture } from "./support/unified-exec-fixture.mjs";
+import { captureNodeGoalCommandDiagnostic } from "./support/node-goal-command-diagnostic.mjs";
 import { createSqliteRunStore, DEFAULT_WORKSPACE_ID, toRuntimeAttemptFence } from "@ling-stack/core";
 import { appendChannelItemFixture } from "../../core/test/channel-conversation-fixture.mjs";
 import { createDelegatedStepRequirement } from "../dist/features/channels/workflow-step-requirement.js";
@@ -99,31 +100,39 @@ test("Host-generated Node commands really invoke ESM/CJS exports and retain fail
   const task = { id: "node_goal_task", title: "Node goal fixture", prompt: "Execute only temporary fixture functions" };
   const f = createUnifiedExecFixture(t, cwd, task, "node_goal_run");
   let sequence = 0;
+  const diagnostics = [];
   const execute = async (modulePath, invocation) => {
     const command = nodeFunctionGoalCommand({ modulePath, invocation }); assert.ok(command);
-    const result = await executeUnifiedBashTool({ ...f,
-      request: { toolCallId: `node-goal-${++sequence}`, toolName: "bash", cwd, task, input: { command, timeoutMs: 4000 } },
-      appendAndMaybeStream: (type, data) => f.store.appendEvent(f.run.id, type, data) });
+    const result = await captureNodeGoalCommandDiagnostic({ sequence: ++sequence, timeoutMs: 4000,
+      record: diagnostic => diagnostics.push(diagnostic),
+      execute: () => executeUnifiedBashTool({ ...f,
+        request: { toolCallId: `node-goal-${sequence}`, toolName: "bash", cwd, task, input: { command, timeoutMs: 4000 } },
+        appendAndMaybeStream: (type, data) => f.store.appendEvent(f.run.id, type, data) }) });
     assert.equal(result.details.unifiedExec.command.command, command);
     assert.equal(result.details.unifiedExec.runId, f.run.id);
     return result.details.unifiedExec;
   };
-  await writeFile(join(cwd, "proof.mjs"), 'export const formatProof = async value => "proof:" + value;');
-  let result = await execute("proof.mjs", 'formatProof("7319")');
-  assert.equal(result.status, "succeeded"); assert.equal(result.stdout.preview, "proof:7319\n");
-  await writeFile(join(cwd, "proof.cjs"), 'module.exports = { formatProof: value => "common:" + value };');
-  result = await execute("proof.cjs", 'formatProof("7319")');
-  assert.equal(result.status, "succeeded"); assert.equal(result.stdout.preview, "common:7319\n");
-  await writeFile(join(cwd, "quoted '# %.mjs"), "export const echo = value => value;");
-  result = await execute("quoted '# %.mjs", `echo("'$(touch must-not-exist)")`);
-  assert.equal(result.status, "succeeded"); assert.equal(result.stdout.preview, "'$(touch must-not-exist)\n");
-  await assert.rejects(readFile(join(cwd, "must-not-exist")), { code: "ENOENT" });
-  result = await execute("proof.mjs", "missing()");
-  assert.equal(result.status, "failed"); assert.notEqual(result.exitCode, 0); assert.match(result.stderr.preview, /missing export/);
-  await writeFile(join(cwd, "throwing.mjs"), 'export const formatProof = async () => { throw Error("fixture rejection"); };');
-  result = await execute("throwing.mjs", 'formatProof("7319")');
-  assert.equal(result.status, "failed"); assert.notEqual(result.exitCode, 0); assert.match(result.stderr.preview, /fixture rejection/);
-  assert.deepEqual(f.store.listHostCommandWriteBlockers(cwd), []);
+  try {
+    await writeFile(join(cwd, "proof.mjs"), 'export const formatProof = async value => "proof:" + value;');
+    let result = await execute("proof.mjs", 'formatProof("7319")');
+    assert.equal(result.status, "succeeded"); assert.equal(result.stdout.preview, "proof:7319\n");
+    await writeFile(join(cwd, "proof.cjs"), 'module.exports = { formatProof: value => "common:" + value };');
+    result = await execute("proof.cjs", 'formatProof("7319")');
+    assert.equal(result.status, "succeeded"); assert.equal(result.stdout.preview, "common:7319\n");
+    await writeFile(join(cwd, "quoted '# %.mjs"), "export const echo = value => value;");
+    result = await execute("quoted '# %.mjs", `echo("'$(touch must-not-exist)")`);
+    assert.equal(result.status, "succeeded"); assert.equal(result.stdout.preview, "'$(touch must-not-exist)\n");
+    await assert.rejects(readFile(join(cwd, "must-not-exist")), { code: "ENOENT" });
+    result = await execute("proof.mjs", "missing()");
+    assert.equal(result.status, "failed"); assert.notEqual(result.exitCode, 0); assert.match(result.stderr.preview, /missing export/);
+    await writeFile(join(cwd, "throwing.mjs"), 'export const formatProof = async () => { throw Error("fixture rejection"); };');
+    result = await execute("throwing.mjs", 'formatProof("7319")');
+    assert.equal(result.status, "failed"); assert.notEqual(result.exitCode, 0); assert.match(result.stderr.preview, /fixture rejection/);
+    assert.deepEqual(f.store.listHostCommandWriteBlockers(cwd), []);
+  } catch (error) {
+    for (const diagnostic of diagnostics) t.diagnostic(JSON.stringify(diagnostic));
+    throw error;
+  }
 });
 
 test("recomputed plan fingerprints cannot detach generated commands from their accepted goal", async t => {

packages/daemon/test/node-goal-command-diagnostic.test.mjs

@@ -0,0 +1,105 @@
+import assert from "node:assert/strict";
+import { spawn } from "node:child_process";
+import { channel, hasSubscribers } from "node:diagnostics_channel";
+import { EventEmitter } from "node:events";
+import test from "node:test";
+import { captureNodeGoalCommandDiagnostic } from "./support/node-goal-command-diagnostic.mjs";
+
+test("command diagnostics retain failure evidence but exclude command, environment and output payloads", async () => {
+  const secret = "private-diagnostic-sentinel";
+  const result = { status: "timed_out", startedAt: "2026-11-02T00:00:00Z", endedAt: "2026-11-02T00:00:04Z",
+    durationMs: 4001, signal: "SIGTERM", timedOut: true, cancelled: false,
+    command: { command: secret }, cwd: secret, environmentSummary: { keys: [secret], extraEnv: { TOKEN: secret } },
+    processObservation: { scope: "process_group", leaderExited: true, stdioClosed: true,
+      descendants: "unverified", stopReason: "timeout", settledBy: "close", signalFailures: ["SIGTERM:group:EPERM"], private: secret },
+    stdout: { byteCount: 0, truncated: false, preview: secret, artifactId: secret },
+    stderr: { byteCount: 12, truncated: true, preview: secret }, artifacts: [{ path: secret }], reason: secret };
+  const handled = { details: { unifiedExec: result, private: secret }, content: [{ text: secret }] };
+  const records = [];
+  assert.equal(await captureNodeGoalCommandDiagnostic({ sequence: 1, timeoutMs: 4000,
+    execute: async () => handled, record: value => records.push(value) }), handled);
+  assert.equal(records.length, 1);
+  const diagnostic = records[0];
+  assert.equal(diagnostic.kind, "node_goal_command_diagnostic_v1");
+  assert.equal(diagnostic.timeoutMs, 4000);
+  assert.equal(diagnostic.result.status, "timed_out");
+  assert.equal(diagnostic.result.durationMs, 4001);
+  assert.equal(diagnostic.result.exitCode, null);
+  assert.equal(diagnostic.result.signal, "SIGTERM");
+  assert.deepEqual(diagnostic.result.processObservation, { scope: "process_group", leaderExited: true,
+    stdioClosed: true, descendants: "unverified", stopReason: "timeout", settledBy: "close", signalFailures: ["SIGTERM:group:EPERM"] });
+  assert.deepEqual(diagnostic.result.stdout, { byteCount: 0, truncated: false });
+  assert.deepEqual(diagnostic.result.stderr, { byteCount: 12, truncated: true });
+  assert.equal(JSON.stringify(diagnostic).includes(secret), false);
+});
+
+test("diagnostics distinguish no-start denial from a returned process result", async () => {
+  let diagnostic;
+  await captureNodeGoalCommandDiagnostic({ sequence: 2, timeoutMs: 4000,
+    execute: async () => ({ details: { unifiedExec: { status: "failed", denied: true, reason: "private denial" } } }),
+    record: value => { diagnostic = value; } });
+  assert.equal(diagnostic.returned, true);
+  assert.equal(diagnostic.result.denied, true);
+  assert.equal(diagnostic.result.processObservation, null);
+  assert.equal(diagnostic.result.durationMs, null);
+  assert.deepEqual(diagnostic.processes, []);
+  assert.equal(JSON.stringify(diagnostic).includes("private denial"), false);
+});
+
+test("real process lifecycle timings do not consume or alter output", async () => {
+  const subscribed = hasSubscribers("child_process");
+  let diagnostic;
+  let stdout = "", stderr = "";
+  await captureNodeGoalCommandDiagnostic({ sequence: 3, timeoutMs: 4000,
+    execute: () => new Promise((resolve, reject) => {
+      const child = spawn(process.execPath, ["-e", 'process.stdout.write("fixture-out");process.stderr.write("fixture-err");']);
+      child.stdout.on("data", chunk => { stdout += chunk; });
+      child.stderr.on("data", chunk => { stderr += chunk; });
+      child.once("error", reject);
+      child.once("close", code => {
+        if (code === 0) resolve({});
+        else reject(new Error(`fixture exited with code ${code}`));
+      });
+    }), record: value => { diagnostic = value; } });
+  assert.equal(hasSubscribers("child_process"), subscribed);
+  assert.equal(stdout, "fixture-out");
+  assert.equal(stderr, "fixture-err");
+  assert.equal(diagnostic.processes.length, 1);
+  const timing = diagnostic.processes[0];
+  for (const key of ["createdMs", "spawnedMs", "firstStdoutMs", "firstStderrMs", "exitMs", "closeMs"]) {
+    assert.equal(typeof timing[key], "number", key);
+    assert.ok(timing[key] >= 0, key);
+  }
+  assert.ok(timing.createdMs <= timing.spawnedMs);
+  assert.ok(timing.spawnedMs <= timing.firstStdoutMs);
+  assert.ok(timing.spawnedMs <= timing.firstStderrMs);
+  assert.ok(timing.exitMs <= timing.closeMs);
+  assert.ok(timing.closeMs <= diagnostic.elapsedMs);
+  assert.equal(timing.exitCode, 0);
+  assert.equal(timing.signal, null);
+  assert.equal(JSON.stringify(diagnostic).includes("fixture-out"), false);
+  assert.equal(JSON.stringify(diagnostic).includes("fixture-err"), false);
+});
+
+test("thrown execution keeps its original error and detaches every diagnostic listener", async () => {
+  const subscribed = hasSubscribers("child_process");
+  const child = Object.assign(new EventEmitter(), { stdout: new EventEmitter(), stderr: new EventEmitter() });
+  const error = new Error("original private error");
+  let diagnostic;
+  await assert.rejects(captureNodeGoalCommandDiagnostic({ sequence: 4, timeoutMs: 4000,
+    execute: async () => {
+      channel("child_process").publish({ process: child });
+      child.emit("spawn");
+      child.emit("error", { code: "ENOENT", message: "private spawn error" });
+      throw error;
+    }, record: value => { diagnostic = value; } }), cause => cause === error);
+  assert.equal(hasSubscribers("child_process"), subscribed);
+  for (const emitter of [child, child.stdout, child.stderr]) assert.deepEqual(emitter.eventNames(), []);
+  assert.equal(diagnostic.returned, false);
+  assert.equal(diagnostic.result.status, null);
+  assert.equal(diagnostic.processes[0].errorCode, "ENOENT");
+  assert.equal(diagnostic.processes[0].firstStdoutMs, null);
+  assert.equal(diagnostic.processes[0].exitMs, null);
+  assert.equal(diagnostic.processes[0].closeMs, null);
+  assert.equal(JSON.stringify(diagnostic).includes("private"), false);
+});